From 14a9778e1f492d35461c7e04e0f24d695ddd07d7 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:06:12 +0800 Subject: [PATCH 01/10] feat(storage): add semantic transcript position snapshots Generated-by: OpenAI Codex --- .../licenses/npm/THIRD_PARTY_NOTICES.txt | 748 +++++++ package-lock.json | 9 +- packages/cli/THIRD_PARTY_NOTICES.txt | 748 +++++++ packages/storage/package.json | 3 +- .../session-turn-identity-scanner.test.ts | 261 +++ .../session-turn-position-index.test.ts | 1985 +++++++++++++++++ packages/storage/src/agent-run-store.ts | 7 + .../src/conversation-operational-state.ts | 2 + packages/storage/src/execution-stores.ts | 19 + .../storage/src/operational-state-store.ts | 9 + packages/storage/src/session-store.ts | 161 ++ .../src/session-turn-identity-recovery.ts | 497 +++++ .../src/session-turn-identity-scanner.ts | 684 ++++++ .../src/session-turn-position-authority.ts | 257 +++ .../src/session-turn-position-index.ts | 47 + .../src/session-turn-position-snapshots.ts | 785 +++++++ .../src/sqlite-session-metadata-schema.ts | 189 +- .../src/sqlite-session-metadata-store.ts | 453 ++-- .../src/sqlite-session-transcript-slices.ts | 175 ++ scripts/generate-third-party-notices.mjs | 149 +- 20 files changed, 6998 insertions(+), 190 deletions(-) create mode 100644 packages/storage/src/__tests__/session-turn-identity-scanner.test.ts create mode 100644 packages/storage/src/__tests__/session-turn-position-index.test.ts create mode 100644 packages/storage/src/session-turn-identity-recovery.ts create mode 100644 packages/storage/src/session-turn-identity-scanner.ts create mode 100644 packages/storage/src/session-turn-position-authority.ts create mode 100644 packages/storage/src/session-turn-position-index.ts create mode 100644 packages/storage/src/session-turn-position-snapshots.ts create mode 100644 packages/storage/src/sqlite-session-transcript-slices.ts diff --git a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt index 4695c0eeb4..0021d96683 100644 --- a/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt +++ b/apps/desktop/resources/licenses/npm/THIRD_PARTY_NOTICES.txt @@ -9864,6 +9864,53 @@ SOFTWARE. ================================================================================ +Package: hash-wasm@4.12.0 +Declared license: MIT +Selected license: MIT +Repository: git+https://github.com/Daninet/hash-wasm.git + +--- LICENSE --- +MIT License + +Copyright (c) 2020 Dani Biró + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Embedded C implementations might use other, similarly permissive licenses. +Check the beginning of the files from the /src directory. + +Special thank you to the authors of original C algorithms: +- Alexander Peslyak +- Aleksey Kravchenko +- Colin Percival +- Stephan Brumme +- Steve Reid +- Samuel Neves +- Solar Designer +- Project Nayuki +- ARM Limited +- Yanbo Li dreamfly281@gmail.com, goldboar@163.comYanbo Li +- Mark Adler +- Yann Collet + +================================================================================ + Package: hasown@2.0.4 Declared license: MIT Selected license: MIT @@ -14581,3 +14628,704 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + +Embedded component: Go crypto Argon2 implementation basis +Embedded by: hash-wasm@4.12.0 +Selected license: BSD-3-Clause +Repository: https://go.googlesource.com/crypto + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + +Embedded component: Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations +Embedded by: hash-wasm@4.12.0 +Selected license: Zlib-like +Repository: https://create.stephan-brumme.com + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +================================================================================ + +Embedded component: Yanbo Li SM3 implementation +Embedded by: hash-wasm@4.12.0 +Selected license: MIT +Repository: https://github.com/Daninet/hash-wasm + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +MIT License + +Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ + +Embedded source notice: adler32.c +Embedded by: hash-wasm@4.12.0 +Source: src/adler32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + adler32.c -- compute the Adler-32 checksum of a data stream + Copyright (C) 1995-2011, 2016 Mark Adler + + Licensed under the zlib license: + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + Modified for hash-wasm by Nicholas Sherlock and Dani Biro, 2021 +*/ + +================================================================================ + +Embedded source notice: argon2.c +Embedded by: hash-wasm@4.12.0 +Source: src/argon2.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + Based on Golang's Argon2 implementation from crypto package + + Written for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: bcrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/bcrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * The crypt_blowfish homepage is: + * + * http://www.openwall.com/crypt/ + * + * This code comes from John the Ripper password cracker, with reentrant + * and crypt(3) interfaces added, but optimizations specific to password + * cracking removed. + * + * Written by Solar Designer in 1998-2014. + * No copyright is claimed, and the software is hereby placed in the public + * domain. In case this attempt to disclaim copyright and place the software + * in the public domain is deemed null and void, then the software is + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * It is my intent that you should be able to use this on your system, + * as part of a software package, or anywhere else to improve security, + * ensure compatibility, or for any other purpose. I would appreciate + * it if you give credit where it is due and keep your modifications in + * the public domain as well, but I don't require that in order to let + * you place this code and any modifications you make under a license + * of your choice. + * + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos , and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres . For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. + * + * There's a paper on the algorithm that explains its design decisions: + * + * http://www.usenix.org/events/usenix99/provos.html + * + * Some of the tricks in BF_ROUND might be inspired by Eric Young's + * Blowfish library (I can't be sure if I would think of something if I + * hadn't seen his code). + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: blake2b.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2b.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake2s.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2s.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake3.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE3 - reference C implementation + https://github.com/BLAKE3-team/BLAKE3 + This work is released into the public domain with CC0 1.0. Alternatively, it + is licensed under the Apache License 2.0. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: crc32.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// Crc32.cpp +// Copyright (c) 2011-2019 Stephan Brumme. All rights reserved. +// Slicing-by-16 contributed by Bulat Ziganshin +// Tableless bytewise CRC contributed by Hagai Gold +// see http://create.stephan-brumme.com/disclaimer.html +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: crc64.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// Based on crc32.c implementation of Stephan Brumme +// Modified for hash-wasm by Dani Biró + +================================================================================ + +Embedded source notice: md4.c +Embedded by: hash-wasm@4.12.0 +Source: src/md4.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD4 Message-Digest Algorithm (RFC 1320). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: md5.c +Embedded by: hash-wasm@4.12.0 +Source: src/md5.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD5 Message-Digest Algorithm (RFC 1321). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: ripemd160.c +Embedded by: hash-wasm@4.12.0 +Source: src/ripemd160.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * RIPE MD-160 implementation + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: scrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/scrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * Copyright 2009 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha1.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha1.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* +SHA-1 in C +By Steve Reid +100% Public Domain + +Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: sha256.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha256.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha256.c - an implementation of SHA-256/224 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha3.c - an implementation of Secure Hash Algorithm 3 (Keccak). + * based on the + * The Keccak SHA-3 submission. Submission to NIST (Round 3), 2011 + * by Guido Bertoni, Joan Daemen, Michaël Peeters and Gilles Van Assche + * + * Copyright (c) 2013, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha512.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha512.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha512.c - an implementation of SHA-384/512 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sm3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sm3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/******************************************************************************* + * SM3 function implementation + * Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + * MIT License + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: whirlpool.c +Embedded by: hash-wasm@4.12.0 +Source: src/whirlpool.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/** + * Whirlpool hash in C + * + * Copyright (c) 2017 Project Nayuki. (MIT License) + * https://www.nayuki.io/page/fast-whirlpool-hash-in-x86-assembly + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash128.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash128.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash3.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash32.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash32.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (32 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: xxhash64.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash64.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (64 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// diff --git a/package-lock.json b/package-lock.json index c0c1f26999..c5a7f8a2aa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8724,6 +8724,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hash-wasm": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/hash-wasm/-/hash-wasm-4.12.0.tgz", + "integrity": "sha512-+/2B2rYLb48I/evdOIhP+K/DD2ca2fgBjp6O+GBEnCDk2e4rpeXIK8GvIyRPjTezgmWn9gmKwkQjjx6BtqDHVQ==", + "license": "MIT" + }, "node_modules/hasown": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", @@ -14480,7 +14486,8 @@ "license": "Apache-2.0", "dependencies": { "@maka/core": "0.1.0", - "fs-native-extensions": "^1.5.0" + "fs-native-extensions": "^1.5.0", + "hash-wasm": "4.12.0" } }, "packages/ui": { diff --git a/packages/cli/THIRD_PARTY_NOTICES.txt b/packages/cli/THIRD_PARTY_NOTICES.txt index d1b9143775..5e26ecf32b 100644 --- a/packages/cli/THIRD_PARTY_NOTICES.txt +++ b/packages/cli/THIRD_PARTY_NOTICES.txt @@ -4587,6 +4587,53 @@ SOFTWARE. ================================================================================ +Package: hash-wasm@4.12.0 +Declared license: MIT +Selected license: MIT +Repository: git+https://github.com/Daninet/hash-wasm.git + +--- LICENSE --- +MIT License + +Copyright (c) 2020 Dani Biró + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Embedded C implementations might use other, similarly permissive licenses. +Check the beginning of the files from the /src directory. + +Special thank you to the authors of original C algorithms: +- Alexander Peslyak +- Aleksey Kravchenko +- Colin Percival +- Stephan Brumme +- Steve Reid +- Samuel Neves +- Solar Designer +- Project Nayuki +- ARM Limited +- Yanbo Li dreamfly281@gmail.com, goldboar@163.comYanbo Li +- Mark Adler +- Yann Collet + +================================================================================ + Package: hasown@2.0.4 Declared license: MIT Selected license: MIT @@ -7707,3 +7754,704 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================================ + +Embedded component: Go crypto Argon2 implementation basis +Embedded by: hash-wasm@4.12.0 +Selected license: BSD-3-Clause +Repository: https://go.googlesource.com/crypto + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + +Embedded component: Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations +Embedded by: hash-wasm@4.12.0 +Selected license: Zlib-like +Repository: https://create.stephan-brumme.com + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +================================================================================ + +Embedded component: Yanbo Li SM3 implementation +Embedded by: hash-wasm@4.12.0 +Selected license: MIT +Repository: https://github.com/Daninet/hash-wasm + +--- VERSION-PINNED EMBEDDED LICENSE TEXT --- +MIT License + +Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================================ + +Embedded source notice: adler32.c +Embedded by: hash-wasm@4.12.0 +Source: src/adler32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + adler32.c -- compute the Adler-32 checksum of a data stream + Copyright (C) 1995-2011, 2016 Mark Adler + + Licensed under the zlib license: + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + Modified for hash-wasm by Nicholas Sherlock and Dani Biro, 2021 +*/ + +================================================================================ + +Embedded source notice: argon2.c +Embedded by: hash-wasm@4.12.0 +Source: src/argon2.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + Based on Golang's Argon2 implementation from crypto package + + Written for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: bcrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/bcrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * The crypt_blowfish homepage is: + * + * http://www.openwall.com/crypt/ + * + * This code comes from John the Ripper password cracker, with reentrant + * and crypt(3) interfaces added, but optimizations specific to password + * cracking removed. + * + * Written by Solar Designer in 1998-2014. + * No copyright is claimed, and the software is hereby placed in the public + * domain. In case this attempt to disclaim copyright and place the software + * in the public domain is deemed null and void, then the software is + * Copyright (c) 1998-2014 Solar Designer and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * It is my intent that you should be able to use this on your system, + * as part of a software package, or anywhere else to improve security, + * ensure compatibility, or for any other purpose. I would appreciate + * it if you give credit where it is due and keep your modifications in + * the public domain as well, but I don't require that in order to let + * you place this code and any modifications you make under a license + * of your choice. + * + * This implementation is fully compatible with OpenBSD's bcrypt.c for prefix + * "$2b$", originally by Niels Provos , and it uses + * some of his ideas. The password hashing algorithm was designed by David + * Mazieres . For information on the level of + * compatibility for bcrypt hash prefixes other than "$2b$", please refer to + * the comments in BF_set_key() below and to the included crypt(3) man page. + * + * There's a paper on the algorithm that explains its design decisions: + * + * http://www.usenix.org/events/usenix99/provos.html + * + * Some of the tricks in BF_ROUND might be inspired by Eric Young's + * Blowfish library (I can't be sure if I would think of something if I + * hadn't seen his code). + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: blake2b.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2b.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake2s.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake2s.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE2 reference source code package - reference C implementations + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: blake3.c +Embedded by: hash-wasm@4.12.0 +Source: src/blake3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + BLAKE3 - reference C implementation + https://github.com/BLAKE3-team/BLAKE3 + This work is released into the public domain with CC0 1.0. Alternatively, it + is licensed under the Apache License 2.0. + + Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: crc32.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// Crc32.cpp +// Copyright (c) 2011-2019 Stephan Brumme. All rights reserved. +// Slicing-by-16 contributed by Bulat Ziganshin +// Tableless bytewise CRC contributed by Hagai Gold +// see http://create.stephan-brumme.com/disclaimer.html +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: crc64.c +Embedded by: hash-wasm@4.12.0 +Source: src/crc64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// Based on crc32.c implementation of Stephan Brumme +// Modified for hash-wasm by Dani Biró + +================================================================================ + +Embedded source notice: md4.c +Embedded by: hash-wasm@4.12.0 +Source: src/md4.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD4 Message-Digest Algorithm (RFC 1320). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md4 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: md5.c +Embedded by: hash-wasm@4.12.0 +Source: src/md5.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * This is an OpenSSL-compatible implementation of the RSA Data Security, Inc. + * MD5 Message-Digest Algorithm (RFC 1321). + * + * Homepage: + * http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5 + * + * Author: + * Alexander Peslyak, better known as Solar Designer + * + * This software was written by Alexander Peslyak in 2001. No copyright is + * claimed, and the software is hereby placed in the public domain. + * In case this attempt to disclaim copyright and place the software in the + * public domain is deemed null and void, then the software is + * Copyright (c) 2001 Alexander Peslyak and it is hereby released to the + * general public under the following terms: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted. + * + * There's ABSOLUTELY NO WARRANTY, express or implied. + * + * (This is a heavily cut-down "BSD license".) + * + * This differs from Colin Plumb's older public domain implementation in that + * no exactly 32-bit integer data type is required (any 32-bit or wider + * unsigned integer data type will do), there's no compile-time endianness + * configuration, and the function prototypes match OpenSSL's. No code from + * Colin Plumb's implementation has been reused; this comment merely compares + * the properties of the two independent implementations. + * + * The primary goals of this implementation are portability and ease of use. + * It is meant to be fast, but not as fast as possible. Some known + * optimizations are not included to reduce source code size and avoid + * compile-time configuration. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: ripemd160.c +Embedded by: hash-wasm@4.12.0 +Source: src/ripemd160.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * RIPE MD-160 implementation + * + * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: scrypt.c +Embedded by: hash-wasm@4.12.0 +Source: src/scrypt.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * Copyright 2009 Colin Percival + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file was originally written by Colin Percival as part of the Tarsnap + * online backup system. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha1.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha1.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* +SHA-1 in C +By Steve Reid +100% Public Domain + +Modified for hash-wasm by Dani Biró +*/ + +================================================================================ + +Embedded source notice: sha256.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha256.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha256.c - an implementation of SHA-256/224 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha3.c - an implementation of Secure Hash Algorithm 3 (Keccak). + * based on the + * The Keccak SHA-3 submission. Submission to NIST (Round 3), 2011 + * by Guido Bertoni, Joan Daemen, Michaël Peeters and Gilles Van Assche + * + * Copyright (c) 2013, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sha512.c +Embedded by: hash-wasm@4.12.0 +Source: src/sha512.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* sha512.c - an implementation of SHA-384/512 hash functions + * based on FIPS 180-3 (Federal Information Processing Standart). + * + * Copyright (c) 2010, Aleksey Kravchenko + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + * AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: sm3.c +Embedded by: hash-wasm@4.12.0 +Source: src/sm3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/******************************************************************************* + * SM3 function implementation + * Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com + * MIT License + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: whirlpool.c +Embedded by: hash-wasm@4.12.0 +Source: src/whirlpool.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/** + * Whirlpool hash in C + * + * Copyright (c) 2017 Project Nayuki. (MIT License) + * https://www.nayuki.io/page/fast-whirlpool-hash-in-x86-assembly + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash128.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash128.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash3.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash3.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + * + * Modified for hash-wasm by Dani Biró + */ + +================================================================================ + +Embedded source notice: xxhash32.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash32.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash32.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (32 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// + +================================================================================ + +Embedded source notice: xxhash64.c +Embedded by: hash-wasm@4.12.0 +Source: src/xxhash64.c + +--- VERSION-PINNED UPSTREAM SOURCE NOTICE --- +// ////////////////////////////////////////////////////////// +// xxhash64.h +// Copyright (c) 2016 Stephan Brumme. All rights reserved. +// see http://create.stephan-brumme.com/disclaimer.html +// +// XXHash (64 bit), based on Yann Collet's descriptions, see +// http://cyan4973.github.io/xxHash/ +// +// Modified for hash-wasm by Dani Biró +// diff --git a/packages/storage/package.json b/packages/storage/package.json index cbcbaddc18..7323ed5446 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -68,6 +68,7 @@ }, "dependencies": { "@maka/core": "0.1.0", - "fs-native-extensions": "^1.5.0" + "fs-native-extensions": "^1.5.0", + "hash-wasm": "4.12.0" } } diff --git a/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts new file mode 100644 index 0000000000..de9881c89a --- /dev/null +++ b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { decodeStoredMessage } from '@maka/core/session'; +import { + advanceSessionTurnIdentityScanner, + completeSessionTurnIdentityScanner, + createSessionTurnIdentityScannerState, + restoreSessionTurnIdentityScannerState, + serializeSessionTurnIdentityScannerState, + SessionTurnIdentityScannerError, +} from '../session-turn-identity-scanner.js'; + +function scan( + json: string, + splits: readonly number[] = [], + sql = { messageId: 'message-1', messageType: 'user' }, +) { + let state = createSessionTurnIdentityScannerState(); + const bytes = Buffer.from(json); + let offset = 0; + for (const end of [...splits, bytes.length]) { + advanceSessionTurnIdentityScanner(state, bytes.subarray(offset, end)); + state = restoreSessionTurnIdentityScannerState(serializeSessionTurnIdentityScannerState(state)); + offset = end; + } + return completeSessionTurnIdentityScanner(state, sql); +} + +test('extracts only the top-level identity envelope across arbitrary fragment boundaries', () => { + const json = JSON.stringify({ + text: '正文 😀 \\ escaped', + nested: { id: 'fake-id', type: 'system_note', turnId: 'fake-turn', kind: 'step_limit' }, + turnId: 'turn-一', + type: 'user', + id: 'message-1', + ts: 1, + }); + const expected = { kind: 'turn', turnId: 'turn-一' }; + assert.deepEqual(scan(json), expected); + for (let split = 1; split < Buffer.byteLength(json); split += 1) { + assert.deepEqual(scan(json, [split]), expected); + } +}); + +test('handles escaped identity values and visible/invisible turnless notes', () => { + assert.deepEqual( + scan( + '{"kind":"step_limit","id":"note\\u002d1","type":"system_note","text":"x"}', + [2, 7, 19, 31, 47], + { messageId: 'note-1', messageType: 'system_note' }, + ), + { kind: 'note', turnId: 'session-note:note-1' }, + ); + assert.deepEqual( + scan('{"id":"hidden","type":"system_note","kind":"mode_change"}', [], { + messageId: 'hidden', + messageType: 'system_note', + }), + { kind: 'ignored' }, + ); +}); + +test('rejects duplicate, empty, missing, nested, mismatched, and non-string identity', () => { + const invalid = [ + '{"id":"message-1","id":"message-1","type":"user","turnId":"turn"}', + '{"id":"message-1","type":"user","turnId":""}', + '{"id":"message-1","type":"user"}', + '{"id":"message-1","type":"user","nested":{"turnId":"turn"}}', + '{"id":"message-1","type":"user","turnId":1}', + '{"id":"message-1","type":"user","turnId":{"value":"turn"}}', + ]; + for (const json of invalid) { + assert.throws(() => scan(json), SessionTurnIdentityScannerError); + } + assert.throws(() => scan('{"id":"other","type":"user","turnId":"turn"}'), /mismatch/u); +}); + +test('rejects invalid UTF-8, truncated JSON, excessive nesting, and corrupt persisted state', () => { + const state = createSessionTurnIdentityScannerState(); + assert.throws( + () => advanceSessionTurnIdentityScanner(state, Uint8Array.from([0x7b, 0x22, 0xc0])), + /UTF-8/u, + ); + assert.throws(() => scan('{"id":"message-1","type":"user","turnId":"turn"'), /truncated/u); + assert.throws(() => scan('{"id":"message-1","type":"user","turnId":"turn",}'), /JSON/u); + assert.throws( + () => scan('{"id":"message-1","type":"user","turnId":"turn","body":[1,]}'), + /JSON/u, + ); + assert.throws(() => scan('{"id":"message-1",\u00a0"type":"user","turnId":"turn"}'), /JSON/u); + assert.throws( + () => + scan( + `{"id":"message-1","type":"user","turnId":"turn","x":${'['.repeat(4_097)}0${']'.repeat(4_097)}}`, + ), + /nesting/u, + ); + assert.throws(() => restoreSessionTurnIdentityScannerState('{"version":999}'), /unsupported/u); + assert.throws( + () => + restoreSessionTurnIdentityScannerState( + JSON.stringify({ + version: 1, + rootStarted: true, + rootComplete: false, + stack: [{ kind: 'object', expectation: 'invented', pendingKey: null }], + lexical: { kind: 'invented' }, + utf8: { needed: 0, codePoint: 0, minimum: 0 }, + seenIdentityKeys: [], + identity: { turnIdPresent: false, kindPresent: false }, + capturedBytes: 0, + }), + ), + /unsupported/u, + ); +}); + +test('rejects captured identity larger than 32 KiB without retaining body fields', () => { + assert.throws( + () => + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'x'.repeat(32 * 1024 + 1), + text: 'body'.repeat(100_000), + }), + ), + /32 KiB/u, + ); + assert.deepEqual( + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'turn', + text: 'body'.repeat(100_000), + }), + [4 * 1024], + ), + { kind: 'turn', turnId: 'turn' }, + ); + assert.deepEqual( + scan( + JSON.stringify({ + id: 'message-1', + type: 'user', + turnId: 'turn', + ['x'.repeat(100 * 1024)]: 'ignored body field', + }), + [4 * 1024], + ), + { kind: 'turn', turnId: 'turn' }, + ); +}); + +test('differentially extracts every canonical StoredMessage identity under key reordering', () => { + const messages = [ + { type: 'user', id: 'message-1', turnId: 'turn-1', ts: 1, text: 'user' }, + { + type: 'assistant', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + text: 'assistant', + modelId: 'model', + }, + { + type: 'tool_call', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolName: 'tool', + args: { turnId: 'nested-fake', kind: 'step_limit' }, + }, + { + type: 'tool_result', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolUseId: 'call-1', + isError: false, + content: { kind: 'text', text: 'result' }, + }, + { + type: 'permission_decision', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + toolUseId: 'call-1', + toolName: 'tool', + decision: 'allow', + }, + { type: 'token_usage', id: 'message-1', turnId: 'turn-1', ts: 1, input: 1, output: 2 }, + { + type: 'turn_state', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + status: 'completed', + partialOutputRetained: false, + }, + { + type: 'workhub_coordination', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + schemaVersion: 1, + kind: 'delegation_assigned', + actionId: 'action-1', + actionFingerprint: `sha256:${'0'.repeat(64)}`, + coordinationTurnId: 'turn-1', + targetSessionId: 'target-session', + disposition: 'delegate_existing', + userText: 'work', + delegationId: 'delegation-1', + targetTurnId: 'target-turn', + targetMessageId: 'target-message', + targetSessionName: 'target', + }, + { + type: 'system_note', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + kind: 'error', + }, + ]; + for (const message of messages) { + const reordered = Object.fromEntries(Object.entries(message).reverse()); + const json = JSON.stringify(reordered); + const decoded = decodeStoredMessage(JSON.parse(json) as never); + assert.equal(decoded.id, 'message-1'); + assert.deepEqual( + scan(json, [1, 2, 7, Math.floor(Buffer.byteLength(json) / 2)], { + messageId: 'message-1', + messageType: message.type, + }), + { kind: 'turn', turnId: 'turn-1' }, + ); + } +}); diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts new file mode 100644 index 0000000000..7a4a5a1266 --- /dev/null +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -0,0 +1,1985 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { describe, test } from 'node:test'; +import { createSHA256 } from 'hash-wasm'; +import type { CreateSessionInput } from '@maka/core/runtime-inputs'; +import { createSqliteAgentRunStore } from '../agent-run-store.js'; +import { createConversationOperationalStateStore } from '../conversation-operational-state.js'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '../operational-state-store.js'; +import { createSessionStore } from '../session-store.js'; +import { advanceSessionTurnIdentityRecovery } from '../session-turn-identity-recovery.js'; + +describe('Session Turn position snapshots', () => { + test('publishes one stable synthetic position for an empty Session', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-empty-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const page = await readyPage(store, session.id); + + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected an atomically ready page'); + assert.deepEqual(page.snapshotKey, { + throughSequence: null, + authorityRevision: 0, + snapshotGeneration: 1, + }); + assert.equal(page.startOrdinal, 0); + assert.equal(page.totalTurns, 1); + assert.deepEqual(page.positions, [ + { + ordinal: 0, + turnId: `session:${session.id}`, + firstSequence: null, + }, + ]); + assert.equal(page.hasOlder, false); + assert.equal(page.hasNewer, false); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses inclusive predecessor semantics for legacy ordinal, sequence, and turn anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-legacy-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-a', turnId: 'turn-a', ts: 1, text: 'a' }, + { + type: 'assistant', + id: 'assistant-a', + turnId: 'turn-a', + ts: 2, + text: 'A', + modelId: 'test-model', + }, + { type: 'user', id: 'user-b', turnId: 'turn-b', ts: 3, text: 'b' }, + { + type: 'assistant', + id: 'assistant-b', + turnId: 'turn-b', + ts: 4, + text: 'B', + modelId: 'test-model', + }, + { type: 'system_note', id: 'note', ts: 5, kind: 'step_limit' }, + ]); + + const ready = await readyPage(store, session.id); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 1 }, + maxPositions: 2, + }); + assert.equal(first.kind, 'page'); + if (first.kind !== 'page') assert.fail('expected a ready legacy page'); + assert.deepEqual(first.positions, [ + { ordinal: 1, turnId: 'turn-b', firstSequence: 2 }, + { ordinal: 2, turnId: 'session-note:note', firstSequence: 4 }, + ]); + + for (const anchor of [ + { kind: 'sequence' as const, sequence: 3 }, + { kind: 'turn' as const, turnId: 'turn-b' }, + ]) { + const anchored = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: first.snapshotKey, + anchor, + maxPositions: 2, + }); + assert.equal(anchored.kind, 'page'); + if (anchored.kind !== 'page') assert.fail('expected an exact snapshot page'); + assert.equal(anchored.startOrdinal, 1); + assert.deepEqual(anchored.positions, first.positions); + } + const clamped = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 999 }, + maxPositions: 2, + }); + assert.equal(clamped.kind, 'page'); + if (clamped.kind !== 'page') assert.fail('expected a clamped inclusive ordinal'); + assert.deepEqual(clamped.positions, [ + { ordinal: 2, turnId: 'session-note:note', firstSequence: 4 }, + ]); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'turn', turnId: 'missing-turn' }, + maxPositions: 2, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_anchor_not_found', + ); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'turn', turnId: '' }, + maxPositions: 2, + }), + /invalid/iu, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('orders modern Turns only by root admission and keeps steering in its existing position', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-modern-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-c', 'user-c', 5)); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-b', turnId: 'turn-b', ts: 1, text: 'b' }, + { type: 'user', id: 'user-a', turnId: 'turn-a', ts: 2, text: 'a' }, + { type: 'user', id: 'user-c', turnId: 'turn-c', ts: 3, text: 'c' }, + { + type: 'user', + id: 'steering-b', + turnId: 'turn-b', + ts: 4, + text: 'steer', + steeringEventId: 'steering-b', + }, + ]); + + const page = await readyPage(store, session.id); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected modern positions'); + assert.deepEqual(page.positions, [ + { ordinal: 0, turnId: 'turn-c', firstSequence: 2 }, + { ordinal: 1, turnId: 'turn-a', firstSequence: 1 }, + { ordinal: 2, turnId: 'turn-b', firstSequence: 0 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('represents admission-before-body and applies the closed turnless-note visibility policy', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-before-body-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-bodyless', 'future-user', 10)); + const bodyless = await readyPage(store, session.id, 'lease-bodyless'); + assert.deepEqual(bodyless.positions, [ + { ordinal: 0, turnId: 'turn-bodyless', firstSequence: null }, + ]); + + await store.appendMessages(session.id, [ + { type: 'system_note', id: 'hidden-note', ts: 1, kind: 'mode_change' }, + { + type: 'system_note', + id: 'associated-note', + turnId: 'turn-bodyless', + ts: 2, + kind: 'error', + }, + { type: 'user', id: 'future-user', turnId: 'turn-bodyless', ts: 3, text: 'body' }, + { type: 'system_note', id: 'visible-note', ts: 4, kind: 'step_limit' }, + ]); + const materialized = await readyPage(store, session.id, 'lease-materialized'); + assert.deepEqual(materialized.positions, [ + { ordinal: 0, turnId: 'turn-bodyless', firstSequence: 1 }, + { ordinal: 1, turnId: 'session-note:visible-note', firstSequence: 3 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed on an explicitly empty Turn identity before committing the append', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-empty-turn-identity-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await assert.rejects( + store.appendMessage(session.id, { + type: 'user', + id: 'empty-turn', + turnId: '', + ts: 1, + text: 'invalid', + }), + (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'incompatible_identity'); + return true; + }, + ); + assert.equal(await store.readTranscriptHighWaterSnapshot(session.id), null); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed when a generated note identity collides with a real Turn identity', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-note-turn-identity-collision-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + type: 'system_note', + id: 'collision', + ts: 1, + kind: 'step_limit', + }); + await assert.rejects( + store.appendMessage(session.id, { + type: 'user', + id: 'colliding-turn', + turnId: 'session-note:collision', + ts: 2, + text: 'must not fold into the note', + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'incompatible_identity', + ); + assert.equal(await store.readTranscriptHighWaterSnapshot(session.id), 0); + + await store.appendMessage(session.id, user('turn-other', 1)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = ? + WHERE session_id = ? AND sequence = 1`) + .run( + JSON.stringify({ + ...user('session-note:collision', 1), + id: 'user-1', + }), + session.id, + ); + } finally { + database.close(); + } + store = createSessionStore(root); + await assert.rejects(readyPage(store, session.id, 'lease-collision'), (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'incompatible_identity'); + assert.equal((error as { sequence?: unknown }).sequence, 1); + return true; + }); + + const reverse = await store.create(makeInput({ name: 'Reverse collision' })); + await store.appendMessage(reverse.id, user('session-note:reverse', 2)); + await assert.rejects( + store.appendMessage(reverse.id, { + type: 'system_note', + id: 'reverse', + ts: 3, + kind: 'step_limit', + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'incompatible_identity', + ); + assert.equal(await store.readTranscriptHighWaterSnapshot(reverse.id), 0); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('purging admission authority removes bodyless placeholders and downgrades body Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-bodyless', 'future-user', 5)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-body', 'body-user', 10)); + await store.appendMessage(session.id, { + type: 'user', + id: 'body-user', + turnId: 'turn-body', + ts: 1, + text: 'body', + }); + const admitted = await readyPage(store, session.id, 'lease-admitted'); + assert.deepEqual( + admitted.positions.map(({ turnId }) => turnId), + ['turn-bodyless', 'turn-body'], + ); + + await conversation.purge(session.id); + const legacy = await readyPage(store, session.id, 'lease-purged'); + assert.equal( + legacy.snapshotKey.authorityRevision, + admitted.snapshotKey.authorityRevision + 1, + ); + assert.deepEqual(legacy.positions, [{ ordinal: 0, turnId: 'turn-body', firstSequence: 0 }]); + await store.remove(session.id); + await conversation.purge(session.id); + } finally { + conversation.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('shares one exact generation across leases and releases only after the last consumer', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-lease-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const first = await readyPage(store, session.id, 'lease-one'); + const secondFacade = createSessionStore(root); + const shared = await secondFacade.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-two', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(shared.kind, 'page'); + if (shared.kind !== 'page') assert.fail('expected shared ready snapshot'); + assert.deepEqual(shared.snapshotKey, first.snapshotKey); + await secondFacade.close?.(); + + await store.releaseTurnPositionSnapshot(session.id, 'lease-one', first.snapshotKey); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + await store.releaseTurnPositionSnapshot(session.id, 'lease-two', first.snapshotKey); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const rebuilt = await readyPage(store, session.id, 'lease-three'); + assert.ok(rebuilt.snapshotKey.snapshotGeneration > first.snapshotKey.snapshotGeneration); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps a legacy prefix by sequence before admission-ordered modern Turns', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-prefix-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 20)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 10)); + await store.appendMessages(session.id, [ + user('turn-legacy', 0), + user('turn-b', 1), + user('turn-a', 2), + { type: 'system_note', id: 'modern-note', ts: 4, kind: 'step_limit' }, + ]); + + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, turnId: 'turn-legacy', firstSequence: 0 }, + { ordinal: 1, turnId: 'turn-a', firstSequence: 2 }, + { ordinal: 2, turnId: 'turn-b', firstSequence: 1 }, + { ordinal: 3, turnId: 'session-note:modern-note', firstSequence: 3 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed after bounded recovery finds a hybrid Turn without admission', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-turn-position-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-modern', 'user-modern', 10)); + await store.appendMessages(session.id, [ + { type: 'user', id: 'user-modern', turnId: 'turn-modern', ts: 1, text: 'modern' }, + { type: 'user', id: 'user-missing', turnId: 'turn-missing', ts: 2, text: 'missing' }, + ]); + + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'hybrid_missing_admission'); + return true; + }); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('increments authority at a fixed watermark while preserving the old exact snapshot', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-authority-revision-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const legacy = await readyPage(store, session.id); + assert.deepEqual( + legacy.positions.map(({ turnId }) => turnId), + ['turn-a', 'turn-b'], + ); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-1', 10)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-0', 20)); + + const modern = await readyPage(store, session.id, 'lease-modern'); + assert.deepEqual(modern.snapshotKey, { + throughSequence: legacy.snapshotKey.throughSequence, + authorityRevision: legacy.snapshotKey.authorityRevision + 2, + snapshotGeneration: legacy.snapshotKey.snapshotGeneration + 1, + }); + assert.deepEqual( + modern.positions.map(({ turnId }) => turnId), + ['turn-b', 'turn-a'], + ); + + const stableLegacy = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: legacy.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stableLegacy.kind, 'page'); + if (stableLegacy.kind !== 'page') assert.fail('expected retained exact snapshot'); + assert.deepEqual( + stableLegacy.positions.map(({ turnId }) => turnId), + ['turn-a', 'turn-b'], + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('atomically invalidates an unfinished snapshot when admission authority changes', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-building-authority-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected an unfinished position build'); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-1024', 'user-1024', 10)); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: building.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('returns capacity for a third exact snapshot and never reuses a released generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-capacity-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const first = await readyPage(store, session.id, 'lease-first'); + await store.appendMessage(session.id, user('turn-b', 1)); + const second = await readyPage(store, session.id, 'lease-second'); + await store.appendMessage(session.id, user('turn-c', 2)); + + const capacity = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-third', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.deepEqual(capacity, { + kind: 'capacity', + throughSequence: 2, + authorityRevision: 0, + retainedSnapshots: 2, + }); + + const stableFirst = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stableFirst.kind, 'page'); + if (stableFirst.kind !== 'page') assert.fail('expected retained first snapshot'); + assert.equal(stableFirst.totalTurns, 1); + assert.deepEqual( + stableFirst.positions.map(({ turnId }) => turnId), + ['turn-a'], + ); + + await store.releaseTurnPositionSnapshot(session.id, 'lease-first', first.snapshotKey); + const third = await readyPage(store, session.id, 'lease-third'); + assert.equal(first.snapshotKey.snapshotGeneration, 1); + assert.equal(second.snapshotKey.snapshotGeneration, 2); + assert.equal(third.snapshotKey.snapshotGeneration, 3); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('caps one recovery step at 1,024 source records and resumes after reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-recovery-count-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database + .prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?') + .run(session.id); + database.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(session.id); + database + .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, + source_records = 0, source_bytes = 0, + failure_reason = NULL, failure_sequence = NULL WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + + store = createSessionStore(root); + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected bounded recovery progress'); + assert.equal(building.progress.sourceRecords, 1_024); + assert.equal(building.progress.nextSequence, 1_024); + + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: building.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const ready = await readyPage(store, session.id, 'lease-resumed'); + assert.equal(ready.totalTurns, 1_025); + assert.ok(ready.snapshotKey.snapshotGeneration > building.snapshotKey.snapshotGeneration); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('caps one recovery step below 4 MiB of source payload', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-recovery-bytes-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 5 }, (_, index) => ({ + ...user(`turn-${index}`, index), + text: `${index}${'x'.repeat(1024 * 1024)}`, + })), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + const building = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(building.kind, 'building'); + if (building.kind !== 'building') assert.fail('expected byte-bounded recovery'); + assert.equal(building.progress.sourceRecords, 3); + assert.ok(building.progress.sourceBytes < 4 * 1024 * 1024); + assert.equal(building.progress.nextSequence, 3); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('resumes an oversized first source record without a size-only failure', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-oversized-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + ...user('turn-oversized', 0), + text: 'x'.repeat(4 * 1024 * 1024 + 1024), + }); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial scalar recovery'); + assert.equal(first.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(first.progress.lastStepRecords, 0); + assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); + const ready = await readyPage(store, session.id); + assert.equal(ready.totalTurns, 1); + assert.equal(ready.positions[0]?.turnId, 'turn-oversized'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('accounts exact 4 MiB boundaries and resumes identity located after a huge body', async () => { + const targets = [4 * 1024 * 1024 - 1, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1, 9 * 1024 * 1024]; + for (const [index, targetBytes] of targets.entries()) { + const root = await mkdtemp(join(tmpdir(), `maka-turn-position-exact-bytes-${index}-`)); + let store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const message = exactSizeUserMessage(targetBytes, `exact-turn-${index}`, index % 2 === 1); + assert.equal(Buffer.byteLength(JSON.stringify(message), 'utf8'), targetBytes); + await store.appendMessage(session.id, message); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + + let result = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: `lease-exact-${index}`, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + const recoveryBytes: number[] = []; + while (result.kind === 'building') { + if (result.progress.phase === 'recovering') { + recoveryBytes.push(result.progress.lastStepBytes); + assert.ok(result.progress.lastStepBytes <= 4 * 1024 * 1024); + assert.ok(result.progress.lastStepRecords <= 1_024); + } + result = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: `lease-exact-${index}`, + snapshotKey: result.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(result.kind, 'page'); + if (result.kind !== 'page') assert.fail('expected exact-size recovered page'); + assert.equal(result.positions[0]?.turnId, `exact-turn-${index}`); + assert.equal( + recoveryBytes.reduce((total, bytes) => total + bytes, 0), + targetBytes, + ); + assert.equal(Math.max(...recoveryBytes), Math.min(targetBytes, 4 * 1024 * 1024)); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + } + }); + + test('discards corrupt derived state and restarts only the partial source record after reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-derived-recovery-reset-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage( + session.id, + exactSizeUserMessage(9 * 1024 * 1024, 'turn-derived-reset', true), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const partial = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-derived-before', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(partial.kind, 'building'); + if (partial.kind !== 'building') assert.fail('expected partial record'); + assert.equal(partial.progress.currentByteOffset, 4 * 1024 * 1024); + await store.close?.(); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const row = database + .prepare(`SELECT scanner_state FROM session_turn_identity_recovery + WHERE session_id = ?`) + .get(session.id) as { scanner_state: string }; + const scannerState = JSON.parse(row.scanner_state) as { + identity: { id: string }; + }; + scannerState.identity.id = 'x'.repeat(scannerState.identity.id.length); + database + .prepare(`UPDATE session_turn_identity_recovery + SET scanner_state = ? WHERE session_id = ?`) + .run(JSON.stringify(scannerState), session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + const restarted = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-derived-after', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(restarted.kind, 'building'); + if (restarted.kind !== 'building') assert.fail('expected reconstructed partial state'); + assert.equal(restarted.progress.currentByteOffset, 4 * 1024 * 1024); + assert.equal(restarted.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(restarted.progress.sourceBytes, 8 * 1024 * 1024); + const ready = await readyPage(store, session.id, 'lease-derived-after'); + assert.equal(ready.positions[0]?.turnId, 'turn-derived-reset'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('publishes recovery cursor, counters, and membership atomically at commit boundaries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-recovery-transaction-boundary-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-atomic', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON; BEGIN IMMEDIATE'); + const rolledBack = advanceSessionTurnIdentityRecovery(database, { + sessionId: session.id, + throughSequence: 0, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher: await createSHA256(), + }); + assert.equal('failure' in rolledBack, false); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_memberships + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 1, + ); + database.exec('ROLLBACK'); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_memberships + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + database.exec('BEGIN IMMEDIATE'); + const committed = advanceSessionTurnIdentityRecovery(database, { + sessionId: session.id, + throughSequence: 0, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher: await createSHA256(), + }); + assert.equal('failure' in committed, false); + database.exec('COMMIT'); + const state = database + .prepare(`SELECT indexed_through_sequence, source_records + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { indexed_through_sequence: number; source_records: number }; + assert.deepEqual([state.indexed_through_sequence, state.source_records], [0, 1]); + } finally { + database.close(); + } + store = createSessionStore(root); + const ready = await readyPage(store, session.id, 'lease-atomic'); + assert.equal(ready.positions[0]?.turnId, 'turn-atomic'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('persists a partial-source mutation failure until authoritative invalidation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-partial-source-mutation-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage( + session.id, + exactSizeUserMessage(9 * 1024 * 1024, 'turn-source-mutation', false), + ); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const partial = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-source-before', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(partial.kind, 'building'); + await store.close?.(); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET message_id = 'mutated-id' + WHERE session_id = ? AND sequence = 0`) + .run(session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + for (const lease of ['lease-source-failed', 'lease-source-failed']) { + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: lease, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + assert.equal((error as { sequence?: unknown }).sequence, 0); + return true; + }, + ); + } + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const failure = failed + .prepare(`SELECT failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { failure_reason: string; failure_sequence: number }; + assert.equal(failure.failure_reason, 'corrupt_source'); + assert.equal(failure.failure_sequence, 0); + } finally { + failed.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('binds transcript record reads to the exact snapshot key and Turn membership', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-records-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + user('turn-a', 0), + { + type: 'assistant', + id: 'assistant-a', + turnId: 'turn-a', + ts: 2, + text: 'answer', + modelId: 'test-model', + }, + user('turn-b', 1), + ]); + const page = await readyPage(store, session.id); + + const records = await store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + turnIds: ['turn-a'], + maxBytes: 64 * 1024, + maxRecords: 8, + }); + assert.deepEqual(records.snapshotKey, page.snapshotKey); + assert.deepEqual( + records.records.map(({ sequence, message }) => [sequence, message.id]), + [ + [0, 'user-0'], + [1, 'assistant-a'], + ], + ); + for (const [limits, reason] of [ + [{ maxBytes: 64 * 1024, maxRecords: 1 }, 'transcript_record_count'], + [{ maxBytes: 1, maxRecords: 8 }, 'transcript_record_bytes'], + ] as const) { + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + turnIds: ['turn-a'], + ...limits, + }), + (error: unknown) => { + assert.equal( + (error as { code?: unknown }).code, + 'session_turn_position_limit_exceeded', + ); + assert.equal((error as { reason?: unknown }).reason, reason); + return true; + }, + ); + } + for (const snapshotKey of [ + { ...page.snapshotKey, authorityRevision: page.snapshotKey.authorityRevision + 1 }, + { ...page.snapshotKey, snapshotGeneration: 999 }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey, + turnIds: ['turn-a'], + maxBytes: 64 * 1024, + maxRecords: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } + for (const request of [ + { turnIds: ['turn-a', 'turn-a'], maxRecords: 8, maxBytes: 64 * 1024 }, + { turnIds: [''], maxRecords: 8, maxBytes: 64 * 1024 }, + { + turnIds: Array.from({ length: 129 }, (_, index) => `turn-${index}`), + maxRecords: 8, + maxBytes: 64 * 1024, + }, + { turnIds: ['turn-a'], maxRecords: 257, maxBytes: 64 * 1024 }, + { turnIds: ['turn-a'], maxRecords: 8, maxBytes: 16 * 1024 * 1024 + 1 }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + ...request, + }), + /invalid|unique/iu, + ); + } + for (const request of [ + { snapshotLeaseId: 'lease-unknown', turnIds: ['turn-a'] }, + { snapshotLeaseId: 'lease-default', turnIds: ['turn-unknown'] }, + ]) { + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotKey: page.snapshotKey, + maxRecords: 8, + maxBytes: 64 * 1024, + ...request, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = '{' + WHERE session_id = ? AND sequence = 1`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + turnIds: ['turn-a'], + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + turnIds: ['turn-a'], + maxRecords: 2, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed with a typed recovery error for a corrupted durable source record', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-corrupt-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-a', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare( + 'UPDATE session_messages SET record_json = ? WHERE session_id = ? AND sequence = 0', + ) + .run('{', session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + failed + .prepare(`SELECT COUNT(*) AS count FROM session_turn_position_snapshots + WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + } finally { + failed.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-retry-failed-source', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('persists chunk-integrity failure across repair/reopen until authoritative invalidation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-chunk-corrupt-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, { + ...user('turn-chunk-corrupt', 0), + text: 'x'.repeat(1024 * 1024), + }); + await store.close?.(); + resetProjection(root, session.id); + const corrupt = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + corrupt + .prepare(`UPDATE session_message_chunks SET sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .run('0'.repeat(64), session.id); + } finally { + corrupt.close(); + } + store = createSessionStore(root); + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }); + await store.close?.(); + + const repaired = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const chunk = repaired + .prepare(`SELECT data FROM session_message_chunks + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .get(session.id) as { data: Uint8Array }; + repaired + .prepare(`UPDATE session_message_chunks SET sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = 0`) + .run(createHash('sha256').update(chunk.data).digest('hex'), session.id); + } finally { + repaired.close(); + } + store = createSessionStore(root); + await assert.rejects(readyPage(store, session.id, 'lease-still-failed'), (error: unknown) => { + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }); + await store.close?.(); + resetProjection(root, session.id); + store = createSessionStore(root); + const recovered = await readyPage(store, session.id, 'lease-after-reset'); + assert.equal(recovered.positions[0]?.turnId, 'turn-chunk-corrupt'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses the shared physical decoder to reject noncanonical complete-record chunks', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-body-chunk-shape-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, { + ...user('turn-chunk-shape', 0), + text: 'x'.repeat(1024 * 1024), + }); + const page = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const tail = database + .prepare(`SELECT chunk_index, data FROM session_message_chunks + WHERE session_id = ? AND sequence = 0 + ORDER BY chunk_index DESC LIMIT 2`) + .all(session.id) as Array<{ chunk_index: number; data: Uint8Array }>; + assert.equal(tail.length, 2); + const last = tail[0]!; + const prior = tail[1]!; + assert.equal(prior.data.byteLength, 64 * 1024); + const joined = Buffer.concat([Buffer.from(prior.data), Buffer.from(last.data)]); + const shiftedPrior = joined.subarray(0, 64 * 1024 - 1); + const shiftedLast = joined.subarray(64 * 1024 - 1); + const update = database.prepare(`UPDATE session_message_chunks + SET data = ?, sha256 = ? + WHERE session_id = ? AND sequence = 0 AND chunk_index = ?`); + update.run( + shiftedPrior, + createHash('sha256').update(shiftedPrior).digest('hex'), + session.id, + prior.chunk_index, + ); + update.run( + shiftedLast, + createHash('sha256').update(shiftedLast).digest('hex'), + session.id, + last.chunk_index, + ); + } finally { + database.close(); + } + + await assert.rejects( + store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + turnIds: ['turn-chunk-shape'], + maxRecords: 1, + maxBytes: 2 * 1024 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed when recovered Turn identity conflicts with persisted projection metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-identity-conflict-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-a', 0)); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`INSERT INTO session_turn_metadata( + session_id, turn_id, identity_kind, order_source, first_sequence + ) VALUES (?, 'turn-conflict', 'turn', 'legacy', 0)`) + .run(session.id); + } finally { + database.close(); + } + store = createSessionStore(root); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_recovery_failed'); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('shortens a page to keep serialized position metadata below 64 KiB', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-page-bytes-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 128 }, (_, index) => user(`turn-${index}-${'t'.repeat(600)}`, index)), + ); + const ready = await readyPage(store, session.id); + const page = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 0 }, + maxPositions: 128, + }); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected bounded page'); + assert.ok(page.positions.length < 128); + assert.ok(Buffer.byteLength(JSON.stringify(page), 'utf8') <= 64 * 1024); + assert.equal(page.hasNewer, true); + + const tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 128, + }); + assert.equal(tail.kind, 'page'); + if (tail.kind !== 'page') assert.fail('expected bounded tail page'); + assert.ok(tail.positions.length < 128); + assert.equal(tail.positions.at(-1)?.ordinal, 127); + assert.equal(tail.hasNewer, false); + assert.equal(tail.hasOlder, true); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('returns a typed failure when one position alone exceeds the metadata budget', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-single-page-limit-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user(`turn-${'x'.repeat(70 * 1024)}`, 0)); + await assert.rejects(readyPage(store, session.id), (error: unknown) => { + assert.equal((error as { code?: unknown }).code, 'session_turn_position_limit_exceeded'); + assert.equal((error as { reason?: unknown }).reason, 'page_metadata_bytes'); + return true; + }); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('indexes imported transcripts without copying a 16 KiB-plus body into positions', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-imported-')); + const store = createSessionStore(root); + try { + const session = await store.createImportedSession( + makeInput(), + [{ ...user('turn-imported', 0), text: 'x'.repeat(20 * 1024) }, user('turn-second', 1)], + { adapterId: 'test-adapter', sourceSessionId: 'source-session' }, + ); + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, turnId: 'turn-imported', firstSequence: 0 }, + { ordinal: 1, turnId: 'turn-second', firstSequence: 1 }, + ]); + assert.ok(Buffer.byteLength(JSON.stringify(page), 'utf8') < 4 * 1024); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('reclaims ready generations across owner reopen and preserves monotonic generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-ready-reopen-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const before = await readyPage(store, session.id); + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id, 'lease-after-reopen'); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(after.positions, before.positions); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps live snapshot leases when a second facade shares the same database owner', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-same-owner-')); + const first = createSessionStore(root); + let second: ReturnType | undefined; + try { + const session = await first.create(makeInput()); + await first.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + const original = await readyPage(first, session.id, 'lease-first'); + + second = createSessionStore(root); + const shared = await second.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-second', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(shared.kind, 'page'); + if (shared.kind !== 'page') assert.fail('expected a shared ready generation'); + assert.deepEqual(shared.snapshotKey, original.snapshotKey); + + await first.releaseTurnPositionSnapshot(session.id, 'lease-first', original.snapshotKey); + const retained = await second.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-second', + snapshotKey: shared.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + } finally { + await second?.close?.(); + await first.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('database guards reject mutation of ready position rows', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-ready-corrupt-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [ + user('turn-a', 0), + user('turn-b', 1), + user('turn-c', 2), + ]); + const ready = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + for (const mutate of [ + () => + database + .prepare(`INSERT INTO session_turn_snapshot_positions( + session_id, snapshot_generation, ordinal, turn_id, first_sequence + ) VALUES (?, ?, 3, 'turn-extra', 3)`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`UPDATE session_turn_snapshot_positions SET turn_id = 'mutated' + WHERE session_id = ? AND snapshot_generation = ? AND ordinal = 2`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`DELETE FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND ordinal = 2`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + () => + database + .prepare(`UPDATE session_turn_position_snapshots SET ready_total = 2 + WHERE session_id = ? AND snapshot_generation = ?`) + .run(session.id, ready.snapshotKey.snapshotGeneration), + ]) { + assert.throws(mutate, /immutable/u); + } + } finally { + database.close(); + } + const stable = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: ready.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(stable.kind, 'page'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('atomically expires exact snapshots when historical handoff shifts sequence anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-historical-insert-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { + type: 'assistant', + id: 'prior-output', + turnId: 'turn-prior', + ts: 10, + text: 'prior', + modelId: 'test-model', + }, + { + type: 'assistant', + id: 'target-output', + turnId: 'turn-target', + ts: 20, + text: 'target', + modelId: 'test-model', + }, + { type: 'user', id: 'newer-user', turnId: 'turn-newer', ts: 30, text: 'newer' }, + ]); + const before = await readyPage(store, session.id); + + await store.markMessagesHandedOff({ + sessionId: session.id, + messageIds: ['historical-user'], + turnId: 'turn-target', + provenRootMessages: [ + { messageId: 'historical-user', content: { text: 'historical' }, admittedAt: 17 }, + ], + }); + + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual( + after.positions.map(({ turnId, firstSequence }) => [turnId, firstSequence]), + [ + ['turn-prior', 0], + ['turn-target', 1], + ['turn-newer', 3], + ], + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('additively migrates a current v34 database and boundedly rebuilds legacy metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-migration-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + DROP TABLE session_turn_snapshot_leases; + DROP TABLE session_turn_snapshot_positions; + DROP TABLE session_turn_position_snapshots; + DROP TABLE session_turn_identity_recovery; + DROP TABLE session_turn_memberships; + DROP TABLE session_turn_metadata; + DROP TABLE session_turn_index_state; + DROP TABLE session_turn_authority_revisions; + UPDATE session_metadata_schema SET version = 34 WHERE scope = 'session_metadata'; + `); + } finally { + database.close(); + } + + store = createSessionStore(root); + const lazy = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + lazy + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + } finally { + lazy.close(); + } + const page = await readyPage(store, session.id); + assert.deepEqual(page.positions, [ + { ordinal: 0, turnId: 'turn-a', firstSequence: 0 }, + { ordinal: 1, turnId: 'turn-b', firstSequence: 1 }, + ]); + const migrated = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + migrated + .prepare( + `SELECT version FROM session_metadata_schema WHERE scope = 'session_metadata'`, + ) + .get() as { version: number } + ).version, + 35, + ); + } finally { + migrated.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('uses indexed keyset plans for 10,000-Turn steady pages and every anchor', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-plan-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages( + session.id, + Array.from({ length: 10_000 }, (_, index) => user(`turn-${index}`, index)), + ); + const firstStep = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(firstStep.kind, 'building'); + if (firstStep.kind !== 'building') assert.fail('expected bounded position publication'); + assert.equal(firstStep.progress.phase, 'recovering'); + assert.equal(firstStep.progress.sourceRecords, 0); + assert.equal(firstStep.progress.builtPositions, 0); + assert.equal(firstStep.progress.lastStepPositions, 0); + const secondStep = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: firstStep.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(secondStep.kind, 'building'); + if (secondStep.kind !== 'building') assert.fail('expected legacy ordinal build'); + assert.equal(secondStep.progress.phase, 'legacy'); + assert.equal(secondStep.progress.builtPositions, 1_024); + const bounded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, secondStep.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 1_024, + ); + const recoveryPlans = [ + bounded + .prepare(`EXPLAIN QUERY PLAN SELECT MIN(first_sequence) AS boundary + FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission' AND first_sequence <= ?`) + .all(session.id, 9_999), + bounded + .prepare(`EXPLAIN QUERY PLAN SELECT turn_id, first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR first_sequence < ?) AND first_sequence > ? + ORDER BY first_sequence, turn_id LIMIT ?`) + .all(session.id, 9_999, null, null, -1, 1_025), + ].flat() as Array<{ detail: string }>; + assert.equal( + recoveryPlans.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/.test(detail)), + false, + ); + assert.equal( + recoveryPlans.every(({ detail }) => detail.includes('SEARCH session_turn_metadata')), + true, + ); + } finally { + bounded.close(); + } + let tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: secondStep.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + while (tail.kind === 'building') { + tail = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(tail.kind, 'page'); + if (tail.kind !== 'page') assert.fail('expected steady page'); + assert.equal(tail.totalTurns, 10_000); + assert.equal(tail.positions.length, 8); + + const sparseRecords = await store.readTranscriptRecordsByTurnIdsSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + turnIds: ['turn-9999', 'turn-0'], + maxRecords: 2, + maxBytes: 64 * 1024, + }); + assert.deepEqual( + sparseRecords.records.map(({ sequence }) => sequence), + [0, 9_999], + ); + + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const plans = [ + database + .prepare(`EXPLAIN QUERY PLAN SELECT ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND first_sequence <= ? + ORDER BY first_sequence DESC LIMIT 1`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000), + database + .prepare(`EXPLAIN QUERY PLAN SELECT ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ?`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 'turn-5000'), + database + .prepare(`EXPLAIN QUERY PLAN SELECT ordinal, turn_id, first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND ordinal >= ? + ORDER BY ordinal LIMIT ?`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000, 8), + ].flat() as Array<{ detail: string }>; + assert.equal( + plans.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/.test(detail)), + false, + ); + assert.equal( + plans.every(({ detail }) => detail.includes('SEARCH session_turn_snapshot_positions')), + true, + ); + const membershipPlan = database + .prepare(`EXPLAIN QUERY PLAN + SELECT membership.sequence, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) + FROM session_turn_memberships AS membership + INNER JOIN session_messages AS message + ON message.session_id = membership.session_id + AND message.sequence = membership.sequence + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE membership.session_id = ? AND membership.turn_id = ? + AND membership.sequence <= ? + ORDER BY membership.sequence LIMIT ?`) + .all(session.id, 'turn-9999', 9_999, 2) as Array<{ detail: string }>; + assert.equal( + membershipPlan.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/u.test(detail)), + false, + ); + assert.equal( + membershipPlan.some(({ detail }) => detail.includes('session_turn_memberships_by_turn')), + true, + ); + } finally { + database.close(); + } + + for (const anchor of [ + { kind: 'ordinal' as const, ordinal: 5_000 }, + { kind: 'sequence' as const, sequence: 5_000 }, + { kind: 'turn' as const, turnId: 'turn-5000' }, + ]) { + const page = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor, + maxPositions: 8, + }); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected indexed anchored page'); + assert.equal(page.startOrdinal, 5_000); + assert.equal(page.positions.length, 8); + } + const adjacent = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 5_008 }, + maxPositions: 8, + }); + assert.equal(adjacent.kind, 'page'); + if (adjacent.kind !== 'page') assert.fail('expected adjacent keyset page'); + assert.deepEqual( + adjacent.positions.map(({ ordinal }) => ordinal), + Array.from({ length: 8 }, (_, index) => 5_008 + index), + ); + assert.equal(new Set(adjacent.positions.map(({ turnId }) => turnId)).size, 8); + const prior = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-default', + snapshotKey: tail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 5_000 }, + maxPositions: 8, + }); + assert.equal(prior.kind, 'page'); + if (prior.kind !== 'page') assert.fail('expected prior keyset page'); + assert.equal( + prior.positions.some(({ turnId }) => + adjacent.positions.some((position) => position.turnId === turnId), + ), + false, + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); +}); + +type TestStore = ReturnType; + +async function readyPage(store: TestStore, sessionId: string, snapshotLeaseId = 'lease-default') { + let result = await store.readTurnPositionPageSnapshot({ + sessionId, + snapshotLeaseId, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + while (result.kind === 'building') { + result = await store.readTurnPositionPageSnapshot({ + sessionId, + snapshotLeaseId, + snapshotKey: result.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(result.kind, 'page'); + if (result.kind !== 'page') assert.fail('expected a ready page'); + return result; +} + +function user(turnId: string, index: number) { + return { + type: 'user' as const, + id: `user-${index}`, + turnId, + ts: index, + text: turnId, + }; +} + +function exactSizeUserMessage(targetBytes: number, turnId: string, turnIdAfterBody: boolean) { + const message = turnIdAfterBody + ? { type: 'user' as const, id: `user-${turnId}`, ts: 1, text: '', turnId } + : { type: 'user' as const, id: `user-${turnId}`, turnId, ts: 1, text: '' }; + const envelopeBytes = Buffer.byteLength(JSON.stringify(message), 'utf8'); + assert.ok(targetBytes >= envelopeBytes); + message.text = 'x'.repeat(targetBytes - envelopeBytes); + return message; +} + +function rootAdmission( + sessionId: string, + turnId: string, + userMessageId: string, + admittedAt: number, +) { + return { + sessionId, + turnId, + proposedRunId: `run-${turnId}`, + proposedUserMessageId: userMessageId, + execution: { kind: 'external_message' as const }, + previousRootTurnId: null, + normalizedInput: { text: turnId }, + sourceMessages: [], + admittedAt, + }; +} + +function resetProjection(root: string, sessionId: string): void { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database + .prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?') + .run(sessionId); + database.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + database + .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, + source_records = 0, source_bytes = 0, + failure_reason = NULL, failure_sequence = NULL WHERE session_id = ?`) + .run(sessionId); + } finally { + database.close(); + } +} + +function makeInput(overrides: Partial = {}): CreateSessionInput { + return { + cwd: '/tmp/cwd', + llmConnectionSlug: 'test-connection', + model: 'test-model', + permissionMode: 'ask', + name: 'Session', + labels: [], + ...overrides, + }; +} diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 27481f3fec..fb652a375a 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -34,6 +34,7 @@ import { type SubmittedTurnIntent, } from './submitted-turn-intent.js'; import { assertNoReservedWorkspaceAuthorityAppend } from './runtime-event-authority.js'; +import { recordRootTurnAdmissionForPositionIndex } from './session-turn-position-index.js'; import { acquireOperationalStateDatabase, type OperationalStateDatabaseLease, @@ -738,6 +739,12 @@ class SqliteAgentRunStore implements DurableAgentRunStore { admission.admittedAt, JSON.stringify(admission), ); + recordRootTurnAdmissionForPositionIndex( + this.#lease.database, + admission.sessionId, + admission.turnId, + admission.admittedAt, + ); for (const source of admission.sourceMessages) { this.#lease.database .prepare(` diff --git a/packages/storage/src/conversation-operational-state.ts b/packages/storage/src/conversation-operational-state.ts index 2f87bd8921..32303ba7a9 100644 --- a/packages/storage/src/conversation-operational-state.ts +++ b/packages/storage/src/conversation-operational-state.ts @@ -23,6 +23,7 @@ import { type OperationalStateDatabaseLease, } from './operational-state-store.js'; import { isRuntimeStorageSafeId } from './runtime-event-invariants.js'; +import { recordRootTurnAdmissionsPurgedForPositionIndex } from './session-turn-position-index.js'; export interface ConversationOperationalStateStore { purge(sessionId: string): Promise; @@ -82,6 +83,7 @@ class SqliteConversationOperationalStateStore implements ConversationOperational .prepare('DELETE FROM core_agent_run_projections WHERE session_id = ?') .run(sessionId); database.prepare('DELETE FROM core_root_turn_admissions WHERE session_id = ?').run(sessionId); + recordRootTurnAdmissionsPurgedForPositionIndex(database, sessionId); database .prepare('DELETE FROM core_root_turn_start_rejections WHERE session_id = ?') .run(sessionId); diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 33bdf087e7..dc1484ad65 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -85,6 +85,10 @@ export { normalizeRootTurnAdmissionPayload } from './agent-run-store.js'; export { isSessionNotFoundError, SessionReadMarkerMessageNotFoundError, + SessionTurnPositionAnchorNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, + SessionTurnPositionSnapshotMismatchError, } from './session-store.js'; export { SessionMetadataConflictError, @@ -133,6 +137,13 @@ export type { SessionTranscriptRecordScanRequest, SessionTranscriptStoragePage, SessionTranscriptStorageFragment, + SessionTranscriptRecordsByTurnIdsSnapshotRequest, + SessionTranscriptRecordsByTurnIdsSnapshotResult, + SessionTurnPosition, + SessionTurnPositionAnchor, + SessionTurnPositionPageSnapshotRequest, + SessionTurnPositionReadResult, + SessionTurnPositionSnapshotKey, } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; @@ -412,6 +423,14 @@ async function createExecutionStoresForWrite run(() => sessionStore.readTurnLandmarksSnapshot(sessionId, maxLandmarks)), + readTurnPositionPageSnapshot: (request) => + run(() => sessionStore.readTurnPositionPageSnapshot(request)), + readTranscriptRecordsByTurnIdsSnapshot: (request) => + run(() => sessionStore.readTranscriptRecordsByTurnIdsSnapshot(request)), + releaseTurnPositionSnapshot: (sessionId, snapshotLeaseId, snapshotKey) => + run(() => + sessionStore.releaseTurnPositionSnapshot(sessionId, snapshotLeaseId, snapshotKey), + ), readMessagesForRecovery: (sessionId) => run(() => sessionStore.readMessagesForRecovery(sessionId)), listTurnsSnapshot: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), diff --git a/packages/storage/src/operational-state-store.ts b/packages/storage/src/operational-state-store.ts index 5f2772a680..47b2029435 100644 --- a/packages/storage/src/operational-state-store.ts +++ b/packages/storage/src/operational-state-store.ts @@ -57,6 +57,7 @@ import { ensureOperationalSchemaRegistry, isCurrentOperationalTargetSchema, } from './operational-target-schema.js'; +import { reclaimSessionTurnPositionSnapshotsForNewOwner } from './session-turn-position-index.js'; export const OPERATIONAL_STATE_DATABASE_NAME = 'runtime.sqlite'; export const OPERATIONAL_STATE_SCHEMA_VERSION = 2; @@ -201,6 +202,14 @@ class OperationalStateDatabaseOwner { this.database.exec('PRAGMA foreign_keys = ON'); inspectAndMigrateOperationalState(this.database, options.now ?? Date.now); configureSqliteRuntimeDatabase(this.database); + this.database.exec('BEGIN IMMEDIATE'); + try { + reclaimSessionTurnPositionSnapshotsForNewOwner(this.database); + this.database.exec('COMMIT'); + } catch (error) { + rollback(this.database); + throw error; + } } catch (error) { this.database.close(); this.closed = true; diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 55eed4d1d4..32568d32f4 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -301,6 +301,133 @@ export interface SessionTurnLandmarkSnapshot { readonly landmarks: readonly SessionTurnLandmark[]; } +export interface SessionTurnPositionSnapshotKey { + readonly throughSequence: number | null; + readonly authorityRevision: number; + readonly snapshotGeneration: number; +} + +export type SessionTurnPositionAnchor = + /** The final position, returned as the inclusive end of a tail-sized page. */ + | { readonly kind: 'tail' } + /** An inclusive ordinal; values beyond the end clamp to the final position. */ + | { readonly kind: 'ordinal'; readonly ordinal: number } + /** The position with the greatest firstSequence at or before this inclusive sequence. */ + | { readonly kind: 'sequence'; readonly sequence: number } + /** An exact non-empty Turn identity; a missing Turn fails with a typed anchor error. */ + | { readonly kind: 'turn'; readonly turnId: string }; + +export interface SessionTurnPosition { + readonly ordinal: number; + readonly turnId: string; + readonly firstSequence: number | null; +} + +export interface SessionTurnPositionPageSnapshotRequest { + readonly sessionId: string; + readonly snapshotLeaseId: string; + /** Exact continuation identity. Omit only when allocating a new snapshot. */ + readonly snapshotKey?: SessionTurnPositionSnapshotKey; + /** Inclusive durable watermark. Omit to capture the current durable tail. */ + readonly throughSequence?: number | null; + readonly anchor: SessionTurnPositionAnchor; + readonly maxPositions: number; +} + +export type SessionTurnPositionReadResult = + | { + readonly kind: 'building'; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly progress: { + readonly phase: 'recovering' | 'legacy' | 'admission' | 'notes'; + readonly nextSequence: number; + readonly currentByteOffset: number; + readonly sourceRecords: number; + readonly sourceBytes: number; + readonly builtPositions: number; + readonly lastStepRecords: number; + readonly lastStepBytes: number; + readonly lastStepPositions: number; + }; + } + | { + readonly kind: 'capacity'; + readonly throughSequence: number | null; + readonly authorityRevision: number; + readonly retainedSnapshots: 2; + } + | { + readonly kind: 'page'; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly startOrdinal: number; + readonly totalTurns: number; + readonly positions: readonly SessionTurnPosition[]; + readonly hasOlder: boolean; + readonly hasNewer: boolean; + }; + +export interface SessionTranscriptRecordsByTurnIdsSnapshotRequest { + readonly sessionId: string; + readonly snapshotLeaseId: string; + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly turnIds: readonly string[]; + readonly maxBytes: number; + readonly maxRecords: number; +} + +export interface SessionTranscriptRecordsByTurnIdsSnapshotResult { + readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly records: readonly { readonly sequence: number; readonly message: StoredMessage }[]; + readonly rawBytes: number; +} + +export class SessionTurnPositionSnapshotMismatchError extends Error { + readonly name = 'SessionTurnPositionSnapshotMismatchError'; + readonly code = 'session_turn_position_snapshot_mismatch'; + + constructor(readonly sessionId: string) { + super(`Session Turn-position snapshot is stale or mismatched: ${sessionId}`); + } +} + +export class SessionTurnPositionRecoveryError extends Error { + readonly name = 'SessionTurnPositionRecoveryError'; + readonly code = 'session_turn_position_recovery_failed'; + + constructor( + readonly sessionId: string, + readonly reason: 'corrupt_source' | 'incompatible_identity' | 'hybrid_missing_admission', + readonly sequence?: number, + options?: ErrorOptions, + ) { + super(`Session Turn-position recovery failed (${reason}): ${sessionId}`, options); + } +} + +export class SessionTurnPositionAnchorNotFoundError extends Error { + readonly name = 'SessionTurnPositionAnchorNotFoundError'; + readonly code = 'session_turn_position_anchor_not_found'; + + constructor( + readonly sessionId: string, + readonly turnId: string, + ) { + super(`Session Turn-position anchor does not exist: ${sessionId}/${turnId}`); + } +} + +export class SessionTurnPositionLimitError extends Error { + readonly name = 'SessionTurnPositionLimitError'; + readonly code = 'session_turn_position_limit_exceeded'; + + constructor( + readonly sessionId: string, + readonly reason: 'page_metadata_bytes' | 'transcript_record_count' | 'transcript_record_bytes', + ) { + super(`Session Turn-position limit exceeded (${reason}): ${sessionId}`); + } +} + export interface SessionStore { create(input: CreateSessionInput, initialBoundary?: ExecutionBoundary): Promise; list(filter?: SessionListFilter): Promise; @@ -327,6 +454,17 @@ export interface SessionStore { sessionId: string, maxLandmarks: number, ): Promise; + readTurnPositionPageSnapshot( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise; + readTranscriptRecordsByTurnIdsSnapshot( + request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, + ): Promise; + releaseTurnPositionSnapshot( + sessionId: string, + snapshotLeaseId: string, + snapshotKey: SessionTurnPositionSnapshotKey, + ): Promise; /** Read durable messages for startup recovery. */ readMessagesForRecovery(sessionId: string): Promise; /** Derive durable turns without triggering connection-lock self-healing. */ @@ -946,6 +1084,29 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.readTurnLandmarks(sessionId, maxLandmarks); } + async readTurnPositionPageSnapshot( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise { + await this.ensureReady(); + return this.metadata.readTurnPositionPage(request); + } + + async readTranscriptRecordsByTurnIdsSnapshot( + request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, + ): Promise { + await this.ensureReady(); + return this.metadata.readTranscriptRecordsByTurnIds(request); + } + + async releaseTurnPositionSnapshot( + sessionId: string, + snapshotLeaseId: string, + snapshotKey: SessionTurnPositionSnapshotKey, + ): Promise { + await this.ensureReady(); + await this.metadata.releaseTurnPositionSnapshot(sessionId, snapshotLeaseId, snapshotKey); + } + async readMessagesForRecovery(sessionId: string): Promise { await this.ensureReady(); return this.metadata.readMessagesForRecovery(sessionId); diff --git a/packages/storage/src/session-turn-identity-recovery.ts b/packages/storage/src/session-turn-identity-recovery.ts new file mode 100644 index 0000000000..4e8aae4ad6 --- /dev/null +++ b/packages/storage/src/session-turn-identity-recovery.ts @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import type { IHasher } from 'hash-wasm'; +import { + advanceSessionTurnIdentityScanner, + completeSessionTurnIdentityScanner, + createSessionTurnIdentityScannerState, + restoreSessionTurnIdentityScannerState, + serializeSessionTurnIdentityScannerState, + SessionTurnIdentityScannerError, + type SessionTurnIdentityScannerStateV1, + type SessionTurnRecoveredIdentity, +} from './session-turn-identity-scanner.js'; +import { + readTranscriptSlices, + StoredSessionMessageIncompatibleError, +} from './sqlite-session-transcript-slices.js'; + +export const SESSION_TURN_IDENTITY_HASH_STATE_VERSION = 1; +export const SESSION_TURN_IDENTITY_HASH_ALGORITHM = 'sha256'; +export const SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION = 'hash-wasm@4.12.0'; + +export type SessionTurnIdentityRecoveryFailureReason = + | 'corrupt_source' + | 'incompatible_identity' + | 'hybrid_missing_admission'; + +export interface SessionTurnIdentityRecoveryProgress { + readonly complete: boolean; + readonly nextSequence: number; + readonly currentByteOffset: number; + readonly sourceRecords: number; + readonly sourceBytes: number; + readonly lastStepRecords: number; + readonly lastStepBytes: number; +} + +export interface SessionTurnIdentityRecoveryFailureFact { + readonly complete: false; + readonly failure: SessionTurnIdentityRecoveryFailureReason; + readonly failureSequence: number; +} + +export type SessionTurnIdentityRecoveryResult = + | SessionTurnIdentityRecoveryProgress + | SessionTurnIdentityRecoveryFailureFact; + +interface IndexStateRow { + readonly indexed_through_sequence: number; + readonly source_records: number; + readonly source_bytes: number; + readonly failure_reason: SessionTurnIdentityRecoveryFailureReason | null; + readonly failure_sequence: number | null; +} + +interface PartialRow { + readonly sequence: number; + readonly byte_offset: number; + readonly record_bytes: number; + readonly chunked: number; + readonly expected_digest: string | null; + readonly message_id: string; + readonly message_type: string; + readonly hash_state_version: number; + readonly hash_algorithm: string; + readonly hash_implementation: string; + readonly hash_state: Uint8Array; + readonly scanner_state_version: number; + readonly scanner_state: string; + readonly derived_state_digest: string; +} + +interface SourceMetadata { + readonly sequence: number; + readonly message_id: string; + readonly message_type: string; + readonly record_bytes: number; + readonly chunked: number; + readonly expected_digest: string | null; +} + +export class SessionTurnIdentityRecoveryFailure extends Error { + readonly code = 'session_turn_identity_recovery_failed'; + + constructor( + readonly sessionId: string, + readonly reason: SessionTurnIdentityRecoveryFailureReason, + readonly sequence: number, + options?: ErrorOptions, + ) { + super(`Session Turn identity recovery failed (${reason}) at ${sessionId}/${sequence}`, options); + this.name = 'SessionTurnIdentityRecoveryFailure'; + } +} + +/** + * Advance only the scalar identity index. This function never returns source + * bytes, JSON, fragments, or StoredMessage objects to its caller. + */ +export function advanceSessionTurnIdentityRecovery( + db: DatabaseSync, + input: { + readonly sessionId: string; + readonly throughSequence: number; + readonly maxSourceBytes: number; + readonly maxCompletedRecords: number; + readonly hasher: IHasher; + }, +): SessionTurnIdentityRecoveryResult { + const { sessionId, throughSequence, maxSourceBytes, maxCompletedRecords, hasher } = input; + const initial = readIndexState(db, sessionId); + if (initial.failure_reason !== null) { + return { + complete: false, + failure: initial.failure_reason, + failureSequence: initial.failure_sequence!, + }; + } + let nextSequence = initial.indexed_through_sequence + 1; + let stepBytes = 0; + let stepRecords = 0; + while ( + nextSequence <= throughSequence && + stepBytes < maxSourceBytes && + stepRecords < maxCompletedRecords + ) { + const source = readSourceMetadata(db, sessionId, nextSequence); + if (!source) return fail(db, sessionId, 'corrupt_source', nextSequence); + let partial = readPartial(db, sessionId); + let scanner: SessionTurnIdentityScannerStateV1; + let byteOffset: number; + if (partial && partial.sequence !== nextSequence) { + return fail(db, sessionId, 'corrupt_source', nextSequence); + } + if (partial && !sameSource(partial, source)) { + return fail(db, sessionId, 'corrupt_source', nextSequence); + } + try { + if (partial) { + validateDerivedEnvelope(partial); + hasher.load(Uint8Array.from(partial.hash_state)); + scanner = restoreSessionTurnIdentityScannerState(partial.scanner_state); + byteOffset = partial.byte_offset; + } else { + hasher.init(); + scanner = createSessionTurnIdentityScannerState(); + byteOffset = 0; + } + } catch { + // Derived scanner/hash state is reconstructible. Discard only the + // current partial record and restart it from the authoritative bytes. + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + partial = undefined; + hasher.init(); + scanner = createSessionTurnIdentityScannerState(); + byteOffset = 0; + } + + const available = maxSourceBytes - stepBytes; + const byteLength = boundedSliceLength(source, byteOffset, available); + if (byteLength === 0) break; + let data: Buffer; + try { + const bySequence = readTranscriptSlices(db, sessionId, [ + { + sequence: source.sequence, + byteOffset, + totalBytes: source.record_bytes, + byteLength, + chunked: source.chunked === 1, + payloadDigest: + source.expected_digest === null ? null : `sha256:${source.expected_digest}`, + }, + ]); + data = bySequence.get(source.sequence)!; + if (!data || data.byteLength !== byteLength) { + return fail(db, sessionId, 'corrupt_source', source.sequence); + } + hasher.update(data); + advanceSessionTurnIdentityScanner(scanner, data); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnIdentityScannerError ? error.reason : 'corrupt_source', + source.sequence, + error, + ); + } + byteOffset += byteLength; + stepBytes += byteLength; + db.prepare(` + UPDATE session_turn_index_state SET source_bytes = source_bytes + ? WHERE session_id = ? + `).run(byteLength, sessionId); + if (byteOffset < source.record_bytes) { + writePartial(db, sessionId, source, byteOffset, hasher.save(), scanner); + break; + } + + let identity: SessionTurnRecoveredIdentity; + try { + const digest = hasher.digest('hex'); + if (source.expected_digest !== null && digest !== source.expected_digest) { + return fail(db, sessionId, 'corrupt_source', source.sequence); + } + identity = completeSessionTurnIdentityScanner(scanner, { + messageId: source.message_id, + messageType: source.message_type, + }); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnIdentityScannerError ? error.reason : 'corrupt_source', + source.sequence, + error, + ); + } + try { + publishRecoveredMembership(db, sessionId, source.sequence, identity); + } catch (error) { + return fail( + db, + sessionId, + error instanceof SessionTurnIdentityRecoveryFailure && + error.reason === 'incompatible_identity' + ? 'incompatible_identity' + : 'corrupt_source', + source.sequence, + error, + ); + } + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + nextSequence += 1; + stepRecords += 1; + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = ?, source_records = source_records + 1 + WHERE session_id = ? + `).run(source.sequence, sessionId); + } + const final = readIndexState(db, sessionId); + const partial = readPartial(db, sessionId); + return { + complete: final.indexed_through_sequence >= throughSequence, + nextSequence: final.indexed_through_sequence + 1, + currentByteOffset: partial?.byte_offset ?? 0, + sourceRecords: final.source_records, + sourceBytes: final.source_bytes, + lastStepRecords: stepRecords, + lastStepBytes: stepBytes, + }; +} + +function readIndexState(db: DatabaseSync, sessionId: string): IndexStateRow { + return db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes, + failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ? + `) + .get(sessionId) as unknown as IndexStateRow; +} + +function readPartial(db: DatabaseSync, sessionId: string): PartialRow | undefined { + return db + .prepare('SELECT * FROM session_turn_identity_recovery WHERE session_id = ?') + .get(sessionId) as PartialRow | undefined; +} + +function readSourceMetadata( + db: DatabaseSync, + sessionId: string, + sequence: number, +): SourceMetadata | undefined { + const row = db + .prepare(` + SELECT message.sequence, message.message_id, message.message_type, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS record_bytes, + payload.record_bytes IS NOT NULL AS chunked, payload.sha256 AS expected_digest + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? AND message.sequence = ? + `) + .get(sessionId, sequence) as Partial | undefined; + if ( + !row || + row.sequence !== sequence || + typeof row.message_id !== 'string' || + row.message_id.length === 0 || + typeof row.message_type !== 'string' || + row.message_type.length === 0 || + !Number.isSafeInteger(row.record_bytes) || + (row.record_bytes ?? 0) < 1 || + (row.chunked !== 0 && row.chunked !== 1) || + (row.expected_digest !== null && + (typeof row.expected_digest !== 'string' || !/^[0-9a-f]{64}$/u.test(row.expected_digest))) + ) { + return undefined; + } + return row as SourceMetadata; +} + +function boundedSliceLength(source: SourceMetadata, byteOffset: number, available: number): number { + const remaining = source.record_bytes - byteOffset; + if (remaining <= available) return remaining; + if (source.chunked === 0) return 0; + const chunkBytes = 64 * 1024; + return Math.floor(available / chunkBytes) * chunkBytes; +} + +function sameSource(partial: PartialRow, source: SourceMetadata): boolean { + return ( + partial.sequence === source.sequence && + partial.record_bytes === source.record_bytes && + partial.chunked === source.chunked && + partial.expected_digest === source.expected_digest && + partial.message_id === source.message_id && + partial.message_type === source.message_type && + partial.byte_offset >= 0 && + partial.byte_offset < source.record_bytes + ); +} + +function validateDerivedEnvelope(partial: PartialRow): void { + if ( + partial.hash_state_version !== SESSION_TURN_IDENTITY_HASH_STATE_VERSION || + partial.hash_algorithm !== SESSION_TURN_IDENTITY_HASH_ALGORITHM || + partial.hash_implementation !== SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION || + partial.scanner_state_version !== 1 || + !(partial.hash_state instanceof Uint8Array) || + partial.hash_state.byteLength < 1 || + partial.hash_state.byteLength > 64 * 1024 || + partial.derived_state_digest !== derivedStateDigest(partial.hash_state, partial.scanner_state) + ) { + throw new Error('Unsupported derived recovery state'); + } +} + +function writePartial( + db: DatabaseSync, + sessionId: string, + source: SourceMetadata, + byteOffset: number, + hashState: Uint8Array, + scanner: SessionTurnIdentityScannerStateV1, +): void { + const scannerState = serializeSessionTurnIdentityScannerState(scanner); + const derivedDigest = derivedStateDigest(hashState, scannerState); + db.prepare(` + INSERT INTO session_turn_identity_recovery( + session_id, sequence, byte_offset, record_bytes, chunked, expected_digest, + message_id, message_type, hash_state_version, hash_algorithm, + hash_implementation, hash_state, scanner_state_version, scanner_state + , derived_state_digest + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, 'sha256', 'hash-wasm@4.12.0', ?, 1, ?, ?) + ON CONFLICT(session_id) DO UPDATE SET + sequence = excluded.sequence, byte_offset = excluded.byte_offset, + record_bytes = excluded.record_bytes, chunked = excluded.chunked, + expected_digest = excluded.expected_digest, message_id = excluded.message_id, + message_type = excluded.message_type, hash_state_version = excluded.hash_state_version, + hash_algorithm = excluded.hash_algorithm, + hash_implementation = excluded.hash_implementation, hash_state = excluded.hash_state, + scanner_state_version = excluded.scanner_state_version, + scanner_state = excluded.scanner_state, + derived_state_digest = excluded.derived_state_digest + `).run( + sessionId, + source.sequence, + byteOffset, + source.record_bytes, + source.chunked, + source.expected_digest, + source.message_id, + source.message_type, + hashState, + scannerState, + derivedDigest, + ); +} + +function derivedStateDigest(hashState: Uint8Array, scannerState: string): string { + return createHash('sha256') + .update('session-turn-identity-recovery-v1\0') + .update(SESSION_TURN_IDENTITY_HASH_ALGORITHM) + .update('\0') + .update(SESSION_TURN_IDENTITY_HASH_IMPLEMENTATION) + .update('\0') + .update(hashState) + .update('\0') + .update(scannerState, 'utf8') + .digest('hex'); +} + +function publishRecoveredMembership( + db: DatabaseSync, + sessionId: string, + sequence: number, + identity: SessionTurnRecoveredIdentity, +): void { + if (identity.kind === 'ignored') return; + const admission = + identity.kind === 'turn' && tableExists(db, 'core_root_turn_admissions') + ? (db + .prepare(` + SELECT admitted_at FROM core_root_turn_admissions + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, identity.turnId) as { admitted_at: number } | undefined) + : undefined; + const existingIdentity = db + .prepare(` + SELECT identity_kind FROM session_turn_metadata + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, identity.turnId) as { identity_kind: string } | undefined; + if (existingIdentity && existingIdentity.identity_kind !== identity.kind) { + throw new SessionTurnIdentityRecoveryFailure(sessionId, 'incompatible_identity', sequence); + } + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, turn_id) DO UPDATE SET + order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' + ELSE session_turn_metadata.order_source END, + admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), + first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL + THEN excluded.first_sequence + ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END + `).run( + sessionId, + identity.turnId, + identity.kind, + admission ? 'admission' : 'legacy', + admission?.admitted_at ?? null, + sequence, + ); + const inserted = db + .prepare(` + INSERT INTO session_turn_memberships(session_id, sequence, turn_id) + VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING + `) + .run(sessionId, sequence, identity.turnId); + if (inserted.changes !== 1) { + const existing = db + .prepare(` + SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? + `) + .get(sessionId, sequence) as { turn_id: string }; + if (existing.turn_id !== identity.turnId) { + throw new SessionTurnIdentityRecoveryFailure(sessionId, 'corrupt_source', sequence); + } + } +} + +function fail( + db: DatabaseSync, + sessionId: string, + reason: 'corrupt_source' | 'incompatible_identity', + sequence: number, + cause?: unknown, +): SessionTurnIdentityRecoveryFailureFact { + db.prepare(` + UPDATE session_turn_index_state SET failure_reason = ?, failure_sequence = ? + WHERE session_id = ? + `).run(reason, sequence, sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + void cause; + return { complete: false, failure: reason, failureSequence: sequence }; +} + +function tableExists(db: DatabaseSync, table: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), + ); +} diff --git a/packages/storage/src/session-turn-identity-scanner.ts b/packages/storage/src/session-turn-identity-scanner.ts new file mode 100644 index 0000000000..aecf05aad0 --- /dev/null +++ b/packages/storage/src/session-turn-identity-scanner.ts @@ -0,0 +1,684 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { isUserVisibleSessionSystemNote } from '@maka/core/session'; + +export const SESSION_TURN_IDENTITY_SCANNER_VERSION = 1; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES = 64 * 1024; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES = 32 * 1024; +export const SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH = 4_096; + +type ObjectExpectation = 'key_or_end' | 'key' | 'colon' | 'value' | 'comma_or_end'; +type ArrayExpectation = 'value_or_end' | 'value' | 'comma_or_end'; +type ContainerState = + | { readonly kind: 'object'; expectation: ObjectExpectation; pendingKey: string | null } + | { readonly kind: 'array'; expectation: ArrayExpectation }; + +type LexicalState = + | { readonly kind: 'default' } + | { + readonly kind: 'string'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + } + | { + readonly kind: 'escape'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + } + | { + readonly kind: 'unicode'; + readonly role: 'key' | 'value'; + capture: boolean; + value: string; + digits: string; + } + | { readonly kind: 'number'; value: string } + | { readonly kind: 'literal'; readonly expected: 'true' | 'false' | 'null'; offset: number }; + +interface Utf8State { + needed: number; + codePoint: number; + minimum: number; +} + +export interface SessionTurnIdentityScannerStateV1 { + readonly version: 1; + rootStarted: boolean; + rootComplete: boolean; + stack: ContainerState[]; + lexical: LexicalState; + utf8: Utf8State; + seenIdentityKeys: string[]; + identity: { + id?: string; + type?: string; + turnId?: string; + turnIdPresent: boolean; + kind?: string; + kindPresent: boolean; + }; + capturedBytes: number; +} + +export type SessionTurnRecoveredIdentity = + | { readonly kind: 'turn'; readonly turnId: string } + | { readonly kind: 'note'; readonly turnId: string } + | { readonly kind: 'ignored' }; + +export class SessionTurnIdentityScannerError extends Error { + readonly code = 'session_turn_identity_incompatible'; + + constructor( + readonly detail: string, + readonly reason: 'corrupt_source' | 'incompatible_identity' = 'corrupt_source', + ) { + super(`Session Turn identity envelope is incompatible: ${detail}`); + this.name = 'SessionTurnIdentityScannerError'; + } +} + +const IDENTITY_KEYS = new Set(['id', 'type', 'turnId', 'kind']); + +export function createSessionTurnIdentityScannerState(): SessionTurnIdentityScannerStateV1 { + return { + version: SESSION_TURN_IDENTITY_SCANNER_VERSION, + rootStarted: false, + rootComplete: false, + stack: [], + lexical: { kind: 'default' }, + utf8: { needed: 0, codePoint: 0, minimum: 0 }, + seenIdentityKeys: [], + identity: { turnIdPresent: false, kindPresent: false }, + capturedBytes: 0, + }; +} + +export function restoreSessionTurnIdentityScannerState( + encoded: string, +): SessionTurnIdentityScannerStateV1 { + if (Buffer.byteLength(encoded, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner state exceeds 64 KiB', + 'incompatible_identity', + ); + } + let value: unknown; + try { + value = JSON.parse(encoded); + } catch (error) { + throw new SessionTurnIdentityScannerError(`invalid persisted scanner state: ${String(error)}`); + } + if (!isScannerState(value)) { + throw new SessionTurnIdentityScannerError('unsupported persisted scanner state'); + } + return value; +} + +export function serializeSessionTurnIdentityScannerState( + state: SessionTurnIdentityScannerStateV1, +): string { + const encoded = JSON.stringify(state); + if (Buffer.byteLength(encoded, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner state exceeds 64 KiB', + 'incompatible_identity', + ); + } + return encoded; +} + +export function advanceSessionTurnIdentityScanner( + state: SessionTurnIdentityScannerStateV1, + fragment: Uint8Array, +): void { + for (const byte of fragment) { + const codePoint = decodeUtf8Byte(state.utf8, byte); + if (codePoint === null) continue; + consumeCodePoint(state, codePoint); + } + serializeSessionTurnIdentityScannerState(state); +} + +export function completeSessionTurnIdentityScanner( + state: SessionTurnIdentityScannerStateV1, + sqlIdentity: { readonly messageId: string; readonly messageType: string }, +): SessionTurnRecoveredIdentity { + if (state.utf8.needed !== 0) throw new SessionTurnIdentityScannerError('truncated UTF-8'); + if (state.lexical.kind === 'number') { + finishNumber(state); + } else if (state.lexical.kind !== 'default') { + throw new SessionTurnIdentityScannerError('truncated JSON token'); + } + if (!state.rootComplete || state.stack.length !== 0) { + throw new SessionTurnIdentityScannerError('truncated JSON structure'); + } + const { id, type, turnId, turnIdPresent, kind, kindPresent } = state.identity; + if (typeof id !== 'string' || id.length === 0 || typeof type !== 'string' || type.length === 0) { + throw new SessionTurnIdentityScannerError('missing id or type', 'incompatible_identity'); + } + if (id !== sqlIdentity.messageId || type !== sqlIdentity.messageType) { + throw new SessionTurnIdentityScannerError( + 'SQL/body id or type mismatch', + 'incompatible_identity', + ); + } + if (turnIdPresent) { + if (typeof turnId !== 'string' || turnId.length === 0) { + throw new SessionTurnIdentityScannerError( + 'turnId must be a non-empty string', + 'incompatible_identity', + ); + } + return { kind: 'turn', turnId }; + } + if (type !== 'system_note') { + throw new SessionTurnIdentityScannerError( + 'non-system message is missing turnId', + 'incompatible_identity', + ); + } + if (!kindPresent || typeof kind !== 'string' || kind.length === 0) { + throw new SessionTurnIdentityScannerError( + 'turnless system note is missing kind', + 'incompatible_identity', + ); + } + return isUserVisibleSessionSystemNote(kind) + ? { kind: 'note', turnId: `session-note:${id}` } + : { kind: 'ignored' }; +} + +function consumeCodePoint(state: SessionTurnIdentityScannerStateV1, codePoint: number): void { + const character = String.fromCodePoint(codePoint); + switch (state.lexical.kind) { + case 'string': + if (codePoint === 0x22) { + const token = state.lexical; + state.lexical = { kind: 'default' }; + acceptString(state, token.role, token.value); + } else if (codePoint === 0x5c) { + state.lexical = { ...state.lexical, kind: 'escape' }; + } else { + if (codePoint < 0x20) throw new SessionTurnIdentityScannerError('control byte in string'); + appendCaptured(state, character); + } + return; + case 'escape': { + const escaped = escapeCharacter(codePoint); + if (escaped === null) { + if (codePoint !== 0x75) throw new SessionTurnIdentityScannerError('invalid JSON escape'); + state.lexical = { ...state.lexical, kind: 'unicode', digits: '' }; + } else { + const previous = state.lexical; + state.lexical = { + kind: 'string', + role: previous.role, + capture: previous.capture, + value: previous.value, + }; + appendCaptured(state, escaped); + } + return; + } + case 'unicode': { + if (!/[0-9a-f]/iu.test(character)) { + throw new SessionTurnIdentityScannerError('invalid Unicode escape'); + } + state.lexical.digits += character; + if (state.lexical.digits.length === 4) { + const previous = state.lexical; + state.lexical = { + kind: 'string', + role: previous.role, + capture: previous.capture, + value: previous.value, + }; + appendCaptured(state, String.fromCharCode(Number.parseInt(previous.digits, 16))); + } + return; + } + case 'number': + if (/[0-9eE+.-]/u.test(character)) { + if (state.lexical.value.length >= 128) { + throw new SessionTurnIdentityScannerError( + 'number token is too long', + 'incompatible_identity', + ); + } + state.lexical.value += character; + return; + } + finishNumber(state); + consumeCodePoint(state, codePoint); + return; + case 'literal': + if (character !== state.lexical.expected[state.lexical.offset]) { + throw new SessionTurnIdentityScannerError('invalid JSON literal'); + } + state.lexical.offset += 1; + if (state.lexical.offset === state.lexical.expected.length) { + state.lexical = { kind: 'default' }; + acceptPrimitive(state, 'non-string'); + } + return; + case 'default': + break; + } + + if (codePoint === 0x20 || codePoint === 0x09 || codePoint === 0x0a || codePoint === 0x0d) return; + if (state.rootComplete) throw new SessionTurnIdentityScannerError('trailing JSON data'); + if (!state.rootStarted) { + if (character !== '{') { + throw new SessionTurnIdentityScannerError('root must be an object', 'incompatible_identity'); + } + state.rootStarted = true; + pushContainer(state, { kind: 'object', expectation: 'key_or_end', pendingKey: null }); + return; + } + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('invalid JSON structure'); + if (character === '}' || character === ']') { + closeContainer(state, character); + return; + } + if (character === ',') { + if (current.expectation !== 'comma_or_end') { + throw new SessionTurnIdentityScannerError('unexpected comma'); + } + current.expectation = current.kind === 'object' ? 'key' : 'value'; + if (current.kind === 'object') current.pendingKey = null; + return; + } + if (character === ':') { + if (current.kind !== 'object' || current.expectation !== 'colon') { + throw new SessionTurnIdentityScannerError('unexpected colon'); + } + current.expectation = 'value'; + return; + } + if (character === '"') { + const role = + current.kind === 'object' && + (current.expectation === 'key_or_end' || current.expectation === 'key') + ? 'key' + : 'value'; + if (role === 'value') requireValueExpectation(current); + const capture = + (role === 'key' && state.stack.length === 1) || + (role === 'value' && + state.stack.length === 1 && + current.kind === 'object' && + current.pendingKey !== null && + IDENTITY_KEYS.has(current.pendingKey)); + state.lexical = { kind: 'string', role, capture, value: '' }; + return; + } + if (character === '{' || character === '[') { + requireValueExpectation(current); + acceptPrimitive(state, 'container'); + pushContainer( + state, + character === '{' + ? { kind: 'object', expectation: 'key_or_end', pendingKey: null } + : { kind: 'array', expectation: 'value_or_end' }, + ); + return; + } + requireValueExpectation(current); + if (character === '-' || /[0-9]/u.test(character)) { + state.lexical = { kind: 'number', value: character }; + return; + } + if (character === 't' || character === 'f' || character === 'n') { + state.lexical = { + kind: 'literal', + expected: character === 't' ? 'true' : character === 'f' ? 'false' : 'null', + offset: 1, + }; + return; + } + throw new SessionTurnIdentityScannerError('invalid JSON token'); +} + +function acceptString( + state: SessionTurnIdentityScannerStateV1, + role: 'key' | 'value', + value: string, +): void { + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('string outside root'); + if (role === 'key') { + if ( + current.kind !== 'object' || + (current.expectation !== 'key_or_end' && current.expectation !== 'key') + ) { + throw new SessionTurnIdentityScannerError('unexpected object key'); + } + current.pendingKey = state.stack.length === 1 ? value : null; + current.expectation = 'colon'; + return; + } + if (current.kind === 'object' && state.stack.length === 1 && current.pendingKey) { + recordIdentityValue(state, current.pendingKey, value); + } + acceptPrimitive(state, 'string'); +} + +function recordIdentityValue( + state: SessionTurnIdentityScannerStateV1, + key: string, + value: string, +): void { + if (!IDENTITY_KEYS.has(key)) return; + if (state.seenIdentityKeys.includes(key)) { + throw new SessionTurnIdentityScannerError( + `duplicate top-level ${key}`, + 'incompatible_identity', + ); + } + state.seenIdentityKeys.push(key); + state.capturedBytes += Buffer.byteLength(value, 'utf8'); + if (state.capturedBytes > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'captured identity exceeds 32 KiB', + 'incompatible_identity', + ); + } + if (key === 'turnId') state.identity.turnIdPresent = true; + if (key === 'kind') state.identity.kindPresent = true; + state.identity[key as 'id' | 'type' | 'turnId' | 'kind'] = value; +} + +function acceptPrimitive( + state: SessionTurnIdentityScannerStateV1, + token: 'string' | 'non-string' | 'container', +): void { + const current = state.stack.at(-1); + if (!current) throw new SessionTurnIdentityScannerError('value outside root'); + requireValueExpectation(current); + if ( + token !== 'string' && + current.kind === 'object' && + state.stack.length === 1 && + current.pendingKey !== null && + IDENTITY_KEYS.has(current.pendingKey) + ) { + throw new SessionTurnIdentityScannerError( + `${current.pendingKey} must be a string`, + 'incompatible_identity', + ); + } + current.expectation = 'comma_or_end'; +} + +function requireValueExpectation(current: ContainerState): void { + const valid = + (current.kind === 'object' && current.expectation === 'value') || + (current.kind === 'array' && + (current.expectation === 'value_or_end' || current.expectation === 'value')); + if (!valid) throw new SessionTurnIdentityScannerError('unexpected JSON value'); +} + +function pushContainer(state: SessionTurnIdentityScannerStateV1, container: ContainerState): void { + if (state.stack.length >= SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH) { + throw new SessionTurnIdentityScannerError('JSON nesting exceeds 4096', 'incompatible_identity'); + } + state.stack.push(container); +} + +function closeContainer(state: SessionTurnIdentityScannerStateV1, character: string): void { + const current = state.stack.at(-1); + if (!current || (character === '}' ? current.kind !== 'object' : current.kind !== 'array')) { + throw new SessionTurnIdentityScannerError('mismatched JSON container'); + } + const mayClose = + current.kind === 'object' + ? current.expectation === 'key_or_end' || current.expectation === 'comma_or_end' + : current.expectation === 'value_or_end' || current.expectation === 'comma_or_end'; + if (!mayClose) throw new SessionTurnIdentityScannerError('incomplete JSON container'); + state.stack.pop(); + if (state.stack.length === 0) state.rootComplete = true; +} + +function finishNumber(state: SessionTurnIdentityScannerStateV1): void { + if (state.lexical.kind !== 'number') return; + if (!/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/u.test(state.lexical.value)) { + throw new SessionTurnIdentityScannerError('invalid JSON number'); + } + state.lexical = { kind: 'default' }; + acceptPrimitive(state, 'non-string'); +} + +function appendCaptured(state: SessionTurnIdentityScannerStateV1, value: string): void { + const lexical = state.lexical; + if (lexical.kind !== 'string' && lexical.kind !== 'escape' && lexical.kind !== 'unicode') { + throw new SessionTurnIdentityScannerError('invalid string scanner state'); + } + if (!lexical.capture) return; + if (lexical.role === 'key' && state.stack.length === 1) { + const candidate = lexical.value + value; + if (![...IDENTITY_KEYS].some((key) => key.startsWith(candidate))) { + lexical.capture = false; + lexical.value = ''; + return; + } + } + lexical.value += value; + if (Buffer.byteLength(lexical.value, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES) { + throw new SessionTurnIdentityScannerError( + 'scanner capture exceeds 32 KiB', + 'incompatible_identity', + ); + } +} + +function escapeCharacter(codePoint: number): string | null { + switch (codePoint) { + case 0x22: + return '"'; + case 0x5c: + return '\\'; + case 0x2f: + return '/'; + case 0x62: + return '\b'; + case 0x66: + return '\f'; + case 0x6e: + return '\n'; + case 0x72: + return '\r'; + case 0x74: + return '\t'; + default: + return null; + } +} + +function decodeUtf8Byte(state: Utf8State, byte: number): number | null { + if (state.needed === 0) { + if (byte <= 0x7f) return byte; + if (byte >= 0xc2 && byte <= 0xdf) { + state.needed = 1; + state.codePoint = byte & 0x1f; + state.minimum = 0x80; + return null; + } + if (byte >= 0xe0 && byte <= 0xef) { + state.needed = 2; + state.codePoint = byte & 0x0f; + state.minimum = 0x800; + return null; + } + if (byte >= 0xf0 && byte <= 0xf4) { + state.needed = 3; + state.codePoint = byte & 0x07; + state.minimum = 0x10000; + return null; + } + throw new SessionTurnIdentityScannerError('invalid UTF-8 lead byte'); + } + if (byte < 0x80 || byte > 0xbf) { + throw new SessionTurnIdentityScannerError('invalid UTF-8 continuation byte'); + } + state.codePoint = (state.codePoint << 6) | (byte & 0x3f); + state.needed -= 1; + if (state.needed > 0) return null; + const codePoint = state.codePoint; + const minimum = state.minimum; + state.codePoint = 0; + state.minimum = 0; + if (codePoint < minimum || codePoint > 0x10ffff || (codePoint >= 0xd800 && codePoint <= 0xdfff)) { + throw new SessionTurnIdentityScannerError('invalid UTF-8 scalar'); + } + return codePoint; +} + +function isScannerState(value: unknown): value is SessionTurnIdentityScannerStateV1 { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if ( + !( + candidate.version === SESSION_TURN_IDENTITY_SCANNER_VERSION && + typeof candidate.rootStarted === 'boolean' && + typeof candidate.rootComplete === 'boolean' && + Array.isArray(candidate.stack) && + candidate.stack.length <= SESSION_TURN_IDENTITY_SCANNER_MAX_DEPTH && + candidate.stack.every(isContainerState) && + isLexicalState(candidate.lexical) && + isUtf8State(candidate.utf8) && + Array.isArray(candidate.seenIdentityKeys) && + candidate.seenIdentityKeys.every( + (key): key is string => typeof key === 'string' && IDENTITY_KEYS.has(key), + ) && + new Set(candidate.seenIdentityKeys).size === candidate.seenIdentityKeys.length && + typeof candidate.identity === 'object' && + candidate.identity !== null && + isOptionalString(candidate.identity.id) && + isOptionalString(candidate.identity.type) && + isOptionalString(candidate.identity.turnId) && + typeof candidate.identity.turnIdPresent === 'boolean' && + isOptionalString(candidate.identity.kind) && + typeof candidate.identity.kindPresent === 'boolean' && + Number.isSafeInteger(candidate.capturedBytes) && + (candidate.capturedBytes ?? -1) >= 0 && + (candidate.capturedBytes ?? 0) <= SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES + ) + ) { + return false; + } + const state = candidate as SessionTurnIdentityScannerStateV1; + if ( + (!state.rootStarted && (state.rootComplete || state.stack.length !== 0)) || + (state.rootComplete && state.stack.length !== 0) || + (!state.rootComplete && state.rootStarted && state.stack.length === 0) || + state.identity.turnIdPresent !== (state.identity.turnId !== undefined) || + state.identity.kindPresent !== (state.identity.kind !== undefined) + ) { + return false; + } + const capturedBytes = ['id', 'type', 'turnId', 'kind'].reduce((total, key) => { + const captured = state.identity[key as keyof typeof state.identity]; + return total + (typeof captured === 'string' ? Buffer.byteLength(captured, 'utf8') : 0); + }, 0); + return capturedBytes === state.capturedBytes; +} + +function isContainerState(value: unknown): value is ContainerState { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if (candidate.kind === 'array') { + return ( + candidate.expectation === 'value_or_end' || + candidate.expectation === 'value' || + candidate.expectation === 'comma_or_end' + ); + } + return ( + candidate.kind === 'object' && + (candidate.expectation === 'key_or_end' || + candidate.expectation === 'key' || + candidate.expectation === 'colon' || + candidate.expectation === 'value' || + candidate.expectation === 'comma_or_end') && + (candidate.pendingKey === null || typeof candidate.pendingKey === 'string') + ); +} + +function isLexicalState(value: unknown): value is LexicalState { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial & Record; + if (candidate.kind === 'default') return true; + if (candidate.kind === 'number') { + return typeof candidate.value === 'string' && candidate.value.length <= 128; + } + if (candidate.kind === 'literal') { + return ( + (candidate.expected === 'true' || + candidate.expected === 'false' || + candidate.expected === 'null') && + Number.isSafeInteger(candidate.offset) && + (candidate.offset as number) >= 1 && + (candidate.offset as number) < candidate.expected.length + ); + } + if (candidate.kind !== 'string' && candidate.kind !== 'escape' && candidate.kind !== 'unicode') { + return false; + } + if ( + (candidate.role !== 'key' && candidate.role !== 'value') || + typeof candidate.capture !== 'boolean' || + typeof candidate.value !== 'string' || + Buffer.byteLength(candidate.value, 'utf8') > SESSION_TURN_IDENTITY_SCANNER_MAX_CAPTURE_BYTES + ) { + return false; + } + return ( + candidate.kind !== 'unicode' || + (typeof candidate.digits === 'string' && + candidate.digits.length <= 3 && + /^[0-9a-f]*$/iu.test(candidate.digits)) + ); +} + +function isUtf8State(value: unknown): value is Utf8State { + if (typeof value !== 'object' || value === null) return false; + const candidate = value as Partial; + if ( + !Number.isSafeInteger(candidate.needed) || + (candidate.needed ?? -1) < 0 || + (candidate.needed ?? 4) > 3 || + !Number.isSafeInteger(candidate.codePoint) || + (candidate.codePoint ?? -1) < 0 || + !Number.isSafeInteger(candidate.minimum) || + (candidate.minimum ?? -1) < 0 + ) { + return false; + } + return candidate.needed === 0 + ? candidate.codePoint === 0 && candidate.minimum === 0 + : candidate.minimum === 0x80 || candidate.minimum === 0x800 || candidate.minimum === 0x10000; +} + +function isOptionalString(value: unknown): value is string | undefined { + return value === undefined || typeof value === 'string'; +} diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts new file mode 100644 index 0000000000..7388a0d12b --- /dev/null +++ b/packages/storage/src/session-turn-position-authority.ts @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { isUserVisibleSessionSystemNote, type StoredMessage } from '@maka/core/session'; +import { SessionTurnPositionRecoveryError } from './session-store.js'; + +export function recordAppendedSessionTurnMetadata( + db: DatabaseSync, + sessionId: string, + firstSequence: number, + messages: readonly StoredMessage[], + encodedByteLengths: readonly number[], +): void { + if (!tableExists(db, 'session_turn_metadata') || messages.length === 0) return; + if (messages.length !== encodedByteLengths.length) { + throw new Error('Session Turn metadata byte evidence is incomplete'); + } + ensureTurnIndexRows(db, sessionId); + const fillsBodylessAdmission = messages.some((message) => { + const turnId = (message as { turnId?: unknown }).turnId; + if (typeof turnId !== 'string' || turnId.length === 0) return false; + return Boolean( + db + .prepare(` + SELECT 1 FROM session_turn_metadata + WHERE session_id = ? AND turn_id = ? AND order_source = 'admission' + AND first_sequence IS NULL + `) + .get(sessionId, turnId), + ); + }); + for (let index = 0; index < messages.length; index += 1) { + publishCanonicalMembership(db, sessionId, firstSequence + index, messages[index]!); + } + const state = db + .prepare('SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ?') + .get(sessionId) as { indexed_through_sequence: number }; + if (state.indexed_through_sequence === firstSequence - 1) { + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = ? + WHERE session_id = ? + `).run(firstSequence + messages.length - 1, sessionId); + } + if (fillsBodylessAdmission) invalidateBuildingSnapshots(db, sessionId); +} + +export function recordRootTurnAdmissionForPositionIndex( + db: DatabaseSync, + sessionId: string, + turnId: string, + admittedAt: number, +): void { + if (!tableExists(db, 'session_turn_metadata')) return; + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; + if (!turnId || !Number.isSafeInteger(admittedAt) || admittedAt < 0) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity'); + } + ensureTurnIndexRows(db, sessionId); + const current = db + .prepare(` + SELECT identity_kind, order_source, admitted_at, first_sequence + FROM session_turn_metadata WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, turnId) as + | { + identity_kind: string; + order_source: string; + admitted_at: number | null; + first_sequence: number | null; + } + | undefined; + if (current && current.identity_kind !== 'turn') { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); + } + if (current?.order_source === 'admission' && current.admitted_at !== admittedAt) { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); + } + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence + ) VALUES (?, ?, 'turn', 'admission', ?, NULL) + ON CONFLICT(session_id, turn_id) DO UPDATE SET + order_source = 'admission', admitted_at = excluded.admitted_at + `).run(sessionId, turnId, admittedAt); + db.prepare(` + UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' + `).run(sessionId); + invalidateBuildingSnapshots(db, sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function recordRootTurnAdmissionsPurgedForPositionIndex( + db: DatabaseSync, + sessionId: string, +): void { + if (!tableExists(db, 'session_turn_metadata')) return; + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; + ensureTurnIndexRows(db, sessionId); + const removed = db + .prepare(` + DELETE FROM session_turn_metadata + WHERE session_id = ? AND identity_kind = 'turn' + AND order_source = 'admission' AND first_sequence IS NULL + `) + .run(sessionId); + const downgraded = db + .prepare(` + UPDATE session_turn_metadata SET order_source = 'legacy', admitted_at = NULL + WHERE session_id = ? AND identity_kind = 'turn' + AND order_source = 'admission' AND first_sequence IS NOT NULL + `) + .run(sessionId); + if (removed.changes === 0 && downgraded.changes === 0) return; + db.prepare(` + UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' + `).run(sessionId); + invalidateBuildingSnapshots(db, sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: string): void { + if (!tableExists(db, 'session_turn_metadata')) return; + ensureTurnIndexRows(db, sessionId); + db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? + `).run(sessionId); + advanceAuthorityRevision(db, sessionId); +} + +export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { + db.prepare(` + INSERT INTO session_turn_authority_revisions(session_id) + VALUES (?) ON CONFLICT(session_id) DO NOTHING + `).run(sessionId); + db.prepare(` + INSERT INTO session_turn_index_state(session_id) + VALUES (?) ON CONFLICT(session_id) DO NOTHING + `).run(sessionId); +} + +function publishCanonicalMembership( + db: DatabaseSync, + sessionId: string, + sequence: number, + message: StoredMessage, +): void { + const explicitTurnId = (message as { turnId?: unknown }).turnId; + let turnId: string; + let identityKind: 'turn' | 'note'; + if (typeof explicitTurnId === 'string') { + if (explicitTurnId.length === 0) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); + } + turnId = explicitTurnId; + identityKind = 'turn'; + } else if (message.type === 'system_note') { + if (!message.id || !isUserVisibleSessionSystemNote(message.kind)) return; + turnId = `session-note:${message.id}`; + identityKind = 'note'; + } else { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); + } + const admission = + identityKind === 'turn' && tableExists(db, 'core_root_turn_admissions') + ? (db + .prepare(` + SELECT admitted_at FROM core_root_turn_admissions + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, turnId) as { admitted_at: number } | undefined) + : undefined; + const existingIdentity = db + .prepare(` + SELECT identity_kind FROM session_turn_metadata + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, turnId) as { identity_kind: string } | undefined; + if (existingIdentity && existingIdentity.identity_kind !== identityKind) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); + } + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, turn_id) DO UPDATE SET + order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' + ELSE session_turn_metadata.order_source END, + admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), + first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL + THEN excluded.first_sequence + ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END + `).run( + sessionId, + turnId, + identityKind, + admission ? 'admission' : 'legacy', + admission?.admitted_at ?? null, + sequence, + ); + const existing = db + .prepare(` + SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? + `) + .get(sessionId, sequence) as { turn_id: string } | undefined; + if (existing && existing.turn_id !== turnId) { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source', sequence); + } + db.prepare(` + INSERT INTO session_turn_memberships(session_id, sequence, turn_id) + VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING + `).run(sessionId, sequence, turnId); +} + +function invalidateBuildingSnapshots(db: DatabaseSync, sessionId: string): void { + db.prepare(` + DELETE FROM session_turn_position_snapshots WHERE session_id = ? AND state = 'building' + `).run(sessionId); +} + +function advanceAuthorityRevision(db: DatabaseSync, sessionId: string): void { + db.prepare(` + UPDATE session_turn_authority_revisions + SET authority_revision = authority_revision + 1 WHERE session_id = ? + `).run(sessionId); +} + +function tableExists(db: DatabaseSync, table: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), + ); +} diff --git a/packages/storage/src/session-turn-position-index.ts b/packages/storage/src/session-turn-position-index.ts new file mode 100644 index 0000000000..95c452bc32 --- /dev/null +++ b/packages/storage/src/session-turn-position-index.ts @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Compatibility barrel for the package-private PR1 seam. Responsibilities +// live in separate authority, recovery, and immutable snapshot modules. +export { + ensureTurnIndexRows, + invalidateSessionTurnPositionIndex, + recordAppendedSessionTurnMetadata, + recordRootTurnAdmissionForPositionIndex, + recordRootTurnAdmissionsPurgedForPositionIndex, +} from './session-turn-position-authority.js'; +export { + SESSION_TURN_POSITION_BODY_MAX_BYTES, + SESSION_TURN_POSITION_BODY_MAX_RECORDS, + SESSION_TURN_POSITION_BODY_MAX_TURNS, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + SESSION_TURN_POSITION_MAX_PAGE_BYTES, + SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, + advanceSessionTurnPositionOrdinalBuild, + allocateOrRequireSessionTurnPositionSnapshot, + markSessionTurnRecoveryComplete, + pageReadySessionTurnPositionSnapshot, + readSessionTurnMembershipPreflight, + reclaimSessionTurnPositionSnapshotsForNewOwner, + releaseSessionTurnPositionSnapshot, + requireSnapshot, + snapshotKeyFromRow, + type SessionTurnPositionBuildPhase, + type SessionTurnPositionSnapshotRow, +} from './session-turn-position-snapshots.js'; diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts new file mode 100644 index 0000000000..d644394264 --- /dev/null +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -0,0 +1,785 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { + SessionTurnPositionAnchorNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, + SessionTurnPositionSnapshotMismatchError, + type SessionTranscriptRecordsByTurnIdsSnapshotRequest, + type SessionTurnPositionPageSnapshotRequest, + type SessionTurnPositionReadResult, + type SessionTurnPositionSnapshotKey, +} from './session-store.js'; +import { ensureTurnIndexRows } from './session-turn-position-authority.js'; + +export const SESSION_TURN_POSITION_MAX_PAGE_POSITIONS = 128; +export const SESSION_TURN_POSITION_MAX_PAGE_BYTES = 64 * 1024; +export const SESSION_TURN_POSITION_BUILD_MAX_POSITIONS = 1_024; +export const SESSION_TURN_POSITION_BODY_MAX_TURNS = 128; +export const SESSION_TURN_POSITION_BODY_MAX_RECORDS = 256; +export const SESSION_TURN_POSITION_BODY_MAX_BYTES = 16 * 1024 * 1024; + +export type SessionTurnPositionBuildPhase = + | 'recovering' + | 'legacy' + | 'admission' + | 'notes' + | 'ready'; + +export interface SessionTurnPositionSnapshotRow { + readonly slot: number; + readonly through_sequence: number | null; + readonly authority_revision: number; + readonly snapshot_generation: number; + readonly state: 'building' | 'ready'; + readonly build_phase: SessionTurnPositionBuildPhase; + readonly build_next_ordinal: number; + readonly build_cursor_sequence: number; + readonly build_cursor_admitted_at: number | null; + readonly build_cursor_turn_id: string | null; + readonly ready_total: number | null; +} + +export type SessionTurnPositionAllocation = + | { readonly kind: 'snapshot'; readonly snapshot: SessionTurnPositionSnapshotRow } + | Extract; + +export interface SessionTurnPositionBuildStep { + readonly snapshot: SessionTurnPositionSnapshotRow; + readonly executedPhase: Exclude; + readonly lastStepPositions: number; +} + +export interface SessionTurnMembershipPreflight { + readonly sequences: readonly number[]; + readonly storedBytes: number; +} + +export function allocateOrRequireSessionTurnPositionSnapshot( + db: DatabaseSync, + request: SessionTurnPositionPageSnapshotRequest, +): SessionTurnPositionAllocation { + validatePageRequest(request); + ensureSessionExists(db, request.sessionId); + ensureTurnIndexRows(db, request.sessionId); + const actualThrough = readHighWater(db, request.sessionId); + if (request.snapshotKey) { + return { + kind: 'snapshot', + snapshot: requireLeasedSnapshot( + db, + request.sessionId, + request.snapshotLeaseId, + request.snapshotKey, + ), + }; + } + const throughSequence = + request.throughSequence === undefined ? actualThrough : request.throughSequence; + if ( + throughSequence !== null && + (!Number.isSafeInteger(throughSequence) || + throughSequence < 0 || + actualThrough === null || + throughSequence > actualThrough) + ) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const authorityRevision = readAuthorityRevision(db, request.sessionId); + const existing = findSnapshotByAuthority( + db, + request.sessionId, + throughSequence, + authorityRevision, + ); + if (existing) { + acquireLease(db, request.sessionId, existing.snapshot_generation, request.snapshotLeaseId); + return { kind: 'snapshot', snapshot: existing }; + } + const occupied = db + .prepare('SELECT slot FROM session_turn_position_snapshots WHERE session_id = ? ORDER BY slot') + .all(request.sessionId) as Array<{ slot: number }>; + if (occupied.length >= 2) { + return { kind: 'capacity', throughSequence, authorityRevision, retainedSnapshots: 2 }; + } + const used = new Set(occupied.map(({ slot }) => slot)); + const slot = used.has(0) ? 1 : 0; + const generation = ( + db + .prepare(` + UPDATE session_turn_authority_revisions + SET next_snapshot_generation = next_snapshot_generation + 1 + WHERE session_id = ? + RETURNING next_snapshot_generation - 1 AS generation + `) + .get(request.sessionId) as { generation: number } + ).generation; + db.prepare(` + INSERT INTO session_turn_position_snapshots( + session_id, slot, through_sequence, authority_revision, snapshot_generation, + state, build_phase + ) VALUES (?, ?, ?, ?, ?, 'building', 'recovering') + `).run(request.sessionId, slot, throughSequence, authorityRevision, generation); + acquireLease(db, request.sessionId, generation, request.snapshotLeaseId); + return { + kind: 'snapshot', + snapshot: requireSnapshot(db, request.sessionId, { + throughSequence, + authorityRevision, + snapshotGeneration: generation, + }), + }; +} + +export function markSessionTurnRecoveryComplete( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnPositionSnapshotRow { + if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') return snapshot; + const indexed = ( + db + .prepare(` + SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ? + `) + .get(sessionId) as { indexed_through_sequence: number } + ).indexed_through_sequence; + if (snapshot.through_sequence !== null && indexed < snapshot.through_sequence) return snapshot; + const updated = db + .prepare(` + UPDATE session_turn_position_snapshots SET build_phase = 'legacy' + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = 'recovering' + `) + .run(sessionId, snapshot.slot); + if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + return requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)); +} + +export function advanceSessionTurnPositionOrdinalBuild( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnPositionBuildStep { + if (snapshot.state !== 'building' || snapshot.build_phase === 'recovering') { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + const executedPhase = snapshot.build_phase; + if (executedPhase === 'ready') throw new SessionTurnPositionSnapshotMismatchError(sessionId); + const boundary = validateHybridBoundary(db, sessionId, snapshot.through_sequence); + const rows = readBuildRows(db, sessionId, snapshot, boundary, executedPhase); + const insert = db.prepare(` + INSERT INTO session_turn_snapshot_positions( + session_id, snapshot_generation, ordinal, turn_id, first_sequence + ) VALUES (?, ?, ?, ?, ?) + `); + let nextOrdinal = snapshot.build_next_ordinal; + let sequenceCursor = snapshot.build_cursor_sequence; + let admittedAtCursor = snapshot.build_cursor_admitted_at; + let turnIdCursor = snapshot.build_cursor_turn_id; + for (const row of rows) { + insert.run( + sessionId, + snapshot.snapshot_generation, + nextOrdinal, + row.turn_id, + row.first_sequence, + ); + nextOrdinal += 1; + sequenceCursor = row.first_sequence ?? sequenceCursor; + admittedAtCursor = row.admitted_at; + turnIdCursor = row.turn_id; + } + const exhausted = rows.length < SESSION_TURN_POSITION_BUILD_MAX_POSITIONS; + const nextPhase = exhausted ? followingPhase(executedPhase) : executedPhase; + if (nextPhase === 'ready') { + if (nextOrdinal === 0) { + insert.run(sessionId, snapshot.snapshot_generation, 0, `session:${sessionId}`, null); + nextOrdinal = 1; + } + const published = db + .prepare(` + UPDATE session_turn_position_snapshots + SET state = 'ready', build_phase = 'ready', build_next_ordinal = ?, ready_total = ? + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? + `) + .run(nextOrdinal, nextOrdinal, sessionId, snapshot.slot, executedPhase); + if (published.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } else { + const phaseChanged = nextPhase !== executedPhase; + const updated = db + .prepare(` + UPDATE session_turn_position_snapshots + SET build_phase = ?, build_next_ordinal = ?, build_cursor_sequence = ?, + build_cursor_admitted_at = ?, build_cursor_turn_id = ? + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? + `) + .run( + nextPhase, + nextOrdinal, + phaseChanged ? -1 : sequenceCursor, + phaseChanged ? null : admittedAtCursor, + phaseChanged ? null : turnIdCursor, + sessionId, + snapshot.slot, + executedPhase, + ); + if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + return { + snapshot: requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)), + executedPhase, + lastStepPositions: rows.length, + }; +} + +export function pageReadySessionTurnPositionSnapshot( + db: DatabaseSync, + request: SessionTurnPositionPageSnapshotRequest, + snapshot: SessionTurnPositionSnapshotRow, +): Extract { + validatePageRequest(request); + requireLease(db, request.sessionId, request.snapshotLeaseId, snapshot.snapshot_generation); + const totalTurns = snapshot.ready_total; + if (snapshot.state !== 'ready' || totalTurns === null || totalTurns < 1) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + let selectedOrdinal: number; + switch (request.anchor.kind) { + case 'tail': + selectedOrdinal = totalTurns - 1; + break; + case 'ordinal': + selectedOrdinal = Math.min(request.anchor.ordinal, totalTurns - 1); + break; + case 'sequence': { + const row = db + .prepare(` + SELECT ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND first_sequence <= ? + ORDER BY first_sequence DESC LIMIT 1 + `) + .get(request.sessionId, snapshot.snapshot_generation, request.anchor.sequence) as + | { ordinal: number } + | undefined; + selectedOrdinal = row?.ordinal ?? 0; + break; + } + case 'turn': { + const row = db + .prepare(` + SELECT ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ? + `) + .get(request.sessionId, snapshot.snapshot_generation, request.anchor.turnId) as + | { ordinal: number } + | undefined; + if (!row) { + throw new SessionTurnPositionAnchorNotFoundError(request.sessionId, request.anchor.turnId); + } + selectedOrdinal = row.ordinal; + break; + } + } + let startOrdinal = + request.anchor.kind === 'tail' + ? Math.max(0, totalTurns - request.maxPositions) + : selectedOrdinal; + const rows = db + .prepare(` + SELECT ordinal, turn_id, first_sequence FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND ordinal >= ? + ORDER BY ordinal LIMIT ? + `) + .all( + request.sessionId, + snapshot.snapshot_generation, + startOrdinal, + request.maxPositions, + ) as Array<{ ordinal: number; turn_id: string; first_sequence: number | null }>; + const expectedRows = Math.min(request.maxPositions, totalTurns - startOrdinal); + if ( + rows.length !== expectedRows || + rows.some((row, index) => row.ordinal !== startOrdinal + index) + ) { + throw new SessionTurnPositionRecoveryError(request.sessionId, 'corrupt_source'); + } + let positions = rows.map((row) => ({ + ordinal: row.ordinal, + turnId: row.turn_id, + firstSequence: row.first_sequence, + })); + const key = snapshotKeyFromRow(snapshot); + while ( + positions.length > 0 && + Buffer.byteLength( + JSON.stringify({ + kind: 'page', + snapshotKey: key, + startOrdinal, + totalTurns, + positions, + hasOlder: startOrdinal > 0, + hasNewer: startOrdinal + positions.length < totalTurns, + }), + 'utf8', + ) > SESSION_TURN_POSITION_MAX_PAGE_BYTES + ) { + if (request.anchor.kind === 'tail') { + positions = positions.slice(1); + startOrdinal += 1; + } else positions = positions.slice(0, -1); + } + if (positions.length === 0) { + throw new SessionTurnPositionLimitError(request.sessionId, 'page_metadata_bytes'); + } + return { + kind: 'page', + snapshotKey: key, + startOrdinal, + totalTurns, + positions, + hasOlder: startOrdinal > 0, + hasNewer: startOrdinal + positions.length < totalTurns, + }; +} + +export function readSessionTurnMembershipPreflight( + db: DatabaseSync, + request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, +): SessionTurnMembershipPreflight { + validateRecordRequest(request); + const snapshot = requireLeasedSnapshot( + db, + request.sessionId, + request.snapshotLeaseId, + request.snapshotKey, + ); + if (snapshot.state !== 'ready') + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + const unique = new Set(request.turnIds); + if (unique.size !== request.turnIds.length) { + throw new Error('Session Turn-position record Turn ids must be unique'); + } + const sequences: number[] = []; + let storedBytes = 0; + for (const turnId of request.turnIds) { + const projected = db + .prepare(` + SELECT 1 FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ? + `) + .get(request.sessionId, snapshot.snapshot_generation, turnId); + if (!projected) throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + if (request.snapshotKey.throughSequence === null) continue; + const remaining = request.maxRecords - sequences.length; + const rows = db + .prepare(` + SELECT membership.sequence, + coalesce(payload.record_bytes, length(CAST(message.record_json AS BLOB))) AS stored_bytes + FROM session_turn_memberships AS membership + INNER JOIN session_messages AS message + ON message.session_id = membership.session_id AND message.sequence = membership.sequence + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE membership.session_id = ? AND membership.turn_id = ? + AND membership.sequence <= ? + ORDER BY membership.sequence LIMIT ? + `) + .all(request.sessionId, turnId, request.snapshotKey.throughSequence, remaining + 1) as Array<{ + sequence: number; + stored_bytes: number; + }>; + if (rows.length > remaining) { + throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_count'); + } + for (const row of rows) { + if (!Number.isSafeInteger(row.stored_bytes) || row.stored_bytes < 1) { + throw new SessionTurnPositionRecoveryError( + request.sessionId, + 'corrupt_source', + row.sequence, + ); + } + storedBytes += row.stored_bytes; + if (storedBytes > request.maxBytes) { + throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_bytes'); + } + sequences.push(row.sequence); + } + } + sequences.sort((left, right) => left - right); + if (new Set(sequences).size !== sequences.length) { + throw new SessionTurnPositionRecoveryError(request.sessionId, 'corrupt_source'); + } + return { sequences, storedBytes }; +} + +export function releaseSessionTurnPositionSnapshot( + db: DatabaseSync, + sessionId: string, + snapshotLeaseId: string, + key: SessionTurnPositionSnapshotKey, +): void { + validateLeaseId(snapshotLeaseId); + const snapshot = requireLeasedSnapshot(db, sessionId, snapshotLeaseId, key); + const released = db + .prepare(` + DELETE FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? AND lease_id = ? + `) + .run(sessionId, snapshot.snapshot_generation, snapshotLeaseId); + if (released.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + const retained = db + .prepare(` + SELECT 1 FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? LIMIT 1 + `) + .get(sessionId, snapshot.snapshot_generation); + if (!retained) { + db.prepare(` + DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ? + `).run(sessionId, snapshot.snapshot_generation); + } +} + +export function reclaimSessionTurnPositionSnapshotsForNewOwner(db: DatabaseSync): void { + if (!tableExists(db, 'session_turn_position_snapshots')) return; + db.prepare('DELETE FROM session_turn_position_snapshots').run(); +} + +export function snapshotKeyFromRow( + row: SessionTurnPositionSnapshotRow, +): SessionTurnPositionSnapshotKey { + return { + throughSequence: row.through_sequence, + authorityRevision: row.authority_revision, + snapshotGeneration: row.snapshot_generation, + }; +} + +export function requireSnapshot( + db: DatabaseSync, + sessionId: string, + key: SessionTurnPositionSnapshotKey, +): SessionTurnPositionSnapshotRow { + validateSnapshotKey(key); + const row = db + .prepare(` + SELECT slot, through_sequence, authority_revision, snapshot_generation, state, + build_phase, build_next_ordinal, build_cursor_sequence, + build_cursor_admitted_at, build_cursor_turn_id, ready_total + FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ? + `) + .get(sessionId, key.snapshotGeneration) as SessionTurnPositionSnapshotRow | undefined; + if ( + !row || + row.through_sequence !== key.throughSequence || + row.authority_revision !== key.authorityRevision + ) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + return row; +} + +function requireLeasedSnapshot( + db: DatabaseSync, + sessionId: string, + leaseId: string, + key: SessionTurnPositionSnapshotKey, +): SessionTurnPositionSnapshotRow { + validateLeaseId(leaseId); + const snapshot = requireSnapshot(db, sessionId, key); + requireLease(db, sessionId, leaseId, snapshot.snapshot_generation); + return snapshot; +} + +function requireLease( + db: DatabaseSync, + sessionId: string, + leaseId: string, + generation: number, +): void { + if ( + !db + .prepare(` + SELECT 1 FROM session_turn_snapshot_leases + WHERE session_id = ? AND snapshot_generation = ? AND lease_id = ? + `) + .get(sessionId, generation, leaseId) + ) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} + +function acquireLease( + db: DatabaseSync, + sessionId: string, + generation: number, + leaseId: string, +): void { + validateLeaseId(leaseId); + try { + db.prepare(` + INSERT INTO session_turn_snapshot_leases(session_id, snapshot_generation, lease_id) + VALUES (?, ?, ?) ON CONFLICT(session_id, snapshot_generation, lease_id) DO NOTHING + `).run(sessionId, generation, leaseId); + } catch (error) { + void error; + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} + +function findSnapshotByAuthority( + db: DatabaseSync, + sessionId: string, + throughSequence: number | null, + authorityRevision: number, +): SessionTurnPositionSnapshotRow | undefined { + return db + .prepare(` + SELECT slot, through_sequence, authority_revision, snapshot_generation, state, + build_phase, build_next_ordinal, build_cursor_sequence, + build_cursor_admitted_at, build_cursor_turn_id, ready_total + FROM session_turn_position_snapshots + WHERE session_id = ? + AND ((through_sequence = ?) OR (through_sequence IS NULL AND ? IS NULL)) + AND authority_revision = ? + `) + .get(sessionId, throughSequence, throughSequence, authorityRevision) as + | SessionTurnPositionSnapshotRow + | undefined; +} + +interface BuildRow { + readonly turn_id: string; + readonly first_sequence: number | null; + readonly admitted_at: number | null; +} + +function readBuildRows( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, + boundary: number | null, + phase: Exclude, +): BuildRow[] { + const through = snapshot.through_sequence; + if (phase === 'legacy') { + if (through === null) return []; + return db + .prepare(` + SELECT turn_id, first_sequence, NULL AS admitted_at FROM session_turn_metadata + WHERE session_id = ? AND first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR first_sequence < ?) + AND first_sequence > ? + ORDER BY first_sequence, turn_id LIMIT ? + `) + .all( + sessionId, + through, + boundary, + boundary, + snapshot.build_cursor_sequence, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ) as unknown as BuildRow[]; + } + if (phase === 'admission') { + const watermarkSql = + through === null + ? 'first_sequence IS NULL' + : '(first_sequence IS NULL OR first_sequence <= ?)'; + const parameters: Array = [sessionId]; + if (through !== null) parameters.push(through); + parameters.push( + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_admitted_at, + snapshot.build_cursor_turn_id, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ); + return db + .prepare(` + SELECT turn_id, first_sequence, admitted_at FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission' AND ${watermarkSql} + AND (? IS NULL OR admitted_at > ? OR (admitted_at = ? AND turn_id > ?)) + ORDER BY admitted_at, turn_id LIMIT ? + `) + .all(...parameters) as unknown as BuildRow[]; + } + if (through === null || boundary === null) return []; + return db + .prepare(` + SELECT turn_id, first_sequence, NULL AS admitted_at FROM session_turn_metadata + WHERE session_id = ? AND first_sequence <= ? AND identity_kind = 'note' + AND order_source = 'legacy' AND first_sequence >= ? AND first_sequence > ? + ORDER BY first_sequence, turn_id LIMIT ? + `) + .all( + sessionId, + through, + boundary, + snapshot.build_cursor_sequence, + SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, + ) as unknown as BuildRow[]; +} + +function validateHybridBoundary( + db: DatabaseSync, + sessionId: string, + throughSequence: number | null, +): number | null { + if (throughSequence === null) return null; + const boundary = ( + db + .prepare(` + SELECT MIN(first_sequence) AS boundary FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission' AND first_sequence <= ? + `) + .get(sessionId, throughSequence) as { boundary: number | null } + ).boundary; + if (boundary !== null) { + const hybrid = db + .prepare(` + SELECT first_sequence FROM session_turn_metadata + WHERE session_id = ? AND identity_kind = 'turn' AND order_source = 'legacy' + AND first_sequence >= ? AND first_sequence <= ? + ORDER BY first_sequence LIMIT 1 + `) + .get(sessionId, boundary, throughSequence) as { first_sequence: number } | undefined; + if (hybrid) { + db.prepare(` + UPDATE session_turn_index_state + SET failure_reason = 'hybrid_missing_admission', failure_sequence = ? + WHERE session_id = ? + `).run(hybrid.first_sequence, sessionId); + throw new SessionTurnPositionRecoveryError( + sessionId, + 'hybrid_missing_admission', + hybrid.first_sequence, + ); + } + } + return boundary; +} + +function followingPhase( + phase: Exclude, +): Exclude { + if (phase === 'legacy') return 'admission'; + if (phase === 'admission') return 'notes'; + return 'ready'; +} + +function validatePageRequest(request: SessionTurnPositionPageSnapshotRequest): void { + validateLeaseId(request.snapshotLeaseId); + if ( + !Number.isSafeInteger(request.maxPositions) || + request.maxPositions < 1 || + request.maxPositions > SESSION_TURN_POSITION_MAX_PAGE_POSITIONS || + (request.snapshotKey !== undefined && request.throughSequence !== undefined) + ) { + throw new Error('Invalid Session Turn-position page request'); + } + if ( + request.anchor.kind === 'ordinal' && + (!Number.isSafeInteger(request.anchor.ordinal) || request.anchor.ordinal < 0) + ) { + throw new Error('Invalid Session Turn-position ordinal anchor'); + } + if ( + request.anchor.kind === 'sequence' && + (!Number.isSafeInteger(request.anchor.sequence) || request.anchor.sequence < 0) + ) { + throw new Error('Invalid Session Turn-position sequence anchor'); + } + if (request.anchor.kind === 'turn' && request.anchor.turnId.length === 0) { + throw new Error('Invalid Session Turn-position Turn anchor'); + } +} + +function validateRecordRequest(request: SessionTranscriptRecordsByTurnIdsSnapshotRequest): void { + validateLeaseId(request.snapshotLeaseId); + if ( + request.turnIds.length < 1 || + request.turnIds.length > SESSION_TURN_POSITION_BODY_MAX_TURNS || + request.turnIds.some((turnId) => turnId.length === 0) || + !Number.isSafeInteger(request.maxRecords) || + request.maxRecords < 1 || + request.maxRecords > SESSION_TURN_POSITION_BODY_MAX_RECORDS || + !Number.isSafeInteger(request.maxBytes) || + request.maxBytes < 1 || + request.maxBytes > SESSION_TURN_POSITION_BODY_MAX_BYTES + ) { + throw new Error('Invalid Session Turn-position record request'); + } + validateSnapshotKey(request.snapshotKey); +} + +function validateLeaseId(leaseId: string): void { + if (!/^[A-Za-z0-9_-]{1,128}$/u.test(leaseId)) { + throw new Error('Invalid Session Turn-position snapshot lease id'); + } +} + +function validateSnapshotKey(key: SessionTurnPositionSnapshotKey): void { + if ( + (key.throughSequence !== null && + (!Number.isSafeInteger(key.throughSequence) || key.throughSequence < 0)) || + !Number.isSafeInteger(key.authorityRevision) || + key.authorityRevision < 0 || + !Number.isSafeInteger(key.snapshotGeneration) || + key.snapshotGeneration < 1 + ) { + throw new Error('Invalid Session Turn-position snapshot key'); + } +} + +function readHighWater(db: DatabaseSync, sessionId: string): number | null { + return ( + db + .prepare('SELECT MAX(sequence) AS high_water FROM session_messages WHERE session_id = ?') + .get(sessionId) as { high_water: number | null } + ).high_water; +} + +function readAuthorityRevision(db: DatabaseSync, sessionId: string): number { + return ( + db + .prepare(` + SELECT authority_revision FROM session_turn_authority_revisions WHERE session_id = ? + `) + .get(sessionId) as { authority_revision: number } + ).authority_revision; +} + +function ensureSessionExists(db: DatabaseSync, sessionId: string): void { + if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } +} + +function tableExists(db: DatabaseSync, table: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), + ); +} diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index da49f651a7..a8916a3c59 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -19,7 +19,7 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 34; +export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 35; export const SQLITE_SESSION_MESSAGE_CHUNK_BYTES = 64 * 1024; export const SQLITE_SESSION_MESSAGE_CHUNK_MARKER = '{"$maka":"session-message-chunks-v1"}'; @@ -1225,6 +1225,193 @@ const MIGRATIONS: ReadonlyMap = new Map([ SELECT 1; `, ], + [ + 35, + ` + CREATE TABLE IF NOT EXISTS session_turn_authority_revisions ( + session_id TEXT PRIMARY KEY, + authority_revision INTEGER NOT NULL DEFAULT 0 CHECK (authority_revision >= 0), + next_snapshot_generation INTEGER NOT NULL DEFAULT 1 + CHECK (next_snapshot_generation >= 1), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_index_state ( + session_id TEXT PRIMARY KEY, + indexed_through_sequence INTEGER NOT NULL DEFAULT -1 + CHECK (indexed_through_sequence >= -1), + source_records INTEGER NOT NULL DEFAULT 0 CHECK (source_records >= 0), + source_bytes INTEGER NOT NULL DEFAULT 0 CHECK (source_bytes >= 0), + failure_reason TEXT + CHECK (failure_reason IN ('corrupt_source', 'incompatible_identity', + 'hybrid_missing_admission')), + failure_sequence INTEGER CHECK (failure_sequence >= 0), + CHECK ((failure_reason IS NULL) = (failure_sequence IS NULL)), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_identity_recovery ( + session_id TEXT PRIMARY KEY, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + byte_offset INTEGER NOT NULL CHECK (byte_offset >= 0), + record_bytes INTEGER NOT NULL CHECK (record_bytes > 0), + chunked INTEGER NOT NULL CHECK (chunked IN (0, 1)), + expected_digest TEXT CHECK (expected_digest IS NULL OR length(expected_digest) = 64), + message_id TEXT NOT NULL, + message_type TEXT NOT NULL, + hash_state_version INTEGER NOT NULL CHECK (hash_state_version = 1), + hash_algorithm TEXT NOT NULL CHECK (hash_algorithm = 'sha256'), + hash_implementation TEXT NOT NULL CHECK (hash_implementation = 'hash-wasm@4.12.0'), + hash_state BLOB NOT NULL CHECK (length(hash_state) BETWEEN 1 AND 65536), + scanner_state_version INTEGER NOT NULL CHECK (scanner_state_version = 1), + scanner_state TEXT NOT NULL CHECK (length(CAST(scanner_state AS BLOB)) <= 65536), + derived_state_digest TEXT NOT NULL CHECK (length(derived_state_digest) = 64), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE, + FOREIGN KEY(session_id, sequence) + REFERENCES session_messages(session_id, sequence) ON DELETE CASCADE ON UPDATE CASCADE + ) WITHOUT ROWID; + + CREATE TABLE IF NOT EXISTS session_turn_metadata ( + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + identity_kind TEXT NOT NULL CHECK (identity_kind IN ('turn', 'note', 'synthetic')), + order_source TEXT NOT NULL CHECK (order_source IN ('legacy', 'admission', 'synthetic')), + admitted_at INTEGER CHECK (admitted_at >= 0), + first_sequence INTEGER CHECK (first_sequence >= 0), + PRIMARY KEY(session_id, turn_id), + UNIQUE(session_id, first_sequence), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_legacy_order + ON session_turn_metadata(session_id, first_sequence, turn_id) + WHERE order_source = 'legacy'; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_order + ON session_turn_metadata(session_id, admitted_at, turn_id) + WHERE order_source = 'admission'; + + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_sequence + ON session_turn_metadata(session_id, first_sequence) + WHERE order_source = 'admission'; + + CREATE TABLE IF NOT EXISTS session_turn_memberships ( + session_id TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + turn_id TEXT NOT NULL, + PRIMARY KEY(session_id, sequence), + FOREIGN KEY(session_id, turn_id) + REFERENCES session_turn_metadata(session_id, turn_id) ON DELETE CASCADE, + FOREIGN KEY(session_id, sequence) + REFERENCES session_messages(session_id, sequence) ON DELETE CASCADE ON UPDATE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS session_turn_memberships_by_turn + ON session_turn_memberships(session_id, turn_id, sequence); + + CREATE TABLE IF NOT EXISTS session_turn_position_snapshots ( + session_id TEXT NOT NULL, + slot INTEGER NOT NULL CHECK (slot IN (0, 1)), + through_sequence INTEGER CHECK (through_sequence >= 0), + authority_revision INTEGER NOT NULL CHECK (authority_revision >= 0), + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + state TEXT NOT NULL CHECK (state IN ('building', 'ready')), + build_phase TEXT NOT NULL DEFAULT 'recovering' + CHECK (build_phase IN ('recovering', 'legacy', 'admission', 'notes', 'ready')), + build_next_ordinal INTEGER NOT NULL DEFAULT 0 CHECK (build_next_ordinal >= 0), + build_cursor_sequence INTEGER NOT NULL DEFAULT -1 CHECK (build_cursor_sequence >= -1), + build_cursor_admitted_at INTEGER CHECK (build_cursor_admitted_at >= 0), + build_cursor_turn_id TEXT, + ready_total INTEGER CHECK (ready_total >= 0), + PRIMARY KEY(session_id, slot), + UNIQUE(session_id, snapshot_generation), + UNIQUE(session_id, through_sequence, authority_revision), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_position_snapshots_by_exact_authority + ON session_turn_position_snapshots( + session_id, + COALESCE(through_sequence, -1), + authority_revision + ); + + CREATE TABLE IF NOT EXISTS session_turn_snapshot_positions ( + session_id TEXT NOT NULL, + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + turn_id TEXT NOT NULL, + first_sequence INTEGER CHECK (first_sequence >= 0), + PRIMARY KEY(session_id, snapshot_generation, ordinal), + UNIQUE(session_id, snapshot_generation, turn_id), + FOREIGN KEY(session_id, snapshot_generation) + REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) + ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_sequence + ON session_turn_snapshot_positions( + session_id, + snapshot_generation, + first_sequence, + ordinal + ); + + CREATE TABLE IF NOT EXISTS session_turn_snapshot_leases ( + session_id TEXT NOT NULL, + snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + lease_id TEXT NOT NULL CHECK (length(CAST(lease_id AS BLOB)) BETWEEN 1 AND 128), + PRIMARY KEY(session_id, snapshot_generation, lease_id), + UNIQUE(session_id, lease_id), + FOREIGN KEY(session_id, snapshot_generation) + REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) + ON DELETE CASCADE + ) WITHOUT ROWID; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_insert + BEFORE INSERT ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = NEW.session_id + AND snapshot_generation = NEW.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_update + BEFORE UPDATE ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = OLD.session_id + AND snapshot_generation = OLD.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_positions_no_delete + BEFORE DELETE ON session_turn_snapshot_positions + WHEN EXISTS ( + SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = OLD.session_id + AND snapshot_generation = OLD.snapshot_generation + AND state = 'ready' + ) + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot positions are immutable'); + END; + + CREATE TRIGGER IF NOT EXISTS session_turn_ready_snapshot_no_update + BEFORE UPDATE ON session_turn_position_snapshots + WHEN OLD.state = 'ready' + BEGIN + SELECT RAISE(ABORT, 'ready Session Turn snapshot is immutable'); + END; + `, + ], ]); if (MIGRATIONS.size !== SQLITE_SESSION_METADATA_SCHEMA_VERSION) { diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index aeaa008467..db88e5c900 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -19,6 +19,7 @@ import { createRequire } from 'node:module'; import { createHash } from 'node:crypto'; +import { createSHA256 } from 'hash-wasm'; import { dirname, resolve } from 'node:path'; import { tmpdir } from 'node:os'; import { existsSync, mkdirSync } from 'node:fs'; @@ -134,6 +135,8 @@ import { decodePersistedSessionHeader, normalizeSessionHeader, SessionNotFoundError, + SessionTurnPositionLimitError, + SessionTurnPositionRecoveryError, type ExternalSessionImportLookupResult, type SessionTranscriptMessageLookupRequest, type SessionTranscriptPageRequest, @@ -143,6 +146,11 @@ import { type SessionTurnContribution, type SessionTurnContributionPage, type SessionTurnLandmarkSnapshot, + type SessionTranscriptRecordsByTurnIdsSnapshotRequest, + type SessionTranscriptRecordsByTurnIdsSnapshotResult, + type SessionTurnPositionPageSnapshotRequest, + type SessionTurnPositionReadResult, + type SessionTurnPositionSnapshotKey, } from './session-store.js'; import { isDiscardableConversationCopy, @@ -157,6 +165,19 @@ import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES, SQLITE_SESSION_MESSAGE_CHUNK_MARKER, } from './sqlite-session-metadata-schema.js'; +import { + advanceSessionTurnPositionOrdinalBuild, + allocateOrRequireSessionTurnPositionSnapshot, + invalidateSessionTurnPositionIndex, + markSessionTurnRecoveryComplete, + pageReadySessionTurnPositionSnapshot, + readSessionTurnMembershipPreflight, + recordAppendedSessionTurnMetadata, + releaseSessionTurnPositionSnapshot, + requireSnapshot, + snapshotKeyFromRow, +} from './session-turn-position-index.js'; +import { advanceSessionTurnIdentityRecovery } from './session-turn-identity-recovery.js'; import type { OperationalStateDatabaseLease } from './operational-state-store.js'; import { buildSqliteSessionCatalogPageQuery, @@ -166,6 +187,13 @@ import { sqliteOrdinarySessionRolePredicate, sqliteRecoverableSessionRolePredicate, } from './sqlite-session-role-scope.js'; +import { + readTranscriptSlices, + StoredSessionMessageIncompatibleError, + type TranscriptRecordSlice, +} from './sqlite-session-transcript-slices.js'; + +export { StoredSessionMessageIncompatibleError } from './sqlite-session-transcript-slices.js'; export { SQLITE_SESSION_METADATA_SCHEMA_VERSION } from './sqlite-session-metadata-schema.js'; @@ -405,19 +433,6 @@ export class SessionMetadataConflictError extends Error { readonly name: string = 'SessionMetadataConflictError'; } -export class StoredSessionMessageIncompatibleError extends Error { - readonly name = 'StoredSessionMessageIncompatibleError'; - readonly code = 'stored_session_message_incompatible'; - - constructor( - readonly sessionId: string, - readonly sequence: number, - options?: ErrorOptions, - ) { - super(`Stored Session message ${sequence} for ${sessionId} is incompatible`, options); - } -} - export class SessionMetadataVersionConflictError extends SessionMetadataConflictError { readonly name = 'SessionMetadataVersionConflictError'; @@ -1476,6 +1491,13 @@ export class SqliteSessionMetadataStore { if (!inserted) return 'existing'; if (encoded.length > 0) { this.insertSessionMessagesSync(normalized.id, 0, encoded); + recordAppendedSessionTurnMetadata( + this.db, + normalized.id, + 0, + encoded.map(({ message }) => message), + encoded.map(({ json }) => Buffer.byteLength(json, 'utf8')), + ); // Align with appendMessages' connection-lock semantics: a session // with any user message is treated as connection-locked, even when // the legacy header did not record it. @@ -1595,6 +1617,13 @@ export class SqliteSessionMetadataStore { } const sequence = row.last_sequence + 1; this.insertSessionMessagesSync(sessionId, sequence, encoded); + recordAppendedSessionTurnMetadata( + this.db, + sessionId, + sequence, + encoded.map(({ message }) => message), + encoded.map(({ json }) => Buffer.byteLength(json, 'utf8')), + ); this.updateCatalogProjectionSync(sessionId, projection, false, lockConnection); }); } @@ -1823,6 +1852,13 @@ export class SqliteSessionMetadataStore { sequenceRow.last_sequence + 1, [{ message: assignment, json: assignmentJson }], ); + recordAppendedSessionTurnMetadata( + this.db, + WORKHUB_COORDINATION_SESSION_ID, + sequenceRow.last_sequence + 1, + [assignment], + [Buffer.byteLength(assignmentJson, 'utf8')], + ); this.updateCatalogProjectionSync(WORKHUB_COORDINATION_SESSION_ID, request.projection, false); return { kind: 'assigned' as const, targetCreated, assignment }; }); @@ -2182,6 +2218,13 @@ export class SqliteSessionMetadataStore { this.insertSessionMessagesSync(input.sessionId, currentLastSequence + 1, ordinaryEntries); tailLatest = ordinaryEntries.at(-1)?.message; } + if (historicalMissingMessages.size > 0 || ordinaryMissingMessages.size > 0) { + // Handoff recovery can insert into the historical prefix and shift + // durable sequence anchors. Expire every exact snapshot atomically + // with that mutation; an old key must fail closed rather than observe + // a position list whose body anchors have moved. + invalidateSessionTurnPositionIndex(this.db, input.sessionId); + } if (tailLatest?.type === 'user') { this.updateCatalogProjectionSync( input.sessionId, @@ -2671,6 +2714,174 @@ export class SqliteSessionMetadataStore { return nullableStoredMessageSequence(row.high_water, sessionId); } + async readTurnPositionPage( + request: SessionTurnPositionPageSnapshotRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(request.sessionId); + if (request.snapshotKey) { + const ready = this.readTransaction(() => { + const snapshot = requireSnapshot(this.db, request.sessionId, request.snapshotKey!); + return snapshot.state === 'ready' + ? pageReadySessionTurnPositionSnapshot(this.db, request, snapshot) + : null; + }); + if (ready) return ready; + } + const hasher = await createSHA256(); + const outcome = this.transaction(() => { + const allocation = allocateOrRequireSessionTurnPositionSnapshot(this.db, request); + if (allocation.kind === 'capacity') return allocation; + let snapshot = allocation.snapshot; + if (snapshot.state === 'ready') { + return pageReadySessionTurnPositionSnapshot(this.db, request, snapshot); + } + if (snapshot.build_phase === 'recovering') { + let recovery: ReturnType | undefined; + if (snapshot.through_sequence !== null) { + const indexed = this.db + .prepare(` + SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { indexed_through_sequence: number }; + if (indexed.indexed_through_sequence < snapshot.through_sequence) { + recovery = advanceSessionTurnIdentityRecovery(this.db, { + sessionId: request.sessionId, + throughSequence: snapshot.through_sequence, + maxSourceBytes: 4 * 1024 * 1024, + maxCompletedRecords: 1_024, + hasher, + }); + if ('failure' in recovery) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return recovery; + } + } + } + snapshot = markSessionTurnRecoveryComplete(this.db, request.sessionId, snapshot); + const state = this.db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + const partial = this.db + .prepare(` + SELECT byte_offset FROM session_turn_identity_recovery WHERE session_id = ? + `) + .get(request.sessionId) as { byte_offset: number } | undefined; + return { + kind: 'building' as const, + snapshotKey: snapshotKeyFromRow(snapshot), + progress: { + phase: 'recovering' as const, + nextSequence: state.indexed_through_sequence + 1, + currentByteOffset: partial?.byte_offset ?? 0, + sourceRecords: state.source_records, + sourceBytes: state.source_bytes, + builtPositions: snapshot.build_next_ordinal, + lastStepRecords: recovery && !('failure' in recovery) ? recovery.lastStepRecords : 0, + lastStepBytes: recovery && !('failure' in recovery) ? recovery.lastStepBytes : 0, + lastStepPositions: 0, + }, + }; + } + let step: ReturnType; + try { + step = advanceSessionTurnPositionOrdinalBuild(this.db, request.sessionId, snapshot); + } catch (error) { + if (error instanceof SessionTurnPositionRecoveryError) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return { + complete: false as const, + failure: error.reason, + failureSequence: error.sequence ?? 0, + }; + } + throw error; + } + const state = this.db + .prepare(` + SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ? + `) + .get(request.sessionId) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + return { + kind: 'building' as const, + snapshotKey: snapshotKeyFromRow(step.snapshot), + progress: { + phase: step.executedPhase, + nextSequence: state.indexed_through_sequence + 1, + currentByteOffset: 0, + sourceRecords: state.source_records, + sourceBytes: state.source_bytes, + builtPositions: step.snapshot.build_next_ordinal, + lastStepRecords: 0, + lastStepBytes: 0, + lastStepPositions: step.lastStepPositions, + }, + }; + }); + if ('failure' in outcome) { + throw new SessionTurnPositionRecoveryError( + request.sessionId, + outcome.failure, + outcome.failureSequence, + ); + } + return outcome; + } + + async readTranscriptRecordsByTurnIds( + request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, + ): Promise { + this.assertOpen(); + assertSafeSessionId(request.sessionId); + return this.readTransaction(() => { + const preflight = readSessionTurnMembershipPreflight(this.db, request); + const records: Array<{ sequence: number; message: StoredMessage }> = []; + for (const row of readStoredMessageRows(this.db, request.sessionId, preflight.sequences)) { + try { + records.push({ + sequence: row.sequence, + message: decodeStoredMessage(JSON.parse(row.recordJson) as unknown), + }); + } catch (error) { + throw new StoredSessionMessageIncompatibleError(request.sessionId, row.sequence, { + cause: error, + }); + } + } + return { snapshotKey: request.snapshotKey, records, rawBytes: preflight.storedBytes }; + }); + } + + async releaseTurnPositionSnapshot( + sessionId: string, + snapshotLeaseId: string, + snapshotKey: SessionTurnPositionSnapshotKey, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + this.transaction(() => + releaseSessionTurnPositionSnapshot(this.db, sessionId, snapshotLeaseId, snapshotKey), + ); + } + async readTurnContributions( sessionId: string, throughSequence: number | null, @@ -2708,21 +2919,13 @@ export class SqliteSessionMetadataStore { let sourceMessages = 0; let sourceBytes = 0; while (nextPosition <= fixedThrough) { - const rows = this.db - .prepare( - ` - SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 - FROM session_messages AS message - LEFT JOIN session_message_payloads AS payload - ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE message.session_id = ? - AND message.sequence >= ? - AND message.sequence <= ? - ORDER BY message.sequence ASC - LIMIT 128 - `, - ) - .all(sessionId, nextPosition, fixedThrough) as StoredSessionMessagePayloadRow[]; + const rows = readStoredMessagePayloadRange( + this.db, + sessionId, + nextPosition, + fixedThrough, + 128, + ); if (rows.length === 0) { throw new StoredSessionMessageIncompatibleError(sessionId, nextPosition); } @@ -6799,8 +7002,9 @@ function readStoredMessageRecordJson( ) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } - const data = readChunkedTranscriptRecord(db, sessionId, sequence, recordBytes); - if (createHash('sha256').update(data).digest('hex') !== row.sha256) { + const payloadDigest = requireTranscriptPayloadDigest(row.sha256, sessionId, sequence); + const data = readChunkedTranscriptRecord(db, sessionId, sequence, recordBytes, payloadDigest); + if (`sha256:${createHash('sha256').update(data).digest('hex')}` !== payloadDigest) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } recordJson = data.toString('utf8'); @@ -6868,6 +7072,28 @@ function foldTurnContribution( }; } +function readStoredMessagePayloadRange( + db: DatabaseSync, + sessionId: string, + startSequence: number, + throughSequence: number, + limit: number, +): StoredSessionMessagePayloadRow[] { + return db + .prepare(` + SELECT message.sequence, message.record_json, payload.record_bytes, payload.sha256 + FROM session_messages AS message + LEFT JOIN session_message_payloads AS payload + ON payload.session_id = message.session_id AND payload.sequence = message.sequence + WHERE message.session_id = ? + AND message.sequence >= ? + AND message.sequence <= ? + ORDER BY message.sequence ASC + LIMIT ? + `) + .all(sessionId, startSequence, throughSequence, limit) as StoredSessionMessagePayloadRow[]; +} + function readStoredMessageRows( db: DatabaseSync, sessionId: string, @@ -6904,41 +7130,19 @@ function readChunkedTranscriptRecord( sessionId: string, sequence: number, recordBytes: number, + payloadDigest: `sha256:${string}`, ): Buffer { - const rows = db - .prepare( - ` - SELECT chunk_index, data, sha256 - FROM session_message_chunks - WHERE session_id = ? AND sequence = ? - ORDER BY chunk_index - `, - ) - .all(sessionId, sequence) as Array<{ - chunk_index?: unknown; - data?: unknown; - sha256?: unknown; - }>; - const expectedChunks = Math.ceil(recordBytes / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); - if (rows.length !== expectedChunks) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const chunks = rows.map((row, index) => { - if ( - row.chunk_index !== index || - !(row.data instanceof Uint8Array) || - typeof row.sha256 !== 'string' - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - const chunk = Buffer.from(row.data); - if (createHash('sha256').update(chunk).digest('hex') !== row.sha256) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - return chunk; - }); - const data = Buffer.concat(chunks, recordBytes); - if (data.byteLength !== recordBytes) { + const data = readTranscriptSlices(db, sessionId, [ + { + sequence, + byteOffset: 0, + totalBytes: recordBytes, + byteLength: recordBytes, + chunked: true, + payloadDigest, + }, + ]).get(sequence); + if (!data || data.byteLength !== recordBytes) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } return data; @@ -6956,15 +7160,6 @@ function nullableStoredMessageSequence(value: unknown, sessionId: string): numbe return requireStoredMessageSequence(value, sessionId); } -interface TranscriptRecordSlice { - readonly sequence: number; - readonly byteOffset: number; - readonly totalBytes: number; - readonly byteLength: number; - readonly chunked: boolean; - readonly payloadDigest: `sha256:${string}` | null; -} - function requireTranscriptPayloadDigest( value: unknown, sessionId: string, @@ -6987,118 +7182,6 @@ function requireTranscriptRecordByteLength( return value as number; } -function readTranscriptSlices( - db: DatabaseSync, - sessionId: string, - slices: readonly TranscriptRecordSlice[], -): Map { - if (slices.length === 0) return new Map(); - const chunkedSlices = slices.filter((slice) => slice.chunked); - const values = chunkedSlices.map(() => '(?, ?, ?)').join(', '); - const parameters = chunkedSlices.flatMap((slice) => [ - slice.sequence, - Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - Math.floor((slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), - ]); - const rows = - chunkedSlices.length === 0 - ? [] - : (db - .prepare( - ` - WITH requested(sequence, first_chunk, last_chunk) AS (VALUES ${values}) - SELECT requested.sequence, chunk.chunk_index, chunk.data, chunk.sha256 - FROM requested - INNER JOIN session_message_chunks AS chunk - ON chunk.session_id = ? - AND chunk.sequence = requested.sequence - AND chunk.chunk_index BETWEEN requested.first_chunk AND requested.last_chunk - ORDER BY requested.sequence, chunk.chunk_index - `, - ) - .all(...parameters, sessionId) as Array<{ - sequence?: unknown; - chunk_index?: unknown; - data?: unknown; - sha256?: unknown; - }>); - const rowsBySequence = new Map(); - for (const row of rows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - const grouped = rowsBySequence.get(sequence); - if (grouped) grouped.push(row); - else rowsBySequence.set(sequence, [row]); - } - const result = new Map(); - for (const slice of slices) { - if (!slice.chunked) continue; - const selected = rowsBySequence.get(slice.sequence) ?? []; - const firstChunk = Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); - const lastChunk = Math.floor( - (slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, - ); - if (selected.length !== lastChunk - firstChunk + 1) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunks: Buffer[] = []; - for (let index = 0; index < selected.length; index += 1) { - const row = selected[index]!; - if ( - row.chunk_index !== firstChunk + index || - !(row.data instanceof Uint8Array) || - typeof row.sha256 !== 'string' - ) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - const chunk = Buffer.from(row.data); - if (createHash('sha256').update(chunk).digest('hex') !== row.sha256) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - chunks.push(chunk); - } - const joined = Buffer.concat(chunks); - const start = slice.byteOffset - firstChunk * SQLITE_SESSION_MESSAGE_CHUNK_BYTES; - const data = joined.subarray(start, start + slice.byteLength); - if (data.byteLength !== slice.byteLength) { - throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); - } - result.set(slice.sequence, data); - } - const inlineSlices = slices.filter((slice) => !slice.chunked); - if (inlineSlices.length > 0) { - const inlineValues = inlineSlices.map(() => '(?, ?, ?)').join(', '); - const inlineParameters = inlineSlices.flatMap((slice) => [ - slice.sequence, - slice.byteOffset + 1, - slice.byteLength, - ]); - const inlineRows = db - .prepare( - ` - WITH requested(sequence, byte_start, byte_length) AS (VALUES ${inlineValues}) - SELECT requested.sequence, - substr(CAST(message.record_json AS BLOB), requested.byte_start, requested.byte_length) - AS data - FROM requested - INNER JOIN session_messages AS message - ON message.session_id = ? AND message.sequence = requested.sequence - `, - ) - .all(...inlineParameters, sessionId) as Array<{ - sequence?: unknown; - data?: unknown; - }>; - for (const row of inlineRows) { - const sequence = requireStoredMessageSequence(row.sequence, sessionId); - if (!(row.data instanceof Uint8Array)) { - throw new StoredSessionMessageIncompatibleError(sessionId, sequence); - } - result.set(sequence, Buffer.from(row.data)); - } - } - return result; -} - function validateTranscriptRecord( data: string | Buffer, sessionId: string, diff --git a/packages/storage/src/sqlite-session-transcript-slices.ts b/packages/storage/src/sqlite-session-transcript-slices.ts new file mode 100644 index 0000000000..9347c32e98 --- /dev/null +++ b/packages/storage/src/sqlite-session-transcript-slices.ts @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createHash } from 'node:crypto'; +import type { DatabaseSync } from 'node:sqlite'; +import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES } from './sqlite-session-metadata-schema.js'; + +export class StoredSessionMessageIncompatibleError extends Error { + readonly name = 'StoredSessionMessageIncompatibleError'; + readonly code = 'stored_session_message_incompatible'; + + constructor( + readonly sessionId: string, + readonly sequence: number, + options?: ErrorOptions, + ) { + super(`Stored Session message ${sequence} for ${sessionId} is incompatible`, options); + } +} + +export interface TranscriptRecordSlice { + readonly sequence: number; + readonly byteOffset: number; + readonly totalBytes: number; + readonly byteLength: number; + readonly chunked: boolean; + readonly payloadDigest: `sha256:${string}` | null; +} + +/** + * The single physical byte-slice primitive shared by the public transcript + * pager and the package-private scalar identity recovery scanner. + */ +export function readTranscriptSlices( + db: DatabaseSync, + sessionId: string, + slices: readonly TranscriptRecordSlice[], +): Map { + if (slices.length === 0) return new Map(); + const chunkedSlices = slices.filter((slice) => slice.chunked); + const values = chunkedSlices.map(() => '(?, ?, ?)').join(', '); + const parameters = chunkedSlices.flatMap((slice) => [ + slice.sequence, + Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), + Math.floor((slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES), + ]); + const rows = + chunkedSlices.length === 0 + ? [] + : (db + .prepare( + ` + WITH requested(sequence, first_chunk, last_chunk) AS (VALUES ${values}) + SELECT requested.sequence, chunk.chunk_index, chunk.data, chunk.sha256 + FROM requested + INNER JOIN session_message_chunks AS chunk + ON chunk.session_id = ? + AND chunk.sequence = requested.sequence + AND chunk.chunk_index BETWEEN requested.first_chunk AND requested.last_chunk + ORDER BY requested.sequence, chunk.chunk_index + `, + ) + .all(...parameters, sessionId) as Array<{ + sequence?: unknown; + chunk_index?: unknown; + data?: unknown; + sha256?: unknown; + }>); + const rowsBySequence = new Map(); + for (const row of rows) { + const sequence = requireSequence(row.sequence, sessionId); + const grouped = rowsBySequence.get(sequence); + if (grouped) grouped.push(row); + else rowsBySequence.set(sequence, [row]); + } + const result = new Map(); + for (const slice of slices) { + if (!slice.chunked) continue; + const selected = rowsBySequence.get(slice.sequence) ?? []; + const firstChunk = Math.floor(slice.byteOffset / SQLITE_SESSION_MESSAGE_CHUNK_BYTES); + const lastChunk = Math.floor( + (slice.byteOffset + slice.byteLength - 1) / SQLITE_SESSION_MESSAGE_CHUNK_BYTES, + ); + if (selected.length !== lastChunk - firstChunk + 1) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + const chunks: Buffer[] = []; + for (let index = 0; index < selected.length; index += 1) { + const row = selected[index]!; + if ( + row.chunk_index !== firstChunk + index || + !(row.data instanceof Uint8Array) || + typeof row.sha256 !== 'string' || + !/^[0-9a-f]{64}$/u.test(row.sha256) + ) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + const chunk = Buffer.from(row.data); + const expectedLength = + row.chunk_index === Math.ceil(slice.totalBytes / SQLITE_SESSION_MESSAGE_CHUNK_BYTES) - 1 + ? slice.totalBytes - row.chunk_index * SQLITE_SESSION_MESSAGE_CHUNK_BYTES + : SQLITE_SESSION_MESSAGE_CHUNK_BYTES; + if ( + chunk.byteLength !== expectedLength || + createHash('sha256').update(chunk).digest('hex') !== row.sha256 + ) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + chunks.push(chunk); + } + const joined = Buffer.concat(chunks); + const start = slice.byteOffset - firstChunk * SQLITE_SESSION_MESSAGE_CHUNK_BYTES; + const data = joined.subarray(start, start + slice.byteLength); + if (data.byteLength !== slice.byteLength) { + throw new StoredSessionMessageIncompatibleError(sessionId, slice.sequence); + } + result.set(slice.sequence, data); + } + const inlineSlices = slices.filter((slice) => !slice.chunked); + if (inlineSlices.length > 0) { + const inlineValues = inlineSlices.map(() => '(?, ?, ?)').join(', '); + const inlineParameters = inlineSlices.flatMap((slice) => [ + slice.sequence, + slice.byteOffset + 1, + slice.byteLength, + ]); + const inlineRows = db + .prepare( + ` + WITH requested(sequence, byte_start, byte_length) AS (VALUES ${inlineValues}) + SELECT requested.sequence, + substr(CAST(message.record_json AS BLOB), requested.byte_start, requested.byte_length) + AS data + FROM requested + INNER JOIN session_messages AS message + ON message.session_id = ? AND message.sequence = requested.sequence + `, + ) + .all(...inlineParameters, sessionId) as Array<{ + sequence?: unknown; + data?: unknown; + }>; + for (const row of inlineRows) { + const sequence = requireSequence(row.sequence, sessionId); + if (!(row.data instanceof Uint8Array)) { + throw new StoredSessionMessageIncompatibleError(sessionId, sequence); + } + result.set(sequence, Buffer.from(row.data)); + } + } + return result; +} + +function requireSequence(value: unknown, sessionId: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new StoredSessionMessageIncompatibleError(sessionId, -1); + } + return value as number; +} diff --git a/scripts/generate-third-party-notices.mjs b/scripts/generate-third-party-notices.mjs index bd4d713427..b8da5b8c00 100644 --- a/scripts/generate-third-party-notices.mjs +++ b/scripts/generate-third-party-notices.mjs @@ -109,6 +109,42 @@ const APACHE_TEXT_OVERRIDE_KEYS = new Set([ '@ai-sdk/provider-utils@5.0.32', '@sigstore/verify@4.1.2', ]); +const BRUMME_ZLIB_TEXT = `Copyright (c) Stephan Brumme + +This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. +2. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.`; +const GO_BSD3_TEXT = `Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.`; const EMBEDDED_COMPONENT_LICENSES = new Map([ [ '@ai-sdk/code-mode', @@ -131,6 +167,64 @@ const EMBEDDED_COMPONENT_LICENSES = new Map([ ], }, ], + [ + 'hash-wasm', + { + version: '4.12.0', + components: [ + { + name: 'Go crypto Argon2 implementation basis', + repository: 'https://go.googlesource.com/crypto', + license: 'BSD-3-Clause', + text: GO_BSD3_TEXT, + }, + { + name: 'Stephan Brumme CRC32/CRC64/xxHash32/xxHash64 implementations', + repository: 'https://create.stephan-brumme.com', + license: 'Zlib-like', + text: BRUMME_ZLIB_TEXT, + }, + { + name: 'Yanbo Li SM3 implementation', + repository: 'https://github.com/Daninet/hash-wasm', + copyright: 'Copyright 2016 Yanbo Li dreamfly281@gmail.com, goldboar@163.com', + }, + ], + }, + ], +]); +const EMBEDDED_SOURCE_NOTICES = new Map([ + [ + 'hash-wasm', + { + version: '4.12.0', + sourceDirectory: 'src', + files: [ + 'adler32.c', + 'argon2.c', + 'bcrypt.c', + 'blake2b.c', + 'blake2s.c', + 'blake3.c', + 'crc32.c', + 'crc64.c', + 'md4.c', + 'md5.c', + 'ripemd160.c', + 'scrypt.c', + 'sha1.c', + 'sha256.c', + 'sha3.c', + 'sha512.c', + 'sm3.c', + 'whirlpool.c', + 'xxhash128.c', + 'xxhash3.c', + 'xxhash32.c', + 'xxhash64.c', + ], + }, + ], ]); const MIT_COPYRIGHT_OVERRIDES = new Map([ // The published tarball omits the monorepo-root LICENSE. @@ -270,6 +364,24 @@ function readLicenseFiles(directory) { .sort((left, right) => left.name.localeCompare(right.name)); } +function readLeadingSourceNotice(path) { + const source = readFileSync(path, 'utf8').replace(/\r\n?/g, '\n'); + if (source.startsWith('/*')) { + const end = source.indexOf('*/'); + if (end < 0) throw new Error(`${path}: unterminated embedded source notice`); + return normalizeText(source.slice(0, end + 2)); + } + if (source.startsWith('//')) { + const lines = source.split('\n'); + let end = 0; + while (end < lines.length && (lines[end].startsWith('//') || lines[end].trim() === '')) { + end += 1; + } + return normalizeText(lines.slice(0, end).join('\n')); + } + throw new Error(`${path}: embedded source has no leading license/attribution notice`); +} + function overrideLicenseText(packageKey, selectedLicense) { if (selectedLicense === 'Apache-2.0' && APACHE_TEXT_OVERRIDE_KEYS.has(packageKey)) { const rootLicense = readFileSync(join(repoRoot, 'LICENSE'), 'utf8'); @@ -436,15 +548,48 @@ function renderNotice() { throw new Error(`${owner}: embedded component licenses require exact-version review`); } for (const component of inventory.components) { + const selectedLicense = component.license ?? 'MIT'; + const licenseText = component.text ?? MIT_TEXT(component.copyright); sections.push( [ `Embedded component: ${component.name}`, `Embedded by: ${owner}`, - 'Selected license: MIT', + `Selected license: ${selectedLicense}`, `Repository: ${component.repository}`, '', '--- VERSION-PINNED EMBEDDED LICENSE TEXT ---', - MIT_TEXT(component.copyright), + licenseText, + ].join('\n'), + ); + } + } + } + for (const [packageName, inventory] of EMBEDDED_SOURCE_NOTICES) { + const matchingDependencies = dependencies.filter((candidate) => candidate.name === packageName); + for (const dependency of matchingDependencies) { + const owner = `${dependency.name}@${dependency.version}`; + if (dependency.version !== inventory.version) { + throw new Error(`${owner}: embedded source notices require exact-version review`); + } + const candidates = lockIndex.get(owner); + const directory = packageDirectory(owner, candidates); + const sourceDirectory = join(directory, inventory.sourceDirectory); + const actualFiles = readdirSync(sourceDirectory) + .filter((name) => name.endsWith('.c')) + .sort(); + const expectedFiles = [...inventory.files].sort(); + if (actualFiles.join('\n') !== expectedFiles.join('\n')) { + throw new Error(`${owner}: embedded source inventory changed and requires review`); + } + for (const file of inventory.files) { + sections.push( + [ + `Embedded source notice: ${file}`, + `Embedded by: ${owner}`, + `Source: ${inventory.sourceDirectory}/${file}`, + '', + '--- VERSION-PINNED UPSTREAM SOURCE NOTICE ---', + readLeadingSourceNotice(join(sourceDirectory, file)), ].join('\n'), ); } From 2ef27587d86d4d9d773600529cde4acb3afabaa3 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:40:50 +0800 Subject: [PATCH 02/10] chore(ci): retrigger checks after base repair Generated-by: OpenAI Codex From 21137b50e4f7172b6fdadcac96f8ab2ba2089b1c Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:16:35 +0800 Subject: [PATCH 03/10] fix(storage): unify turn position recovery primitives Generated-by: OpenAI Codex --- .../session-turn-position-index.test.ts | 49 ++++++ .../src/session-turn-identity-recovery.ts | 111 ++------------ .../src/session-turn-identity-scanner.ts | 46 +++--- .../storage/src/session-turn-membership.ts | 142 ++++++++++++++++++ .../src/session-turn-position-authority.ts | 108 ++++--------- .../src/session-turn-position-snapshots.ts | 9 +- .../storage/src/sqlite-legacy-scheduling.ts | 18 +-- .../src/sqlite-schema-introspection.ts | 26 ++++ .../src/sqlite-session-metadata-store.ts | 14 +- .../src/sqlite-session-transcript-slices.ts | 17 ++- 10 files changed, 306 insertions(+), 234 deletions(-) create mode 100644 packages/storage/src/session-turn-membership.ts create mode 100644 packages/storage/src/sqlite-schema-introspection.ts diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index 7a4a5a1266..a0c0b29a51 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -739,6 +739,55 @@ describe('Session Turn position snapshots', () => { } }); + test('resumes legacy inline records larger than one recovery step', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-legacy-inline-source-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + const message = { + ...user('turn-legacy-inline', 0), + text: 'x'.repeat(4 * 1024 * 1024 + 1024), + }; + try { + await store.appendMessage(session.id, message); + await store.close?.(); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec('PRAGMA foreign_keys = ON'); + database.exec('BEGIN IMMEDIATE'); + database + .prepare(`UPDATE session_messages SET record_json = ? + WHERE session_id = ? AND sequence = 0`) + .run(JSON.stringify(message), session.id); + database + .prepare('DELETE FROM session_message_payloads WHERE session_id = ? AND sequence = 0') + .run(session.id); + database.exec('COMMIT'); + } finally { + database.close(); + } + resetProjection(root, session.id); + store = createSessionStore(root); + + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + snapshotLeaseId: 'lease-legacy-inline', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial inline scalar recovery'); + assert.equal(first.progress.lastStepBytes, 4 * 1024 * 1024); + assert.equal(first.progress.lastStepRecords, 0); + assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); + + const ready = await readyPage(store, session.id, 'lease-legacy-inline'); + assert.equal(ready.positions[0]?.turnId, 'turn-legacy-inline'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('accounts exact 4 MiB boundaries and resumes identity located after a huge body', async () => { const targets = [4 * 1024 * 1024 - 1, 4 * 1024 * 1024, 4 * 1024 * 1024 + 1, 9 * 1024 * 1024]; for (const [index, targetBytes] of targets.entries()) { diff --git a/packages/storage/src/session-turn-identity-recovery.ts b/packages/storage/src/session-turn-identity-recovery.ts index 4e8aae4ad6..fd64416e5e 100644 --- a/packages/storage/src/session-turn-identity-recovery.ts +++ b/packages/storage/src/session-turn-identity-recovery.ts @@ -20,6 +20,10 @@ import { createHash } from 'node:crypto'; import type { DatabaseSync } from 'node:sqlite'; import type { IHasher } from 'hash-wasm'; +import { + publishSessionTurnMembership, + SessionTurnMembershipPublicationError, +} from './session-turn-membership.js'; import { advanceSessionTurnIdentityScanner, completeSessionTurnIdentityScanner, @@ -31,9 +35,10 @@ import { type SessionTurnRecoveredIdentity, } from './session-turn-identity-scanner.js'; import { + planForwardTranscriptSlice, readTranscriptSlices, - StoredSessionMessageIncompatibleError, } from './sqlite-session-transcript-slices.js'; +import { SQLITE_SESSION_MESSAGE_CHUNK_BYTES } from './sqlite-session-metadata-schema.js'; export const SESSION_TURN_IDENTITY_HASH_STATE_VERSION = 1; export const SESSION_TURN_IDENTITY_HASH_ALGORITHM = 'sha256'; @@ -98,20 +103,6 @@ interface SourceMetadata { readonly expected_digest: string | null; } -export class SessionTurnIdentityRecoveryFailure extends Error { - readonly code = 'session_turn_identity_recovery_failed'; - - constructor( - readonly sessionId: string, - readonly reason: SessionTurnIdentityRecoveryFailureReason, - readonly sequence: number, - options?: ErrorOptions, - ) { - super(`Session Turn identity recovery failed (${reason}) at ${sessionId}/${sequence}`, options); - this.name = 'SessionTurnIdentityRecoveryFailure'; - } -} - /** * Advance only the scalar identity index. This function never returns source * bytes, JSON, fragments, or StoredMessage objects to its caller. @@ -176,7 +167,12 @@ export function advanceSessionTurnIdentityRecovery( } const available = maxSourceBytes - stepBytes; - const byteLength = boundedSliceLength(source, byteOffset, available); + const byteLength = planForwardTranscriptSlice( + source.record_bytes, + byteOffset, + available, + source.chunked === 1 ? SQLITE_SESSION_MESSAGE_CHUNK_BYTES : 1, + ); if (byteLength === 0) break; let data: Buffer; try { @@ -236,15 +232,12 @@ export function advanceSessionTurnIdentityRecovery( ); } try { - publishRecoveredMembership(db, sessionId, source.sequence, identity); + publishSessionTurnMembership(db, sessionId, source.sequence, identity); } catch (error) { return fail( db, sessionId, - error instanceof SessionTurnIdentityRecoveryFailure && - error.reason === 'incompatible_identity' - ? 'incompatible_identity' - : 'corrupt_source', + error instanceof SessionTurnMembershipPublicationError ? error.reason : 'corrupt_source', source.sequence, error, ); @@ -321,14 +314,6 @@ function readSourceMetadata( return row as SourceMetadata; } -function boundedSliceLength(source: SourceMetadata, byteOffset: number, available: number): number { - const remaining = source.record_bytes - byteOffset; - if (remaining <= available) return remaining; - if (source.chunked === 0) return 0; - const chunkBytes = 64 * 1024; - return Math.floor(available / chunkBytes) * chunkBytes; -} - function sameSource(partial: PartialRow, source: SourceMetadata): boolean { return ( partial.sequence === source.sequence && @@ -412,68 +397,6 @@ function derivedStateDigest(hashState: Uint8Array, scannerState: string): string .digest('hex'); } -function publishRecoveredMembership( - db: DatabaseSync, - sessionId: string, - sequence: number, - identity: SessionTurnRecoveredIdentity, -): void { - if (identity.kind === 'ignored') return; - const admission = - identity.kind === 'turn' && tableExists(db, 'core_root_turn_admissions') - ? (db - .prepare(` - SELECT admitted_at FROM core_root_turn_admissions - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, identity.turnId) as { admitted_at: number } | undefined) - : undefined; - const existingIdentity = db - .prepare(` - SELECT identity_kind FROM session_turn_metadata - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, identity.turnId) as { identity_kind: string } | undefined; - if (existingIdentity && existingIdentity.identity_kind !== identity.kind) { - throw new SessionTurnIdentityRecoveryFailure(sessionId, 'incompatible_identity', sequence); - } - db.prepare(` - INSERT INTO session_turn_metadata( - session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence - ) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, turn_id) DO UPDATE SET - order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' - ELSE session_turn_metadata.order_source END, - admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), - first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL - THEN excluded.first_sequence - ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END - `).run( - sessionId, - identity.turnId, - identity.kind, - admission ? 'admission' : 'legacy', - admission?.admitted_at ?? null, - sequence, - ); - const inserted = db - .prepare(` - INSERT INTO session_turn_memberships(session_id, sequence, turn_id) - VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING - `) - .run(sessionId, sequence, identity.turnId); - if (inserted.changes !== 1) { - const existing = db - .prepare(` - SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? - `) - .get(sessionId, sequence) as { turn_id: string }; - if (existing.turn_id !== identity.turnId) { - throw new SessionTurnIdentityRecoveryFailure(sessionId, 'corrupt_source', sequence); - } - } -} - function fail( db: DatabaseSync, sessionId: string, @@ -489,9 +412,3 @@ function fail( void cause; return { complete: false, failure: reason, failureSequence: sequence }; } - -function tableExists(db: DatabaseSync, table: string): boolean { - return Boolean( - db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), - ); -} diff --git a/packages/storage/src/session-turn-identity-scanner.ts b/packages/storage/src/session-turn-identity-scanner.ts index aecf05aad0..87cb4ce045 100644 --- a/packages/storage/src/session-turn-identity-scanner.ts +++ b/packages/storage/src/session-turn-identity-scanner.ts @@ -17,7 +17,11 @@ * under the License. */ -import { isUserVisibleSessionSystemNote } from '@maka/core/session'; +import { + classifySessionTurnIdentity, + SessionTurnIdentityClassificationError, + type SessionTurnIdentity, +} from './session-turn-membership.js'; export const SESSION_TURN_IDENTITY_SCANNER_VERSION = 1; export const SESSION_TURN_IDENTITY_SCANNER_MAX_STATE_BYTES = 64 * 1024; @@ -79,10 +83,7 @@ export interface SessionTurnIdentityScannerStateV1 { capturedBytes: number; } -export type SessionTurnRecoveredIdentity = - | { readonly kind: 'turn'; readonly turnId: string } - | { readonly kind: 'note'; readonly turnId: string } - | { readonly kind: 'ignored' }; +export type SessionTurnRecoveredIdentity = SessionTurnIdentity; export class SessionTurnIdentityScannerError extends Error { readonly code = 'session_turn_identity_incompatible'; @@ -181,30 +182,21 @@ export function completeSessionTurnIdentityScanner( 'incompatible_identity', ); } - if (turnIdPresent) { - if (typeof turnId !== 'string' || turnId.length === 0) { - throw new SessionTurnIdentityScannerError( - 'turnId must be a non-empty string', - 'incompatible_identity', - ); + try { + return classifySessionTurnIdentity({ + id, + type, + turnIdPresent, + turnId, + kindPresent, + kind, + }); + } catch (error) { + if (error instanceof SessionTurnIdentityClassificationError) { + throw new SessionTurnIdentityScannerError(error.detail, 'incompatible_identity'); } - return { kind: 'turn', turnId }; - } - if (type !== 'system_note') { - throw new SessionTurnIdentityScannerError( - 'non-system message is missing turnId', - 'incompatible_identity', - ); - } - if (!kindPresent || typeof kind !== 'string' || kind.length === 0) { - throw new SessionTurnIdentityScannerError( - 'turnless system note is missing kind', - 'incompatible_identity', - ); + throw error; } - return isUserVisibleSessionSystemNote(kind) - ? { kind: 'note', turnId: `session-note:${id}` } - : { kind: 'ignored' }; } function consumeCodePoint(state: SessionTurnIdentityScannerStateV1, codePoint: number): void { diff --git a/packages/storage/src/session-turn-membership.ts b/packages/storage/src/session-turn-membership.ts new file mode 100644 index 0000000000..cf87834244 --- /dev/null +++ b/packages/storage/src/session-turn-membership.ts @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { isUserVisibleSessionSystemNote, type StoredMessage } from '@maka/core/session'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; + +export type SessionTurnIdentity = + | { readonly kind: 'turn'; readonly turnId: string } + | { readonly kind: 'note'; readonly turnId: string } + | { readonly kind: 'ignored' }; + +export class SessionTurnIdentityClassificationError extends Error { + constructor(readonly detail: string) { + super(`Session Turn identity is incompatible: ${detail}`); + this.name = 'SessionTurnIdentityClassificationError'; + } +} + +export class SessionTurnMembershipPublicationError extends Error { + constructor( + readonly reason: 'corrupt_source' | 'incompatible_identity', + readonly sequence: number, + ) { + super(`Session Turn membership publication failed (${reason}) at sequence ${sequence}`); + this.name = 'SessionTurnMembershipPublicationError'; + } +} + +export function classifySessionTurnIdentity(input: { + readonly id: string; + readonly type: string; + readonly turnIdPresent: boolean; + readonly turnId?: string; + readonly kindPresent: boolean; + readonly kind?: string; +}): SessionTurnIdentity { + if (input.turnIdPresent) { + if (typeof input.turnId !== 'string' || input.turnId.length === 0) { + throw new SessionTurnIdentityClassificationError('turnId must be a non-empty string'); + } + return { kind: 'turn', turnId: input.turnId }; + } + if (input.type !== 'system_note') { + throw new SessionTurnIdentityClassificationError('non-system message is missing turnId'); + } + if (!input.kindPresent || typeof input.kind !== 'string' || input.kind.length === 0) { + throw new SessionTurnIdentityClassificationError('turnless system note is missing kind'); + } + return input.id.length > 0 && isUserVisibleSessionSystemNote(input.kind) + ? { kind: 'note', turnId: `session-note:${input.id}` } + : { kind: 'ignored' }; +} + +export function classifyStoredMessageTurnIdentity(message: StoredMessage): SessionTurnIdentity { + const turnId = (message as { turnId?: unknown }).turnId; + return classifySessionTurnIdentity({ + id: message.id, + type: message.type, + turnIdPresent: typeof turnId === 'string', + turnId: typeof turnId === 'string' ? turnId : undefined, + kindPresent: message.type === 'system_note', + kind: message.type === 'system_note' ? message.kind : undefined, + }); +} + +export function publishSessionTurnMembership( + db: DatabaseSync, + sessionId: string, + sequence: number, + identity: SessionTurnIdentity, +): void { + if (identity.kind === 'ignored') return; + const admission = + identity.kind === 'turn' && sqliteTableExists(db, 'core_root_turn_admissions') + ? (db + .prepare(` + SELECT admitted_at FROM core_root_turn_admissions + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, identity.turnId) as { admitted_at: number } | undefined) + : undefined; + const existingIdentity = db + .prepare(` + SELECT identity_kind FROM session_turn_metadata + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, identity.turnId) as { identity_kind: string } | undefined; + if (existingIdentity && existingIdentity.identity_kind !== identity.kind) { + throw new SessionTurnMembershipPublicationError('incompatible_identity', sequence); + } + db.prepare(` + INSERT INTO session_turn_metadata( + session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence + ) VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, turn_id) DO UPDATE SET + order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' + ELSE session_turn_metadata.order_source END, + admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), + first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL + THEN excluded.first_sequence + ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END + `).run( + sessionId, + identity.turnId, + identity.kind, + admission ? 'admission' : 'legacy', + admission?.admitted_at ?? null, + sequence, + ); + const inserted = db + .prepare(` + INSERT INTO session_turn_memberships(session_id, sequence, turn_id) + VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING + `) + .run(sessionId, sequence, identity.turnId); + if (inserted.changes === 1) return; + const existing = db + .prepare(` + SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? + `) + .get(sessionId, sequence) as { turn_id: string } | undefined; + if (existing?.turn_id !== identity.turnId) { + throw new SessionTurnMembershipPublicationError('corrupt_source', sequence); + } +} diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts index 7388a0d12b..e14dfc5310 100644 --- a/packages/storage/src/session-turn-position-authority.ts +++ b/packages/storage/src/session-turn-position-authority.ts @@ -18,20 +18,23 @@ */ import type { DatabaseSync } from 'node:sqlite'; -import { isUserVisibleSessionSystemNote, type StoredMessage } from '@maka/core/session'; +import type { StoredMessage } from '@maka/core/session'; import { SessionTurnPositionRecoveryError } from './session-store.js'; +import { + classifyStoredMessageTurnIdentity, + publishSessionTurnMembership, + SessionTurnIdentityClassificationError, + SessionTurnMembershipPublicationError, +} from './session-turn-membership.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; export function recordAppendedSessionTurnMetadata( db: DatabaseSync, sessionId: string, firstSequence: number, messages: readonly StoredMessage[], - encodedByteLengths: readonly number[], ): void { - if (!tableExists(db, 'session_turn_metadata') || messages.length === 0) return; - if (messages.length !== encodedByteLengths.length) { - throw new Error('Session Turn metadata byte evidence is incomplete'); - } + if (!sqliteTableExists(db, 'session_turn_metadata') || messages.length === 0) return; ensureTurnIndexRows(db, sessionId); const fillsBodylessAdmission = messages.some((message) => { const turnId = (message as { turnId?: unknown }).turnId; @@ -68,7 +71,7 @@ export function recordRootTurnAdmissionForPositionIndex( turnId: string, admittedAt: number, ): void { - if (!tableExists(db, 'session_turn_metadata')) return; + if (!sqliteTableExists(db, 'session_turn_metadata')) return; if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; if (!turnId || !Number.isSafeInteger(admittedAt) || admittedAt < 0) { throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity'); @@ -112,7 +115,7 @@ export function recordRootTurnAdmissionsPurgedForPositionIndex( db: DatabaseSync, sessionId: string, ): void { - if (!tableExists(db, 'session_turn_metadata')) return; + if (!sqliteTableExists(db, 'session_turn_metadata')) return; if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; ensureTurnIndexRows(db, sessionId); const removed = db @@ -139,7 +142,7 @@ export function recordRootTurnAdmissionsPurgedForPositionIndex( } export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: string): void { - if (!tableExists(db, 'session_turn_metadata')) return; + if (!sqliteTableExists(db, 'session_turn_metadata')) return; ensureTurnIndexRows(db, sessionId); db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); @@ -170,71 +173,26 @@ function publishCanonicalMembership( sequence: number, message: StoredMessage, ): void { - const explicitTurnId = (message as { turnId?: unknown }).turnId; - let turnId: string; - let identityKind: 'turn' | 'note'; - if (typeof explicitTurnId === 'string') { - if (explicitTurnId.length === 0) { - throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); + try { + publishSessionTurnMembership( + db, + sessionId, + sequence, + classifyStoredMessageTurnIdentity(message), + ); + } catch (error) { + if (error instanceof SessionTurnIdentityClassificationError) { + throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence, { + cause: error, + }); } - turnId = explicitTurnId; - identityKind = 'turn'; - } else if (message.type === 'system_note') { - if (!message.id || !isUserVisibleSessionSystemNote(message.kind)) return; - turnId = `session-note:${message.id}`; - identityKind = 'note'; - } else { - throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); - } - const admission = - identityKind === 'turn' && tableExists(db, 'core_root_turn_admissions') - ? (db - .prepare(` - SELECT admitted_at FROM core_root_turn_admissions - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, turnId) as { admitted_at: number } | undefined) - : undefined; - const existingIdentity = db - .prepare(` - SELECT identity_kind FROM session_turn_metadata - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, turnId) as { identity_kind: string } | undefined; - if (existingIdentity && existingIdentity.identity_kind !== identityKind) { - throw new SessionTurnPositionRecoveryError(sessionId, 'incompatible_identity', sequence); - } - db.prepare(` - INSERT INTO session_turn_metadata( - session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence - ) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, turn_id) DO UPDATE SET - order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' - ELSE session_turn_metadata.order_source END, - admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), - first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL - THEN excluded.first_sequence - ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END - `).run( - sessionId, - turnId, - identityKind, - admission ? 'admission' : 'legacy', - admission?.admitted_at ?? null, - sequence, - ); - const existing = db - .prepare(` - SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? - `) - .get(sessionId, sequence) as { turn_id: string } | undefined; - if (existing && existing.turn_id !== turnId) { - throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source', sequence); + if (error instanceof SessionTurnMembershipPublicationError) { + throw new SessionTurnPositionRecoveryError(sessionId, error.reason, sequence, { + cause: error, + }); + } + throw error; } - db.prepare(` - INSERT INTO session_turn_memberships(session_id, sequence, turn_id) - VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING - `).run(sessionId, sequence, turnId); } function invalidateBuildingSnapshots(db: DatabaseSync, sessionId: string): void { @@ -249,9 +207,3 @@ function advanceAuthorityRevision(db: DatabaseSync, sessionId: string): void { SET authority_revision = authority_revision + 1 WHERE session_id = ? `).run(sessionId); } - -function tableExists(db: DatabaseSync, table: string): boolean { - return Boolean( - db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), - ); -} diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts index d644394264..8bac6facad 100644 --- a/packages/storage/src/session-turn-position-snapshots.ts +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -29,6 +29,7 @@ import { type SessionTurnPositionSnapshotKey, } from './session-store.js'; import { ensureTurnIndexRows } from './session-turn-position-authority.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; export const SESSION_TURN_POSITION_MAX_PAGE_POSITIONS = 128; export const SESSION_TURN_POSITION_MAX_PAGE_BYTES = 64 * 1024; @@ -462,7 +463,7 @@ export function releaseSessionTurnPositionSnapshot( } export function reclaimSessionTurnPositionSnapshotsForNewOwner(db: DatabaseSync): void { - if (!tableExists(db, 'session_turn_position_snapshots')) return; + if (!sqliteTableExists(db, 'session_turn_position_snapshots')) return; db.prepare('DELETE FROM session_turn_position_snapshots').run(); } @@ -777,9 +778,3 @@ function ensureSessionExists(db: DatabaseSync, sessionId: string): void { throw new SessionTurnPositionSnapshotMismatchError(sessionId); } } - -function tableExists(db: DatabaseSync, table: string): boolean { - return Boolean( - db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), - ); -} diff --git a/packages/storage/src/sqlite-legacy-scheduling.ts b/packages/storage/src/sqlite-legacy-scheduling.ts index c24042aa06..4bf736cdba 100644 --- a/packages/storage/src/sqlite-legacy-scheduling.ts +++ b/packages/storage/src/sqlite-legacy-scheduling.ts @@ -26,6 +26,7 @@ import type { } from '@maka/core/scheduled-task'; import type { DatabaseSync } from 'node:sqlite'; import { canonicalizeLegacyPlanReminderCronExpression } from './legacy-cron-expression.js'; +import { sqliteTableExists } from './sqlite-schema-introspection.js'; const LEGACY_AUTOMATION_TABLES = [ 'automation_authority_state', @@ -40,7 +41,9 @@ export function assertLegacySchedulingSchema( versions: ReadonlyMap, ): void { const automationVersion = versions.get('automation'); - const automationTables = LEGACY_AUTOMATION_TABLES.filter((table) => hasTable(database, table)); + const automationTables = LEGACY_AUTOMATION_TABLES.filter((table) => + sqliteTableExists(database, table), + ); if (automationTables.length > 0 && automationVersion === undefined) { throw new Error('Legacy Automation schema registry is missing'); } @@ -55,14 +58,14 @@ export function assertLegacySchedulingSchema( if ( workflowVersion !== undefined && workflowVersion <= LAST_RELEASED_PLAN_REMINDER_WORKFLOW_VERSION && - !hasTable(database, 'workflow_plan_reminders') + !sqliteTableExists(database, 'workflow_plan_reminders') ) { throw new Error('The released Workflow schema is missing workflow_plan_reminders'); } if ( workflowVersion !== undefined && workflowVersion > LAST_RELEASED_PLAN_REMINDER_WORKFLOW_VERSION && - hasTable(database, 'workflow_plan_reminders') + sqliteTableExists(database, 'workflow_plan_reminders') ) { throw new Error('The current Workflow schema still contains released Plan Reminder state'); } @@ -113,7 +116,7 @@ function assertLegacyAutomationEmpty(database: DatabaseSync): void { } function readLegacyPlanReminders(database: DatabaseSync): ScheduledTask[] { - if (!hasTable(database, 'workflow_plan_reminders')) { + if (!sqliteTableExists(database, 'workflow_plan_reminders')) { return []; } return database @@ -286,13 +289,6 @@ function enumValue(value: unknown, values: T) return value as T[number]; } -function hasTable(database: DatabaseSync, name: string): boolean { - return ( - database.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(name) !== - undefined - ); -} - function hasColumn(database: DatabaseSync, table: string, column: string): boolean { return ( database.prepare('SELECT 1 FROM pragma_table_info(?) WHERE name = ?').get(table, column) !== diff --git a/packages/storage/src/sqlite-schema-introspection.ts b/packages/storage/src/sqlite-schema-introspection.ts new file mode 100644 index 0000000000..d08015968a --- /dev/null +++ b/packages/storage/src/sqlite-schema-introspection.ts @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; + +export function sqliteTableExists(db: DatabaseSync, table: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get(table), + ); +} diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index db88e5c900..2983d96069 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -188,7 +188,9 @@ import { sqliteRecoverableSessionRolePredicate, } from './sqlite-session-role-scope.js'; import { + planForwardTranscriptSlice, readTranscriptSlices, + requireStoredMessageSequence, StoredSessionMessageIncompatibleError, type TranscriptRecordSlice, } from './sqlite-session-transcript-slices.js'; @@ -1496,7 +1498,6 @@ export class SqliteSessionMetadataStore { normalized.id, 0, encoded.map(({ message }) => message), - encoded.map(({ json }) => Buffer.byteLength(json, 'utf8')), ); // Align with appendMessages' connection-lock semantics: a session // with any user message is treated as connection-locked, even when @@ -1622,7 +1623,6 @@ export class SqliteSessionMetadataStore { sessionId, sequence, encoded.map(({ message }) => message), - encoded.map(({ json }) => Buffer.byteLength(json, 'utf8')), ); this.updateCatalogProjectionSync(sessionId, projection, false, lockConnection); }); @@ -1857,7 +1857,6 @@ export class SqliteSessionMetadataStore { WORKHUB_COORDINATION_SESSION_ID, sequenceRow.last_sequence + 1, [assignment], - [Buffer.byteLength(assignmentJson, 'utf8')], ); this.updateCatalogProjectionSync(WORKHUB_COORDINATION_SESSION_ID, request.projection, false); return { kind: 'assigned' as const, targetCreated, assignment }; @@ -2473,7 +2472,7 @@ export class SqliteSessionMetadataStore { const byteLength = request.direction === 'older' ? edge - byteOffset - : Math.min(totalBytes - edge, available); + : planForwardTranscriptSlice(totalBytes, edge, available); const complete = request.direction === 'older' ? byteOffset === 0 : byteOffset + byteLength === totalBytes; slices.push({ @@ -7148,13 +7147,6 @@ function readChunkedTranscriptRecord( return data; } -function requireStoredMessageSequence(value: unknown, sessionId: string): number { - if (!Number.isSafeInteger(value) || (value as number) < 0) { - throw new StoredSessionMessageIncompatibleError(sessionId, -1); - } - return value as number; -} - function nullableStoredMessageSequence(value: unknown, sessionId: string): number | null { if (value === null || value === undefined) return null; return requireStoredMessageSequence(value, sessionId); diff --git a/packages/storage/src/sqlite-session-transcript-slices.ts b/packages/storage/src/sqlite-session-transcript-slices.ts index 9347c32e98..ca0bff84ed 100644 --- a/packages/storage/src/sqlite-session-transcript-slices.ts +++ b/packages/storage/src/sqlite-session-transcript-slices.ts @@ -43,6 +43,17 @@ export interface TranscriptRecordSlice { readonly payloadDigest: `sha256:${string}` | null; } +export function planForwardTranscriptSlice( + totalBytes: number, + byteOffset: number, + availableBytes: number, + alignmentBytes = 1, +): number { + const remainingBytes = totalBytes - byteOffset; + if (remainingBytes <= availableBytes) return remainingBytes; + return Math.floor(availableBytes / alignmentBytes) * alignmentBytes; +} + /** * The single physical byte-slice primitive shared by the public transcript * pager and the package-private scalar identity recovery scanner. @@ -84,7 +95,7 @@ export function readTranscriptSlices( }>); const rowsBySequence = new Map(); for (const row of rows) { - const sequence = requireSequence(row.sequence, sessionId); + const sequence = requireStoredMessageSequence(row.sequence, sessionId); const grouped = rowsBySequence.get(sequence); if (grouped) grouped.push(row); else rowsBySequence.set(sequence, [row]); @@ -157,7 +168,7 @@ export function readTranscriptSlices( data?: unknown; }>; for (const row of inlineRows) { - const sequence = requireSequence(row.sequence, sessionId); + const sequence = requireStoredMessageSequence(row.sequence, sessionId); if (!(row.data instanceof Uint8Array)) { throw new StoredSessionMessageIncompatibleError(sessionId, sequence); } @@ -167,7 +178,7 @@ export function readTranscriptSlices( return result; } -function requireSequence(value: unknown, sessionId: string): number { +export function requireStoredMessageSequence(value: unknown, sessionId: string): number { if (!Number.isSafeInteger(value) || (value as number) < 0) { throw new StoredSessionMessageIncompatibleError(sessionId, -1); } From 421bb7cc29fb83485ed234f5f2c2b65d67953fbd Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:31:25 +0800 Subject: [PATCH 04/10] feat(storage): add projection-aware semantic positions Generated-by: OpenAI Codex --- ...ared-session-transcript-visibility.test.ts | 74 ++ packages/core/src/session.ts | 100 +- .../session-transcript-pager.test.ts | 77 ++ .../src/server/shared-session-transcript.ts | 24 +- .../session-turn-identity-scanner.test.ts | 38 +- .../session-turn-position-index.test.ts | 879 +++++++++++++++--- packages/storage/src/execution-stores.ts | 18 +- packages/storage/src/session-store.ts | 67 +- .../storage/src/session-turn-membership.ts | 99 +- .../src/session-turn-position-authority.ts | 86 +- .../src/session-turn-position-index.ts | 2 +- .../src/session-turn-position-snapshots.ts | 423 ++++++--- .../src/sqlite-session-metadata-schema.ts | 114 ++- .../src/sqlite-session-metadata-store.ts | 120 ++- 14 files changed, 1646 insertions(+), 475 deletions(-) create mode 100644 packages/core/src/__tests__/shared-session-transcript-visibility.test.ts diff --git a/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts b/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts new file mode 100644 index 0000000000..5c7d896f0f --- /dev/null +++ b/packages/core/src/__tests__/shared-session-transcript-visibility.test.ts @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + classifySharedSessionTranscriptVisibility, + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, + type StoredMessage, + type SystemNoteMessage, +} from '../session.js'; + +test('shared transcript visibility is closed over every StoredMessage type', () => { + const expected = { + user: 'visible', + assistant: 'visible', + tool_call: 'visible', + tool_result: 'visible', + turn_state: 'visible', + token_usage: 'visible', + permission_decision: 'hidden', + workhub_coordination: 'hidden', + } as const satisfies Record, 'visible' | 'hidden'>; + + for (const [type, visibility] of Object.entries(expected)) { + assert.equal( + classifySharedSessionTranscriptVisibility({ + type: type as Exclude, + }), + visibility, + ); + } + assert.equal(SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, 1); +}); + +test('shared transcript visibility makes an explicit decision for every system-note kind', () => { + const expected = { + session_start: 'hidden', + session_resume: 'hidden', + mode_change: 'hidden', + model_change: 'hidden', + context_compacted: 'visible', + context_compaction_failed_open: 'visible', + step_limit: 'visible', + error: 'hidden', + abort: 'hidden', + } as const satisfies Record; + + for (const [kind, visibility] of Object.entries(expected)) { + assert.equal( + classifySharedSessionTranscriptVisibility({ + type: 'system_note', + kind: kind as SystemNoteMessage['kind'], + }), + visibility, + ); + } +}); diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index cbfc7efec3..e60c37ff35 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -780,17 +780,6 @@ export function userFacingText(message: Pick; + }; + +const SHARED_SESSION_SYSTEM_NOTE_VISIBILITY = { + session_start: 'hidden', + session_resume: 'hidden', + mode_change: 'hidden', + model_change: 'hidden', + context_compacted: 'visible', + context_compaction_failed_open: 'visible', + step_limit: 'visible', + error: 'hidden', + abort: 'hidden', +} as const satisfies Record; + +/** Closed scalar policy for records admitted to a shared transcript projection. */ +export function classifySharedSessionTranscriptVisibility( + envelope: SharedSessionTranscriptVisibilityEnvelope, +): SharedSessionTranscriptVisibility { + switch (envelope.type) { + case 'user': + case 'assistant': + case 'tool_call': + case 'tool_result': + case 'turn_state': + case 'token_usage': + return 'visible'; + case 'permission_decision': + case 'workhub_coordination': + return 'hidden'; + case 'system_note': + return SHARED_SESSION_SYSTEM_NOTE_VISIBILITY[envelope.kind]; + default: { + const unreachable: never = envelope; + return unreachable; + } + } +} + +export function isSessionSystemNoteKind(value: unknown): value is SystemNoteMessage['kind'] { + return typeof value === 'string' && Object.hasOwn(SHARED_SESSION_SYSTEM_NOTE_VISIBILITY, value); +} + +export function isStoredMessageType(value: unknown): value is StoredMessage['type'] { + switch (value) { + case 'user': + case 'assistant': + case 'tool_call': + case 'tool_result': + case 'permission_decision': + case 'token_usage': + case 'turn_state': + case 'workhub_coordination': + case 'system_note': + return true; + default: + return false; + } +} + +/** Closed policy for system notes that are part of the user-visible transcript. */ +export function isUserVisibleSessionSystemNote(kind: string): boolean { + return isSessionSystemNoteKind(kind) && SHARED_SESSION_SYSTEM_NOTE_VISIBILITY[kind] === 'visible'; +} + const USER_MESSAGE_SHAPE = defineObjectShape()( ['type', 'id', 'turnId', 'ts', 'text'], ['displayText', 'attachments', 'quotes', 'inlineReferences', 'steeringEventId', 'origin'], @@ -1135,18 +1199,6 @@ const ASSISTANT_THINKING_SHAPE = defineObjectShape()( ['text'], ['signature', 'providerOptions', 'parts'], ); -const SYSTEM_NOTE_KINDS = new Set([ - 'session_start', - 'session_resume', - 'mode_change', - 'model_change', - 'context_compacted', - 'context_compaction_failed_open', - 'step_limit', - 'error', - 'abort', -]); - export function decodeCanonicalMessage(value: unknown): StoredMessage { return decodeMessage(value, decodeCanonicalToolResultContent); } @@ -1280,7 +1332,7 @@ function decodeMessage( hasExactShape(message, SYSTEM_NOTE_MESSAGE_SHAPE) && hasMessageEnvelope(message, false) && isOptionalString(message.turnId) && - SYSTEM_NOTE_KINDS.has(message.kind as string) + isSessionSystemNoteKind(message.kind) ) return message as unknown as SystemNoteMessage; break; diff --git a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts index fd35f9fb67..e430796b0b 100644 --- a/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts +++ b/packages/runtime-host/src/__tests__/session-transcript-pager.test.ts @@ -20,6 +20,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + classifySharedSessionTranscriptVisibility, decodeStoredMessage as decodePersistedStoredMessage, type StoredMessage, } from '@maka/core/session'; @@ -223,6 +224,82 @@ test('projects durable and active transcript records before sharing them', async } }); +test('shares the Core visibility decision while retaining Host-only body sanitization', () => { + const messages: StoredMessage[] = [ + userMessage(0), + assistantMessage(1), + { + type: 'tool_call', + id: 'call', + turnId: 'turn-0', + ts: 2, + toolName: 'Read', + args: {}, + modelVisibility: 'hidden', + }, + { + type: 'tool_result', + id: 'result', + turnId: 'turn-0', + ts: 3, + toolUseId: 'call', + isError: false, + content: { kind: 'text', text: 'ok' }, + modelVisibility: 'hidden', + }, + { + type: 'permission_decision', + id: 'permission', + turnId: 'turn-0', + ts: 4, + toolUseId: 'call', + toolName: 'Read', + decision: 'allow', + }, + { type: 'token_usage', id: 'usage', turnId: 'turn-0', ts: 5, input: 1, output: 1 }, + { + type: 'turn_state', + id: 'state', + turnId: 'turn-0', + ts: 6, + status: 'completed', + partialOutputRetained: false, + }, + { + type: 'workhub_coordination', + id: 'coordination', + turnId: 'turn-0', + ts: 7, + schemaVersion: 1, + kind: 'delegation_assigned', + actionId: 'action', + actionFingerprint: `sha256:${'0'.repeat(64)}`, + coordinationTurnId: 'turn-0', + targetSessionId: 'target', + disposition: 'delegate_existing', + userText: 'work', + delegationId: 'delegation', + targetTurnId: 'target-turn', + targetMessageId: 'target-message', + targetSessionName: 'target', + }, + { type: 'system_note', id: 'visible-note', ts: 8, kind: 'step_limit' }, + { type: 'system_note', id: 'hidden-note', ts: 9, kind: 'mode_change' }, + ]; + for (const message of messages) { + const visibility = classifySharedSessionTranscriptVisibility( + message.type === 'system_note' + ? { type: message.type, kind: message.kind } + : { type: message.type }, + ); + assert.equal( + projectSharedSessionTranscriptMessage(message, 'session-1') === null, + visibility === 'hidden', + message.type, + ); + } +}); + test('rejects cursor tampering and cross-subscription replay', async () => { const reader = transcriptReader([userMessage(0, 'x'.repeat(2_000))]); const first = await createSessionTranscriptBootstrap({ diff --git a/packages/runtime-host/src/server/shared-session-transcript.ts b/packages/runtime-host/src/server/shared-session-transcript.ts index 5c59dd8b0e..3d8bab879d 100644 --- a/packages/runtime-host/src/server/shared-session-transcript.ts +++ b/packages/runtime-host/src/server/shared-session-transcript.ts @@ -20,7 +20,7 @@ import type { AttachmentRef, MessageContent } from '@maka/core/events'; import { projectToolActivityArgs } from '@maka/core/tool-activity-args'; import { - isUserVisibleSessionSystemNote, + classifySharedSessionTranscriptVisibility, type AssistantThinking, type StoredMessage, userFacingText, @@ -49,6 +49,12 @@ export function projectSharedSessionTranscriptMessage( message: StoredMessage, sessionId: string, ): StoredMessage | null { + const visibility = classifySharedSessionTranscriptVisibility( + message.type === 'system_note' + ? { type: message.type, kind: message.kind } + : { type: message.type }, + ); + if (visibility === 'hidden') return null; switch (message.type) { case 'user': { return { @@ -153,15 +159,13 @@ export function projectSharedSessionTranscriptMessage( ...(message.costUsd === undefined ? {} : { costUsd: message.costUsd }), }; case 'system_note': - return isUserVisibleSessionSystemNote(message.kind) - ? { - type: message.type, - id: message.id, - ...(message.turnId === undefined ? {} : { turnId: message.turnId }), - ts: message.ts, - kind: message.kind, - } - : null; + return { + type: message.type, + id: message.id, + ...(message.turnId === undefined ? {} : { turnId: message.turnId }), + ts: message.ts, + kind: message.kind, + }; case 'permission_decision': case 'workhub_coordination': return null; diff --git a/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts index de9881c89a..b6952bf4a7 100644 --- a/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts +++ b/packages/storage/src/__tests__/session-turn-identity-scanner.test.ts @@ -19,7 +19,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { decodeStoredMessage } from '@maka/core/session'; +import { classifySharedSessionTranscriptVisibility, decodeStoredMessage } from '@maka/core/session'; import { advanceSessionTurnIdentityScanner, completeSessionTurnIdentityScanner, @@ -54,7 +54,7 @@ test('extracts only the top-level identity envelope across arbitrary fragment bo id: 'message-1', ts: 1, }); - const expected = { kind: 'turn', turnId: 'turn-一' }; + const expected = { kind: 'turn', positionId: 'turn-一', sharedVisibility: true }; assert.deepEqual(scan(json), expected); for (let split = 1; split < Buffer.byteLength(json); split += 1) { assert.deepEqual(scan(json, [split]), expected); @@ -68,7 +68,7 @@ test('handles escaped identity values and visible/invisible turnless notes', () [2, 7, 19, 31, 47], { messageId: 'note-1', messageType: 'system_note' }, ), - { kind: 'note', turnId: 'session-note:note-1' }, + { kind: 'note', positionId: 'note-1', sharedVisibility: true }, ); assert.deepEqual( scan('{"id":"hidden","type":"system_note","kind":"mode_change"}', [], { @@ -87,11 +87,28 @@ test('rejects duplicate, empty, missing, nested, mismatched, and non-string iden '{"id":"message-1","type":"user","nested":{"turnId":"turn"}}', '{"id":"message-1","type":"user","turnId":1}', '{"id":"message-1","type":"user","turnId":{"value":"turn"}}', + '{"id":"message-1","type":"future_message","turnId":"turn"}', ]; for (const json of invalid) { assert.throws(() => scan(json), SessionTurnIdentityScannerError); } assert.throws(() => scan('{"id":"other","type":"user","turnId":"turn"}'), /mismatch/u); + assert.throws( + () => + scan('{"id":"note","type":"system_note","kind":"future_note"}', [], { + messageId: 'note', + messageType: 'system_note', + }), + SessionTurnIdentityScannerError, + ); + assert.throws( + () => + scan('{"id":"note","type":"system_note","kind":"step_limit","turnId":""}', [], { + messageId: 'note', + messageType: 'system_note', + }), + SessionTurnIdentityScannerError, + ); }); test('rejects invalid UTF-8, truncated JSON, excessive nesting, and corrupt persisted state', () => { @@ -157,7 +174,7 @@ test('rejects captured identity larger than 32 KiB without retaining body fields }), [4 * 1024], ), - { kind: 'turn', turnId: 'turn' }, + { kind: 'turn', positionId: 'turn', sharedVisibility: true }, ); assert.deepEqual( scan( @@ -169,7 +186,7 @@ test('rejects captured identity larger than 32 KiB without retaining body fields }), [4 * 1024], ), - { kind: 'turn', turnId: 'turn' }, + { kind: 'turn', positionId: 'turn', sharedVisibility: true }, ); }); @@ -255,7 +272,16 @@ test('differentially extracts every canonical StoredMessage identity under key r messageId: 'message-1', messageType: message.type, }), - { kind: 'turn', turnId: 'turn-1' }, + { + kind: 'turn', + positionId: 'turn-1', + sharedVisibility: + classifySharedSessionTranscriptVisibility( + decoded.type === 'system_note' + ? { type: decoded.type, kind: decoded.kind } + : { type: decoded.type }, + ) === 'visible', + }, ); } }); diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index a0c0b29a51..14c2c3afff 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -33,6 +33,171 @@ import { createSessionStore } from '../session-store.js'; import { advanceSessionTurnIdentityRecovery } from '../session-turn-identity-recovery.js'; describe('Session Turn position snapshots', () => { + test('materializes owner and shared tagged positions in one exact generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-dual-turn-position-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessages(session.id, [ + { + type: 'permission_decision', + id: 'hidden-first', + turnId: 'same-id', + ts: 1, + toolUseId: 'tool-use', + toolName: 'Bash', + decision: 'deny', + }, + { type: 'user', id: 'visible-second', turnId: 'same-id', ts: 2, text: 'visible' }, + { type: 'system_note', id: 'same-id', ts: 3, kind: 'step_limit' }, + ]); + + const owner = await readyPage(store, session.id, 'owner-lease', 'owner'); + const shared = await readyPage(store, session.id, 'shared-lease', 'shared'); + assert.equal(owner.snapshotKey.snapshotGeneration, shared.snapshotKey.snapshotGeneration); + assert.equal(owner.totalPositions, 2); + assert.equal(shared.totalPositions, 2); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'same-id' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'note', id: 'same-id' }, firstSequence: 2 }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'same-id' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'note', id: 'same-id' }, firstSequence: 2 }, + ]); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('makes empty and all-hidden shared projections observably identical', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-empty-position-')); + const store = createSessionStore(root); + try { + const emptySession = await store.create(makeInput({ name: 'Empty' })); + const hiddenSession = await store.create(makeInput({ name: 'Hidden' })); + await store.appendMessages(hiddenSession.id, [ + hiddenPermission('hidden-turn', 0), + { + type: 'system_note', + id: 'hidden-note', + ts: 1, + kind: 'mode_change', + }, + ]); + const empty = await readyPage(store, emptySession.id, 'empty-shared', 'shared'); + const hidden = await readyPage(store, hiddenSession.id, 'hidden-shared', 'shared'); + assert.equal(empty.totalPositions, 1); + assert.equal(hidden.totalPositions, 1); + assert.deepEqual(empty.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + assert.deepEqual(hidden.positions, empty.positions); + const owner = await readyPage(store, hiddenSession.id, 'hidden-owner', 'owner'); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'hidden-turn' }, firstSequence: 0 }, + ]); + assert.equal(owner.snapshotKey.snapshotGeneration, hidden.snapshotKey.snapshotGeneration); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: hiddenSession.id, + projection: 'shared', + snapshotLeaseId: 'hidden-shared', + snapshotKey: hidden.snapshotKey, + positionKeys: bodyKeys('hidden-turn'), + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('binds leases to a projection while sharing one retained generation', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-projection-lease-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const owner = await readyPage(store, session.id, 'owner-only', 'owner'); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'owner-only', + snapshotKey: owner.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const shared = await readyPage(store, session.id, 'shared-only', 'shared'); + assert.deepEqual(shared.snapshotKey, owner.snapshotKey); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'owner-only', + snapshotKey: owner.snapshotKey, + }); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'shared-only', + snapshotKey: shared.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed and lazily rebuilds when persisted shared policy version drifts', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-policy-version-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, user('turn-a', 0)); + const before = await readyPage(store, session.id, 'policy-before', 'shared'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'policy-before', + snapshotKey: before.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const after = await readyPage(store, session.id, 'policy-after', 'shared'); + assert.equal(after.snapshotKey.authorityRevision, before.snapshotKey.authorityRevision + 1); + assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(after.positions, before.positions); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('publishes one stable synthetic position for an empty Session', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-empty-turn-position-')); const store = createSessionStore(root); @@ -48,11 +213,11 @@ describe('Session Turn position snapshots', () => { snapshotGeneration: 1, }); assert.equal(page.startOrdinal, 0); - assert.equal(page.totalTurns, 1); + assert.equal(page.totalPositions, 1); assert.deepEqual(page.positions, [ { ordinal: 0, - turnId: `session:${session.id}`, + key: { kind: 'empty' }, firstSequence: null, }, ]); @@ -94,6 +259,7 @@ describe('Session Turn position snapshots', () => { const ready = await readyPage(store, session.id); const first = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'ordinal', ordinal: 1 }, @@ -102,8 +268,8 @@ describe('Session Turn position snapshots', () => { assert.equal(first.kind, 'page'); if (first.kind !== 'page') assert.fail('expected a ready legacy page'); assert.deepEqual(first.positions, [ - { ordinal: 1, turnId: 'turn-b', firstSequence: 2 }, - { ordinal: 2, turnId: 'session-note:note', firstSequence: 4 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 2 }, + { ordinal: 2, key: { kind: 'note', id: 'note' }, firstSequence: 4 }, ]); for (const anchor of [ @@ -112,6 +278,7 @@ describe('Session Turn position snapshots', () => { ]) { const anchored = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: first.snapshotKey, anchor, @@ -124,6 +291,7 @@ describe('Session Turn position snapshots', () => { } const clamped = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'ordinal', ordinal: 999 }, @@ -132,11 +300,12 @@ describe('Session Turn position snapshots', () => { assert.equal(clamped.kind, 'page'); if (clamped.kind !== 'page') assert.fail('expected a clamped inclusive ordinal'); assert.deepEqual(clamped.positions, [ - { ordinal: 2, turnId: 'session-note:note', firstSequence: 4 }, + { ordinal: 2, key: { kind: 'note', id: 'note' }, firstSequence: 4 }, ]); await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'turn', turnId: 'missing-turn' }, @@ -148,6 +317,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'turn', turnId: '' }, @@ -188,9 +358,9 @@ describe('Session Turn position snapshots', () => { assert.equal(page.kind, 'page'); if (page.kind !== 'page') assert.fail('expected modern positions'); assert.deepEqual(page.positions, [ - { ordinal: 0, turnId: 'turn-c', firstSequence: 2 }, - { ordinal: 1, turnId: 'turn-a', firstSequence: 1 }, - { ordinal: 2, turnId: 'turn-b', firstSequence: 0 }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-c' }, firstSequence: 2 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 1 }, + { ordinal: 2, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 0 }, ]); } finally { runs.close?.(); @@ -208,7 +378,26 @@ describe('Session Turn position snapshots', () => { await runs.admitRootTurn(rootAdmission(session.id, 'turn-bodyless', 'future-user', 10)); const bodyless = await readyPage(store, session.id, 'lease-bodyless'); assert.deepEqual(bodyless.positions, [ - { ordinal: 0, turnId: 'turn-bodyless', firstSequence: null }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: null }, + ]); + const bodylessRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-bodyless', + snapshotKey: bodyless.snapshotKey, + positionKeys: bodyKeys('turn-bodyless'), + maxRecords: 1, + maxBytes: 1, + }); + assert.deepEqual(bodylessRecords.records, []); + assert.equal(bodylessRecords.rawBytes, 0); + const sharedBodyless = await readyPage(store, session.id, 'lease-bodyless-shared', 'shared'); + assert.equal( + sharedBodyless.snapshotKey.snapshotGeneration, + bodyless.snapshotKey.snapshotGeneration, + ); + assert.deepEqual(sharedBodyless.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, ]); await store.appendMessages(session.id, [ @@ -225,8 +414,18 @@ describe('Session Turn position snapshots', () => { ]); const materialized = await readyPage(store, session.id, 'lease-materialized'); assert.deepEqual(materialized.positions, [ - { ordinal: 0, turnId: 'turn-bodyless', firstSequence: 1 }, - { ordinal: 1, turnId: 'session-note:visible-note', firstSequence: 3 }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'note', id: 'visible-note' }, firstSequence: 3 }, + ]); + const sharedMaterialized = await readyPage( + store, + session.id, + 'lease-materialized-shared', + 'shared', + ); + assert.deepEqual(sharedMaterialized.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-bodyless' }, firstSequence: 2 }, + { ordinal: 1, key: { kind: 'note', id: 'visible-note' }, firstSequence: 3 }, ]); } finally { runs.close?.(); @@ -260,66 +459,70 @@ describe('Session Turn position snapshots', () => { } }); - test('fails closed when a generated note identity collides with a real Turn identity', async () => { + test('keeps equal note and Turn strings distinct through composite identity', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-note-turn-identity-collision-')); let store = createSessionStore(root); const session = await store.create(makeInput()); try { await store.appendMessage(session.id, { type: 'system_note', - id: 'collision', + id: 'session-note:collision', ts: 1, kind: 'step_limit', }); - await assert.rejects( - store.appendMessage(session.id, { - type: 'user', - id: 'colliding-turn', - turnId: 'session-note:collision', - ts: 2, - text: 'must not fold into the note', - }), - (error: unknown) => (error as { reason?: unknown }).reason === 'incompatible_identity', + await store.appendMessage(session.id, { + type: 'user', + id: 'colliding-turn', + turnId: 'session-note:collision', + ts: 2, + text: 'separate Turn namespace', + }); + await store.appendMessage(session.id, { + type: 'user', + id: 'prefixed-turn', + turnId: 'session:real-turn', + ts: 3, + text: 'prefix is opaque', + }); + const page = await readyPage(store, session.id, 'lease-collision'); + assert.deepEqual( + page.positions.map(({ key }) => key), + [ + { kind: 'note', id: 'session-note:collision' }, + { kind: 'turn', id: 'session-note:collision' }, + { kind: 'turn', id: 'session:real-turn' }, + ], ); - assert.equal(await store.readTranscriptHighWaterSnapshot(session.id), 0); - - await store.appendMessage(session.id, user('turn-other', 1)); - await store.close?.(); - resetProjection(root, session.id); - const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); - try { - database - .prepare(`UPDATE session_messages SET record_json = ? - WHERE session_id = ? AND sequence = 1`) - .run( - JSON.stringify({ - ...user('session-note:collision', 1), - id: 'user-1', - }), - session.id, - ); - } finally { - database.close(); - } - store = createSessionStore(root); - await assert.rejects(readyPage(store, session.id, 'lease-collision'), (error: unknown) => { - assert.equal((error as { reason?: unknown }).reason, 'incompatible_identity'); - assert.equal((error as { sequence?: unknown }).sequence, 1); - return true; + const turnAnchor = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-collision', + snapshotKey: page.snapshotKey, + anchor: { kind: 'turn', turnId: 'session-note:collision' }, + maxPositions: 1, }); - - const reverse = await store.create(makeInput({ name: 'Reverse collision' })); - await store.appendMessage(reverse.id, user('session-note:reverse', 2)); - await assert.rejects( - store.appendMessage(reverse.id, { - type: 'system_note', - id: 'reverse', - ts: 3, - kind: 'step_limit', - }), - (error: unknown) => (error as { reason?: unknown }).reason === 'incompatible_identity', + assert.equal(turnAnchor.kind, 'page'); + if (turnAnchor.kind !== 'page') assert.fail('expected exact Turn anchor'); + assert.equal(turnAnchor.startOrdinal, 1); + const bodies = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-collision', + snapshotKey: page.snapshotKey, + positionKeys: [ + { kind: 'turn', id: 'session-note:collision' }, + { kind: 'note', id: 'session-note:collision' }, + ], + maxRecords: 2, + maxBytes: 64 * 1024, + }); + assert.deepEqual( + bodies.records.map(({ positionKey, message }) => [positionKey, message.id]), + [ + [{ kind: 'note', id: 'session-note:collision' }, 'session-note:collision'], + [{ kind: 'turn', id: 'session-note:collision' }, 'colliding-turn'], + ], ); - assert.equal(await store.readTranscriptHighWaterSnapshot(reverse.id), 0); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -344,7 +547,7 @@ describe('Session Turn position snapshots', () => { }); const admitted = await readyPage(store, session.id, 'lease-admitted'); assert.deepEqual( - admitted.positions.map(({ turnId }) => turnId), + admitted.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), ['turn-bodyless', 'turn-body'], ); @@ -354,7 +557,9 @@ describe('Session Turn position snapshots', () => { legacy.snapshotKey.authorityRevision, admitted.snapshotKey.authorityRevision + 1, ); - assert.deepEqual(legacy.positions, [{ ordinal: 0, turnId: 'turn-body', firstSequence: 0 }]); + assert.deepEqual(legacy.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-body' }, firstSequence: 0 }, + ]); await store.remove(session.id); await conversation.purge(session.id); } finally { @@ -375,6 +580,7 @@ describe('Session Turn position snapshots', () => { const secondFacade = createSessionStore(root); const shared = await secondFacade.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-two', anchor: { kind: 'tail' }, maxPositions: 8, @@ -384,19 +590,31 @@ describe('Session Turn position snapshots', () => { assert.deepEqual(shared.snapshotKey, first.snapshotKey); await secondFacade.close?.(); - await store.releaseTurnPositionSnapshot(session.id, 'lease-one', first.snapshotKey); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-one', + snapshotKey: first.snapshotKey, + }); const retained = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-two', snapshotKey: first.snapshotKey, anchor: { kind: 'tail' }, maxPositions: 8, }); assert.equal(retained.kind, 'page'); - await store.releaseTurnPositionSnapshot(session.id, 'lease-two', first.snapshotKey); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-two', + snapshotKey: first.snapshotKey, + }); await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-two', snapshotKey: first.snapshotKey, anchor: { kind: 'tail' }, @@ -430,10 +648,10 @@ describe('Session Turn position snapshots', () => { const page = await readyPage(store, session.id); assert.deepEqual(page.positions, [ - { ordinal: 0, turnId: 'turn-legacy', firstSequence: 0 }, - { ordinal: 1, turnId: 'turn-a', firstSequence: 2 }, - { ordinal: 2, turnId: 'turn-b', firstSequence: 1 }, - { ordinal: 3, turnId: 'session-note:modern-note', firstSequence: 3 }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-legacy' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 2 }, + { ordinal: 2, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 1 }, + { ordinal: 3, key: { kind: 'note', id: 'modern-note' }, firstSequence: 3 }, ]); } finally { runs.close?.(); @@ -475,7 +693,7 @@ describe('Session Turn position snapshots', () => { await store.appendMessages(session.id, [user('turn-a', 0), user('turn-b', 1)]); const legacy = await readyPage(store, session.id); assert.deepEqual( - legacy.positions.map(({ turnId }) => turnId), + legacy.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), ['turn-a', 'turn-b'], ); @@ -489,12 +707,13 @@ describe('Session Turn position snapshots', () => { snapshotGeneration: legacy.snapshotKey.snapshotGeneration + 1, }); assert.deepEqual( - modern.positions.map(({ turnId }) => turnId), + modern.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), ['turn-b', 'turn-a'], ); const stableLegacy = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: legacy.snapshotKey, anchor: { kind: 'tail' }, @@ -503,7 +722,7 @@ describe('Session Turn position snapshots', () => { assert.equal(stableLegacy.kind, 'page'); if (stableLegacy.kind !== 'page') assert.fail('expected retained exact snapshot'); assert.deepEqual( - stableLegacy.positions.map(({ turnId }) => turnId), + stableLegacy.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), ['turn-a', 'turn-b'], ); } finally { @@ -525,6 +744,7 @@ describe('Session Turn position snapshots', () => { ); const building = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -537,6 +757,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: building.snapshotKey, anchor: { kind: 'tail' }, @@ -560,17 +781,19 @@ describe('Session Turn position snapshots', () => { await store.appendMessage(session.id, user('turn-a', 0)); const first = await readyPage(store, session.id, 'lease-first'); await store.appendMessage(session.id, user('turn-b', 1)); - const second = await readyPage(store, session.id, 'lease-second'); + const second = await readyPage(store, session.id, 'lease-second', 'shared'); await store.appendMessage(session.id, user('turn-c', 2)); const capacity = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-third', anchor: { kind: 'tail' }, maxPositions: 8, }); assert.deepEqual(capacity, { kind: 'capacity', + projection: 'owner', throughSequence: 2, authorityRevision: 0, retainedSnapshots: 2, @@ -578,6 +801,7 @@ describe('Session Turn position snapshots', () => { const stableFirst = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-first', snapshotKey: first.snapshotKey, anchor: { kind: 'tail' }, @@ -585,13 +809,18 @@ describe('Session Turn position snapshots', () => { }); assert.equal(stableFirst.kind, 'page'); if (stableFirst.kind !== 'page') assert.fail('expected retained first snapshot'); - assert.equal(stableFirst.totalTurns, 1); + assert.equal(stableFirst.totalPositions, 1); assert.deepEqual( - stableFirst.positions.map(({ turnId }) => turnId), + stableFirst.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id)), ['turn-a'], ); - await store.releaseTurnPositionSnapshot(session.id, 'lease-first', first.snapshotKey); + await store.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: first.snapshotKey, + }); const third = await readyPage(store, session.id, 'lease-third'); assert.equal(first.snapshotKey.snapshotGeneration, 1); assert.equal(second.snapshotKey.snapshotGeneration, 2); @@ -599,6 +828,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-first', snapshotKey: first.snapshotKey, anchor: { kind: 'tail' }, @@ -642,6 +872,7 @@ describe('Session Turn position snapshots', () => { store = createSessionStore(root); const building = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -656,6 +887,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: building.snapshotKey, anchor: { kind: 'tail' }, @@ -665,7 +897,7 @@ describe('Session Turn position snapshots', () => { (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', ); const ready = await readyPage(store, session.id, 'lease-resumed'); - assert.equal(ready.totalTurns, 1_025); + assert.equal(ready.totalPositions, 1_025); assert.ok(ready.snapshotKey.snapshotGeneration > building.snapshotKey.snapshotGeneration); } finally { await store.close?.(); @@ -691,6 +923,7 @@ describe('Session Turn position snapshots', () => { const building = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -721,6 +954,7 @@ describe('Session Turn position snapshots', () => { const first = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -731,8 +965,11 @@ describe('Session Turn position snapshots', () => { assert.equal(first.progress.lastStepRecords, 0); assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); const ready = await readyPage(store, session.id); - assert.equal(ready.totalTurns, 1); - assert.equal(ready.positions[0]?.turnId, 'turn-oversized'); + assert.equal(ready.totalPositions, 1); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-oversized', + ); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -770,6 +1007,7 @@ describe('Session Turn position snapshots', () => { const first = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-legacy-inline', anchor: { kind: 'tail' }, maxPositions: 8, @@ -781,7 +1019,10 @@ describe('Session Turn position snapshots', () => { assert.equal(first.progress.currentByteOffset, 4 * 1024 * 1024); const ready = await readyPage(store, session.id, 'lease-legacy-inline'); - assert.equal(ready.positions[0]?.turnId, 'turn-legacy-inline'); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-legacy-inline', + ); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -804,6 +1045,7 @@ describe('Session Turn position snapshots', () => { let result = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: `lease-exact-${index}`, anchor: { kind: 'tail' }, maxPositions: 8, @@ -817,6 +1059,7 @@ describe('Session Turn position snapshots', () => { } result = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: `lease-exact-${index}`, snapshotKey: result.snapshotKey, anchor: { kind: 'tail' }, @@ -825,7 +1068,10 @@ describe('Session Turn position snapshots', () => { } assert.equal(result.kind, 'page'); if (result.kind !== 'page') assert.fail('expected exact-size recovered page'); - assert.equal(result.positions[0]?.turnId, `exact-turn-${index}`); + assert.equal( + result.positions[0] ? positionId(result.positions[0]) : undefined, + `exact-turn-${index}`, + ); assert.equal( recoveryBytes.reduce((total, bytes) => total + bytes, 0), targetBytes, @@ -852,6 +1098,7 @@ describe('Session Turn position snapshots', () => { store = createSessionStore(root); const partial = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-derived-before', anchor: { kind: 'tail' }, maxPositions: 8, @@ -881,6 +1128,7 @@ describe('Session Turn position snapshots', () => { store = createSessionStore(root); const restarted = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-derived-after', anchor: { kind: 'tail' }, maxPositions: 8, @@ -891,7 +1139,10 @@ describe('Session Turn position snapshots', () => { assert.equal(restarted.progress.lastStepBytes, 4 * 1024 * 1024); assert.equal(restarted.progress.sourceBytes, 8 * 1024 * 1024); const ready = await readyPage(store, session.id, 'lease-derived-after'); - assert.equal(ready.positions[0]?.turnId, 'turn-derived-reset'); + assert.equal( + ready.positions[0] ? positionId(ready.positions[0]) : undefined, + 'turn-derived-reset', + ); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -956,7 +1207,7 @@ describe('Session Turn position snapshots', () => { } store = createSessionStore(root); const ready = await readyPage(store, session.id, 'lease-atomic'); - assert.equal(ready.positions[0]?.turnId, 'turn-atomic'); + assert.equal(ready.positions[0] ? positionId(ready.positions[0]) : undefined, 'turn-atomic'); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -977,6 +1228,7 @@ describe('Session Turn position snapshots', () => { store = createSessionStore(root); const partial = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-source-before', anchor: { kind: 'tail' }, maxPositions: 8, @@ -998,6 +1250,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: lease, anchor: { kind: 'tail' }, maxPositions: 8, @@ -1034,7 +1287,7 @@ describe('Session Turn position snapshots', () => { try { const session = await store.create(makeInput()); await store.appendMessages(session.id, [ - user('turn-a', 0), + hiddenPermission('turn-a', 0), { type: 'assistant', id: 'assistant-a', @@ -1046,33 +1299,73 @@ describe('Session Turn position snapshots', () => { user('turn-b', 1), ]); const page = await readyPage(store, session.id); + const sharedPage = await readyPage(store, session.id, 'lease-shared-records', 'shared'); + assert.equal(sharedPage.positions[0]?.firstSequence, 1); - const records = await store.readTranscriptRecordsByTurnIdsSnapshot({ + const records = await store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, - turnIds: ['turn-a'], + positionKeys: bodyKeys('turn-a'), maxBytes: 64 * 1024, maxRecords: 8, }); assert.deepEqual(records.snapshotKey, page.snapshotKey); assert.deepEqual( - records.records.map(({ sequence, message }) => [sequence, message.id]), + records.records.map(({ positionKey, sequence, message }) => [ + positionKey, + sequence, + message.id, + ]), + [ + [{ kind: 'turn', id: 'turn-a' }, 0, 'permission-0'], + [{ kind: 'turn', id: 'turn-a' }, 1, 'assistant-a'], + ], + ); + const sharedRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-records', + snapshotKey: sharedPage.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 8, + }); + assert.deepEqual( + sharedRecords.records.map(({ positionKey, sequence, message }) => [ + positionKey, + sequence, + message.id, + ]), [ - [0, 'user-0'], - [1, 'assistant-a'], + [{ kind: 'turn', id: 'turn-a' }, 0, 'permission-0'], + [{ kind: 'turn', id: 'turn-a' }, 1, 'assistant-a'], ], ); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-records', + snapshotKey: sharedPage.snapshotKey, + positionKeys: bodyKeys('turn-a'), + maxBytes: 64 * 1024, + maxRecords: 1, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); for (const [limits, reason] of [ [{ maxBytes: 64 * 1024, maxRecords: 1 }, 'transcript_record_count'], [{ maxBytes: 1, maxRecords: 8 }, 'transcript_record_bytes'], ] as const) { await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, - turnIds: ['turn-a'], + positionKeys: bodyKeys('turn-a'), ...limits, }), (error: unknown) => { @@ -1090,11 +1383,12 @@ describe('Session Turn position snapshots', () => { { ...page.snapshotKey, snapshotGeneration: 999 }, ]) { await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey, - turnIds: ['turn-a'], + positionKeys: bodyKeys('turn-a'), maxBytes: 64 * 1024, maxRecords: 8, }), @@ -1103,33 +1397,43 @@ describe('Session Turn position snapshots', () => { ); } for (const request of [ - { turnIds: ['turn-a', 'turn-a'], maxRecords: 8, maxBytes: 64 * 1024 }, - { turnIds: [''], maxRecords: 8, maxBytes: 64 * 1024 }, + { positionKeys: bodyKeys('turn-a', 'turn-a'), maxRecords: 8, maxBytes: 64 * 1024 }, + { positionKeys: bodyKeys(''), maxRecords: 8, maxBytes: 64 * 1024 }, { - turnIds: Array.from({ length: 129 }, (_, index) => `turn-${index}`), + positionKeys: Array.from({ length: 129 }, (_, index) => `turn-${index}`).map((id) => ({ + kind: 'turn' as const, + id, + })), maxRecords: 8, maxBytes: 64 * 1024, }, - { turnIds: ['turn-a'], maxRecords: 257, maxBytes: 64 * 1024 }, - { turnIds: ['turn-a'], maxRecords: 8, maxBytes: 16 * 1024 * 1024 + 1 }, + { positionKeys: bodyKeys('turn-a'), maxRecords: 257, maxBytes: 64 * 1024 }, + { positionKeys: bodyKeys('turn-a'), maxRecords: 8, maxBytes: 16 * 1024 * 1024 + 1 }, + { positionKeys: [{ kind: 'empty' }] as never, maxRecords: 8, maxBytes: 64 * 1024 }, ]) { await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, ...request, }), - /invalid|unique/iu, + /invalid|unique|stale|mismatched/iu, ); } for (const request of [ - { snapshotLeaseId: 'lease-unknown', turnIds: ['turn-a'] }, - { snapshotLeaseId: 'lease-default', turnIds: ['turn-unknown'] }, + { snapshotLeaseId: 'lease-unknown', positionKeys: bodyKeys('turn-a') }, + { snapshotLeaseId: 'lease-default', positionKeys: bodyKeys('turn-unknown') }, + { + snapshotLeaseId: 'lease-default', + positionKeys: [{ kind: 'note' as const, id: 'turn-a' }], + }, ]) { await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotKey: page.snapshotKey, maxRecords: 8, maxBytes: 64 * 1024, @@ -1150,22 +1454,24 @@ describe('Session Turn position snapshots', () => { database.close(); } await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, - turnIds: ['turn-a'], + positionKeys: bodyKeys('turn-a'), maxRecords: 1, maxBytes: 64 * 1024, }), (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', ); await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, - turnIds: ['turn-a'], + positionKeys: bodyKeys('turn-a'), maxRecords: 2, maxBytes: 64 * 1024, }), @@ -1201,6 +1507,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -1230,6 +1537,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-retry-failed-source', anchor: { kind: 'tail' }, maxPositions: 8, @@ -1242,6 +1550,116 @@ describe('Session Turn position snapshots', () => { } }); + test('fails closed when a valid stored body no longer resolves to its exact position key', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-body-identity-drift-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const original = user('turn-before', 0); + await store.appendMessage(session.id, original); + const page = await readyPage(store, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = ? + WHERE session_id = ? AND sequence = 0`) + .run(JSON.stringify({ ...original, turnId: 'turn-after' }), session.id); + } finally { + database.close(); + } + + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-default', + snapshotKey: page.snapshotKey, + positionKeys: bodyKeys('turn-before'), + maxRecords: 1, + maxBytes: 64 * 1024, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'stored_session_message_incompatible', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('enforces exact shared raw record-count and stored-byte boundaries', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-body-limits-')); + const store = createSessionStore(root); + try { + const countSession = await store.create(makeInput({ name: 'Count boundary' })); + await store.appendMessages(countSession.id, [ + user('turn-count-boundary', 0), + ...Array.from({ length: 255 }, (_, index) => + hiddenPermission('turn-count-boundary', index + 1), + ), + ]); + const countAtLimit = await readyPage(store, countSession.id, 'count-at-limit', 'shared'); + const exactCount = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: countSession.id, + projection: 'shared', + snapshotLeaseId: 'count-at-limit', + snapshotKey: countAtLimit.snapshotKey, + positionKeys: bodyKeys('turn-count-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }); + assert.equal(exactCount.records.length, 256); + await store.appendMessage(countSession.id, hiddenPermission('turn-count-boundary', 256)); + const countOverLimit = await readyPage(store, countSession.id, 'count-over-limit', 'shared'); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: countSession.id, + projection: 'shared', + snapshotLeaseId: 'count-over-limit', + snapshotKey: countOverLimit.snapshotKey, + positionKeys: bodyKeys('turn-count-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_count', + ); + + const byteSession = await store.create(makeInput({ name: 'Byte boundary' })); + await store.appendMessage( + byteSession.id, + exactSizeUserMessage(16 * 1024 * 1024, 'turn-byte-boundary', false), + ); + const bytesAtLimit = await readyPage(store, byteSession.id, 'bytes-at-limit', 'shared'); + const exactBytes = await store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: byteSession.id, + projection: 'shared', + snapshotLeaseId: 'bytes-at-limit', + snapshotKey: bytesAtLimit.snapshotKey, + positionKeys: bodyKeys('turn-byte-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }); + assert.equal(exactBytes.rawBytes, 16 * 1024 * 1024); + await store.appendMessage(byteSession.id, hiddenPermission('turn-byte-boundary', 1)); + const bytesOverLimit = await readyPage(store, byteSession.id, 'bytes-over-limit', 'shared'); + await assert.rejects( + store.readTranscriptRecordsByPositionKeysSnapshot({ + sessionId: byteSession.id, + projection: 'shared', + snapshotLeaseId: 'bytes-over-limit', + snapshotKey: bytesOverLimit.snapshotKey, + positionKeys: bodyKeys('turn-byte-boundary'), + maxRecords: 256, + maxBytes: 16 * 1024 * 1024, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'transcript_record_bytes', + ); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('persists chunk-integrity failure across repair/reopen until authoritative invalidation', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-chunk-corrupt-')); let store = createSessionStore(root); @@ -1291,7 +1709,10 @@ describe('Session Turn position snapshots', () => { resetProjection(root, session.id); store = createSessionStore(root); const recovered = await readyPage(store, session.id, 'lease-after-reset'); - assert.equal(recovered.positions[0]?.turnId, 'turn-chunk-corrupt'); + assert.equal( + recovered.positions[0] ? positionId(recovered.positions[0]) : undefined, + 'turn-chunk-corrupt', + ); } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -1342,11 +1763,12 @@ describe('Session Turn position snapshots', () => { } await assert.rejects( - store.readTranscriptRecordsByTurnIdsSnapshot({ + store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, - turnIds: ['turn-chunk-shape'], + positionKeys: bodyKeys('turn-chunk-shape'), maxRecords: 1, maxBytes: 2 * 1024 * 1024, }), @@ -1371,8 +1793,9 @@ describe('Session Turn position snapshots', () => { try { database .prepare(`INSERT INTO session_turn_metadata( - session_id, turn_id, identity_kind, order_source, first_sequence - ) VALUES (?, 'turn-conflict', 'turn', 'legacy', 0)`) + session_id, position_kind, position_id, order_source, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', 'turn-conflict', 'legacy', 0, 0)`) .run(session.id); } finally { database.close(); @@ -1382,6 +1805,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -1410,6 +1834,7 @@ describe('Session Turn position snapshots', () => { const ready = await readyPage(store, session.id); const page = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'ordinal', ordinal: 0 }, @@ -1423,6 +1848,7 @@ describe('Session Turn position snapshots', () => { const tail = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: page.snapshotKey, anchor: { kind: 'tail' }, @@ -1461,17 +1887,44 @@ describe('Session Turn position snapshots', () => { const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-imported-')); const store = createSessionStore(root); try { - const session = await store.createImportedSession( - makeInput(), - [{ ...user('turn-imported', 0), text: 'x'.repeat(20 * 1024) }, user('turn-second', 1)], - { adapterId: 'test-adapter', sourceSessionId: 'source-session' }, - ); + const messages = [ + { ...user('turn-imported', 0), text: 'x'.repeat(20 * 1024) }, + user('turn-second', 1), + ]; + const session = await store.createImportedSession(makeInput(), messages, { + adapterId: 'test-adapter', + sourceSessionId: 'source-session', + }); const page = await readyPage(store, session.id); assert.deepEqual(page.positions, [ - { ordinal: 0, turnId: 'turn-imported', firstSequence: 0 }, - { ordinal: 1, turnId: 'turn-second', firstSequence: 1 }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-imported' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-second' }, firstSequence: 1 }, ]); assert.ok(Buffer.byteLength(JSON.stringify(page), 'utf8') < 4 * 1024); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const state = database + .prepare(`SELECT indexed_through_sequence, source_records, source_bytes + FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as { + indexed_through_sequence: number; + source_records: number; + source_bytes: number; + }; + assert.equal(state.indexed_through_sequence, 1); + assert.equal(state.source_records, 2); + assert.equal( + state.source_bytes, + messages.reduce( + (total, message) => total + Buffer.byteLength(JSON.stringify(message), 'utf8'), + 0, + ), + ); + } finally { + database.close(); + } } finally { await store.close?.(); await rm(root, { recursive: true, force: true }); @@ -1490,6 +1943,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: before.snapshotKey, anchor: { kind: 'tail' }, @@ -1519,6 +1973,7 @@ describe('Session Turn position snapshots', () => { second = createSessionStore(root); const shared = await second.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-second', anchor: { kind: 'tail' }, maxPositions: 8, @@ -1527,9 +1982,15 @@ describe('Session Turn position snapshots', () => { if (shared.kind !== 'page') assert.fail('expected a shared ready generation'); assert.deepEqual(shared.snapshotKey, original.snapshotKey); - await first.releaseTurnPositionSnapshot(session.id, 'lease-first', original.snapshotKey); + await first.releaseTurnPositionSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-first', + snapshotKey: original.snapshotKey, + }); const retained = await second.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-second', snapshotKey: shared.snapshotKey, anchor: { kind: 'tail' }, @@ -1560,22 +2021,23 @@ describe('Session Turn position snapshots', () => { () => database .prepare(`INSERT INTO session_turn_snapshot_positions( - session_id, snapshot_generation, ordinal, turn_id, first_sequence - ) VALUES (?, ?, 3, 'turn-extra', 3)`) + session_id, snapshot_generation, position_kind, position_id, + owner_ordinal, shared_ordinal, owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, 'turn', 'turn-extra', 3, 3, 3, 3)`) .run(session.id, ready.snapshotKey.snapshotGeneration), () => database - .prepare(`UPDATE session_turn_snapshot_positions SET turn_id = 'mutated' - WHERE session_id = ? AND snapshot_generation = ? AND ordinal = 2`) + .prepare(`UPDATE session_turn_snapshot_positions SET position_id = 'mutated' + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal = 2`) .run(session.id, ready.snapshotKey.snapshotGeneration), () => database .prepare(`DELETE FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND ordinal = 2`) + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal = 2`) .run(session.id, ready.snapshotKey.snapshotGeneration), () => database - .prepare(`UPDATE session_turn_position_snapshots SET ready_total = 2 + .prepare(`UPDATE session_turn_position_snapshots SET ready_owner_total = 2 WHERE session_id = ? AND snapshot_generation = ?`) .run(session.id, ready.snapshotKey.snapshotGeneration), ]) { @@ -1586,6 +2048,7 @@ describe('Session Turn position snapshots', () => { } const stable = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: ready.snapshotKey, anchor: { kind: 'tail' }, @@ -1636,6 +2099,7 @@ describe('Session Turn position snapshots', () => { await assert.rejects( store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: before.snapshotKey, anchor: { kind: 'tail' }, @@ -1647,7 +2111,7 @@ describe('Session Turn position snapshots', () => { const after = await readyPage(store, session.id); assert.ok(after.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); assert.deepEqual( - after.positions.map(({ turnId, firstSequence }) => [turnId, firstSequence]), + after.positions.map((position) => [positionId(position), position.firstSequence]), [ ['turn-prior', 0], ['turn-target', 1], @@ -1703,8 +2167,8 @@ describe('Session Turn position snapshots', () => { } const page = await readyPage(store, session.id); assert.deepEqual(page.positions, [ - { ordinal: 0, turnId: 'turn-a', firstSequence: 0 }, - { ordinal: 1, turnId: 'turn-b', firstSequence: 1 }, + { ordinal: 0, key: { kind: 'turn', id: 'turn-a' }, firstSequence: 0 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-b' }, firstSequence: 1 }, ]); const migrated = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { readOnly: true, @@ -1720,6 +2184,29 @@ describe('Session Turn position snapshots', () => { ).version, 35, ); + const metadataColumns = new Set( + ( + migrated.prepare(`PRAGMA table_info(session_turn_metadata)`).all() as Array<{ + name: string; + }> + ).map(({ name }) => name), + ); + assert.deepEqual( + ['position_kind', 'position_id', 'owner_first_sequence', 'shared_first_sequence'].map( + (name) => metadataColumns.has(name), + ), + [true, true, true, true], + ); + assert.equal(metadataColumns.has('turn_id'), false); + const snapshotColumns = new Set( + ( + migrated.prepare(`PRAGMA table_info(session_turn_position_snapshots)`).all() as Array<{ + name: string; + }> + ).map(({ name }) => name), + ); + assert.equal(snapshotColumns.has('ready_owner_total'), true); + assert.equal(snapshotColumns.has('ready_shared_total'), true); } finally { migrated.close(); } @@ -1729,17 +2216,20 @@ describe('Session Turn position snapshots', () => { } }); - test('uses indexed keyset plans for 10,000-Turn steady pages and every anchor', async () => { + test('uses indexed owner/shared keyset plans for 10,000 alternating-visibility Turns', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-plan-')); const store = createSessionStore(root); try { const session = await store.create(makeInput()); await store.appendMessages( session.id, - Array.from({ length: 10_000 }, (_, index) => user(`turn-${index}`, index)), + Array.from({ length: 10_000 }, (_, index) => + index % 2 === 0 ? user(`turn-${index}`, index) : hiddenPermission(`turn-${index}`, index), + ), ); const firstStep = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', anchor: { kind: 'tail' }, maxPositions: 8, @@ -1747,11 +2237,13 @@ describe('Session Turn position snapshots', () => { assert.equal(firstStep.kind, 'building'); if (firstStep.kind !== 'building') assert.fail('expected bounded position publication'); assert.equal(firstStep.progress.phase, 'recovering'); - assert.equal(firstStep.progress.sourceRecords, 0); + assert.equal(firstStep.progress.sourceRecords, 10_000); + assert.ok(firstStep.progress.sourceBytes > 0); assert.equal(firstStep.progress.builtPositions, 0); assert.equal(firstStep.progress.lastStepPositions, 0); const secondStep = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: firstStep.snapshotKey, anchor: { kind: 'tail' }, @@ -1761,6 +2253,18 @@ describe('Session Turn position snapshots', () => { if (secondStep.kind !== 'building') assert.fail('expected legacy ordinal build'); assert.equal(secondStep.progress.phase, 'legacy'); assert.equal(secondStep.progress.builtPositions, 1_024); + const sharedProgress = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-progress', + throughSequence: secondStep.snapshotKey.throughSequence, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(sharedProgress.kind, 'building'); + if (sharedProgress.kind !== 'building') assert.fail('expected shared ordinal progress'); + assert.equal(sharedProgress.progress.builtPositions, 1_024); + assert.equal(sharedProgress.progress.lastStepPositions, 512); const bounded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { readOnly: true, }); @@ -1772,20 +2276,21 @@ describe('Session Turn position snapshots', () => { WHERE session_id = ? AND snapshot_generation = ?`) .get(session.id, secondStep.snapshotKey.snapshotGeneration) as { count: number } ).count, - 1_024, + 2_048, ); const recoveryPlans = [ bounded - .prepare(`EXPLAIN QUERY PLAN SELECT MIN(first_sequence) AS boundary + .prepare(`EXPLAIN QUERY PLAN SELECT MIN(owner_first_sequence) AS boundary FROM session_turn_metadata - WHERE session_id = ? AND order_source = 'admission' AND first_sequence <= ?`) + WHERE session_id = ? AND order_source = 'admission' AND owner_first_sequence <= ?`) .all(session.id, 9_999), bounded - .prepare(`EXPLAIN QUERY PLAN SELECT turn_id, first_sequence, NULL AS admitted_at + .prepare(`EXPLAIN QUERY PLAN SELECT position_kind, position_id, + owner_first_sequence, NULL AS admitted_at FROM session_turn_metadata - WHERE session_id = ? AND first_sequence <= ? AND order_source = 'legacy' - AND (? IS NULL OR first_sequence < ?) AND first_sequence > ? - ORDER BY first_sequence, turn_id LIMIT ?`) + WHERE session_id = ? AND owner_first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR owner_first_sequence < ?) AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_kind, position_id LIMIT ?`) .all(session.id, 9_999, null, null, -1, 1_025), ].flat() as Array<{ detail: string }>; assert.equal( @@ -1801,6 +2306,7 @@ describe('Session Turn position snapshots', () => { } let tail = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: secondStep.snapshotKey, anchor: { kind: 'tail' }, @@ -1809,6 +2315,7 @@ describe('Session Turn position snapshots', () => { while (tail.kind === 'building') { tail = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: tail.snapshotKey, anchor: { kind: 'tail' }, @@ -1817,14 +2324,30 @@ describe('Session Turn position snapshots', () => { } assert.equal(tail.kind, 'page'); if (tail.kind !== 'page') assert.fail('expected steady page'); - assert.equal(tail.totalTurns, 10_000); + assert.equal(tail.totalPositions, 10_000); assert.equal(tail.positions.length, 8); + const sharedTail = await readyPage(store, session.id, 'lease-shared-10k', 'shared'); + assert.equal(sharedTail.snapshotKey.snapshotGeneration, tail.snapshotKey.snapshotGeneration); + assert.equal(sharedTail.totalPositions, 5_000); + assert.equal(sharedTail.positions.length, 8); + const sharedMiddle = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'shared', + snapshotLeaseId: 'lease-shared-10k', + snapshotKey: sharedTail.snapshotKey, + anchor: { kind: 'ordinal', ordinal: 2_500 }, + maxPositions: 8, + }); + assert.equal(sharedMiddle.kind, 'page'); + if (sharedMiddle.kind !== 'page') assert.fail('expected indexed shared page'); + assert.deepEqual(sharedMiddle.positions[0]?.key, { kind: 'turn', id: 'turn-5000' }); - const sparseRecords = await store.readTranscriptRecordsByTurnIdsSnapshot({ + const sparseRecords = await store.readTranscriptRecordsByPositionKeysSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: tail.snapshotKey, - turnIds: ['turn-9999', 'turn-0'], + positionKeys: bodyKeys('turn-9999', 'turn-0'), maxRecords: 2, maxBytes: 64 * 1024, }); @@ -1839,20 +2362,32 @@ describe('Session Turn position snapshots', () => { try { const plans = [ database - .prepare(`EXPLAIN QUERY PLAN SELECT ordinal FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND first_sequence <= ? - ORDER BY first_sequence DESC LIMIT 1`) + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND owner_ordinal IS NOT NULL AND owner_first_sequence <= ? + ORDER BY owner_first_sequence DESC LIMIT 1`) .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000), database - .prepare(`EXPLAIN QUERY PLAN SELECT ordinal FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ?`) + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = 'turn' AND position_id = ? AND owner_ordinal IS NOT NULL`) .all(session.id, tail.snapshotKey.snapshotGeneration, 'turn-5000'), database - .prepare(`EXPLAIN QUERY PLAN SELECT ordinal, turn_id, first_sequence + .prepare(`EXPLAIN QUERY PLAN SELECT owner_ordinal, position_kind, position_id, + owner_first_sequence FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND ordinal >= ? - ORDER BY ordinal LIMIT ?`) + WHERE session_id = ? AND snapshot_generation = ? + AND owner_ordinal IS NOT NULL AND owner_ordinal >= ? + ORDER BY owner_ordinal LIMIT ?`) .all(session.id, tail.snapshotKey.snapshotGeneration, 5_000, 8), + database + .prepare(`EXPLAIN QUERY PLAN SELECT shared_ordinal, position_kind, position_id, + shared_first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND shared_ordinal IS NOT NULL AND shared_ordinal >= ? + ORDER BY shared_ordinal LIMIT ?`) + .all(session.id, tail.snapshotKey.snapshotGeneration, 2_500, 8), ].flat() as Array<{ detail: string }>; assert.equal( plans.some(({ detail }) => /\bSCAN\b|USE TEMP B-TREE/.test(detail)), @@ -1872,7 +2407,8 @@ describe('Session Turn position snapshots', () => { AND message.sequence = membership.sequence LEFT JOIN session_message_payloads AS payload ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE membership.session_id = ? AND membership.turn_id = ? + WHERE membership.session_id = ? AND membership.position_kind = 'turn' + AND membership.position_id = ? AND membership.sequence <= ? ORDER BY membership.sequence LIMIT ?`) .all(session.id, 'turn-9999', 9_999, 2) as Array<{ detail: string }>; @@ -1881,7 +2417,9 @@ describe('Session Turn position snapshots', () => { false, ); assert.equal( - membershipPlan.some(({ detail }) => detail.includes('session_turn_memberships_by_turn')), + membershipPlan.some(({ detail }) => + detail.includes('session_turn_memberships_by_position'), + ), true, ); } finally { @@ -1895,6 +2433,7 @@ describe('Session Turn position snapshots', () => { ]) { const page = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: tail.snapshotKey, anchor, @@ -1907,6 +2446,7 @@ describe('Session Turn position snapshots', () => { } const adjacent = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: tail.snapshotKey, anchor: { kind: 'ordinal', ordinal: 5_008 }, @@ -1918,9 +2458,13 @@ describe('Session Turn position snapshots', () => { adjacent.positions.map(({ ordinal }) => ordinal), Array.from({ length: 8 }, (_, index) => 5_008 + index), ); - assert.equal(new Set(adjacent.positions.map(({ turnId }) => turnId)).size, 8); + assert.equal( + new Set(adjacent.positions.map(({ key }) => (key.kind === 'empty' ? '' : key.id))).size, + 8, + ); const prior = await store.readTurnPositionPageSnapshot({ sessionId: session.id, + projection: 'owner', snapshotLeaseId: 'lease-default', snapshotKey: tail.snapshotKey, anchor: { kind: 'ordinal', ordinal: 5_000 }, @@ -1929,8 +2473,8 @@ describe('Session Turn position snapshots', () => { assert.equal(prior.kind, 'page'); if (prior.kind !== 'page') assert.fail('expected prior keyset page'); assert.equal( - prior.positions.some(({ turnId }) => - adjacent.positions.some((position) => position.turnId === turnId), + prior.positions.some((position) => + adjacent.positions.some((candidate) => positionId(candidate) === positionId(position)), ), false, ); @@ -1943,9 +2487,15 @@ describe('Session Turn position snapshots', () => { type TestStore = ReturnType; -async function readyPage(store: TestStore, sessionId: string, snapshotLeaseId = 'lease-default') { +async function readyPage( + store: TestStore, + sessionId: string, + snapshotLeaseId = 'lease-default', + projection: 'owner' | 'shared' = 'owner', +) { let result = await store.readTurnPositionPageSnapshot({ sessionId, + projection, snapshotLeaseId, anchor: { kind: 'tail' }, maxPositions: 8, @@ -1953,6 +2503,7 @@ async function readyPage(store: TestStore, sessionId: string, snapshotLeaseId = while (result.kind === 'building') { result = await store.readTurnPositionPageSnapshot({ sessionId, + projection, snapshotLeaseId, snapshotKey: result.snapshotKey, anchor: { kind: 'tail' }, @@ -1974,6 +2525,26 @@ function user(turnId: string, index: number) { }; } +function hiddenPermission(turnId: string, index: number) { + return { + type: 'permission_decision' as const, + id: `permission-${index}`, + turnId, + ts: index, + toolUseId: `tool-${index}`, + toolName: 'Read', + decision: 'deny' as const, + }; +} + +function bodyKeys(...ids: string[]) { + return ids.map((id) => ({ kind: 'turn' as const, id })); +} + +function positionId(position: { key: { kind: 'turn' | 'note'; id: string } | { kind: 'empty' } }) { + return position.key.kind === 'empty' ? '' : position.key.id; +} + function exactSizeUserMessage(targetBytes: number, turnId: string, turnIdAfterBody: boolean) { const message = turnIdAfterBody ? { type: 'user' as const, id: `user-${turnId}`, ts: 1, text: '', turnId } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index dc1484ad65..3187c1964e 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -137,13 +137,17 @@ export type { SessionTranscriptRecordScanRequest, SessionTranscriptStoragePage, SessionTranscriptStorageFragment, - SessionTranscriptRecordsByTurnIdsSnapshotRequest, - SessionTranscriptRecordsByTurnIdsSnapshotResult, + SessionTranscriptBodyPositionKey, + SessionTranscriptPositionKey, + SessionTranscriptProjection, + SessionTranscriptRecordsByPositionKeysSnapshotRequest, + SessionTranscriptRecordsByPositionKeysSnapshotResult, SessionTurnPosition, SessionTurnPositionAnchor, SessionTurnPositionPageSnapshotRequest, SessionTurnPositionReadResult, SessionTurnPositionSnapshotKey, + SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; export type ExecutionSessionWriter = SessionAuthorityStore; @@ -425,12 +429,10 @@ async function createExecutionStoresForWrite sessionStore.readTurnLandmarksSnapshot(sessionId, maxLandmarks)), readTurnPositionPageSnapshot: (request) => run(() => sessionStore.readTurnPositionPageSnapshot(request)), - readTranscriptRecordsByTurnIdsSnapshot: (request) => - run(() => sessionStore.readTranscriptRecordsByTurnIdsSnapshot(request)), - releaseTurnPositionSnapshot: (sessionId, snapshotLeaseId, snapshotKey) => - run(() => - sessionStore.releaseTurnPositionSnapshot(sessionId, snapshotLeaseId, snapshotKey), - ), + readTranscriptRecordsByPositionKeysSnapshot: (request) => + run(() => sessionStore.readTranscriptRecordsByPositionKeysSnapshot(request)), + releaseTurnPositionSnapshot: (request) => + run(() => sessionStore.releaseTurnPositionSnapshot(request)), readMessagesForRecovery: (sessionId) => run(() => sessionStore.readMessagesForRecovery(sessionId)), listTurnsSnapshot: (sessionId) => run(() => sessionStore.listTurnsSnapshot(sessionId)), diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 32568d32f4..c23a910407 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -307,6 +307,18 @@ export interface SessionTurnPositionSnapshotKey { readonly snapshotGeneration: number; } +export type SessionTranscriptProjection = 'owner' | 'shared'; + +export type SessionTranscriptPositionKey = + | { readonly kind: 'turn'; readonly id: string } + | { readonly kind: 'note'; readonly id: string } + | { readonly kind: 'empty' }; + +export type SessionTranscriptBodyPositionKey = Exclude< + SessionTranscriptPositionKey, + { readonly kind: 'empty' } +>; + export type SessionTurnPositionAnchor = /** The final position, returned as the inclusive end of a tail-sized page. */ | { readonly kind: 'tail' } @@ -319,12 +331,13 @@ export type SessionTurnPositionAnchor = export interface SessionTurnPosition { readonly ordinal: number; - readonly turnId: string; + readonly key: SessionTranscriptPositionKey; readonly firstSequence: number | null; } export interface SessionTurnPositionPageSnapshotRequest { readonly sessionId: string; + readonly projection: SessionTranscriptProjection; readonly snapshotLeaseId: string; /** Exact continuation identity. Omit only when allocating a new snapshot. */ readonly snapshotKey?: SessionTurnPositionSnapshotKey; @@ -338,6 +351,7 @@ export type SessionTurnPositionReadResult = | { readonly kind: 'building'; readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly projection: SessionTranscriptProjection; readonly progress: { readonly phase: 'recovering' | 'legacy' | 'admission' | 'notes'; readonly nextSequence: number; @@ -352,6 +366,7 @@ export type SessionTurnPositionReadResult = } | { readonly kind: 'capacity'; + readonly projection: SessionTranscriptProjection; readonly throughSequence: number | null; readonly authorityRevision: number; readonly retainedSnapshots: 2; @@ -359,28 +374,42 @@ export type SessionTurnPositionReadResult = | { readonly kind: 'page'; readonly snapshotKey: SessionTurnPositionSnapshotKey; + readonly projection: SessionTranscriptProjection; readonly startOrdinal: number; - readonly totalTurns: number; + readonly totalPositions: number; readonly positions: readonly SessionTurnPosition[]; readonly hasOlder: boolean; readonly hasNewer: boolean; }; -export interface SessionTranscriptRecordsByTurnIdsSnapshotRequest { +export interface SessionTranscriptRecordsByPositionKeysSnapshotRequest { readonly sessionId: string; + readonly projection: SessionTranscriptProjection; readonly snapshotLeaseId: string; readonly snapshotKey: SessionTurnPositionSnapshotKey; - readonly turnIds: readonly string[]; + readonly positionKeys: readonly SessionTranscriptBodyPositionKey[]; readonly maxBytes: number; readonly maxRecords: number; } -export interface SessionTranscriptRecordsByTurnIdsSnapshotResult { +export interface SessionTranscriptRecordsByPositionKeysSnapshotResult { readonly snapshotKey: SessionTurnPositionSnapshotKey; - readonly records: readonly { readonly sequence: number; readonly message: StoredMessage }[]; + readonly projection: SessionTranscriptProjection; + readonly records: readonly { + readonly positionKey: SessionTranscriptBodyPositionKey; + readonly sequence: number; + readonly message: StoredMessage; + }[]; readonly rawBytes: number; } +export interface SessionTurnPositionSnapshotReleaseRequest { + readonly sessionId: string; + readonly projection: SessionTranscriptProjection; + readonly snapshotLeaseId: string; + readonly snapshotKey: SessionTurnPositionSnapshotKey; +} + export class SessionTurnPositionSnapshotMismatchError extends Error { readonly name = 'SessionTurnPositionSnapshotMismatchError'; readonly code = 'session_turn_position_snapshot_mismatch'; @@ -457,14 +486,10 @@ export interface SessionStore { readTurnPositionPageSnapshot( request: SessionTurnPositionPageSnapshotRequest, ): Promise; - readTranscriptRecordsByTurnIdsSnapshot( - request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, - ): Promise; - releaseTurnPositionSnapshot( - sessionId: string, - snapshotLeaseId: string, - snapshotKey: SessionTurnPositionSnapshotKey, - ): Promise; + readTranscriptRecordsByPositionKeysSnapshot( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise; + releaseTurnPositionSnapshot(request: SessionTurnPositionSnapshotReleaseRequest): Promise; /** Read durable messages for startup recovery. */ readMessagesForRecovery(sessionId: string): Promise; /** Derive durable turns without triggering connection-lock self-healing. */ @@ -1091,20 +1116,18 @@ class SqliteSessionStore implements SessionAuthorityStore { return this.metadata.readTurnPositionPage(request); } - async readTranscriptRecordsByTurnIdsSnapshot( - request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, - ): Promise { + async readTranscriptRecordsByPositionKeysSnapshot( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise { await this.ensureReady(); - return this.metadata.readTranscriptRecordsByTurnIds(request); + return this.metadata.readTranscriptRecordsByPositionKeys(request); } async releaseTurnPositionSnapshot( - sessionId: string, - snapshotLeaseId: string, - snapshotKey: SessionTurnPositionSnapshotKey, + request: SessionTurnPositionSnapshotReleaseRequest, ): Promise { await this.ensureReady(); - await this.metadata.releaseTurnPositionSnapshot(sessionId, snapshotLeaseId, snapshotKey); + await this.metadata.releaseTurnPositionSnapshot(request); } async readMessagesForRecovery(sessionId: string): Promise { diff --git a/packages/storage/src/session-turn-membership.ts b/packages/storage/src/session-turn-membership.ts index cf87834244..64583aa50d 100644 --- a/packages/storage/src/session-turn-membership.ts +++ b/packages/storage/src/session-turn-membership.ts @@ -18,12 +18,20 @@ */ import type { DatabaseSync } from 'node:sqlite'; -import { isUserVisibleSessionSystemNote, type StoredMessage } from '@maka/core/session'; +import { + classifySharedSessionTranscriptVisibility, + isSessionSystemNoteKind, + isStoredMessageType, + type StoredMessage, +} from '@maka/core/session'; import { sqliteTableExists } from './sqlite-schema-introspection.js'; export type SessionTurnIdentity = - | { readonly kind: 'turn'; readonly turnId: string } - | { readonly kind: 'note'; readonly turnId: string } + | { + readonly kind: 'turn' | 'note'; + readonly positionId: string; + readonly sharedVisibility: boolean; + } | { readonly kind: 'ignored' }; export class SessionTurnIdentityClassificationError extends Error { @@ -51,20 +59,36 @@ export function classifySessionTurnIdentity(input: { readonly kindPresent: boolean; readonly kind?: string; }): SessionTurnIdentity { + if (input.id.length === 0) { + throw new SessionTurnIdentityClassificationError('message id must be a non-empty string'); + } + if (!isStoredMessageType(input.type)) { + throw new SessionTurnIdentityClassificationError('message type is unknown'); + } + const visibility = (() => { + if (input.type !== 'system_note') { + return classifySharedSessionTranscriptVisibility({ type: input.type }); + } + if (!input.kindPresent || !isSessionSystemNoteKind(input.kind)) { + throw new SessionTurnIdentityClassificationError('system note kind is unknown or missing'); + } + return classifySharedSessionTranscriptVisibility({ type: input.type, kind: input.kind }); + })(); if (input.turnIdPresent) { if (typeof input.turnId !== 'string' || input.turnId.length === 0) { throw new SessionTurnIdentityClassificationError('turnId must be a non-empty string'); } - return { kind: 'turn', turnId: input.turnId }; + return { + kind: 'turn', + positionId: input.turnId, + sharedVisibility: visibility === 'visible', + }; } if (input.type !== 'system_note') { throw new SessionTurnIdentityClassificationError('non-system message is missing turnId'); } - if (!input.kindPresent || typeof input.kind !== 'string' || input.kind.length === 0) { - throw new SessionTurnIdentityClassificationError('turnless system note is missing kind'); - } - return input.id.length > 0 && isUserVisibleSessionSystemNote(input.kind) - ? { kind: 'note', turnId: `session-note:${input.id}` } + return visibility === 'visible' + ? { kind: 'note', positionId: input.id, sharedVisibility: true } : { kind: 'ignored' }; } @@ -94,49 +118,60 @@ export function publishSessionTurnMembership( SELECT admitted_at FROM core_root_turn_admissions WHERE session_id = ? AND turn_id = ? `) - .get(sessionId, identity.turnId) as { admitted_at: number } | undefined) + .get(sessionId, identity.positionId) as { admitted_at: number } | undefined) : undefined; - const existingIdentity = db - .prepare(` - SELECT identity_kind FROM session_turn_metadata - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, identity.turnId) as { identity_kind: string } | undefined; - if (existingIdentity && existingIdentity.identity_kind !== identity.kind) { - throw new SessionTurnMembershipPublicationError('incompatible_identity', sequence); - } db.prepare(` INSERT INTO session_turn_metadata( - session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence - ) VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(session_id, turn_id) DO UPDATE SET + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET order_source = CASE WHEN excluded.order_source = 'admission' THEN 'admission' ELSE session_turn_metadata.order_source END, admitted_at = COALESCE(excluded.admitted_at, session_turn_metadata.admitted_at), - first_sequence = CASE WHEN session_turn_metadata.first_sequence IS NULL - THEN excluded.first_sequence - ELSE MIN(session_turn_metadata.first_sequence, excluded.first_sequence) END + owner_first_sequence = CASE WHEN session_turn_metadata.owner_first_sequence IS NULL + THEN excluded.owner_first_sequence + ELSE MIN(session_turn_metadata.owner_first_sequence, excluded.owner_first_sequence) END, + shared_first_sequence = CASE + WHEN excluded.shared_first_sequence IS NULL THEN session_turn_metadata.shared_first_sequence + WHEN session_turn_metadata.shared_first_sequence IS NULL THEN excluded.shared_first_sequence + ELSE MIN(session_turn_metadata.shared_first_sequence, excluded.shared_first_sequence) END `).run( sessionId, - identity.turnId, identity.kind, + identity.positionId, admission ? 'admission' : 'legacy', admission?.admitted_at ?? null, sequence, + identity.sharedVisibility ? sequence : null, ); const inserted = db .prepare(` - INSERT INTO session_turn_memberships(session_id, sequence, turn_id) - VALUES (?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING + INSERT INTO session_turn_memberships( + session_id, sequence, position_kind, position_id, shared_visibility + ) VALUES (?, ?, ?, ?, ?) ON CONFLICT(session_id, sequence) DO NOTHING `) - .run(sessionId, sequence, identity.turnId); + .run( + sessionId, + sequence, + identity.kind, + identity.positionId, + identity.sharedVisibility ? 1 : 0, + ); if (inserted.changes === 1) return; const existing = db .prepare(` - SELECT turn_id FROM session_turn_memberships WHERE session_id = ? AND sequence = ? + SELECT position_kind, position_id, shared_visibility + FROM session_turn_memberships WHERE session_id = ? AND sequence = ? `) - .get(sessionId, sequence) as { turn_id: string } | undefined; - if (existing?.turn_id !== identity.turnId) { + .get(sessionId, sequence) as + | { position_kind: string; position_id: string; shared_visibility: number } + | undefined; + if ( + existing?.position_kind !== identity.kind || + existing.position_id !== identity.positionId || + existing.shared_visibility !== (identity.sharedVisibility ? 1 : 0) + ) { throw new SessionTurnMembershipPublicationError('corrupt_source', sequence); } } diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts index e14dfc5310..799ee02891 100644 --- a/packages/storage/src/session-turn-position-authority.ts +++ b/packages/storage/src/session-turn-position-authority.ts @@ -18,7 +18,10 @@ */ import type { DatabaseSync } from 'node:sqlite'; -import type { StoredMessage } from '@maka/core/session'; +import { + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, + type StoredMessage, +} from '@maka/core/session'; import { SessionTurnPositionRecoveryError } from './session-store.js'; import { classifyStoredMessageTurnIdentity, @@ -32,35 +35,43 @@ export function recordAppendedSessionTurnMetadata( db: DatabaseSync, sessionId: string, firstSequence: number, - messages: readonly StoredMessage[], + entries: readonly { + readonly message: StoredMessage; + readonly recordBytes: number; + }[], ): void { - if (!sqliteTableExists(db, 'session_turn_metadata') || messages.length === 0) return; + if (!sqliteTableExists(db, 'session_turn_metadata') || entries.length === 0) return; ensureTurnIndexRows(db, sessionId); - const fillsBodylessAdmission = messages.some((message) => { + const fillsBodylessAdmission = entries.some(({ message }) => { const turnId = (message as { turnId?: unknown }).turnId; if (typeof turnId !== 'string' || turnId.length === 0) return false; return Boolean( db .prepare(` SELECT 1 FROM session_turn_metadata - WHERE session_id = ? AND turn_id = ? AND order_source = 'admission' - AND first_sequence IS NULL + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? + AND order_source = 'admission' AND owner_first_sequence IS NULL `) .get(sessionId, turnId), ); }); - for (let index = 0; index < messages.length; index += 1) { - publishCanonicalMembership(db, sessionId, firstSequence + index, messages[index]!); + for (let index = 0; index < entries.length; index += 1) { + publishCanonicalMembership(db, sessionId, firstSequence + index, entries[index]!.message); } const state = db .prepare('SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ?') .get(sessionId) as { indexed_through_sequence: number }; if (state.indexed_through_sequence === firstSequence - 1) { + const sourceBytes = entries.reduce((total, { recordBytes }) => total + recordBytes, 0); + if (!Number.isSafeInteger(sourceBytes) || sourceBytes < 0) { + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); + } db.prepare(` UPDATE session_turn_index_state - SET indexed_through_sequence = ? + SET indexed_through_sequence = ?, + source_records = source_records + ?, source_bytes = source_bytes + ? WHERE session_id = ? - `).run(firstSequence + messages.length - 1, sessionId); + `).run(firstSequence + entries.length - 1, entries.length, sourceBytes, sessionId); } if (fillsBodylessAdmission) invalidateBuildingSnapshots(db, sessionId); } @@ -79,28 +90,26 @@ export function recordRootTurnAdmissionForPositionIndex( ensureTurnIndexRows(db, sessionId); const current = db .prepare(` - SELECT identity_kind, order_source, admitted_at, first_sequence - FROM session_turn_metadata WHERE session_id = ? AND turn_id = ? + SELECT order_source, admitted_at, owner_first_sequence + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? `) .get(sessionId, turnId) as | { - identity_kind: string; order_source: string; admitted_at: number | null; - first_sequence: number | null; + owner_first_sequence: number | null; } | undefined; - if (current && current.identity_kind !== 'turn') { - throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); - } if (current?.order_source === 'admission' && current.admitted_at !== admittedAt) { throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source'); } db.prepare(` INSERT INTO session_turn_metadata( - session_id, turn_id, identity_kind, order_source, admitted_at, first_sequence - ) VALUES (?, ?, 'turn', 'admission', ?, NULL) - ON CONFLICT(session_id, turn_id) DO UPDATE SET + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', ?, 'admission', ?, NULL, NULL) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET order_source = 'admission', admitted_at = excluded.admitted_at `).run(sessionId, turnId, admittedAt); db.prepare(` @@ -121,15 +130,15 @@ export function recordRootTurnAdmissionsPurgedForPositionIndex( const removed = db .prepare(` DELETE FROM session_turn_metadata - WHERE session_id = ? AND identity_kind = 'turn' - AND order_source = 'admission' AND first_sequence IS NULL + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence IS NULL `) .run(sessionId); const downgraded = db .prepare(` UPDATE session_turn_metadata SET order_source = 'legacy', admitted_at = NULL - WHERE session_id = ? AND identity_kind = 'turn' - AND order_source = 'admission' AND first_sequence IS NOT NULL + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence IS NOT NULL `) .run(sessionId); if (removed.changes === 0 && downgraded.changes === 0) return; @@ -158,13 +167,36 @@ export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { db.prepare(` - INSERT INTO session_turn_authority_revisions(session_id) - VALUES (?) ON CONFLICT(session_id) DO NOTHING - `).run(sessionId); + INSERT INTO session_turn_authority_revisions(session_id, visibility_policy_version) + VALUES (?, ?) ON CONFLICT(session_id) DO NOTHING + `).run(sessionId, SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION); db.prepare(` INSERT INTO session_turn_index_state(session_id) VALUES (?) ON CONFLICT(session_id) DO NOTHING `).run(sessionId); + const authority = db + .prepare(` + SELECT visibility_policy_version FROM session_turn_authority_revisions + WHERE session_id = ? + `) + .get(sessionId) as { visibility_policy_version: number }; + if (authority.visibility_policy_version === SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION) { + return; + } + db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); + db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + db.prepare(` + UPDATE session_turn_index_state + SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? + `).run(sessionId); + db.prepare(` + UPDATE session_turn_authority_revisions + SET visibility_policy_version = ?, authority_revision = authority_revision + 1 + WHERE session_id = ? + `).run(SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, sessionId); } function publishCanonicalMembership( diff --git a/packages/storage/src/session-turn-position-index.ts b/packages/storage/src/session-turn-position-index.ts index 95c452bc32..184649a914 100644 --- a/packages/storage/src/session-turn-position-index.ts +++ b/packages/storage/src/session-turn-position-index.ts @@ -29,7 +29,7 @@ export { export { SESSION_TURN_POSITION_BODY_MAX_BYTES, SESSION_TURN_POSITION_BODY_MAX_RECORDS, - SESSION_TURN_POSITION_BODY_MAX_TURNS, + SESSION_TURN_POSITION_BODY_MAX_KEYS, SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, SESSION_TURN_POSITION_MAX_PAGE_BYTES, SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts index 8bac6facad..71e9f891b8 100644 --- a/packages/storage/src/session-turn-position-snapshots.ts +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -23,10 +23,13 @@ import { SessionTurnPositionLimitError, SessionTurnPositionRecoveryError, SessionTurnPositionSnapshotMismatchError, - type SessionTranscriptRecordsByTurnIdsSnapshotRequest, + type SessionTranscriptBodyPositionKey, + type SessionTranscriptProjection, + type SessionTranscriptRecordsByPositionKeysSnapshotRequest, type SessionTurnPositionPageSnapshotRequest, type SessionTurnPositionReadResult, type SessionTurnPositionSnapshotKey, + type SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; import { ensureTurnIndexRows } from './session-turn-position-authority.js'; import { sqliteTableExists } from './sqlite-schema-introspection.js'; @@ -34,7 +37,7 @@ import { sqliteTableExists } from './sqlite-schema-introspection.js'; export const SESSION_TURN_POSITION_MAX_PAGE_POSITIONS = 128; export const SESSION_TURN_POSITION_MAX_PAGE_BYTES = 64 * 1024; export const SESSION_TURN_POSITION_BUILD_MAX_POSITIONS = 1_024; -export const SESSION_TURN_POSITION_BODY_MAX_TURNS = 128; +export const SESSION_TURN_POSITION_BODY_MAX_KEYS = 128; export const SESSION_TURN_POSITION_BODY_MAX_RECORDS = 256; export const SESSION_TURN_POSITION_BODY_MAX_BYTES = 16 * 1024 * 1024; @@ -52,11 +55,14 @@ export interface SessionTurnPositionSnapshotRow { readonly snapshot_generation: number; readonly state: 'building' | 'ready'; readonly build_phase: SessionTurnPositionBuildPhase; - readonly build_next_ordinal: number; + readonly build_next_owner_ordinal: number; + readonly build_next_shared_ordinal: number; readonly build_cursor_sequence: number; readonly build_cursor_admitted_at: number | null; - readonly build_cursor_turn_id: string | null; - readonly ready_total: number | null; + readonly build_cursor_position_kind: 'turn' | 'note' | null; + readonly build_cursor_position_id: string | null; + readonly ready_owner_total: number | null; + readonly ready_shared_total: number | null; } export type SessionTurnPositionAllocation = @@ -66,11 +72,13 @@ export type SessionTurnPositionAllocation = export interface SessionTurnPositionBuildStep { readonly snapshot: SessionTurnPositionSnapshotRow; readonly executedPhase: Exclude; - readonly lastStepPositions: number; } export interface SessionTurnMembershipPreflight { - readonly sequences: readonly number[]; + readonly records: readonly { + readonly positionKey: SessionTranscriptBodyPositionKey; + readonly sequence: number; + }[]; readonly storedBytes: number; } @@ -88,6 +96,7 @@ export function allocateOrRequireSessionTurnPositionSnapshot( snapshot: requireLeasedSnapshot( db, request.sessionId, + request.projection, request.snapshotLeaseId, request.snapshotKey, ), @@ -112,14 +121,26 @@ export function allocateOrRequireSessionTurnPositionSnapshot( authorityRevision, ); if (existing) { - acquireLease(db, request.sessionId, existing.snapshot_generation, request.snapshotLeaseId); + acquireLease( + db, + request.sessionId, + existing.snapshot_generation, + request.projection, + request.snapshotLeaseId, + ); return { kind: 'snapshot', snapshot: existing }; } const occupied = db .prepare('SELECT slot FROM session_turn_position_snapshots WHERE session_id = ? ORDER BY slot') .all(request.sessionId) as Array<{ slot: number }>; if (occupied.length >= 2) { - return { kind: 'capacity', throughSequence, authorityRevision, retainedSnapshots: 2 }; + return { + kind: 'capacity', + projection: request.projection, + throughSequence, + authorityRevision, + retainedSnapshots: 2, + }; } const used = new Set(occupied.map(({ slot }) => slot)); const slot = used.has(0) ? 1 : 0; @@ -139,7 +160,7 @@ export function allocateOrRequireSessionTurnPositionSnapshot( state, build_phase ) VALUES (?, ?, ?, ?, ?, 'building', 'recovering') `).run(request.sessionId, slot, throughSequence, authorityRevision, generation); - acquireLease(db, request.sessionId, generation, request.snapshotLeaseId); + acquireLease(db, request.sessionId, generation, request.projection, request.snapshotLeaseId); return { kind: 'snapshot', snapshot: requireSnapshot(db, request.sessionId, { @@ -158,9 +179,7 @@ export function markSessionTurnRecoveryComplete( if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') return snapshot; const indexed = ( db - .prepare(` - SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ? - `) + .prepare('SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ?') .get(sessionId) as { indexed_through_sequence: number } ).indexed_through_sequence; if (snapshot.through_sequence !== null && indexed < snapshot.through_sequence) return snapshot; @@ -188,56 +207,85 @@ export function advanceSessionTurnPositionOrdinalBuild( const rows = readBuildRows(db, sessionId, snapshot, boundary, executedPhase); const insert = db.prepare(` INSERT INTO session_turn_snapshot_positions( - session_id, snapshot_generation, ordinal, turn_id, first_sequence - ) VALUES (?, ?, ?, ?, ?) + session_id, snapshot_generation, position_kind, position_id, + owner_ordinal, shared_ordinal, owner_first_sequence, shared_first_sequence + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) `); - let nextOrdinal = snapshot.build_next_ordinal; + let nextOwnerOrdinal = snapshot.build_next_owner_ordinal; + let nextSharedOrdinal = snapshot.build_next_shared_ordinal; let sequenceCursor = snapshot.build_cursor_sequence; let admittedAtCursor = snapshot.build_cursor_admitted_at; - let turnIdCursor = snapshot.build_cursor_turn_id; + let positionKindCursor = snapshot.build_cursor_position_kind; + let positionIdCursor = snapshot.build_cursor_position_id; for (const row of rows) { + const sharedVisible = + row.shared_first_sequence !== null && + snapshot.through_sequence !== null && + row.shared_first_sequence <= snapshot.through_sequence; insert.run( sessionId, snapshot.snapshot_generation, - nextOrdinal, - row.turn_id, - row.first_sequence, + row.position_kind, + row.position_id, + nextOwnerOrdinal, + sharedVisible ? nextSharedOrdinal : null, + row.owner_first_sequence, + sharedVisible ? row.shared_first_sequence : null, ); - nextOrdinal += 1; - sequenceCursor = row.first_sequence ?? sequenceCursor; + nextOwnerOrdinal += 1; + if (sharedVisible) nextSharedOrdinal += 1; + sequenceCursor = row.owner_first_sequence ?? sequenceCursor; admittedAtCursor = row.admitted_at; - turnIdCursor = row.turn_id; + positionKindCursor = row.position_kind; + positionIdCursor = row.position_id; } const exhausted = rows.length < SESSION_TURN_POSITION_BUILD_MAX_POSITIONS; const nextPhase = exhausted ? followingPhase(executedPhase) : executedPhase; if (nextPhase === 'ready') { - if (nextOrdinal === 0) { - insert.run(sessionId, snapshot.snapshot_generation, 0, `session:${sessionId}`, null); - nextOrdinal = 1; + if (nextOwnerOrdinal === 0) { + insert.run(sessionId, snapshot.snapshot_generation, 'empty', '', 0, 0, null, null); + nextOwnerOrdinal = 1; + nextSharedOrdinal = 1; + } else if (nextSharedOrdinal === 0) { + insert.run(sessionId, snapshot.snapshot_generation, 'empty', '', null, 0, null, null); + nextSharedOrdinal = 1; } const published = db .prepare(` UPDATE session_turn_position_snapshots - SET state = 'ready', build_phase = 'ready', build_next_ordinal = ?, ready_total = ? + SET state = 'ready', build_phase = 'ready', + build_next_owner_ordinal = ?, build_next_shared_ordinal = ?, + ready_owner_total = ?, ready_shared_total = ? WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? `) - .run(nextOrdinal, nextOrdinal, sessionId, snapshot.slot, executedPhase); + .run( + nextOwnerOrdinal, + nextSharedOrdinal, + nextOwnerOrdinal, + nextSharedOrdinal, + sessionId, + snapshot.slot, + executedPhase, + ); if (published.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); } else { const phaseChanged = nextPhase !== executedPhase; const updated = db .prepare(` UPDATE session_turn_position_snapshots - SET build_phase = ?, build_next_ordinal = ?, build_cursor_sequence = ?, - build_cursor_admitted_at = ?, build_cursor_turn_id = ? + SET build_phase = ?, build_next_owner_ordinal = ?, build_next_shared_ordinal = ?, + build_cursor_sequence = ?, build_cursor_admitted_at = ?, + build_cursor_position_kind = ?, build_cursor_position_id = ? WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = ? `) .run( nextPhase, - nextOrdinal, + nextOwnerOrdinal, + nextSharedOrdinal, phaseChanged ? -1 : sequenceCursor, phaseChanged ? null : admittedAtCursor, - phaseChanged ? null : turnIdCursor, + phaseChanged ? null : positionKindCursor, + phaseChanged ? null : positionIdCursor, sessionId, snapshot.slot, executedPhase, @@ -247,7 +295,6 @@ export function advanceSessionTurnPositionOrdinalBuild( return { snapshot: requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)), executedPhase, - lastStepPositions: rows.length, }; } @@ -257,25 +304,34 @@ export function pageReadySessionTurnPositionSnapshot( snapshot: SessionTurnPositionSnapshotRow, ): Extract { validatePageRequest(request); - requireLease(db, request.sessionId, request.snapshotLeaseId, snapshot.snapshot_generation); - const totalTurns = snapshot.ready_total; - if (snapshot.state !== 'ready' || totalTurns === null || totalTurns < 1) { + requireLease( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + snapshot.snapshot_generation, + ); + const columns = projectionColumns(request.projection); + const totalPositions = + request.projection === 'owner' ? snapshot.ready_owner_total : snapshot.ready_shared_total; + if (snapshot.state !== 'ready' || totalPositions === null || totalPositions < 1) { throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); } let selectedOrdinal: number; switch (request.anchor.kind) { case 'tail': - selectedOrdinal = totalTurns - 1; + selectedOrdinal = totalPositions - 1; break; case 'ordinal': - selectedOrdinal = Math.min(request.anchor.ordinal, totalTurns - 1); + selectedOrdinal = Math.min(request.anchor.ordinal, totalPositions - 1); break; case 'sequence': { const row = db .prepare(` - SELECT ordinal FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND first_sequence <= ? - ORDER BY first_sequence DESC LIMIT 1 + SELECT ${columns.ordinal} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND ${columns.ordinal} IS NOT NULL AND ${columns.firstSequence} <= ? + ORDER BY ${columns.firstSequence} DESC LIMIT 1 `) .get(request.sessionId, snapshot.snapshot_generation, request.anchor.sequence) as | { ordinal: number } @@ -286,8 +342,10 @@ export function pageReadySessionTurnPositionSnapshot( case 'turn': { const row = db .prepare(` - SELECT ordinal FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ? + SELECT ${columns.ordinal} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = 'turn' AND position_id = ? + AND ${columns.ordinal} IS NOT NULL `) .get(request.sessionId, snapshot.snapshot_generation, request.anchor.turnId) as | { ordinal: number } @@ -301,21 +359,28 @@ export function pageReadySessionTurnPositionSnapshot( } let startOrdinal = request.anchor.kind === 'tail' - ? Math.max(0, totalTurns - request.maxPositions) + ? Math.max(0, totalPositions - request.maxPositions) : selectedOrdinal; const rows = db .prepare(` - SELECT ordinal, turn_id, first_sequence FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND ordinal >= ? - ORDER BY ordinal LIMIT ? + SELECT ${columns.ordinal} AS ordinal, position_kind, position_id, + ${columns.firstSequence} AS first_sequence + FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND ${columns.ordinal} >= ? + ORDER BY ${columns.ordinal} LIMIT ? `) .all( request.sessionId, snapshot.snapshot_generation, startOrdinal, request.maxPositions, - ) as Array<{ ordinal: number; turn_id: string; first_sequence: number | null }>; - const expectedRows = Math.min(request.maxPositions, totalTurns - startOrdinal); + ) as Array<{ + ordinal: number; + position_kind: 'turn' | 'note' | 'empty'; + position_id: string; + first_sequence: number | null; + }>; + const expectedRows = Math.min(request.maxPositions, totalPositions - startOrdinal); if ( rows.length !== expectedRows || rows.some((row, index) => row.ordinal !== startOrdinal + index) @@ -324,7 +389,10 @@ export function pageReadySessionTurnPositionSnapshot( } let positions = rows.map((row) => ({ ordinal: row.ordinal, - turnId: row.turn_id, + key: + row.position_kind === 'empty' + ? ({ kind: 'empty' } as const) + : ({ kind: row.position_kind, id: row.position_id } as const), firstSequence: row.first_sequence, })); const key = snapshotKeyFromRow(snapshot); @@ -334,11 +402,12 @@ export function pageReadySessionTurnPositionSnapshot( JSON.stringify({ kind: 'page', snapshotKey: key, + projection: request.projection, startOrdinal, - totalTurns, + totalPositions, positions, hasOlder: startOrdinal > 0, - hasNewer: startOrdinal + positions.length < totalTurns, + hasNewer: startOrdinal + positions.length < totalPositions, }), 'utf8', ) > SESSION_TURN_POSITION_MAX_PAGE_BYTES @@ -346,7 +415,9 @@ export function pageReadySessionTurnPositionSnapshot( if (request.anchor.kind === 'tail') { positions = positions.slice(1); startOrdinal += 1; - } else positions = positions.slice(0, -1); + } else { + positions = positions.slice(0, -1); + } } if (positions.length === 0) { throw new SessionTurnPositionLimitError(request.sessionId, 'page_metadata_bytes'); @@ -354,43 +425,57 @@ export function pageReadySessionTurnPositionSnapshot( return { kind: 'page', snapshotKey: key, + projection: request.projection, startOrdinal, - totalTurns, + totalPositions, positions, hasOlder: startOrdinal > 0, - hasNewer: startOrdinal + positions.length < totalTurns, + hasNewer: startOrdinal + positions.length < totalPositions, }; } export function readSessionTurnMembershipPreflight( db: DatabaseSync, - request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, ): SessionTurnMembershipPreflight { validateRecordRequest(request); const snapshot = requireLeasedSnapshot( db, request.sessionId, + request.projection, request.snapshotLeaseId, request.snapshotKey, ); - if (snapshot.state !== 'ready') + if (snapshot.state !== 'ready') { throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); - const unique = new Set(request.turnIds); - if (unique.size !== request.turnIds.length) { - throw new Error('Session Turn-position record Turn ids must be unique'); } - const sequences: number[] = []; - let storedBytes = 0; - for (const turnId of request.turnIds) { - const projected = db + const unique = new Set(request.positionKeys.map(positionKeyIdentity)); + if (unique.size !== request.positionKeys.length) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } + const ordinalColumn = projectionColumns(request.projection).ordinal; + const selected = request.positionKeys.map((positionKey) => { + const row = db .prepare(` - SELECT 1 FROM session_turn_snapshot_positions - WHERE session_id = ? AND snapshot_generation = ? AND turn_id = ? + SELECT ${ordinalColumn} AS ordinal FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? + AND position_kind = ? AND position_id = ? AND ${ordinalColumn} IS NOT NULL `) - .get(request.sessionId, snapshot.snapshot_generation, turnId); - if (!projected) throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + .get(request.sessionId, snapshot.snapshot_generation, positionKey.kind, positionKey.id) as + | { ordinal: number } + | undefined; + if (!row) throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + return { positionKey, ordinal: row.ordinal }; + }); + selected.sort((left, right) => left.ordinal - right.ordinal); + const records: Array<{ + positionKey: SessionTranscriptBodyPositionKey; + sequence: number; + }> = []; + let storedBytes = 0; + for (const selectedPosition of selected) { if (request.snapshotKey.throughSequence === null) continue; - const remaining = request.maxRecords - sequences.length; + const remaining = request.maxRecords - records.length; const rows = db .prepare(` SELECT membership.sequence, @@ -400,14 +485,17 @@ export function readSessionTurnMembershipPreflight( ON message.session_id = membership.session_id AND message.sequence = membership.sequence LEFT JOIN session_message_payloads AS payload ON payload.session_id = message.session_id AND payload.sequence = message.sequence - WHERE membership.session_id = ? AND membership.turn_id = ? - AND membership.sequence <= ? + WHERE membership.session_id = ? AND membership.position_kind = ? + AND membership.position_id = ? AND membership.sequence <= ? ORDER BY membership.sequence LIMIT ? `) - .all(request.sessionId, turnId, request.snapshotKey.throughSequence, remaining + 1) as Array<{ - sequence: number; - stored_bytes: number; - }>; + .all( + request.sessionId, + selectedPosition.positionKey.kind, + selectedPosition.positionKey.id, + request.snapshotKey.throughSequence, + remaining + 1, + ) as Array<{ sequence: number; stored_bytes: number }>; if (rows.length > remaining) { throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_count'); } @@ -423,42 +511,54 @@ export function readSessionTurnMembershipPreflight( if (storedBytes > request.maxBytes) { throw new SessionTurnPositionLimitError(request.sessionId, 'transcript_record_bytes'); } - sequences.push(row.sequence); + records.push({ positionKey: selectedPosition.positionKey, sequence: row.sequence }); } } - sequences.sort((left, right) => left - right); - if (new Set(sequences).size !== sequences.length) { + if (new Set(records.map((record) => record.sequence)).size !== records.length) { throw new SessionTurnPositionRecoveryError(request.sessionId, 'corrupt_source'); } - return { sequences, storedBytes }; + return { records, storedBytes }; } export function releaseSessionTurnPositionSnapshot( db: DatabaseSync, - sessionId: string, - snapshotLeaseId: string, - key: SessionTurnPositionSnapshotKey, + request: SessionTurnPositionSnapshotReleaseRequest, ): void { - validateLeaseId(snapshotLeaseId); - const snapshot = requireLeasedSnapshot(db, sessionId, snapshotLeaseId, key); + validateProjection(request.projection); + validateLeaseId(request.snapshotLeaseId); + ensureTurnIndexRows(db, request.sessionId); + const snapshot = requireLeasedSnapshot( + db, + request.sessionId, + request.projection, + request.snapshotLeaseId, + request.snapshotKey, + ); const released = db .prepare(` DELETE FROM session_turn_snapshot_leases - WHERE session_id = ? AND snapshot_generation = ? AND lease_id = ? + WHERE session_id = ? AND snapshot_generation = ? AND projection = ? AND lease_id = ? `) - .run(sessionId, snapshot.snapshot_generation, snapshotLeaseId); - if (released.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + .run( + request.sessionId, + snapshot.snapshot_generation, + request.projection, + request.snapshotLeaseId, + ); + if (released.changes !== 1) { + throw new SessionTurnPositionSnapshotMismatchError(request.sessionId); + } const retained = db .prepare(` SELECT 1 FROM session_turn_snapshot_leases WHERE session_id = ? AND snapshot_generation = ? LIMIT 1 `) - .get(sessionId, snapshot.snapshot_generation); + .get(request.sessionId, snapshot.snapshot_generation); if (!retained) { db.prepare(` DELETE FROM session_turn_position_snapshots WHERE session_id = ? AND snapshot_generation = ? - `).run(sessionId, snapshot.snapshot_generation); + `).run(request.sessionId, snapshot.snapshot_generation); } } @@ -486,8 +586,9 @@ export function requireSnapshot( const row = db .prepare(` SELECT slot, through_sequence, authority_revision, snapshot_generation, state, - build_phase, build_next_ordinal, build_cursor_sequence, - build_cursor_admitted_at, build_cursor_turn_id, ready_total + build_phase, build_next_owner_ordinal, build_next_shared_ordinal, + build_cursor_sequence, build_cursor_admitted_at, build_cursor_position_kind, + build_cursor_position_id, ready_owner_total, ready_shared_total FROM session_turn_position_snapshots WHERE session_id = ? AND snapshot_generation = ? `) @@ -505,18 +606,21 @@ export function requireSnapshot( function requireLeasedSnapshot( db: DatabaseSync, sessionId: string, + projection: SessionTranscriptProjection, leaseId: string, key: SessionTurnPositionSnapshotKey, ): SessionTurnPositionSnapshotRow { + validateProjection(projection); validateLeaseId(leaseId); const snapshot = requireSnapshot(db, sessionId, key); - requireLease(db, sessionId, leaseId, snapshot.snapshot_generation); + requireLease(db, sessionId, projection, leaseId, snapshot.snapshot_generation); return snapshot; } function requireLease( db: DatabaseSync, sessionId: string, + projection: SessionTranscriptProjection, leaseId: string, generation: number, ): void { @@ -524,9 +628,9 @@ function requireLease( !db .prepare(` SELECT 1 FROM session_turn_snapshot_leases - WHERE session_id = ? AND snapshot_generation = ? AND lease_id = ? + WHERE session_id = ? AND snapshot_generation = ? AND projection = ? AND lease_id = ? `) - .get(sessionId, generation, leaseId) + .get(sessionId, generation, projection, leaseId) ) { throw new SessionTurnPositionSnapshotMismatchError(sessionId); } @@ -536,16 +640,27 @@ function acquireLease( db: DatabaseSync, sessionId: string, generation: number, + projection: SessionTranscriptProjection, leaseId: string, ): void { + validateProjection(projection); validateLeaseId(leaseId); - try { - db.prepare(` - INSERT INTO session_turn_snapshot_leases(session_id, snapshot_generation, lease_id) - VALUES (?, ?, ?) ON CONFLICT(session_id, snapshot_generation, lease_id) DO NOTHING - `).run(sessionId, generation, leaseId); - } catch (error) { - void error; + const inserted = db + .prepare(` + INSERT INTO session_turn_snapshot_leases( + session_id, snapshot_generation, projection, lease_id + ) VALUES (?, ?, ?, ?) + ON CONFLICT(session_id, projection, lease_id) DO NOTHING + `) + .run(sessionId, generation, projection, leaseId); + if (inserted.changes === 0) { + const existing = db + .prepare(` + SELECT snapshot_generation FROM session_turn_snapshot_leases + WHERE session_id = ? AND projection = ? AND lease_id = ? + `) + .get(sessionId, projection, leaseId) as { snapshot_generation: number } | undefined; + if (existing?.snapshot_generation === generation) return; throw new SessionTurnPositionSnapshotMismatchError(sessionId); } } @@ -559,8 +674,9 @@ function findSnapshotByAuthority( return db .prepare(` SELECT slot, through_sequence, authority_revision, snapshot_generation, state, - build_phase, build_next_ordinal, build_cursor_sequence, - build_cursor_admitted_at, build_cursor_turn_id, ready_total + build_phase, build_next_owner_ordinal, build_next_shared_ordinal, + build_cursor_sequence, build_cursor_admitted_at, build_cursor_position_kind, + build_cursor_position_id, ready_owner_total, ready_shared_total FROM session_turn_position_snapshots WHERE session_id = ? AND ((through_sequence = ?) OR (through_sequence IS NULL AND ? IS NULL)) @@ -572,8 +688,10 @@ function findSnapshotByAuthority( } interface BuildRow { - readonly turn_id: string; - readonly first_sequence: number | null; + readonly position_kind: 'turn' | 'note'; + readonly position_id: string; + readonly owner_first_sequence: number | null; + readonly shared_first_sequence: number | null; readonly admitted_at: number | null; } @@ -589,11 +707,13 @@ function readBuildRows( if (through === null) return []; return db .prepare(` - SELECT turn_id, first_sequence, NULL AS admitted_at FROM session_turn_metadata - WHERE session_id = ? AND first_sequence <= ? AND order_source = 'legacy' - AND (? IS NULL OR first_sequence < ?) - AND first_sequence > ? - ORDER BY first_sequence, turn_id LIMIT ? + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND owner_first_sequence <= ? AND order_source = 'legacy' + AND (? IS NULL OR owner_first_sequence < ?) + AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_kind, position_id LIMIT ? `) .all( sessionId, @@ -607,33 +727,39 @@ function readBuildRows( if (phase === 'admission') { const watermarkSql = through === null - ? 'first_sequence IS NULL' - : '(first_sequence IS NULL OR first_sequence <= ?)'; + ? 'owner_first_sequence IS NULL' + : '(owner_first_sequence IS NULL OR owner_first_sequence <= ?)'; const parameters: Array = [sessionId]; if (through !== null) parameters.push(through); parameters.push( snapshot.build_cursor_admitted_at, snapshot.build_cursor_admitted_at, snapshot.build_cursor_admitted_at, - snapshot.build_cursor_turn_id, + snapshot.build_cursor_position_id, SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, ); return db .prepare(` - SELECT turn_id, first_sequence, admitted_at FROM session_turn_metadata - WHERE session_id = ? AND order_source = 'admission' AND ${watermarkSql} - AND (? IS NULL OR admitted_at > ? OR (admitted_at = ? AND turn_id > ?)) - ORDER BY admitted_at, turn_id LIMIT ? + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND ${watermarkSql} + AND (? IS NULL OR admitted_at > ? OR (admitted_at = ? AND position_id > ?)) + ORDER BY admitted_at, position_id LIMIT ? `) .all(...parameters) as unknown as BuildRow[]; } if (through === null || boundary === null) return []; return db .prepare(` - SELECT turn_id, first_sequence, NULL AS admitted_at FROM session_turn_metadata - WHERE session_id = ? AND first_sequence <= ? AND identity_kind = 'note' - AND order_source = 'legacy' AND first_sequence >= ? AND first_sequence > ? - ORDER BY first_sequence, turn_id LIMIT ? + SELECT position_kind, position_id, owner_first_sequence, + shared_first_sequence, NULL AS admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND owner_first_sequence <= ? AND position_kind = 'note' + AND order_source = 'legacy' AND owner_first_sequence >= ? + AND owner_first_sequence > ? + ORDER BY owner_first_sequence, position_id LIMIT ? `) .all( sessionId, @@ -653,30 +779,31 @@ function validateHybridBoundary( const boundary = ( db .prepare(` - SELECT MIN(first_sequence) AS boundary FROM session_turn_metadata - WHERE session_id = ? AND order_source = 'admission' AND first_sequence <= ? + SELECT MIN(owner_first_sequence) AS boundary FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' + AND order_source = 'admission' AND owner_first_sequence <= ? `) .get(sessionId, throughSequence) as { boundary: number | null } ).boundary; if (boundary !== null) { const hybrid = db .prepare(` - SELECT first_sequence FROM session_turn_metadata - WHERE session_id = ? AND identity_kind = 'turn' AND order_source = 'legacy' - AND first_sequence >= ? AND first_sequence <= ? - ORDER BY first_sequence LIMIT 1 + SELECT owner_first_sequence FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND order_source = 'legacy' + AND owner_first_sequence >= ? AND owner_first_sequence <= ? + ORDER BY owner_first_sequence LIMIT 1 `) - .get(sessionId, boundary, throughSequence) as { first_sequence: number } | undefined; + .get(sessionId, boundary, throughSequence) as { owner_first_sequence: number } | undefined; if (hybrid) { db.prepare(` UPDATE session_turn_index_state SET failure_reason = 'hybrid_missing_admission', failure_sequence = ? WHERE session_id = ? - `).run(hybrid.first_sequence, sessionId); + `).run(hybrid.owner_first_sequence, sessionId); throw new SessionTurnPositionRecoveryError( sessionId, 'hybrid_missing_admission', - hybrid.first_sequence, + hybrid.owner_first_sequence, ); } } @@ -692,6 +819,7 @@ function followingPhase( } function validatePageRequest(request: SessionTurnPositionPageSnapshotRequest): void { + validateProjection(request.projection); validateLeaseId(request.snapshotLeaseId); if ( !Number.isSafeInteger(request.maxPositions) || @@ -718,12 +846,20 @@ function validatePageRequest(request: SessionTurnPositionPageSnapshotRequest): v } } -function validateRecordRequest(request: SessionTranscriptRecordsByTurnIdsSnapshotRequest): void { +function validateRecordRequest( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, +): void { + validateProjection(request.projection); validateLeaseId(request.snapshotLeaseId); if ( - request.turnIds.length < 1 || - request.turnIds.length > SESSION_TURN_POSITION_BODY_MAX_TURNS || - request.turnIds.some((turnId) => turnId.length === 0) || + request.positionKeys.length < 1 || + request.positionKeys.length > SESSION_TURN_POSITION_BODY_MAX_KEYS || + request.positionKeys.some( + (positionKey) => + (positionKey.kind !== 'turn' && positionKey.kind !== 'note') || + typeof positionKey.id !== 'string' || + positionKey.id.length === 0, + ) || !Number.isSafeInteger(request.maxRecords) || request.maxRecords < 1 || request.maxRecords > SESSION_TURN_POSITION_BODY_MAX_RECORDS || @@ -736,6 +872,25 @@ function validateRecordRequest(request: SessionTranscriptRecordsByTurnIdsSnapsho validateSnapshotKey(request.snapshotKey); } +function projectionColumns(projection: SessionTranscriptProjection): { + readonly ordinal: 'owner_ordinal' | 'shared_ordinal'; + readonly firstSequence: 'owner_first_sequence' | 'shared_first_sequence'; +} { + return projection === 'owner' + ? { ordinal: 'owner_ordinal', firstSequence: 'owner_first_sequence' } + : { ordinal: 'shared_ordinal', firstSequence: 'shared_first_sequence' }; +} + +function positionKeyIdentity(positionKey: SessionTranscriptBodyPositionKey): string { + return `${positionKey.kind}\0${positionKey.id}`; +} + +function validateProjection(projection: string): asserts projection is SessionTranscriptProjection { + if (projection !== 'owner' && projection !== 'shared') { + throw new Error('Invalid Session transcript projection'); + } +} + function validateLeaseId(leaseId: string): void { if (!/^[A-Za-z0-9_-]{1,128}$/u.test(leaseId)) { throw new Error('Invalid Session Turn-position snapshot lease id'); diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index a8916a3c59..ef21b05d95 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -1231,6 +1231,8 @@ const MIGRATIONS: ReadonlyMap = new Map([ CREATE TABLE IF NOT EXISTS session_turn_authority_revisions ( session_id TEXT PRIMARY KEY, authority_revision INTEGER NOT NULL DEFAULT 0 CHECK (authority_revision >= 0), + visibility_policy_version INTEGER NOT NULL DEFAULT 1 + CHECK (visibility_policy_version >= 1), next_snapshot_generation INTEGER NOT NULL DEFAULT 1 CHECK (next_snapshot_generation >= 1), FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE @@ -1273,41 +1275,49 @@ const MIGRATIONS: ReadonlyMap = new Map([ CREATE TABLE IF NOT EXISTS session_turn_metadata ( session_id TEXT NOT NULL, - turn_id TEXT NOT NULL, - identity_kind TEXT NOT NULL CHECK (identity_kind IN ('turn', 'note', 'synthetic')), - order_source TEXT NOT NULL CHECK (order_source IN ('legacy', 'admission', 'synthetic')), + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note')), + position_id TEXT NOT NULL CHECK (length(CAST(position_id AS BLOB)) > 0), + order_source TEXT NOT NULL CHECK (order_source IN ('legacy', 'admission')), admitted_at INTEGER CHECK (admitted_at >= 0), - first_sequence INTEGER CHECK (first_sequence >= 0), - PRIMARY KEY(session_id, turn_id), - UNIQUE(session_id, first_sequence), + owner_first_sequence INTEGER CHECK (owner_first_sequence >= 0), + shared_first_sequence INTEGER CHECK (shared_first_sequence >= 0), + PRIMARY KEY(session_id, position_kind, position_id), + UNIQUE(session_id, owner_first_sequence), + UNIQUE(session_id, shared_first_sequence), FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS session_turn_metadata_by_legacy_order - ON session_turn_metadata(session_id, first_sequence, turn_id) + ON session_turn_metadata(session_id, owner_first_sequence, position_kind, position_id) WHERE order_source = 'legacy'; CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_order - ON session_turn_metadata(session_id, admitted_at, turn_id) + ON session_turn_metadata(session_id, admitted_at, position_id) WHERE order_source = 'admission'; CREATE INDEX IF NOT EXISTS session_turn_metadata_by_admission_sequence - ON session_turn_metadata(session_id, first_sequence) + ON session_turn_metadata(session_id, owner_first_sequence) WHERE order_source = 'admission'; + CREATE INDEX IF NOT EXISTS session_turn_metadata_by_shared_sequence + ON session_turn_metadata(session_id, shared_first_sequence, position_kind, position_id) + WHERE shared_first_sequence IS NOT NULL; + CREATE TABLE IF NOT EXISTS session_turn_memberships ( session_id TEXT NOT NULL, sequence INTEGER NOT NULL CHECK (sequence >= 0), - turn_id TEXT NOT NULL, + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note')), + position_id TEXT NOT NULL CHECK (length(CAST(position_id AS BLOB)) > 0), + shared_visibility INTEGER NOT NULL CHECK (shared_visibility IN (0, 1)), PRIMARY KEY(session_id, sequence), - FOREIGN KEY(session_id, turn_id) - REFERENCES session_turn_metadata(session_id, turn_id) ON DELETE CASCADE, + FOREIGN KEY(session_id, position_kind, position_id) + REFERENCES session_turn_metadata(session_id, position_kind, position_id) ON DELETE CASCADE, FOREIGN KEY(session_id, sequence) REFERENCES session_messages(session_id, sequence) ON DELETE CASCADE ON UPDATE CASCADE ) WITHOUT ROWID; - CREATE INDEX IF NOT EXISTS session_turn_memberships_by_turn - ON session_turn_memberships(session_id, turn_id, sequence); + CREATE INDEX IF NOT EXISTS session_turn_memberships_by_position + ON session_turn_memberships(session_id, position_kind, position_id, sequence); CREATE TABLE IF NOT EXISTS session_turn_position_snapshots ( session_id TEXT NOT NULL, @@ -1318,11 +1328,26 @@ const MIGRATIONS: ReadonlyMap = new Map([ state TEXT NOT NULL CHECK (state IN ('building', 'ready')), build_phase TEXT NOT NULL DEFAULT 'recovering' CHECK (build_phase IN ('recovering', 'legacy', 'admission', 'notes', 'ready')), - build_next_ordinal INTEGER NOT NULL DEFAULT 0 CHECK (build_next_ordinal >= 0), + build_next_owner_ordinal INTEGER NOT NULL DEFAULT 0 + CHECK (build_next_owner_ordinal >= 0), + build_next_shared_ordinal INTEGER NOT NULL DEFAULT 0 + CHECK (build_next_shared_ordinal >= 0), build_cursor_sequence INTEGER NOT NULL DEFAULT -1 CHECK (build_cursor_sequence >= -1), build_cursor_admitted_at INTEGER CHECK (build_cursor_admitted_at >= 0), - build_cursor_turn_id TEXT, - ready_total INTEGER CHECK (ready_total >= 0), + build_cursor_position_kind TEXT CHECK ( + build_cursor_position_kind IS NULL OR build_cursor_position_kind IN ('turn', 'note') + ), + build_cursor_position_id TEXT, + ready_owner_total INTEGER CHECK (ready_owner_total >= 0), + ready_shared_total INTEGER CHECK (ready_shared_total >= 0), + CHECK ( + (state = 'building' AND build_phase <> 'ready' + AND ready_owner_total IS NULL AND ready_shared_total IS NULL) + OR + (state = 'ready' AND build_phase = 'ready' + AND ready_owner_total >= 1 AND ready_shared_total >= 1) + ), + CHECK ((build_cursor_position_kind IS NULL) = (build_cursor_position_id IS NULL)), PRIMARY KEY(session_id, slot), UNIQUE(session_id, snapshot_generation), UNIQUE(session_id, through_sequence, authority_revision), @@ -1339,30 +1364,63 @@ const MIGRATIONS: ReadonlyMap = new Map([ CREATE TABLE IF NOT EXISTS session_turn_snapshot_positions ( session_id TEXT NOT NULL, snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), - ordinal INTEGER NOT NULL CHECK (ordinal >= 0), - turn_id TEXT NOT NULL, - first_sequence INTEGER CHECK (first_sequence >= 0), - PRIMARY KEY(session_id, snapshot_generation, ordinal), - UNIQUE(session_id, snapshot_generation, turn_id), + position_kind TEXT NOT NULL CHECK (position_kind IN ('turn', 'note', 'empty')), + position_id TEXT NOT NULL, + owner_ordinal INTEGER CHECK (owner_ordinal >= 0), + shared_ordinal INTEGER CHECK (shared_ordinal >= 0), + owner_first_sequence INTEGER CHECK (owner_first_sequence >= 0), + shared_first_sequence INTEGER CHECK (shared_first_sequence >= 0), + CHECK ((position_kind = 'empty' AND position_id = '') OR + (position_kind <> 'empty' AND length(CAST(position_id AS BLOB)) > 0)), + CHECK (owner_ordinal IS NOT NULL OR shared_ordinal IS NOT NULL), + CHECK (position_kind = 'empty' OR owner_ordinal IS NOT NULL), + CHECK (shared_ordinal IS NULL OR position_kind = 'empty' + OR shared_first_sequence IS NOT NULL), + CHECK (shared_ordinal IS NOT NULL OR shared_first_sequence IS NULL), + CHECK (position_kind <> 'empty' OR + (owner_first_sequence IS NULL AND shared_first_sequence IS NULL)), + PRIMARY KEY(session_id, snapshot_generation, position_kind, position_id), FOREIGN KEY(session_id, snapshot_generation) REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) ON DELETE CASCADE ) WITHOUT ROWID; - CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_sequence + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_owner_ordinal + ON session_turn_snapshot_positions(session_id, snapshot_generation, owner_ordinal) + WHERE owner_ordinal IS NOT NULL; + + CREATE UNIQUE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_shared_ordinal + ON session_turn_snapshot_positions(session_id, snapshot_generation, shared_ordinal) + WHERE shared_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_owner_sequence ON session_turn_snapshot_positions( session_id, snapshot_generation, - first_sequence, - ordinal - ); + owner_first_sequence, + owner_ordinal + ) WHERE owner_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_shared_sequence + ON session_turn_snapshot_positions( + session_id, + snapshot_generation, + shared_first_sequence, + shared_ordinal + ) WHERE shared_ordinal IS NOT NULL; + + CREATE INDEX IF NOT EXISTS session_turn_snapshot_positions_by_turn_anchor + ON session_turn_snapshot_positions( + session_id, snapshot_generation, position_kind, position_id + ) WHERE position_kind = 'turn'; CREATE TABLE IF NOT EXISTS session_turn_snapshot_leases ( session_id TEXT NOT NULL, snapshot_generation INTEGER NOT NULL CHECK (snapshot_generation >= 1), + projection TEXT NOT NULL CHECK (projection IN ('owner', 'shared')), lease_id TEXT NOT NULL CHECK (length(CAST(lease_id AS BLOB)) BETWEEN 1 AND 128), - PRIMARY KEY(session_id, snapshot_generation, lease_id), - UNIQUE(session_id, lease_id), + PRIMARY KEY(session_id, snapshot_generation, projection, lease_id), + UNIQUE(session_id, projection, lease_id), FOREIGN KEY(session_id, snapshot_generation) REFERENCES session_turn_position_snapshots(session_id, snapshot_generation) ON DELETE CASCADE diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 2983d96069..05d0f2af95 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -89,6 +89,7 @@ import { isSubagentSessionParent, isSubagentSessionRuntime, isSubagentSessionSpawn, + SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION, type SessionHeader, type SessionHeaderPatch, type StoredMessage, @@ -146,17 +147,19 @@ import { type SessionTurnContribution, type SessionTurnContributionPage, type SessionTurnLandmarkSnapshot, - type SessionTranscriptRecordsByTurnIdsSnapshotRequest, - type SessionTranscriptRecordsByTurnIdsSnapshotResult, + type SessionTranscriptRecordsByPositionKeysSnapshotRequest, + type SessionTranscriptRecordsByPositionKeysSnapshotResult, type SessionTurnPositionPageSnapshotRequest, type SessionTurnPositionReadResult, type SessionTurnPositionSnapshotKey, + type SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; import { isDiscardableConversationCopy, isValidConversationCopyTransition, } from './session-conversation-copy.js'; import { catalogPreviewForUserMessage } from './session-message-projection.js'; +import { classifyStoredMessageTurnIdentity } from './session-turn-membership.js'; import { configureSqliteSessionMetadataDatabase, migrateSqliteSessionMetadataDatabase, @@ -168,6 +171,7 @@ import { import { advanceSessionTurnPositionOrdinalBuild, allocateOrRequireSessionTurnPositionSnapshot, + ensureTurnIndexRows, invalidateSessionTurnPositionIndex, markSessionTurnRecoveryComplete, pageReadySessionTurnPositionSnapshot, @@ -1492,12 +1496,12 @@ export class SqliteSessionMetadataStore { const inserted = this.tryInsertHeader(normalized, 1, normalized.createdAt, true); if (!inserted) return 'existing'; if (encoded.length > 0) { - this.insertSessionMessagesSync(normalized.id, 0, encoded); + const recordBytes = this.insertSessionMessagesSync(normalized.id, 0, encoded); recordAppendedSessionTurnMetadata( this.db, normalized.id, 0, - encoded.map(({ message }) => message), + encoded.map(({ message }, index) => ({ message, recordBytes: recordBytes[index]! })), ); // Align with appendMessages' connection-lock semantics: a session // with any user message is treated as connection-locked, even when @@ -1617,12 +1621,12 @@ export class SqliteSessionMetadataStore { throw new Error(`Invalid Session message sequence for ${sessionId}`); } const sequence = row.last_sequence + 1; - this.insertSessionMessagesSync(sessionId, sequence, encoded); + const recordBytes = this.insertSessionMessagesSync(sessionId, sequence, encoded); recordAppendedSessionTurnMetadata( this.db, sessionId, sequence, - encoded.map(({ message }) => message), + encoded.map(({ message }, index) => ({ message, recordBytes: recordBytes[index]! })), ); this.updateCatalogProjectionSync(sessionId, projection, false, lockConnection); }); @@ -1847,7 +1851,7 @@ export class SqliteSessionMetadataStore { ) { throw new SessionMetadataConflictError('Invalid WorkHub transcript sequence'); } - this.insertSessionMessagesSync( + const recordBytes = this.insertSessionMessagesSync( WORKHUB_COORDINATION_SESSION_ID, sequenceRow.last_sequence + 1, [{ message: assignment, json: assignmentJson }], @@ -1856,7 +1860,7 @@ export class SqliteSessionMetadataStore { this.db, WORKHUB_COORDINATION_SESSION_ID, sequenceRow.last_sequence + 1, - [assignment], + [{ message: assignment, recordBytes: recordBytes[0]! }], ); this.updateCatalogProjectionSync(WORKHUB_COORDINATION_SESSION_ID, request.projection, false); return { kind: 'assigned' as const, targetCreated, assignment }; @@ -2718,6 +2722,9 @@ export class SqliteSessionMetadataStore { ): Promise { this.assertOpen(); assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } if (request.snapshotKey) { const ready = this.readTransaction(() => { const snapshot = requireSnapshot(this.db, request.sessionId, request.snapshotKey!); @@ -2779,13 +2786,17 @@ export class SqliteSessionMetadataStore { return { kind: 'building' as const, snapshotKey: snapshotKeyFromRow(snapshot), + projection: request.projection, progress: { phase: 'recovering' as const, nextSequence: state.indexed_through_sequence + 1, currentByteOffset: partial?.byte_offset ?? 0, sourceRecords: state.source_records, sourceBytes: state.source_bytes, - builtPositions: snapshot.build_next_ordinal, + builtPositions: + request.projection === 'owner' + ? snapshot.build_next_owner_ordinal + : snapshot.build_next_shared_ordinal, lastStepRecords: recovery && !('failure' in recovery) ? recovery.lastStepRecords : 0, lastStepBytes: recovery && !('failure' in recovery) ? recovery.lastStepBytes : 0, lastStepPositions: 0, @@ -2822,16 +2833,23 @@ export class SqliteSessionMetadataStore { return { kind: 'building' as const, snapshotKey: snapshotKeyFromRow(step.snapshot), + projection: request.projection, progress: { phase: step.executedPhase, nextSequence: state.indexed_through_sequence + 1, currentByteOffset: 0, sourceRecords: state.source_records, sourceBytes: state.source_bytes, - builtPositions: step.snapshot.build_next_ordinal, + builtPositions: + request.projection === 'owner' + ? step.snapshot.build_next_owner_ordinal + : step.snapshot.build_next_shared_ordinal, lastStepRecords: 0, lastStepBytes: 0, - lastStepPositions: step.lastStepPositions, + lastStepPositions: + request.projection === 'owner' + ? step.snapshot.build_next_owner_ordinal - snapshot.build_next_owner_ordinal + : step.snapshot.build_next_shared_ordinal - snapshot.build_next_shared_ordinal, }, }; }); @@ -2845,40 +2863,70 @@ export class SqliteSessionMetadataStore { return outcome; } - async readTranscriptRecordsByTurnIds( - request: SessionTranscriptRecordsByTurnIdsSnapshotRequest, - ): Promise { + async readTranscriptRecordsByPositionKeys( + request: SessionTranscriptRecordsByPositionKeysSnapshotRequest, + ): Promise { this.assertOpen(); assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } return this.readTransaction(() => { const preflight = readSessionTurnMembershipPreflight(this.db, request); - const records: Array<{ sequence: number; message: StoredMessage }> = []; - for (const row of readStoredMessageRows(this.db, request.sessionId, preflight.sequences)) { + const decodedBySequence = new Map(); + for (const row of readStoredMessageRows( + this.db, + request.sessionId, + preflight.records.map((record) => record.sequence), + )) { try { - records.push({ - sequence: row.sequence, - message: decodeStoredMessage(JSON.parse(row.recordJson) as unknown), - }); + decodedBySequence.set( + row.sequence, + decodeStoredMessage(JSON.parse(row.recordJson) as unknown), + ); } catch (error) { throw new StoredSessionMessageIncompatibleError(request.sessionId, row.sequence, { cause: error, }); } } - return { snapshotKey: request.snapshotKey, records, rawBytes: preflight.storedBytes }; + const records = preflight.records.map(({ positionKey, sequence }) => { + const message = decodedBySequence.get(sequence); + if (!message) throw new StoredSessionMessageIncompatibleError(request.sessionId, sequence); + try { + const identity = classifyStoredMessageTurnIdentity(message); + if ( + identity.kind === 'ignored' || + identity.kind !== positionKey.kind || + identity.positionId !== positionKey.id + ) { + throw new Error('decoded transcript identity no longer matches the exact position'); + } + } catch (error) { + throw new StoredSessionMessageIncompatibleError(request.sessionId, sequence, { + cause: error, + }); + } + return { positionKey, sequence, message }; + }); + return { + snapshotKey: request.snapshotKey, + projection: request.projection, + records, + rawBytes: preflight.storedBytes, + }; }); } async releaseTurnPositionSnapshot( - sessionId: string, - snapshotLeaseId: string, - snapshotKey: SessionTurnPositionSnapshotKey, + request: SessionTurnPositionSnapshotReleaseRequest, ): Promise { this.assertOpen(); - assertSafeSessionId(sessionId); - this.transaction(() => - releaseSessionTurnPositionSnapshot(this.db, sessionId, snapshotLeaseId, snapshotKey), - ); + assertSafeSessionId(request.sessionId); + if (!isSessionTurnVisibilityPolicyCurrent(this.db, request.sessionId)) { + this.transaction(() => ensureTurnIndexRows(this.db, request.sessionId)); + } + this.transaction(() => releaseSessionTurnPositionSnapshot(this.db, request)); } async readTurnContributions( @@ -5384,7 +5432,7 @@ export class SqliteSessionMetadataStore { readonly message: StoredMessage; readonly json: string; }[], - ): void { + ): readonly number[] { if ( !Number.isSafeInteger(firstSequence) || firstSequence < 0 || @@ -5405,10 +5453,12 @@ export class SqliteSessionMetadataStore { INSERT INTO session_message_chunks(session_id, sequence, chunk_index, data, sha256) VALUES (?, ?, ?, ?, ?) `); + const recordBytes: number[] = []; for (let index = 0; index < entries.length; index += 1) { const entry = entries[index]!; const sequence = firstSequence + index; const encoded = Buffer.from(entry.json, 'utf8'); + recordBytes.push(encoded.byteLength); const chunked = encoded.byteLength > SQLITE_SESSION_MESSAGE_CHUNK_BYTES; insertMessage.run( sessionId, @@ -5440,6 +5490,7 @@ export class SqliteSessionMetadataStore { ); } } + return recordBytes; } private replaceSessionMessageSync( @@ -7152,6 +7203,17 @@ function nullableStoredMessageSequence(value: unknown, sessionId: string): numbe return requireStoredMessageSequence(value, sessionId); } +function isSessionTurnVisibilityPolicyCurrent(db: DatabaseSync, sessionId: string): boolean { + const row = db + .prepare(`SELECT visibility_policy_version FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(sessionId) as { visibility_policy_version: number } | undefined; + return ( + row === undefined || + row.visibility_policy_version === SHARED_SESSION_TRANSCRIPT_VISIBILITY_POLICY_VERSION + ); +} + function requireTranscriptPayloadDigest( value: unknown, sessionId: string, From 683a268c0a1411decc178484a2eb9f148652d394 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Sun, 30 Aug 2026 23:11:23 +0800 Subject: [PATCH 05/10] fix(storage): recover legacy bodyless admissions Generated-by: OpenAI Codex --- .../session-turn-position-index.test.ts | 405 ++++++++++++++++++ .../src/session-turn-admission-recovery.ts | 138 ++++++ .../src/session-turn-position-index.ts | 1 + .../src/session-turn-position-snapshots.ts | 45 ++ .../src/sqlite-session-metadata-store.ts | 27 +- 5 files changed, 614 insertions(+), 2 deletions(-) create mode 100644 packages/storage/src/session-turn-admission-recovery.ts diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index 14c2c3afff..bdda7dffe6 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -2216,6 +2216,391 @@ describe('Session Turn position snapshots', () => { } }); + test('recovers a v34 bodyless root admission before publishing owner and shared snapshots', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-bodyless-admission-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-v34', 'future-user-v34', 42), + ); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const owner = await readyPage(store, session.id, 'v34-bodyless-owner', 'owner'); + const shared = await readyPage(store, session.id, 'v34-bodyless-shared', 'shared'); + assert.deepEqual(owner.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-bodyless-v34' }, + firstSequence: null, + }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + const migrated = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + ( + migrated + .prepare(`SELECT position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + FROM session_turn_metadata WHERE session_id = ?`) + .all(session.id) as Array> + ).map((row) => ({ ...row })), + [ + { + position_kind: 'turn', + position_id: 'turn-bodyless-v34', + order_source: 'admission', + admitted_at: 42, + owner_first_sequence: null, + shared_first_sequence: null, + }, + ], + ); + } finally { + migrated.close(); + } + + await store.close?.(); + store = createSessionStore(root); + const reopenedOwner = await readyPage(store, session.id, 'v34-bodyless-reopen', 'owner'); + assert.deepEqual(reopenedOwner.positions, owner.positions); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('recovers v34 admission order while preserving owner and shared body anchors', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-v34-admission-bodies-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await runs.admitRootTurn(rootAdmission(session.id, 'turn-late', 'late-user', 20)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-early', 'early-user', 10)); + await store.appendMessages(session.id, [ + hiddenPermission('turn-late', 0), + { ...user('turn-early', 1), id: 'early-user' }, + { ...user('turn-late', 2), id: 'late-user' }, + ]); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const owner = await readyPage(store, session.id, 'v34-bodies-owner', 'owner'); + const shared = await readyPage(store, session.id, 'v34-bodies-shared', 'shared'); + assert.deepEqual(owner.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-early' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-late' }, firstSequence: 0 }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'turn', id: 'turn-early' }, firstSequence: 1 }, + { ordinal: 1, key: { kind: 'turn', id: 'turn-late' }, firstSequence: 2 }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('bounds legacy admission reconciliation and atomically rebuilds after owner reopen', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-recovery-bound-')); + let store = createSessionStore(root); + const session = await store.create(makeInput()); + try { + await store.close?.(); + const seeded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = seeded.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + seeded.exec('BEGIN IMMEDIATE'); + for (let index = 0; index < 1_025; index += 1) { + insert.run(session.id, `turn-${index.toString().padStart(4, '0')}`, index); + } + seeded.exec('COMMIT'); + } finally { + seeded.close(); + } + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-first', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected bounded admission reconciliation'); + const bounded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission'`) + .get(session.id) as { count: number } + ).count, + 1_024, + ); + const snapshotState = bounded + .prepare(`SELECT state, build_phase, build_cursor_admitted_at, + build_cursor_position_id FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { + state: string; + build_phase: string; + build_cursor_admitted_at: number; + build_cursor_position_id: string; + }; + assert.deepEqual( + { ...snapshotState }, + { + state: 'building', + build_phase: 'recovering', + build_cursor_admitted_at: 1_023, + build_cursor_position_id: 'turn-1023', + }, + ); + assert.equal( + ( + bounded + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + const plan = bounded + .prepare(`EXPLAIN QUERY PLAN SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? AND (admitted_at, turn_id) > (?, ?) + ORDER BY admitted_at, turn_id LIMIT ?`) + .all(session.id, 1_023, 'turn-1023', 1_024) + .map((row) => String((row as { detail?: unknown }).detail ?? '')) + .join('\n'); + assert.match(plan, /core_root_turn_admissions_order/u); + assert.match(plan, /\(admitted_at,turn_id\)>\(\?,\?\)/u); + assert.doesNotMatch(plan, /TEMP B-TREE|OFFSET|ROW_NUMBER|COUNT DISTINCT/iu); + } finally { + bounded.close(); + } + + await store.close?.(); + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-first', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const ready = await readyPage(store, session.id, 'admission-bound-reopen', 'owner'); + assert.equal(ready.totalPositions, 1_025); + assert.ok(ready.snapshotKey.snapshotGeneration > first.snapshotKey.snapshotGeneration); + const completed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + const positions = completed + .prepare(`SELECT owner_ordinal, position_id FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ? AND owner_ordinal IS NOT NULL + ORDER BY owner_ordinal`) + .all(session.id, ready.snapshotKey.snapshotGeneration) as Array<{ + owner_ordinal: number; + position_id: string; + }>; + assert.equal(positions.length, 1_025); + for (let index = 0; index < positions.length; index += 1) { + assert.deepEqual( + { ...positions[index] }, + { + owner_ordinal: index, + position_id: `turn-${index.toString().padStart(4, '0')}`, + }, + ); + } + } finally { + completed.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('rehydrates a bodyless admission after shared visibility policy reset', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-bodyless-policy-reset-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-policy', 'future-policy-user', 7), + ); + const before = await readyPage(store, session.id, 'bodyless-policy-before', 'owner'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(session.id); + } finally { + database.close(); + } + + const owner = await readyPage(store, session.id, 'bodyless-policy-after', 'owner'); + const shared = await readyPage(store, session.id, 'bodyless-policy-shared', 'shared'); + assert.ok(owner.snapshotKey.snapshotGeneration > before.snapshotKey.snapshotGeneration); + assert.deepEqual(owner.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-bodyless-policy' }, + firstSequence: null, + }, + ]); + assert.deepEqual(shared.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('fails closed on invalid or contradictory canonical admission scalars', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-invalid-admission-source-')); + let store = createSessionStore(root); + try { + const invalidIdentity = await store.create(makeInput({ name: 'Invalid identity' })); + const invalidTime = await store.create(makeInput({ name: 'Invalid time' })); + const contradiction = await store.create(makeInput({ name: 'Contradiction' })); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = database.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + insert.run(invalidIdentity.id, '', 1); + insert.run(invalidTime.id, 'turn-negative-time', -1); + insert.run(contradiction.id, 'turn-contradiction', 1); + database + .prepare(`INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', 'turn-contradiction', 'admission', 2, NULL, NULL)`) + .run(contradiction.id); + } finally { + database.close(); + } + + for (const sessionId of [invalidIdentity.id, invalidTime.id, contradiction.id]) { + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId, + projection: 'owner', + snapshotLeaseId: 'invalid-admission', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => { + assert.equal( + (error as { code?: unknown }).code, + 'session_turn_position_recovery_failed', + ); + assert.equal((error as { reason?: unknown }).reason, 'corrupt_source'); + return true; + }, + ); + const failed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(failed + .prepare(`SELECT failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as Record), + }, + { failure_reason: 'corrupt_source', failure_sequence: 0 }, + ); + } finally { + failed.close(); + } + } + + await store.close?.(); + const repaired = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + repaired + .prepare(`UPDATE core_root_turn_admissions SET turn_id = 'turn-repaired' + WHERE session_id = ? AND turn_id = ''`) + .run(invalidIdentity.id); + } finally { + repaired.close(); + } + store = createSessionStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: invalidIdentity.id, + projection: 'owner', + snapshotLeaseId: 'invalid-admission-reopen', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + + const reset = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + reset + .prepare(`UPDATE session_turn_authority_revisions + SET visibility_policy_version = 999 WHERE session_id = ?`) + .run(invalidIdentity.id); + } finally { + reset.close(); + } + const recovered = await readyPage( + store, + invalidIdentity.id, + 'invalid-admission-policy-reset', + 'owner', + ); + assert.deepEqual(recovered.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-repaired' }, + firstSequence: null, + }, + ]); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('uses indexed owner/shared keyset plans for 10,000 alternating-visibility Turns', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-plan-')); const store = createSessionStore(root); @@ -2592,6 +2977,26 @@ function resetProjection(root: string, sessionId: string): void { } } +function downgradeTurnProjectionToV34(root: string): void { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + DROP TABLE session_turn_snapshot_leases; + DROP TABLE session_turn_snapshot_positions; + DROP TABLE session_turn_position_snapshots; + DROP TABLE session_turn_identity_recovery; + DROP TABLE session_turn_memberships; + DROP TABLE session_turn_metadata; + DROP TABLE session_turn_index_state; + DROP TABLE session_turn_authority_revisions; + UPDATE session_metadata_schema SET version = 34 WHERE scope = 'session_metadata'; + `); + } finally { + database.close(); + } +} + function makeInput(overrides: Partial = {}): CreateSessionInput { return { cwd: '/tmp/cwd', diff --git a/packages/storage/src/session-turn-admission-recovery.ts b/packages/storage/src/session-turn-admission-recovery.ts new file mode 100644 index 0000000000..c5e055b914 --- /dev/null +++ b/packages/storage/src/session-turn-admission-recovery.ts @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { DatabaseSync } from 'node:sqlite'; +import { SessionTurnPositionRecoveryError } from './session-store.js'; + +export const SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS = 1_024; + +export interface SessionTurnAdmissionRecoveryResult { + readonly complete: boolean; + readonly lastStepAdmissions: number; + readonly cursorAdmittedAt: number | null; + readonly cursorTurnId: string | null; +} + +interface PersistedRecoveryFailure { + readonly failure_reason: 'corrupt_source' | 'incompatible_identity' | 'hybrid_missing_admission'; + readonly failure_sequence: number; +} + +/** + * Reconciles only canonical admission identity/order scalars. Transcript + * membership and body decoding remain owned by their existing recovery path. + */ +export function advanceSessionTurnAdmissionRecovery( + db: DatabaseSync, + input: { + readonly sessionId: string; + readonly cursorAdmittedAt: number | null; + readonly cursorTurnId: string | null; + readonly maxAdmissions: number; + }, +): SessionTurnAdmissionRecoveryResult { + if ( + !Number.isSafeInteger(input.maxAdmissions) || + input.maxAdmissions < 1 || + input.maxAdmissions > SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS || + (input.cursorAdmittedAt === null) !== (input.cursorTurnId === null) + ) { + throw new SessionTurnPositionRecoveryError(input.sessionId, 'corrupt_source'); + } + const persistedFailure = db + .prepare(`SELECT failure_reason, failure_sequence FROM session_turn_index_state + WHERE session_id = ? AND failure_reason IS NOT NULL`) + .get(input.sessionId) as PersistedRecoveryFailure | undefined; + if (persistedFailure) { + throw new SessionTurnPositionRecoveryError( + input.sessionId, + persistedFailure.failure_reason, + persistedFailure.failure_sequence, + ); + } + const rows = ( + input.cursorAdmittedAt === null + ? db + .prepare(` + SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? + ORDER BY admitted_at, turn_id + LIMIT ? + `) + .all(input.sessionId, input.maxAdmissions) + : db + .prepare(` + SELECT turn_id, admitted_at + FROM core_root_turn_admissions + WHERE session_id = ? AND (admitted_at, turn_id) > (?, ?) + ORDER BY admitted_at, turn_id + LIMIT ? + `) + .all(input.sessionId, input.cursorAdmittedAt, input.cursorTurnId, input.maxAdmissions) + ) as Array<{ turn_id?: unknown; admitted_at?: unknown }>; + const readExisting = db.prepare(` + SELECT order_source, admitted_at + FROM session_turn_metadata + WHERE session_id = ? AND position_kind = 'turn' AND position_id = ? + `); + const upsert = db.prepare(` + INSERT INTO session_turn_metadata( + session_id, position_kind, position_id, order_source, admitted_at, + owner_first_sequence, shared_first_sequence + ) VALUES (?, 'turn', ?, 'admission', ?, NULL, NULL) + ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET + order_source = 'admission', admitted_at = excluded.admitted_at + `); + let cursorAdmittedAt = input.cursorAdmittedAt; + let cursorTurnId = input.cursorTurnId; + for (const row of rows) { + if ( + typeof row.turn_id !== 'string' || + row.turn_id.length === 0 || + typeof row.admitted_at !== 'number' || + !Number.isSafeInteger(row.admitted_at) || + row.admitted_at < 0 + ) { + failAdmissionRecovery(db, input.sessionId); + } + const existing = readExisting.get(input.sessionId, row.turn_id) as + | { order_source: string; admitted_at: number | null } + | undefined; + if (existing?.order_source === 'admission' && existing.admitted_at !== row.admitted_at) { + failAdmissionRecovery(db, input.sessionId); + } + upsert.run(input.sessionId, row.turn_id, row.admitted_at); + cursorAdmittedAt = row.admitted_at; + cursorTurnId = row.turn_id; + } + return { + complete: rows.length < input.maxAdmissions, + lastStepAdmissions: rows.length, + cursorAdmittedAt, + cursorTurnId, + }; +} + +function failAdmissionRecovery(db: DatabaseSync, sessionId: string): never { + db.prepare(`UPDATE session_turn_index_state + SET failure_reason = 'corrupt_source', failure_sequence = 0 + WHERE session_id = ?`).run(sessionId); + throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source', 0); +} diff --git a/packages/storage/src/session-turn-position-index.ts b/packages/storage/src/session-turn-position-index.ts index 184649a914..e729e33412 100644 --- a/packages/storage/src/session-turn-position-index.ts +++ b/packages/storage/src/session-turn-position-index.ts @@ -33,6 +33,7 @@ export { SESSION_TURN_POSITION_BUILD_MAX_POSITIONS, SESSION_TURN_POSITION_MAX_PAGE_BYTES, SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, + advanceSessionTurnAdmissionRecoveryForSnapshot, advanceSessionTurnPositionOrdinalBuild, allocateOrRequireSessionTurnPositionSnapshot, markSessionTurnRecoveryComplete, diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts index 71e9f891b8..0877fd4c67 100644 --- a/packages/storage/src/session-turn-position-snapshots.ts +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -32,6 +32,10 @@ import { type SessionTurnPositionSnapshotReleaseRequest, } from './session-store.js'; import { ensureTurnIndexRows } from './session-turn-position-authority.js'; +import { + advanceSessionTurnAdmissionRecovery, + SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS, +} from './session-turn-admission-recovery.js'; import { sqliteTableExists } from './sqlite-schema-introspection.js'; export const SESSION_TURN_POSITION_MAX_PAGE_POSITIONS = 128; @@ -74,6 +78,11 @@ export interface SessionTurnPositionBuildStep { readonly executedPhase: Exclude; } +export interface SessionTurnAdmissionRecoveryStep { + readonly snapshot: SessionTurnPositionSnapshotRow; + readonly lastStepAdmissions: number; +} + export interface SessionTurnMembershipPreflight { readonly records: readonly { readonly positionKey: SessionTranscriptBodyPositionKey; @@ -193,6 +202,42 @@ export function markSessionTurnRecoveryComplete( return requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)); } +export function advanceSessionTurnAdmissionRecoveryForSnapshot( + db: DatabaseSync, + sessionId: string, + snapshot: SessionTurnPositionSnapshotRow, +): SessionTurnAdmissionRecoveryStep { + if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') { + throw new SessionTurnPositionSnapshotMismatchError(sessionId); + } + const recovery = advanceSessionTurnAdmissionRecovery(db, { + sessionId, + cursorAdmittedAt: snapshot.build_cursor_admitted_at, + cursorTurnId: snapshot.build_cursor_position_id, + maxAdmissions: SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS, + }); + const updated = db + .prepare(` + UPDATE session_turn_position_snapshots + SET build_cursor_admitted_at = ?, build_cursor_position_kind = ?, + build_cursor_position_id = ? + WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = 'recovering' + `) + .run( + recovery.cursorAdmittedAt, + recovery.cursorTurnId === null ? null : 'turn', + recovery.cursorTurnId, + sessionId, + snapshot.slot, + ); + if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); + const current = requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)); + return { + snapshot: recovery.complete ? markSessionTurnRecoveryComplete(db, sessionId, current) : current, + lastStepAdmissions: recovery.lastStepAdmissions, + }; +} + export function advanceSessionTurnPositionOrdinalBuild( db: DatabaseSync, sessionId: string, diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index 05d0f2af95..b6eb2a2c04 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -169,11 +169,11 @@ import { SQLITE_SESSION_MESSAGE_CHUNK_MARKER, } from './sqlite-session-metadata-schema.js'; import { + advanceSessionTurnAdmissionRecoveryForSnapshot, advanceSessionTurnPositionOrdinalBuild, allocateOrRequireSessionTurnPositionSnapshot, ensureTurnIndexRows, invalidateSessionTurnPositionIndex, - markSessionTurnRecoveryComplete, pageReadySessionTurnPositionSnapshot, readSessionTurnMembershipPreflight, recordAppendedSessionTurnMetadata, @@ -2744,6 +2744,7 @@ export class SqliteSessionMetadataStore { } if (snapshot.build_phase === 'recovering') { let recovery: ReturnType | undefined; + let transcriptRecoveryAdvanced = false; if (snapshot.through_sequence !== null) { const indexed = this.db .prepare(` @@ -2751,6 +2752,7 @@ export class SqliteSessionMetadataStore { `) .get(request.sessionId) as { indexed_through_sequence: number }; if (indexed.indexed_through_sequence < snapshot.through_sequence) { + transcriptRecoveryAdvanced = true; recovery = advanceSessionTurnIdentityRecovery(this.db, { sessionId: request.sessionId, throughSequence: snapshot.through_sequence, @@ -2767,7 +2769,28 @@ export class SqliteSessionMetadataStore { } } } - snapshot = markSessionTurnRecoveryComplete(this.db, request.sessionId, snapshot); + if (!transcriptRecoveryAdvanced) { + try { + snapshot = advanceSessionTurnAdmissionRecoveryForSnapshot( + this.db, + request.sessionId, + snapshot, + ).snapshot; + } catch (error) { + if (error instanceof SessionTurnPositionRecoveryError) { + this.db + .prepare(`DELETE FROM session_turn_position_snapshots + WHERE session_id = ? AND slot = ? AND state = 'building'`) + .run(request.sessionId, snapshot.slot); + return { + complete: false as const, + failure: error.reason, + failureSequence: error.sequence ?? 0, + }; + } + throw error; + } + } const state = this.db .prepare(` SELECT indexed_through_sequence, source_records, source_bytes From 4fbd00ae51dee01e9542a2c98c00c563b2fa7b3f Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:03:30 +0800 Subject: [PATCH 06/10] fix(storage): persist admission recovery fixed point Generated-by: Codex --- .../session-turn-position-index.test.ts | 205 +++++++++++++++++- .../src/session-turn-admission-recovery.ts | 58 +++-- .../src/session-turn-position-authority.ts | 11 + .../src/session-turn-position-snapshots.ts | 50 ++--- .../src/sqlite-session-metadata-schema.ts | 6 + .../src/sqlite-session-metadata-store.ts | 5 +- 6 files changed, 268 insertions(+), 67 deletions(-) diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index bdda7dffe6..f1d452094b 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -22,7 +22,7 @@ import { createHash } from 'node:crypto'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { DatabaseSync } from 'node:sqlite'; +import { constants as sqliteConstants, DatabaseSync } from 'node:sqlite'; import { describe, test } from 'node:test'; import { createSHA256 } from 'hash-wasm'; import type { CreateSessionInput } from '@maka/core/runtime-inputs'; @@ -31,6 +31,7 @@ import { createConversationOperationalStateStore } from '../conversation-operati import { OPERATIONAL_STATE_DATABASE_NAME } from '../operational-state-store.js'; import { createSessionStore } from '../session-store.js'; import { advanceSessionTurnIdentityRecovery } from '../session-turn-identity-recovery.js'; +import { invalidateSessionTurnPositionIndex } from '../session-turn-position-index.js'; describe('Session Turn position snapshots', () => { test('materializes owner and shared tagged positions in one exact generation', async () => { @@ -2315,6 +2316,134 @@ describe('Session Turn position snapshots', () => { } }); + test('reuses normalized admission authority across later exact snapshots', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-fixed-point-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-a', 'user-a', 1)); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-b', 'user-b', 2)); + const first = await readyPage(store, session.id, 'admission-fixed-first', 'owner'); + assert.deepEqual( + first.positions.map((position) => position.key), + [ + { kind: 'turn', id: 'turn-a' }, + { kind: 'turn', id: 'turn-b' }, + ], + ); + + await runs.admitRootTurn(rootAdmission(session.id, 'turn-c', 'user-c', 3)); + const database = (store as unknown as { metadata: { db: DatabaseSync } }).metadata.db; + let admissionReads = 0; + database.setAuthorizer((action, table) => { + if (action === sqliteConstants.SQLITE_READ && table === 'core_root_turn_admissions') { + admissionReads += 1; + } + return sqliteConstants.SQLITE_OK; + }); + try { + const next = await readyPage(store, session.id, 'admission-fixed-next', 'owner'); + assert.ok(next.snapshotKey.authorityRevision > first.snapshotKey.authorityRevision); + assert.deepEqual( + next.positions.map((position) => position.key), + [ + { kind: 'turn', id: 'turn-a' }, + { kind: 'turn', id: 'turn-b' }, + { kind: 'turn', id: 'turn-c' }, + ], + ); + } finally { + database.setAuthorizer(null); + } + assert.equal(admissionReads, 0); + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 3, + admission_cursor_turn_id: 'turn-c', + admission_recovery_complete: 1, + }, + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('explicit invalidation resets and rebuilds persisted admission recovery state', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-reset-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + try { + const session = await store.create(makeInput()); + await runs.admitRootTurn(rootAdmission(session.id, 'turn-reset', 'user-reset', 9)); + await readyPage(store, session.id, 'admission-reset-before', 'owner'); + const database = (store as unknown as { metadata: { db: DatabaseSync } }).metadata.db; + database.exec('BEGIN IMMEDIATE'); + try { + invalidateSessionTurnPositionIndex(database, session.id); + database.exec('COMMIT'); + } catch (error) { + database.exec('ROLLBACK'); + throw error; + } + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: null, + admission_cursor_turn_id: null, + admission_recovery_complete: 0, + }, + ); + assert.equal( + ( + database + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata WHERE session_id = ?`) + .get(session.id) as { count: number } + ).count, + 0, + ); + + const rebuilt = await readyPage(store, session.id, 'admission-reset-after', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { + ordinal: 0, + key: { kind: 'turn', id: 'turn-reset' }, + firstSequence: null, + }, + ]); + assert.deepEqual( + { + ...(database + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 9, + admission_cursor_turn_id: 'turn-reset', + admission_recovery_complete: 1, + }, + ); + } finally { + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('bounds legacy admission reconciliation and atomically rebuilds after owner reopen', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-turn-position-admission-recovery-bound-')); let store = createSessionStore(root); @@ -2360,22 +2489,31 @@ describe('Session Turn position snapshots', () => { 1_024, ); const snapshotState = bounded - .prepare(`SELECT state, build_phase, build_cursor_admitted_at, - build_cursor_position_id FROM session_turn_position_snapshots + .prepare(`SELECT state, build_phase FROM session_turn_position_snapshots WHERE session_id = ? AND snapshot_generation = ?`) .get(session.id, first.snapshotKey.snapshotGeneration) as { state: string; build_phase: string; - build_cursor_admitted_at: number; - build_cursor_position_id: string; }; assert.deepEqual( { ...snapshotState }, { state: 'building', build_phase: 'recovering', - build_cursor_admitted_at: 1_023, - build_cursor_position_id: 'turn-1023', + }, + ); + assert.deepEqual( + { + ...(bounded + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state + WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_023, + admission_cursor_turn_id: 'turn-1023', + admission_recovery_complete: 0, }, ); assert.equal( @@ -2416,7 +2554,58 @@ describe('Session Turn position snapshots', () => { (error: unknown) => (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', ); - const ready = await readyPage(store, session.id, 'admission-bound-reopen', 'owner'); + let resumed = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-reopen', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(resumed.kind, 'building'); + if (resumed.kind !== 'building') assert.fail('expected resumed admission reconciliation'); + const resumedState = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(resumedState + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state + WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_024, + admission_cursor_turn_id: 'turn-1024', + admission_recovery_complete: 1, + }, + ); + assert.equal( + ( + resumedState + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, resumed.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + resumedState.close(); + } + while (resumed.kind === 'building') { + resumed = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-bound-reopen', + snapshotKey: resumed.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(resumed.kind, 'page'); + if (resumed.kind !== 'page') assert.fail('expected ready page after resumed recovery'); + const ready = resumed; assert.equal(ready.totalPositions, 1_025); assert.ok(ready.snapshotKey.snapshotGeneration > first.snapshotKey.snapshotGeneration); const completed = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { diff --git a/packages/storage/src/session-turn-admission-recovery.ts b/packages/storage/src/session-turn-admission-recovery.ts index c5e055b914..98986c0451 100644 --- a/packages/storage/src/session-turn-admission-recovery.ts +++ b/packages/storage/src/session-turn-admission-recovery.ts @@ -24,9 +24,6 @@ export const SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS = 1_024; export interface SessionTurnAdmissionRecoveryResult { readonly complete: boolean; - readonly lastStepAdmissions: number; - readonly cursorAdmittedAt: number | null; - readonly cursorTurnId: string | null; } interface PersistedRecoveryFailure { @@ -34,6 +31,14 @@ interface PersistedRecoveryFailure { readonly failure_sequence: number; } +interface AdmissionRecoveryState { + readonly admission_cursor_admitted_at: number | null; + readonly admission_cursor_turn_id: string | null; + readonly admission_recovery_complete: number; + readonly failure_reason: PersistedRecoveryFailure['failure_reason'] | null; + readonly failure_sequence: number | null; +} + /** * Reconciles only canonical admission identity/order scalars. Transcript * membership and body decoding remain owned by their existing recovery path. @@ -42,32 +47,32 @@ export function advanceSessionTurnAdmissionRecovery( db: DatabaseSync, input: { readonly sessionId: string; - readonly cursorAdmittedAt: number | null; - readonly cursorTurnId: string | null; readonly maxAdmissions: number; }, ): SessionTurnAdmissionRecoveryResult { if ( !Number.isSafeInteger(input.maxAdmissions) || input.maxAdmissions < 1 || - input.maxAdmissions > SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS || - (input.cursorAdmittedAt === null) !== (input.cursorTurnId === null) + input.maxAdmissions > SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS ) { throw new SessionTurnPositionRecoveryError(input.sessionId, 'corrupt_source'); } - const persistedFailure = db - .prepare(`SELECT failure_reason, failure_sequence FROM session_turn_index_state - WHERE session_id = ? AND failure_reason IS NOT NULL`) - .get(input.sessionId) as PersistedRecoveryFailure | undefined; - if (persistedFailure) { + const state = db + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(input.sessionId) as unknown as AdmissionRecoveryState | undefined; + if (!state) throw new SessionTurnPositionRecoveryError(input.sessionId, 'corrupt_source'); + if (state.failure_reason !== null) { throw new SessionTurnPositionRecoveryError( input.sessionId, - persistedFailure.failure_reason, - persistedFailure.failure_sequence, + state.failure_reason, + state.failure_sequence ?? 0, ); } + if (state.admission_recovery_complete === 1) return { complete: true }; const rows = ( - input.cursorAdmittedAt === null + state.admission_cursor_admitted_at === null ? db .prepare(` SELECT turn_id, admitted_at @@ -85,7 +90,12 @@ export function advanceSessionTurnAdmissionRecovery( ORDER BY admitted_at, turn_id LIMIT ? `) - .all(input.sessionId, input.cursorAdmittedAt, input.cursorTurnId, input.maxAdmissions) + .all( + input.sessionId, + state.admission_cursor_admitted_at, + state.admission_cursor_turn_id, + input.maxAdmissions, + ) ) as Array<{ turn_id?: unknown; admitted_at?: unknown }>; const readExisting = db.prepare(` SELECT order_source, admitted_at @@ -100,8 +110,8 @@ export function advanceSessionTurnAdmissionRecovery( ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET order_source = 'admission', admitted_at = excluded.admitted_at `); - let cursorAdmittedAt = input.cursorAdmittedAt; - let cursorTurnId = input.cursorTurnId; + let cursorAdmittedAt = state.admission_cursor_admitted_at; + let cursorTurnId = state.admission_cursor_turn_id; for (const row of rows) { if ( typeof row.turn_id !== 'string' || @@ -122,12 +132,12 @@ export function advanceSessionTurnAdmissionRecovery( cursorAdmittedAt = row.admitted_at; cursorTurnId = row.turn_id; } - return { - complete: rows.length < input.maxAdmissions, - lastStepAdmissions: rows.length, - cursorAdmittedAt, - cursorTurnId, - }; + const complete = rows.length < input.maxAdmissions; + db.prepare(`UPDATE session_turn_index_state + SET admission_cursor_admitted_at = ?, admission_cursor_turn_id = ?, + admission_recovery_complete = ? + WHERE session_id = ?`).run(cursorAdmittedAt, cursorTurnId, complete ? 1 : 0, input.sessionId); + return { complete }; } function failAdmissionRecovery(db: DatabaseSync, sessionId: string): never { diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts index 799ee02891..121bfdfd43 100644 --- a/packages/storage/src/session-turn-position-authority.ts +++ b/packages/storage/src/session-turn-position-authority.ts @@ -112,6 +112,13 @@ export function recordRootTurnAdmissionForPositionIndex( ON CONFLICT(session_id, position_kind, position_id) DO UPDATE SET order_source = 'admission', admitted_at = excluded.admitted_at `).run(sessionId, turnId, admittedAt); + db.prepare(` + UPDATE session_turn_index_state + SET admission_cursor_admitted_at = ?, admission_cursor_turn_id = ? + WHERE session_id = ? AND admission_recovery_complete = 1 + AND (admission_cursor_admitted_at IS NULL + OR (admission_cursor_admitted_at, admission_cursor_turn_id) < (?, ?)) + `).run(admittedAt, turnId, sessionId, admittedAt, turnId); db.prepare(` UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' @@ -159,6 +166,8 @@ export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, + admission_recovery_complete = 0, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); @@ -189,6 +198,8 @@ export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, + admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, + admission_recovery_complete = 0, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts index 0877fd4c67..15d88d9df7 100644 --- a/packages/storage/src/session-turn-position-snapshots.ts +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -78,11 +78,6 @@ export interface SessionTurnPositionBuildStep { readonly executedPhase: Exclude; } -export interface SessionTurnAdmissionRecoveryStep { - readonly snapshot: SessionTurnPositionSnapshotRow; - readonly lastStepAdmissions: number; -} - export interface SessionTurnMembershipPreflight { readonly records: readonly { readonly positionKey: SessionTranscriptBodyPositionKey; @@ -186,12 +181,20 @@ export function markSessionTurnRecoveryComplete( snapshot: SessionTurnPositionSnapshotRow, ): SessionTurnPositionSnapshotRow { if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') return snapshot; - const indexed = ( - db - .prepare('SELECT indexed_through_sequence FROM session_turn_index_state WHERE session_id = ?') - .get(sessionId) as { indexed_through_sequence: number } - ).indexed_through_sequence; - if (snapshot.through_sequence !== null && indexed < snapshot.through_sequence) return snapshot; + const indexState = db + .prepare(`SELECT indexed_through_sequence, admission_recovery_complete + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as { + indexed_through_sequence: number; + admission_recovery_complete: number; + }; + if ( + indexState.admission_recovery_complete !== 1 || + (snapshot.through_sequence !== null && + indexState.indexed_through_sequence < snapshot.through_sequence) + ) { + return snapshot; + } const updated = db .prepare(` UPDATE session_turn_position_snapshots SET build_phase = 'legacy' @@ -206,36 +209,15 @@ export function advanceSessionTurnAdmissionRecoveryForSnapshot( db: DatabaseSync, sessionId: string, snapshot: SessionTurnPositionSnapshotRow, -): SessionTurnAdmissionRecoveryStep { +): SessionTurnPositionSnapshotRow { if (snapshot.state !== 'building' || snapshot.build_phase !== 'recovering') { throw new SessionTurnPositionSnapshotMismatchError(sessionId); } const recovery = advanceSessionTurnAdmissionRecovery(db, { sessionId, - cursorAdmittedAt: snapshot.build_cursor_admitted_at, - cursorTurnId: snapshot.build_cursor_position_id, maxAdmissions: SESSION_TURN_ADMISSION_RECOVERY_MAX_ROWS, }); - const updated = db - .prepare(` - UPDATE session_turn_position_snapshots - SET build_cursor_admitted_at = ?, build_cursor_position_kind = ?, - build_cursor_position_id = ? - WHERE session_id = ? AND slot = ? AND state = 'building' AND build_phase = 'recovering' - `) - .run( - recovery.cursorAdmittedAt, - recovery.cursorTurnId === null ? null : 'turn', - recovery.cursorTurnId, - sessionId, - snapshot.slot, - ); - if (updated.changes !== 1) throw new SessionTurnPositionSnapshotMismatchError(sessionId); - const current = requireSnapshot(db, sessionId, snapshotKeyFromRow(snapshot)); - return { - snapshot: recovery.complete ? markSessionTurnRecoveryComplete(db, sessionId, current) : current, - lastStepAdmissions: recovery.lastStepAdmissions, - }; + return recovery.complete ? markSessionTurnRecoveryComplete(db, sessionId, snapshot) : snapshot; } export function advanceSessionTurnPositionOrdinalBuild( diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index ef21b05d95..8e78cdadc5 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -1244,10 +1244,16 @@ const MIGRATIONS: ReadonlyMap = new Map([ CHECK (indexed_through_sequence >= -1), source_records INTEGER NOT NULL DEFAULT 0 CHECK (source_records >= 0), source_bytes INTEGER NOT NULL DEFAULT 0 CHECK (source_bytes >= 0), + admission_cursor_admitted_at INTEGER CHECK (admission_cursor_admitted_at >= 0), + admission_cursor_turn_id TEXT + CHECK (admission_cursor_turn_id IS NULL OR length(CAST(admission_cursor_turn_id AS BLOB)) > 0), + admission_recovery_complete INTEGER NOT NULL DEFAULT 0 + CHECK (admission_recovery_complete IN (0, 1)), failure_reason TEXT CHECK (failure_reason IN ('corrupt_source', 'incompatible_identity', 'hybrid_missing_admission')), failure_sequence INTEGER CHECK (failure_sequence >= 0), + CHECK ((admission_cursor_admitted_at IS NULL) = (admission_cursor_turn_id IS NULL)), CHECK ((failure_reason IS NULL) = (failure_sequence IS NULL)), FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE ) WITHOUT ROWID; diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index b6eb2a2c04..80331a76f0 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -2775,7 +2775,7 @@ export class SqliteSessionMetadataStore { this.db, request.sessionId, snapshot, - ).snapshot; + ); } catch (error) { if (error instanceof SessionTurnPositionRecoveryError) { this.db @@ -2806,6 +2806,9 @@ export class SqliteSessionMetadataStore { SELECT byte_offset FROM session_turn_identity_recovery WHERE session_id = ? `) .get(request.sessionId) as { byte_offset: number } | undefined; + // Admission reconciliation owns scalar authority, not transcript source + // records/bytes. Its persisted cursor is intentionally package-private; + // callers observe progress when this phase advances to ordinal building. return { kind: 'building' as const, snapshotKey: snapshotKeyFromRow(snapshot), From 7fab43bb13505961ae1d7e1a218c3f58cd4ccd0b Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:22:05 +0800 Subject: [PATCH 07/10] fix(storage): invalidate admission recovery on purge Generated-by: Codex --- .../session-turn-position-index.test.ts | 292 ++++++++++++++++++ .../src/conversation-operational-state.ts | 11 +- .../src/session-turn-position-authority.ts | 19 +- 3 files changed, 315 insertions(+), 7 deletions(-) diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index f1d452094b..55f0b3a803 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -553,6 +553,17 @@ describe('Session Turn position snapshots', () => { ); await conversation.purge(session.id); + const retained = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'lease-admitted', + snapshotKey: admitted.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(retained.kind, 'page'); + if (retained.kind !== 'page') assert.fail('expected leased ready snapshot to survive purge'); + assert.deepEqual(retained.positions, admitted.positions); const legacy = await readyPage(store, session.id, 'lease-purged'); assert.equal( legacy.snapshotKey.authorityRevision, @@ -571,6 +582,287 @@ describe('Session Turn position snapshots', () => { } }); + test('invalidates v34 transcript recovery when purge deletes an unreconciled admission', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-v34-building-')); + let store = createSessionStore(root); + let conversation: ReturnType | undefined; + const runs = createSqliteAgentRunStore(root); + const session = await store.create(makeInput()); + try { + await store.appendMessages( + session.id, + Array.from({ length: 1_025 }, (_, index) => user(`turn-${index}`, index)), + ); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-purged', 'future-user-purged', 2_000), + ); + runs.close?.(); + await store.close?.(); + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-old', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected bounded transcript recovery'); + assert.equal(first.progress.lastStepRecords, 1_024); + const beforePurge = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + beforePurge + .prepare(`SELECT COUNT(*) AS count FROM session_turn_metadata + WHERE session_id = ? AND order_source = 'admission'`) + .get(session.id) as { count: number } + ).count, + 0, + ); + assert.equal( + ( + beforePurge + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + beforePurge.close(); + } + + await conversation.purge(session.id); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-old', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const afterPurge = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + afterPurge + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + first.snapshotKey.authorityRevision + 1, + ); + assert.equal( + afterPurge + .prepare(`SELECT 1 FROM session_turn_position_snapshots + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, first.snapshotKey.snapshotGeneration), + undefined, + ); + } finally { + afterPurge.close(); + } + + let next = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-new', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(next.kind, 'building'); + if (next.kind !== 'building') assert.fail('expected post-purge rebuild'); + const noPartial = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + noPartial + .prepare(`SELECT COUNT(*) AS count FROM session_turn_snapshot_positions + WHERE session_id = ? AND snapshot_generation = ?`) + .get(session.id, next.snapshotKey.snapshotGeneration) as { count: number } + ).count, + 0, + ); + } finally { + noPartial.close(); + } + while (next.kind === 'building') { + next = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-v34-building-new', + snapshotKey: next.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + } + assert.equal(next.kind, 'page'); + if (next.kind !== 'page') assert.fail('expected post-purge ready page'); + assert.equal(next.totalPositions, 1_025); + assert.equal( + next.positions.some(({ key }) => key.kind === 'turn' && key.id === 'turn-bodyless-purged'), + false, + ); + } finally { + conversation?.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('resets partial admission recovery when purge deletes canonical authority', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-partial-recovery-')); + let store = createSessionStore(root); + let conversation: ReturnType | undefined; + const session = await store.create(makeInput()); + try { + await store.close?.(); + const seeded = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + const insert = seeded.prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, ?, ?, '{}')`); + seeded.exec('BEGIN IMMEDIATE'); + for (let index = 0; index < 1_025; index += 1) { + insert.run(session.id, `turn-${index.toString().padStart(4, '0')}`, index); + } + seeded.exec('COMMIT'); + } finally { + seeded.close(); + } + downgradeTurnProjectionToV34(root); + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + const first = await store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-partial-old', + anchor: { kind: 'tail' }, + maxPositions: 8, + }); + assert.equal(first.kind, 'building'); + if (first.kind !== 'building') assert.fail('expected partial admission recovery'); + const partial = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(partial + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: 1_023, + admission_cursor_turn_id: 'turn-1023', + admission_recovery_complete: 0, + }, + ); + } finally { + partial.close(); + } + + await conversation.purge(session.id); + const purged = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.deepEqual( + { + ...(purged + .prepare(`SELECT admission_cursor_admitted_at, admission_cursor_turn_id, + admission_recovery_complete FROM session_turn_index_state WHERE session_id = ?`) + .get(session.id) as Record), + }, + { + admission_cursor_admitted_at: null, + admission_cursor_turn_id: null, + admission_recovery_complete: 0, + }, + ); + assert.equal( + ( + purged + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + first.snapshotKey.authorityRevision + 1, + ); + } finally { + purged.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'purge-partial-old', + snapshotKey: first.snapshotKey, + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => + (error as { code?: unknown }).code === 'session_turn_position_snapshot_mismatch', + ); + const rebuilt = await readyPage(store, session.id, 'purge-partial-new', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + } finally { + conversation?.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('keeps admission authority revision stable for an idempotent empty purge', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-purge-idempotent-')); + const store = createSessionStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + const before = await readyPage(store, session.id, 'purge-idempotent-ready', 'owner'); + await conversation.purge(session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + assert.equal( + ( + database + .prepare(`SELECT authority_revision FROM session_turn_authority_revisions + WHERE session_id = ?`) + .get(session.id) as { authority_revision: number } + ).authority_revision, + before.snapshotKey.authorityRevision, + ); + } finally { + database.close(); + } + } finally { + conversation.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('shares one exact generation across leases and releases only after the last consumer', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-lease-')); const store = createSessionStore(root); diff --git a/packages/storage/src/conversation-operational-state.ts b/packages/storage/src/conversation-operational-state.ts index 32303ba7a9..fbd766ec09 100644 --- a/packages/storage/src/conversation-operational-state.ts +++ b/packages/storage/src/conversation-operational-state.ts @@ -82,8 +82,15 @@ class SqliteConversationOperationalStateStore implements ConversationOperational database .prepare('DELETE FROM core_agent_run_projections WHERE session_id = ?') .run(sessionId); - database.prepare('DELETE FROM core_root_turn_admissions WHERE session_id = ?').run(sessionId); - recordRootTurnAdmissionsPurgedForPositionIndex(database, sessionId); + const canonicalAdmissionsDeleted = + database + .prepare('DELETE FROM core_root_turn_admissions WHERE session_id = ?') + .run(sessionId).changes > 0; + recordRootTurnAdmissionsPurgedForPositionIndex( + database, + sessionId, + canonicalAdmissionsDeleted, + ); database .prepare('DELETE FROM core_root_turn_start_rejections WHERE session_id = ?') .run(sessionId); diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts index 121bfdfd43..830a249f88 100644 --- a/packages/storage/src/session-turn-position-authority.ts +++ b/packages/storage/src/session-turn-position-authority.ts @@ -130,6 +130,7 @@ export function recordRootTurnAdmissionForPositionIndex( export function recordRootTurnAdmissionsPurgedForPositionIndex( db: DatabaseSync, sessionId: string, + canonicalAdmissionsDeleted: boolean, ): void { if (!sqliteTableExists(db, 'session_turn_metadata')) return; if (!db.prepare('SELECT 1 FROM session_metadata WHERE session_id = ?').get(sessionId)) return; @@ -148,7 +149,8 @@ export function recordRootTurnAdmissionsPurgedForPositionIndex( AND order_source = 'admission' AND owner_first_sequence IS NOT NULL `) .run(sessionId); - if (removed.changes === 0 && downgraded.changes === 0) return; + if (!canonicalAdmissionsDeleted && removed.changes === 0 && downgraded.changes === 0) return; + resetSessionTurnAdmissionRecoveryState(db, sessionId); db.prepare(` UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' @@ -163,11 +165,10 @@ export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + resetSessionTurnAdmissionRecoveryState(db, sessionId); db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, - admission_recovery_complete = 0, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); @@ -195,11 +196,10 @@ export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { db.prepare('DELETE FROM session_turn_position_snapshots WHERE session_id = ?').run(sessionId); db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); db.prepare('DELETE FROM session_turn_metadata WHERE session_id = ?').run(sessionId); + resetSessionTurnAdmissionRecoveryState(db, sessionId); db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, - admission_recovery_complete = 0, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); @@ -244,6 +244,15 @@ function invalidateBuildingSnapshots(db: DatabaseSync, sessionId: string): void `).run(sessionId); } +function resetSessionTurnAdmissionRecoveryState(db: DatabaseSync, sessionId: string): void { + db.prepare(` + UPDATE session_turn_index_state + SET admission_cursor_admitted_at = NULL, admission_cursor_turn_id = NULL, + admission_recovery_complete = 0 + WHERE session_id = ? + `).run(sessionId); +} + function advanceAuthorityRevision(db: DatabaseSync, sessionId: string): void { db.prepare(` UPDATE session_turn_authority_revisions From 4f85aecaf60832de69036f860b806ddd8487a978 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 31 Aug 2026 00:46:58 +0800 Subject: [PATCH 08/10] fix(storage): persist recovery failure provenance Generated-by: Codex --- .../session-turn-position-index.test.ts | 292 +++++++++++++++++- .../src/session-turn-admission-recovery.ts | 2 +- .../src/session-turn-identity-recovery.ts | 3 +- .../src/session-turn-position-authority.ts | 23 +- .../src/session-turn-position-snapshots.ts | 3 +- .../src/sqlite-session-metadata-schema.ts | 4 +- 6 files changed, 310 insertions(+), 17 deletions(-) diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index 55f0b3a803..540116b30c 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -863,6 +863,259 @@ describe('Session Turn position snapshots', () => { } }); + test('clears an admission-origin corrupt source after purge removes its cause', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-admission-failure-purge-')); + const store = createSessionStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const session = await store.create(makeInput()); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`INSERT INTO core_root_turn_admissions( + session_id, turn_id, admitted_at, record_json + ) VALUES (?, '', 1, '{}')`) + .run(session.id); + } finally { + database.close(); + } + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'admission-failure-before-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'admission', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + + await conversation.purge(session.id); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const rebuilt = await readyPage(store, session.id, 'admission-failure-after-purge', 'owner'); + assert.deepEqual(rebuilt.positions, [ + { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, + ]); + } finally { + conversation.close(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('retains transcript-origin corrupt source across an admission purge', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-transcript-failure-admission-purge-')); + let store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + let conversation: ReturnType | undefined; + const session = await store.create(makeInput()); + try { + await store.appendMessage(session.id, user('turn-corrupt-transcript', 0)); + await runs.admitRootTurn( + rootAdmission(session.id, 'turn-bodyless-to-purge', 'future-purged-user', 5), + ); + runs.close?.(); + await store.close?.(); + resetProjection(root, session.id); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + database + .prepare(`UPDATE session_messages SET record_json = '{' + WHERE session_id = ? AND sequence = 0`) + .run(session.id); + } finally { + database.close(); + } + + store = createSessionStore(root); + conversation = createConversationOperationalStateStore(root); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'transcript-failure-before-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'transcript', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + + await conversation.purge(session.id); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: 'transcript', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }); + await assert.rejects( + store.readTurnPositionPageSnapshot({ + sessionId: session.id, + projection: 'owner', + snapshotLeaseId: 'transcript-failure-after-purge', + anchor: { kind: 'tail' }, + maxPositions: 8, + }), + (error: unknown) => (error as { reason?: unknown }).reason === 'corrupt_source', + ); + } finally { + conversation?.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('tags hybrid failure as admission and clears it only for resolving authority', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-hybrid-failure-origin-')); + const store = createSessionStore(root); + const runs = createSqliteAgentRunStore(root); + const conversation = createConversationOperationalStateStore(root); + try { + const resolvedSession = await store.create(makeInput({ name: 'Resolved hybrid' })); + await runs.admitRootTurn(rootAdmission(resolvedSession.id, 'turn-modern', 'user-modern', 10)); + await store.appendMessages(resolvedSession.id, [ + { type: 'user', id: 'user-modern', turnId: 'turn-modern', ts: 1, text: 'modern' }, + { type: 'user', id: 'user-missing', turnId: 'turn-missing', ts: 2, text: 'missing' }, + ]); + await assert.rejects(readyPage(store, resolvedSession.id), (error: unknown) => { + return (error as { reason?: unknown }).reason === 'hybrid_missing_admission'; + }); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + + await runs.admitRootTurn( + rootAdmission(resolvedSession.id, 'turn-unrelated', 'future-unrelated-user', 12), + ); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + await runs.admitRootTurn( + rootAdmission(resolvedSession.id, 'turn-missing', 'user-missing', 11), + ); + assert.deepEqual(readTurnIndexFailure(root, resolvedSession.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const resolved = await readyPage(store, resolvedSession.id, 'hybrid-resolved', 'owner'); + assert.deepEqual( + resolved.positions.map(({ key }) => key), + [ + { kind: 'turn', id: 'turn-modern' }, + { kind: 'turn', id: 'turn-missing' }, + { kind: 'turn', id: 'turn-unrelated' }, + ], + ); + + const purgedSession = await store.create(makeInput({ name: 'Purged hybrid' })); + await runs.admitRootTurn( + rootAdmission(purgedSession.id, 'turn-modern-purge', 'user-modern-purge', 20), + ); + await store.appendMessages(purgedSession.id, [ + { + type: 'user', + id: 'user-modern-purge', + turnId: 'turn-modern-purge', + ts: 1, + text: 'modern', + }, + { + type: 'user', + id: 'user-missing-purge', + turnId: 'turn-missing-purge', + ts: 2, + text: 'missing', + }, + ]); + await assert.rejects(readyPage(store, purgedSession.id), (error: unknown) => { + return (error as { reason?: unknown }).reason === 'hybrid_missing_admission'; + }); + assert.deepEqual(readTurnIndexFailure(root, purgedSession.id), { + failure_origin: 'admission', + failure_reason: 'hybrid_missing_admission', + failure_sequence: 1, + }); + await conversation.purge(purgedSession.id); + assert.deepEqual(readTurnIndexFailure(root, purgedSession.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); + const purged = await readyPage(store, purgedSession.id, 'hybrid-purged', 'owner'); + assert.equal(purged.totalPositions, 2); + } finally { + conversation.close(); + runs.close?.(); + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('rejects partial persisted failure provenance triples', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-failure-provenance-check-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + await readyPage(store, session.id, 'failure-provenance-schema', 'owner'); + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); + try { + for (const statement of [ + `UPDATE session_turn_index_state SET failure_origin = 'transcript' + WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_reason = 'corrupt_source', failure_sequence = 0 WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'corrupt_source' + WHERE session_id = ?`, + `UPDATE session_turn_index_state + SET failure_origin = 'other', failure_reason = 'corrupt_source', failure_sequence = 0 + WHERE session_id = ?`, + ]) { + assert.throws( + () => database.prepare(statement).run(session.id), + /CHECK constraint failed/, + ); + } + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = 'transcript', failure_reason = 'corrupt_source', + failure_sequence = 0 WHERE session_id = ?`) + .run(session.id); + assert.throws( + () => + database + .prepare(`UPDATE session_turn_index_state SET failure_origin = NULL + WHERE session_id = ?`) + .run(session.id), + /CHECK constraint failed/, + ); + } finally { + database.close(); + } + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } + }); + test('shares one exact generation across leases and releases only after the last consumer', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-shared-position-lease-')); const store = createSessionStore(root); @@ -1156,7 +1409,8 @@ describe('Session Turn position snapshots', () => { database .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - failure_reason = NULL, failure_sequence = NULL WHERE session_id = ?`) + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ?`) .run(session.id); } finally { database.close(); @@ -1560,9 +1814,14 @@ describe('Session Turn position snapshots', () => { }); try { const failure = failed - .prepare(`SELECT failure_reason, failure_sequence + .prepare(`SELECT failure_origin, failure_reason, failure_sequence FROM session_turn_index_state WHERE session_id = ?`) - .get(session.id) as { failure_reason: string; failure_sequence: number }; + .get(session.id) as { + failure_origin: string; + failure_reason: string; + failure_sequence: number; + }; + assert.equal(failure.failure_origin, 'transcript'); assert.equal(failure.failure_reason, 'corrupt_source'); assert.equal(failure.failure_sequence, 0); } finally { @@ -3021,11 +3280,15 @@ describe('Session Turn position snapshots', () => { assert.deepEqual( { ...(failed - .prepare(`SELECT failure_reason, failure_sequence + .prepare(`SELECT failure_origin, failure_reason, failure_sequence FROM session_turn_index_state WHERE session_id = ?`) .get(sessionId) as Record), }, - { failure_reason: 'corrupt_source', failure_sequence: 0 }, + { + failure_origin: 'admission', + failure_reason: 'corrupt_source', + failure_sequence: 0, + }, ); } finally { failed.close(); @@ -3451,13 +3714,30 @@ function resetProjection(root: string, sessionId: string): void { database .prepare(`UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - failure_reason = NULL, failure_sequence = NULL WHERE session_id = ?`) + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ?`) .run(sessionId); } finally { database.close(); } } +function readTurnIndexFailure(root: string, sessionId: string): Record { + const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME), { + readOnly: true, + }); + try { + return { + ...(database + .prepare(`SELECT failure_origin, failure_reason, failure_sequence + FROM session_turn_index_state WHERE session_id = ?`) + .get(sessionId) as Record), + }; + } finally { + database.close(); + } +} + function downgradeTurnProjectionToV34(root: string): void { const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); try { diff --git a/packages/storage/src/session-turn-admission-recovery.ts b/packages/storage/src/session-turn-admission-recovery.ts index 98986c0451..136c7af466 100644 --- a/packages/storage/src/session-turn-admission-recovery.ts +++ b/packages/storage/src/session-turn-admission-recovery.ts @@ -142,7 +142,7 @@ export function advanceSessionTurnAdmissionRecovery( function failAdmissionRecovery(db: DatabaseSync, sessionId: string): never { db.prepare(`UPDATE session_turn_index_state - SET failure_reason = 'corrupt_source', failure_sequence = 0 + SET failure_origin = 'admission', failure_reason = 'corrupt_source', failure_sequence = 0 WHERE session_id = ?`).run(sessionId); throw new SessionTurnPositionRecoveryError(sessionId, 'corrupt_source', 0); } diff --git a/packages/storage/src/session-turn-identity-recovery.ts b/packages/storage/src/session-turn-identity-recovery.ts index fd64416e5e..fcb760cdac 100644 --- a/packages/storage/src/session-turn-identity-recovery.ts +++ b/packages/storage/src/session-turn-identity-recovery.ts @@ -405,7 +405,8 @@ function fail( cause?: unknown, ): SessionTurnIdentityRecoveryFailureFact { db.prepare(` - UPDATE session_turn_index_state SET failure_reason = ?, failure_sequence = ? + UPDATE session_turn_index_state + SET failure_origin = 'transcript', failure_reason = ?, failure_sequence = ? WHERE session_id = ? `).run(reason, sequence, sessionId); db.prepare('DELETE FROM session_turn_identity_recovery WHERE session_id = ?').run(sessionId); diff --git a/packages/storage/src/session-turn-position-authority.ts b/packages/storage/src/session-turn-position-authority.ts index 830a249f88..d640c2bf39 100644 --- a/packages/storage/src/session-turn-position-authority.ts +++ b/packages/storage/src/session-turn-position-authority.ts @@ -120,9 +120,17 @@ export function recordRootTurnAdmissionForPositionIndex( OR (admission_cursor_admitted_at, admission_cursor_turn_id) < (?, ?)) `).run(admittedAt, turnId, sessionId, admittedAt, turnId); db.prepare(` - UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL - WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' - `).run(sessionId); + UPDATE session_turn_index_state + SET failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_origin = 'admission' + AND failure_reason = 'hybrid_missing_admission' + AND EXISTS ( + SELECT 1 FROM session_turn_memberships AS membership + WHERE membership.session_id = session_turn_index_state.session_id + AND membership.sequence = session_turn_index_state.failure_sequence + AND membership.position_kind = 'turn' AND membership.position_id = ? + ) + `).run(sessionId, turnId); invalidateBuildingSnapshots(db, sessionId); advanceAuthorityRevision(db, sessionId); } @@ -152,8 +160,9 @@ export function recordRootTurnAdmissionsPurgedForPositionIndex( if (!canonicalAdmissionsDeleted && removed.changes === 0 && downgraded.changes === 0) return; resetSessionTurnAdmissionRecoveryState(db, sessionId); db.prepare(` - UPDATE session_turn_index_state SET failure_reason = NULL, failure_sequence = NULL - WHERE session_id = ? AND failure_reason = 'hybrid_missing_admission' + UPDATE session_turn_index_state + SET failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL + WHERE session_id = ? AND failure_origin = 'admission' `).run(sessionId); invalidateBuildingSnapshots(db, sessionId); advanceAuthorityRevision(db, sessionId); @@ -169,7 +178,7 @@ export function invalidateSessionTurnPositionIndex(db: DatabaseSync, sessionId: db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - failure_reason = NULL, failure_sequence = NULL + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); advanceAuthorityRevision(db, sessionId); @@ -200,7 +209,7 @@ export function ensureTurnIndexRows(db: DatabaseSync, sessionId: string): void { db.prepare(` UPDATE session_turn_index_state SET indexed_through_sequence = -1, source_records = 0, source_bytes = 0, - failure_reason = NULL, failure_sequence = NULL + failure_origin = NULL, failure_reason = NULL, failure_sequence = NULL WHERE session_id = ? `).run(sessionId); db.prepare(` diff --git a/packages/storage/src/session-turn-position-snapshots.ts b/packages/storage/src/session-turn-position-snapshots.ts index 15d88d9df7..92dca0ef4f 100644 --- a/packages/storage/src/session-turn-position-snapshots.ts +++ b/packages/storage/src/session-turn-position-snapshots.ts @@ -824,7 +824,8 @@ function validateHybridBoundary( if (hybrid) { db.prepare(` UPDATE session_turn_index_state - SET failure_reason = 'hybrid_missing_admission', failure_sequence = ? + SET failure_origin = 'admission', failure_reason = 'hybrid_missing_admission', + failure_sequence = ? WHERE session_id = ? `).run(hybrid.owner_first_sequence, sessionId); throw new SessionTurnPositionRecoveryError( diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index 8e78cdadc5..cdc97c3d4e 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -1249,12 +1249,14 @@ const MIGRATIONS: ReadonlyMap = new Map([ CHECK (admission_cursor_turn_id IS NULL OR length(CAST(admission_cursor_turn_id AS BLOB)) > 0), admission_recovery_complete INTEGER NOT NULL DEFAULT 0 CHECK (admission_recovery_complete IN (0, 1)), + failure_origin TEXT CHECK (failure_origin IN ('transcript', 'admission')), failure_reason TEXT CHECK (failure_reason IN ('corrupt_source', 'incompatible_identity', 'hybrid_missing_admission')), failure_sequence INTEGER CHECK (failure_sequence >= 0), CHECK ((admission_cursor_admitted_at IS NULL) = (admission_cursor_turn_id IS NULL)), - CHECK ((failure_reason IS NULL) = (failure_sequence IS NULL)), + CHECK ((failure_origin IS NULL) = (failure_reason IS NULL) + AND (failure_reason IS NULL) = (failure_sequence IS NULL)), FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE ) WITHOUT ROWID; From 1c79bad38d12cffabfd329be8bfd83baf8728561 Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 31 Aug 2026 01:00:58 +0800 Subject: [PATCH 09/10] fix(storage): constrain recovery failure pairs Generated-by: Codex --- .../session-turn-position-index.test.ts | 53 ++++++++++++++++--- .../src/sqlite-session-metadata-schema.ts | 5 ++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/packages/storage/src/__tests__/session-turn-position-index.test.ts b/packages/storage/src/__tests__/session-turn-position-index.test.ts index 540116b30c..3d420f3f32 100644 --- a/packages/storage/src/__tests__/session-turn-position-index.test.ts +++ b/packages/storage/src/__tests__/session-turn-position-index.test.ts @@ -1069,7 +1069,7 @@ describe('Session Turn position snapshots', () => { } }); - test('rejects partial persisted failure provenance triples', async () => { + test('enforces the persisted failure provenance pair matrix', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-failure-provenance-check-')); const store = createSessionStore(root); try { @@ -1094,11 +1094,32 @@ describe('Session Turn position snapshots', () => { /CHECK constraint failed/, ); } - database - .prepare(`UPDATE session_turn_index_state - SET failure_origin = 'transcript', failure_reason = 'corrupt_source', - failure_sequence = 0 WHERE session_id = ?`) - .run(session.id); + for (const [origin, reason] of [ + ['transcript', 'corrupt_source'], + ['transcript', 'incompatible_identity'], + ['admission', 'corrupt_source'], + ['admission', 'hybrid_missing_admission'], + ] as const) { + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = ?, failure_reason = ?, failure_sequence = 0 + WHERE session_id = ?`) + .run(origin, reason, session.id); + } + for (const [origin, reason] of [ + ['transcript', 'hybrid_missing_admission'], + ['admission', 'incompatible_identity'], + ] as const) { + assert.throws( + () => + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = ?, failure_reason = ?, failure_sequence = 0 + WHERE session_id = ?`) + .run(origin, reason, session.id), + /CHECK constraint failed/, + ); + } assert.throws( () => database @@ -2937,6 +2958,11 @@ describe('Session Turn position snapshots', () => { await runs.admitRootTurn(rootAdmission(session.id, 'turn-reset', 'user-reset', 9)); await readyPage(store, session.id, 'admission-reset-before', 'owner'); const database = (store as unknown as { metadata: { db: DatabaseSync } }).metadata.db; + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = 'transcript', failure_reason = 'corrupt_source', + failure_sequence = 0 WHERE session_id = ?`) + .run(session.id); database.exec('BEGIN IMMEDIATE'); try { invalidateSessionTurnPositionIndex(database, session.id); @@ -2958,6 +2984,11 @@ describe('Session Turn position snapshots', () => { admission_recovery_complete: 0, }, ); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); assert.equal( ( database @@ -3202,6 +3233,11 @@ describe('Session Turn position snapshots', () => { const before = await readyPage(store, session.id, 'bodyless-policy-before', 'owner'); const database = new DatabaseSync(join(root, OPERATIONAL_STATE_DATABASE_NAME)); try { + database + .prepare(`UPDATE session_turn_index_state + SET failure_origin = 'admission', failure_reason = 'corrupt_source', + failure_sequence = 0 WHERE session_id = ?`) + .run(session.id); database .prepare(`UPDATE session_turn_authority_revisions SET visibility_policy_version = 999 WHERE session_id = ?`) @@ -3223,6 +3259,11 @@ describe('Session Turn position snapshots', () => { assert.deepEqual(shared.positions, [ { ordinal: 0, key: { kind: 'empty' }, firstSequence: null }, ]); + assert.deepEqual(readTurnIndexFailure(root, session.id), { + failure_origin: null, + failure_reason: null, + failure_sequence: null, + }); } finally { runs.close?.(); await store.close?.(); diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index cdc97c3d4e..20439fade4 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -1257,6 +1257,11 @@ const MIGRATIONS: ReadonlyMap = new Map([ CHECK ((admission_cursor_admitted_at IS NULL) = (admission_cursor_turn_id IS NULL)), CHECK ((failure_origin IS NULL) = (failure_reason IS NULL) AND (failure_reason IS NULL) = (failure_sequence IS NULL)), + CHECK (failure_origin IS NULL + OR (failure_origin = 'transcript' + AND failure_reason IN ('corrupt_source', 'incompatible_identity')) + OR (failure_origin = 'admission' + AND failure_reason IN ('corrupt_source', 'hybrid_missing_admission'))), FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE ) WITHOUT ROWID; From b00436209373c0d8453c51f1c0cbad9f53dfeb8d Mon Sep 17 00:00:00 2001 From: sungl <81428141+Sun-GLiang@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:43:36 +0800 Subject: [PATCH 10/10] fix(storage): export semantic transcript position limits Generated-by: OpenAI Codex --- packages/storage/src/execution-stores.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 3187c1964e..0441fa2d45 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -123,6 +123,13 @@ export type { ProvenSteeringMessageHandoff, } from './message-admission-store.js'; export { submittedTurnIntentsEqual } from './submitted-turn-intent.js'; +export { + SESSION_TURN_POSITION_BODY_MAX_BYTES, + SESSION_TURN_POSITION_BODY_MAX_KEYS, + SESSION_TURN_POSITION_BODY_MAX_RECORDS, + SESSION_TURN_POSITION_MAX_PAGE_BYTES, + SESSION_TURN_POSITION_MAX_PAGE_POSITIONS, +} from './session-turn-position-snapshots.js'; export type { SubmittedTurnIntent } from './submitted-turn-intent.js'; export type { ProbeSessionRemovalResult,