From 102e9efa38bf75c2d9ae5348b71e74dccb9ba211 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 09:06:44 +0000 Subject: [PATCH 1/3] Bump the cargo group across 1 directory with 2 updates Bumps the cargo group with 2 updates in the / directory: [rb-sys](https://github.com/oxidize-rb/rb-sys) and [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs). Updates `rb-sys` from 0.9.127 to 0.9.128 - [Release notes](https://github.com/oxidize-rb/rb-sys/releases) - [Commits](https://github.com/oxidize-rb/rb-sys/compare/v0.9.127...v0.9.128) Updates `tiktoken-rs` from 0.11.0 to 0.12.0 - [Release notes](https://github.com/zurawiki/tiktoken-rs/releases) - [Commits](https://github.com/zurawiki/tiktoken-rs/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: rb-sys dependency-version: 0.9.128 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: cargo - dependency-name: tiktoken-rs dependency-version: 0.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: cargo ... Signed-off-by: dependabot[bot] --- Cargo.lock | 22 ++++++++-------------- ext/tiktoken_ruby/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbfdcfd..d9c1f73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,7 +36,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash", "shlex", "syn", ] @@ -218,18 +218,18 @@ dependencies = [ [[package]] name = "rb-sys" -version = "0.9.127" +version = "0.9.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7d7c9560fe42dcffa576941394075f18a17dce89fcf718a2fa90b7dc2134d12" +checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a" dependencies = [ "rb-sys-build", ] [[package]] name = "rb-sys-build" -version = "0.9.127" +version = "0.9.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1688e8f32967ba48c89e4dfa283b57f901075f542fc7ee9c3d7c5f9091ca1d9" +checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a" dependencies = [ "bindgen", "lazy_static", @@ -275,12 +275,6 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.2" @@ -347,9 +341,9 @@ dependencies = [ [[package]] name = "tiktoken-rs" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee" +checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" dependencies = [ "anyhow", "base64", @@ -357,7 +351,7 @@ dependencies = [ "fancy-regex", "lazy_static", "regex", - "rustc-hash 1.1.0", + "rustc-hash", ] [[package]] diff --git a/ext/tiktoken_ruby/Cargo.toml b/ext/tiktoken_ruby/Cargo.toml index 004fea7..27b1f84 100644 --- a/ext/tiktoken_ruby/Cargo.toml +++ b/ext/tiktoken_ruby/Cargo.toml @@ -12,4 +12,4 @@ crate-type = ["cdylib"] [dependencies] magnus = { version = "0.8.2" } rb-sys = { version = "0.9.127", features = ["stable-api-compiled-fallback"] } -tiktoken-rs = { version = "0.11.0" } +tiktoken-rs = { version = "0.12.0" } From 13c2165499a1daf114a90f1adbe3d4969725211d Mon Sep 17 00:00:00 2001 From: "Garen J. Torikian" Date: Fri, 17 Jul 2026 20:16:19 -0400 Subject: [PATCH 2/3] fix(ext): adapt CoreBPE::encode to fallible tiktoken-rs 0.12.0 API tiktoken-rs 0.12.0 changed `CoreBPE::encode` to return `Result<(Vec, usize), EncodeError>` instead of the infallible `(Vec, usize)` tuple. The wrapper accessed `.0` on the result directly, which no longer compiles (E0609: no field `0` on `Result`). Store the encode result as `Result, String>`, keep the token vector on success, and propagate any tokenization failure as a Ruby RuntimeError so the previously-infallible behavior is preserved for normal inputs. Co-Authored-By: Claude Opus 4.8 (1M context) --- ext/tiktoken_ruby/src/core_bpe_wrapper.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs index c054ff6..690f8bb 100644 --- a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs +++ b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs @@ -20,7 +20,7 @@ struct EncodeData { core_bpe: *const tiktoken_rs::CoreBPE, text: String, allowed_special: HashSet, - result: Vec, + result: Result, String>, } struct EncodeSpecialData { @@ -46,7 +46,13 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void { let data = &mut *(data as *mut EncodeData); let core_bpe = &*data.core_bpe; let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect(); - data.result = core_bpe.encode(&data.text, &allowed_special).0; + // tiktoken-rs 0.12.0 made `encode` fallible, returning + // `Result<(Vec, usize), EncodeError>`. Keep only the token vector and + // surface any tokenization failure to Ruby. + data.result = core_bpe + .encode(&data.text, &allowed_special) + .map(|(tokens, _last_piece_token_len)| tokens) + .map_err(|e| e.to_string()); std::ptr::null_mut() } @@ -99,7 +105,7 @@ impl CoreBPEWrapper { core_bpe: &self.core_bpe as *const _, text, allowed_special: HashSet::from_iter(allowed_special), - result: Vec::new(), + result: Ok(Vec::new()), }; unsafe { @@ -111,7 +117,9 @@ impl CoreBPEWrapper { ); } - Ok(data.result) + data.result.map_err(|e| { + magnus::Error::new(magnus::Ruby::get().unwrap().exception_runtime_error(), e) + }) } pub fn encode_with_special_tokens(&self, text: String) -> Vec { From e5b772cb70d56a6ef97a468593e66f6956662beb Mon Sep 17 00:00:00 2001 From: Garen Torikian Date: Sat, 18 Jul 2026 12:05:22 -0400 Subject: [PATCH 3/3] Refine comment in encode_without_gvl function Updated comment to clarify the purpose of the code. --- ext/tiktoken_ruby/src/core_bpe_wrapper.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs index 690f8bb..5feded7 100644 --- a/ext/tiktoken_ruby/src/core_bpe_wrapper.rs +++ b/ext/tiktoken_ruby/src/core_bpe_wrapper.rs @@ -46,9 +46,7 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void { let data = &mut *(data as *mut EncodeData); let core_bpe = &*data.core_bpe; let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect(); - // tiktoken-rs 0.12.0 made `encode` fallible, returning - // `Result<(Vec, usize), EncodeError>`. Keep only the token vector and - // surface any tokenization failure to Ruby. + // Keep only the token vector and surface any tokenization failure to Ruby. data.result = core_bpe .encode(&data.text, &allowed_special) .map(|(tokens, _last_piece_token_len)| tokens)