From 416d1609d36595044bd0935b56b37d17b471cea4 Mon Sep 17 00:00:00 2001 From: xonx <119700621+xonx4l@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:43:11 +0000 Subject: [PATCH] Add stdarch-gen-common --- .github/workflows/main.yml | 2 + Cargo.lock | 62 +++++ crates/core_arch/src/hexagon/v128.rs | 1 + crates/core_arch/src/hexagon/v64.rs | 1 + crates/stdarch-gen-common/Cargo.toml | 7 + crates/stdarch-gen-common/src/lib.rs | 352 +++++++++++++++++++++++++ crates/stdarch-gen-hexagon/Cargo.toml | 1 + crates/stdarch-gen-hexagon/src/main.rs | 34 +-- 8 files changed, 443 insertions(+), 17 deletions(-) create mode 100644 crates/stdarch-gen-common/Cargo.toml create mode 100644 crates/stdarch-gen-common/src/lib.rs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 98f6b842d1..6333b32ab8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -337,6 +337,8 @@ jobs: cargo run --bin=stdarch-gen-loongarch --release -- crates/stdarch-gen-loongarch/lasx.spec git diff --exit-code - name: Check hexagon + env: + STDARCH_GEN_MODE: check run: | cargo run -p stdarch-gen-hexagon --release git diff --exit-code diff --git a/Cargo.lock b/Cargo.lock index 804879c8fd..c8cfc21a5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,6 +268,22 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -446,6 +462,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "log" version = "0.4.29" @@ -458,6 +480,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "once_cell_polyfill" version = "1.70.2" @@ -654,6 +682,19 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "ryu" version = "1.0.23" @@ -793,11 +834,19 @@ dependencies = [ "walkdir", ] +[[package]] +name = "stdarch-gen-common" +version = "0.1.0" +dependencies = [ + "tempfile", +] + [[package]] name = "stdarch-gen-hexagon" version = "0.1.0" dependencies = [ "regex", + "stdarch-gen-common", ] [[package]] @@ -870,6 +919,19 @@ version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d0e35dc7d73976a53c7e6d7d177ef804a0c0ee774ec77bcc520c2216fd7cbe" +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "termcolor" version = "1.4.1" diff --git a/crates/core_arch/src/hexagon/v128.rs b/crates/core_arch/src/hexagon/v128.rs index 1f0566af78..1a1826ddda 100644 --- a/crates/core_arch/src/hexagon/v128.rs +++ b/crates/core_arch/src/hexagon/v128.rs @@ -1,3 +1,4 @@ +// This code is automatically generated. DO NOT MODIFY. //! Hexagon HVX 128-byte vector mode intrinsics //! //! This module provides intrinsics for the Hexagon Vector Extensions (HVX) diff --git a/crates/core_arch/src/hexagon/v64.rs b/crates/core_arch/src/hexagon/v64.rs index e9b18b2fd8..3ac8b47079 100644 --- a/crates/core_arch/src/hexagon/v64.rs +++ b/crates/core_arch/src/hexagon/v64.rs @@ -1,3 +1,4 @@ +// This code is automatically generated. DO NOT MODIFY. //! Hexagon HVX 64-byte vector mode intrinsics //! //! This module provides intrinsics for the Hexagon Vector Extensions (HVX) diff --git a/crates/stdarch-gen-common/Cargo.toml b/crates/stdarch-gen-common/Cargo.toml new file mode 100644 index 0000000000..691be14971 --- /dev/null +++ b/crates/stdarch-gen-common/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "stdarch-gen-common" +version = "0.1.0" +edition = "2024" + +[dependencies] +tempfile = "3" \ No newline at end of file diff --git a/crates/stdarch-gen-common/src/lib.rs b/crates/stdarch-gen-common/src/lib.rs new file mode 100644 index 0000000000..13d788594c --- /dev/null +++ b/crates/stdarch-gen-common/src/lib.rs @@ -0,0 +1,352 @@ +//! Shared check/bless harness for stdarch generators. + +use std::error::Error as StdError; +use std::fmt; +use std::fs; +use std::io; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// First-line marker identifying an auto-generated file. Generators emit this +/// as the first line of every file they produce; the harness uses it to +/// discover which files in `committed` are owned by the generator. +pub const GENERATED_MARKER: &str = "// This code is automatically generated. DO NOT MODIFY."; + +/// Controls what `run_generator` does with the generator's output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// Verify that the `committed` matches the generator's output for owned files. + /// + /// Runs the generator into a temp directory, then compares each produced + /// file against the committed copy. Returns an error on the first mismatch. + Check, + /// Update the `committed` to match the generator's output for owned files. + /// + /// Runs the generator into a temp directory and copies each produced file + /// into `committed`. If the generator no longer produces an owned file, the + /// committed copy is deleted. Files in `committed` that are not owned + /// are left untouched. + Bless, +} + +impl Mode { + /// Read the mode from the `STDARCH_GEN_MODE` environment variable. + /// + /// Recognized values: + /// - `"check"` → [`Mode::Check`] + /// - `"bless"` → [`Mode::Bless`] + /// - unset → [`Mode::Bless`] + /// - any other value → panic + pub fn from_env() -> Self { + match std::env::var("STDARCH_GEN_MODE").as_deref() { + Ok("check") => Mode::Check, + Ok("bless") => Mode::Bless, + Ok(other) => panic!("unknown STDARCH_GEN_MODE value: {other:?}"), + Err(_) => Mode::Bless, + } + } +} + +#[derive(Debug)] +pub enum Error { + Io(io::Error), + Mismatch { path: PathBuf, kind: MismatchKind }, + Generator(Box), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MismatchKind { + /// Owned file produced by the generator but absent from the `committed`. + /// Means the `committed` needs to be regenerated. + MissingInCommitted, + /// Owned file present in the `committed` but the generator no longer + /// produces it. The file must be removed from the `committed` . + ExtraInCommitted, + /// Owned file exists on both sides but contents differ. + ContentsDiffer, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Io(e) => write!(f, "I/O error: {e}"), + Error::Mismatch { path, kind } => match kind { + MismatchKind::MissingInCommitted => { + write!(f, "{}: generated but not committed", path.display()) + } + MismatchKind::ExtraInCommitted => { + write!(f, "{}: committed but no longer generated", path.display()) + } + MismatchKind::ContentsDiffer => write!(f, "{}: contents differ", path.display()), + }, + Error::Generator(e) => write!(f, "generator failed: {e}"), + } + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + match self { + Error::Io(e) => Some(e), + Error::Generator(e) => Some(&**e), + _ => None, + } + } +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Error::Io(e) + } +} + +pub type Result = std::result::Result; + +/// Run a generator under the chosen `mode`, reconciling its output with `committed`. +/// +/// Arguments: +/// - `committed` — the directory holding the in-tree (committed) source files. +/// Files inside `committed` whose first line begins with [`GENERATED_MARKER`] +/// are treated as owned by the generator. Anything else is treated as +/// hand-written and is left untouched by `Bless` and ignored by `Check`. +/// So generated files coexist with hand-written files in the same directory. +/// - `mode` — what to do with the generator's output. +/// - `generate` — closure that writes the generator's output into the +/// directory it is given. Its error is wrapped in [`Error::Generator`]. +/// +/// Behavior per mode: +/// - [`Mode::Check`]: runs the generator into a temp dir and compares owned +/// files against the committed copies. Mismatch returns [`Error::Mismatch`]. +/// - [`Mode::Bless`]: runs the generator into a temp dir and copies owned +/// files into `committed`, or removes `committed`'s copy if the generator no +/// longer produces them. +pub fn run_generator(committed: &Path, mode: Mode, generate: F) -> Result<()> +where + F: FnOnce(&Path) -> std::result::Result<(), E>, + E: Into>, +{ + let scratch = tempfile::tempdir()?; + generate(scratch.path()).map_err(|e| Error::Generator(e.into()))?; + + let owned = discover_owned(committed)?; + let produced = discover_all(scratch.path())?; + + let mut names: Vec<&String> = owned.iter().chain(produced.iter()).collect(); + names.sort(); + names.dedup(); + + for name in names { + match mode { + Mode::Check => compare(scratch.path(), committed, name)?, + Mode::Bless => apply_bless(scratch.path(), committed, name)?, + } + } + Ok(()) +} + +/// Returns the names of files in `dir` whose first line begins with +/// [`GENERATED_MARKER`]. Files without the marker are skipped. +fn discover_owned(dir: &Path) -> Result> { + let mut out = Vec::new(); + for entry in fs::read_dir(dir)? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let Ok(mut file) = fs::File::open(entry.path()) else { + continue; + }; + let mut buf = [0u8; GENERATED_MARKER.len()]; + if file.read_exact(&mut buf).is_err() { + continue; + } + if buf == *GENERATED_MARKER.as_bytes() + && let Some(name) = entry.file_name().to_str() + { + out.push(name.to_owned()); + } + } + out.sort(); + Ok(out) +} + +/// Returns every file name in `dir` (scratch dir — all files are generator output). +fn discover_all(dir: &Path) -> Result> { + let mut out = Vec::new(); + for entry in fs::read_dir(dir)? { + let entry = entry?; + if entry.file_type()?.is_file() + && let Some(name) = entry.file_name().to_str() + { + out.push(name.to_string()); + } + } + Ok(out) +} + +fn compare(generated_dir: &Path, committed_dir: &Path, filename: &str) -> Result<()> { + let rel_path = PathBuf::from(filename); + let gen_path = generated_dir.join(&rel_path); + let comm_path = committed_dir.join(&rel_path); + match (gen_path.exists(), comm_path.exists()) { + (true, false) => Err(Error::Mismatch { + path: rel_path, + kind: MismatchKind::MissingInCommitted, + }), + (false, true) => Err(Error::Mismatch { + path: rel_path, + kind: MismatchKind::ExtraInCommitted, + }), + (false, false) => Ok(()), + (true, true) => { + if fs::read(&gen_path)? != fs::read(&comm_path)? { + Err(Error::Mismatch { + path: rel_path, + kind: MismatchKind::ContentsDiffer, + }) + } else { + Ok(()) + } + } + } +} + +fn apply_bless(generated_dir: &Path, committed_dir: &Path, filename: &str) -> Result<()> { + fs::create_dir_all(committed_dir)?; + let rel_path = PathBuf::from(filename); + let from = generated_dir.join(&rel_path); + let to = committed_dir.join(&rel_path); + if from.exists() { + if let Some(parent) = to.parent() { + fs::create_dir_all(parent)?; + } + fs::copy(&from, &to)?; + } else if to.exists() { + fs::remove_file(&to)?; + } + Ok(()) +} + +// Skipped on iOS because these tests fail on the `x86_64-apple-ios-macabi` CI runner +// with `Os { code: 17, kind: AlreadyExists, message: "File exists" }`. +#[cfg(all(test, not(target_os = "ios")))] +mod tests { + use super::*; + + fn write_marker(p: &Path, body: &[u8]) { + if let Some(d) = p.parent() { + fs::create_dir_all(d).unwrap(); + } + let mut bytes = Vec::new(); + bytes.extend_from_slice(GENERATED_MARKER.as_bytes()); + bytes.push(b'\n'); + bytes.extend_from_slice(body); + fs::write(p, bytes).unwrap(); + } + + #[test] + fn check_fails_on_byte_diff() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + write_marker(&committed.join("a.txt"), b"hi"); + let e = run_generator( + &committed, + Mode::Check, + |out| -> std::result::Result<(), io::Error> { + write_marker(&out.join("a.txt"), b"HI"); + Ok(()) + }, + ) + .unwrap_err(); + assert!(matches!( + e, + Error::Mismatch { + kind: MismatchKind::ContentsDiffer, + .. + } + )); + } + + #[test] + fn check_fails_when_owned_file_missing_from_generated() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + write_marker(&committed.join("a.txt"), b"hi"); + let e = run_generator( + &committed, + Mode::Check, + |_| -> std::result::Result<(), io::Error> { Ok(()) }, + ) + .unwrap_err(); + assert!(matches!( + e, + Error::Mismatch { + kind: MismatchKind::ExtraInCommitted, + .. + } + )); + } + + #[test] + fn check_fails_when_owned_file_missing_from_committed() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + fs::create_dir_all(&committed).unwrap(); + let e = run_generator( + &committed, + Mode::Check, + |out| -> std::result::Result<(), io::Error> { + write_marker(&out.join("a.txt"), b"hi"); + Ok(()) + }, + ) + .unwrap_err(); + assert!(matches!( + e, + Error::Mismatch { + kind: MismatchKind::MissingInCommitted, + .. + } + )); + } + + #[test] + fn bless_deletes_files_no_longer_produced() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + write_marker(&committed.join("keep.txt"), b""); + write_marker(&committed.join("stale.txt"), b""); + run_generator( + &committed, + Mode::Bless, + |out| -> std::result::Result<(), io::Error> { + write_marker(&out.join("keep.txt"), b""); + Ok(()) + }, + ) + .unwrap(); + assert!(committed.join("keep.txt").exists()); + assert!(!committed.join("stale.txt").exists()); + } + + #[test] + fn bless_preserves_unowned_files() { + let tmp = tempfile::tempdir().unwrap(); + let committed = tmp.path().join("c"); + fs::create_dir_all(&committed).unwrap(); + fs::write(committed.join("mod.rs"), b"hand-written").unwrap(); + fs::write(committed.join("old.txt"), b"old").unwrap(); + run_generator( + &committed, + Mode::Bless, + |out| -> std::result::Result<(), io::Error> { + write_marker(&out.join("new.txt"), b"new"); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(fs::read(committed.join("mod.rs")).unwrap(), b"hand-written"); + assert_eq!(fs::read(committed.join("old.txt")).unwrap(), b"old"); + assert!(committed.join("new.txt").exists()); + } +} diff --git a/crates/stdarch-gen-hexagon/Cargo.toml b/crates/stdarch-gen-hexagon/Cargo.toml index 397c7816f8..c7dfce2c0f 100644 --- a/crates/stdarch-gen-hexagon/Cargo.toml +++ b/crates/stdarch-gen-hexagon/Cargo.toml @@ -7,3 +7,4 @@ edition = "2021" [dependencies] regex = "1.10" +stdarch-gen-common = { path = "../stdarch-gen-common" } diff --git a/crates/stdarch-gen-hexagon/src/main.rs b/crates/stdarch-gen-hexagon/src/main.rs index 7a1c3030c0..c3ad153ab0 100644 --- a/crates/stdarch-gen-hexagon/src/main.rs +++ b/crates/stdarch-gen-hexagon/src/main.rs @@ -21,6 +21,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::Write; use std::path::Path; +use stdarch_gen_common::{run_generator, Mode, GENERATED_MARKER}; /// Mappings from HVX intrinsics to architecture-independent SIMD intrinsics. /// These intrinsics have equivalent semantics and can be lowered to the generic form. @@ -1609,6 +1610,7 @@ fn generate_module_file( let mut output = File::create(output_path).map_err(|e| format!("Failed to create output: {}", e))?; + writeln!(output, "{}", GENERATED_MARKER).map_err(|e| e.to_string())?; writeln!(output, "{}", generate_module_doc(mode)).map_err(|e| e.to_string())?; writeln!(output, "{}", generate_types(mode)).map_err(|e| e.to_string())?; writeln!(output, "{}", generate_extern_block(intrinsics, mode)).map_err(|e| e.to_string())?; @@ -1691,23 +1693,21 @@ fn main() -> Result<(), String> { } // Generate output files - let hexagon_dir = std::env::args() - .nth(1) - .map(std::path::PathBuf::from) - .unwrap_or_else(|| crate_dir.join("../core_arch/src/hexagon")); - std::fs::create_dir_all(&hexagon_dir).map_err(|e| e.to_string())?; - - // Generate v64.rs (64-byte vector mode) - let v64_path = hexagon_dir.join("v64.rs"); - println!("\nStep 3: Generating v64.rs (64-byte mode)..."); - generate_module_file(&intrinsics, &v64_path, VectorMode::V64)?; - println!(" Output: {}", v64_path.display()); - - // Generate v128.rs (128-byte vector mode) - let v128_path = hexagon_dir.join("v128.rs"); - println!("\nStep 4: Generating v128.rs (128-byte mode)..."); - generate_module_file(&intrinsics, &v128_path, VectorMode::V128)?; - println!(" Output: {}", v128_path.display()); + let hexagon_dir = crate_dir.join("../core_arch/src/hexagon"); + + // Either "check" to check the output versus the committed output, or "bless" + // to update the output. + let mode = Mode::from_env(); + println!("\nStep 3: Generating v64.rs and v128.rs (mode: {mode:?})..."); + run_generator(&hexagon_dir, mode, |out_dir| -> Result<(), String> { + for (filename, vmode) in [("v64.rs", VectorMode::V64), ("v128.rs", VectorMode::V128)] { + let path = out_dir.join(filename); + generate_module_file(&intrinsics, &path, vmode)?; + println!(" Output: {}", hexagon_dir.join(filename).display()); + } + Ok(()) + }) + .map_err(|e| e.to_string())?; println!("\n=== Results ==="); println!(