From ebf5f1f99415fc985c303c802de8cb2791a6ff71 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:00:36 +0100 Subject: [PATCH 01/28] wip: project validation backend --- .../routes/v3/{projects.rs => projects/mod.rs} | 2 ++ .../src/routes/v3/projects/validate.rs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) rename apps/labrinth/src/routes/v3/{projects.rs => projects/mod.rs} (99%) create mode 100644 apps/labrinth/src/routes/v3/projects/validate.rs diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects/mod.rs similarity index 99% rename from apps/labrinth/src/routes/v3/projects.rs rename to apps/labrinth/src/routes/v3/projects/mod.rs index ee99c2b059..f663e23025 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -50,6 +50,8 @@ use serde::{Deserialize, Serialize}; use validator::Validate; use xredis::RedisPool; +pub mod validate; + pub fn config(cfg: &mut actix_web::web::ServiceConfig) { cfg.service(project_search) .service(project_search_post) diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs new file mode 100644 index 0000000000..bbec1ebb4d --- /dev/null +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -0,0 +1,18 @@ +use actix_web::{HttpRequest, web}; +use xredis::RedisPool; + +use crate::database::PgPool; + +pub fn config(cfg: &mut actix_web::web::ServiceConfig) { + cfg.service(validate) +} + +#[derive(Debug)] +pub enum ValidateError {} + +pub async fn validate( + req: HttpRequest, + pool: web::Data, + redis: web::Data, +) -> Result<(), ApiError> { +} From e32985297ee2b1fc09de4e7e22d43f5cc78de8cd Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:11:57 +0100 Subject: [PATCH 02/28] wip: validate with subsecond --- Cargo.lock | 225 ++++++++++++++++-- Cargo.toml | 2 + apps/labrinth/Cargo.toml | 2 + apps/labrinth/src/main.rs | 2 + apps/labrinth/src/routes/v3/projects/mod.rs | 3 +- .../src/routes/v3/projects/validate.rs | 13 +- 6 files changed, 225 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 459048aa57..c93f4e9c9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,7 +834,7 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", - "tungstenite", + "tungstenite 0.27.0", "webpki-roots 1.0.3", ] @@ -3001,6 +3001,85 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dioxus-cli-config" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322d10f47effebd85ee0662a2cb083ff920d40dd6295a4afd8cfa4f9bc2e05c6" + +[[package]] +name = "dioxus-core" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b232228ada232c0adc151e41ebd9ef70cc04683db097fa0dc6015979fa01722" +dependencies = [ + "anyhow", + "const_format", + "dioxus-core-types", + "futures-channel", + "futures-util", + "generational-box", + "longest-increasing-subsequence", + "rustc-hash", + "rustversion", + "serde", + "slab", + "slotmap", + "subsecond", + "tracing", +] + +[[package]] +name = "dioxus-core-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96007ed6cbad951dfed82d83c581aabd95bc591f351f5666d454ddbe7845b324" + +[[package]] +name = "dioxus-devtools" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434cebb282b3f820a341582ebfd37dab907d5a25c3eeafd87802468afcd2a959" +dependencies = [ + "dioxus-cli-config", + "dioxus-core", + "dioxus-devtools-types", + "dioxus-signals", + "serde", + "serde_json", + "subsecond", + "thiserror 2.0.17", + "tracing", + "tungstenite 0.28.0", +] + +[[package]] +name = "dioxus-devtools-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfd6f3475e38c93be245a664a9ce11679a8f2b99832620705cf7837f42aa39ce" +dependencies = [ + "dioxus-core", + "serde", + "subsecond-types", +] + +[[package]] +name = "dioxus-signals" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95d85e36fcaf7abdc986836801bf248776da7d5961f5fac65fdbf5ed491afb4f" +dependencies = [ + "dioxus-core", + "futures-channel", + "futures-util", + "generational-box", + "parking_lot", + "rustc-hash", + "tracing", + "warnings", +] + [[package]] name = "directories" version = "6.0.0" @@ -3785,9 +3864,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -3795,9 +3874,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" @@ -3823,9 +3902,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -3857,26 +3936,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -3886,9 +3965,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -3898,7 +3977,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -4032,6 +4110,16 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "generational-box" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b2dc9b873cd1a8adb80aa5dcef9a8205f781a8a313d5ffb867248cb3bcb764" +dependencies = [ + "parking_lot", + "tracing", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -5494,6 +5582,7 @@ dependencies = [ "const_format", "dashmap", "derive_more 2.1.1", + "dioxus-devtools", "dotenv-build", "dotenvy", "either", @@ -5545,6 +5634,7 @@ dependencies = [ "sqlx", "sqlx-tracing", "strum", + "subsecond", "thiserror 2.0.17", "tikv-jemalloc-ctl", "tikv-jemallocator", @@ -5811,6 +5901,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "longest-increasing-subsequence" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" + [[package]] name = "loop9" version = "0.1.5" @@ -5971,6 +6067,24 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix 1.1.2", +] + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -9825,6 +9939,16 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "serde", + "version_check", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -10336,6 +10460,34 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "subsecond" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d350d5788fa94d560d92269266a50efc77b036bfca6675e420b64df7b3211f37" +dependencies = [ + "js-sys", + "libc", + "libloading 0.8.8", + "memfd", + "memmap2", + "serde", + "subsecond-types", + "thiserror 2.0.17", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "subsecond-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf32d66269b5fbb8558334e8d22b657b2f7af0dd2ef56c231541099a862e464" +dependencies = [ + "serde", +] + [[package]] name = "subtle" version = "2.6.1" @@ -11863,6 +12015,23 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http 1.3.1", + "httparse", + "log", + "rand 0.9.2", + "sha1 0.10.6", + "thiserror 2.0.17", + "utf-8", +] + [[package]] name = "typed-path" version = "0.12.0" @@ -12296,6 +12465,28 @@ dependencies = [ "try-lock", ] +[[package]] +name = "warnings" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f68998838dab65727c9b30465595c6f7c953313559371ca8bf31759b3680ad" +dependencies = [ + "pin-project", + "tracing", + "warnings-macro", +] + +[[package]] +name = "warnings-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59195a1db0e95b920366d949ba5e0d3fc0e70b67c09be15ce5abb790106b0571" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 1377c4b047..cfcb0e8c2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,8 @@ hex = "0.4.3" hickory-resolver = "0.25.2" hmac = "0.12.1" httpdate = "1.0.3" +subsecond = "0.7.10" +dioxus-devtools = "0.7.10" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ "aws-lc-rs", diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 03eb3ecae6..d0fedde2ec 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -29,8 +29,10 @@ base64 = { workspace = true } bitflags = { workspace = true } bytes = { workspace = true } censor = { workspace = true } +dioxus-devtools = { workspace = true } chrono = { workspace = true, features = ["serde"] } chumsky = { workspace = true } +subsecond = { workspace = true } clap = { workspace = true, features = ["derive"] } clickhouse = { workspace = true, features = ["time", "uuid"] } color-eyre = { workspace = true } diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index b456b77a67..68e0d8ef4c 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -94,6 +94,8 @@ async fn app() -> std::io::Result<()> { .install_default() .unwrap(); + dioxus_devtools::connect_subsecond(); + if args.run_background_task.is_none() { info!("Starting labrinth on {}", &ENV.BIND_ADDR); diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index f663e23025..c85ebae776 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -76,7 +76,8 @@ pub fn project_config(cfg: &mut actix_web::web::ServiceConfig) { .service(super::teams::team_members_get_project) .service(super::versions::version_list) .service(super::versions::version_project_get) - .service(dependency_list); + .service(dependency_list) + .service(validate::validate); } pub async fn clear_project_cache_and_queue_search( diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index bbec1ebb4d..88a5608630 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -1,18 +1,23 @@ -use actix_web::{HttpRequest, web}; +use actix_web::{HttpRequest, get, web}; use xredis::RedisPool; -use crate::database::PgPool; +use crate::{database::PgPool, routes::ApiError}; pub fn config(cfg: &mut actix_web::web::ServiceConfig) { - cfg.service(validate) + cfg.service(validate); } #[derive(Debug)] pub enum ValidateError {} +/// Validate that a project is ready to be submitted for review. +#[utoipa::path(tag = "projects")] +#[get("/{id}/validate")] pub async fn validate( req: HttpRequest, + info: web::Path<(String,)>, pool: web::Data, redis: web::Data, -) -> Result<(), ApiError> { +) -> Result { + Ok(subsecond::call(|| String::from("foobar world"))) } From 6caf07d5e1077a442ff5aad35f624c54f62b9b1e Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:20:24 +0100 Subject: [PATCH 03/28] wip: project validation pass --- apps/labrinth/src/validate/mod.rs | 1 + .../src/validate/project/description.rs | 92 +++++ .../src/validate/project/disclosures.rs | 12 + apps/labrinth/src/validate/project/gallery.rs | 63 ++++ apps/labrinth/src/validate/project/icon.rs | 14 + apps/labrinth/src/validate/project/license.rs | 92 +++++ apps/labrinth/src/validate/project/links.rs | 243 +++++++++++++ apps/labrinth/src/validate/project/mod.rs | 157 ++++++++ .../src/validate/project/moderation.rs | 19 + apps/labrinth/src/validate/project/name.rs | 51 +++ .../src/validate/project/permissions.rs | 16 + .../src/validate/project/server_settings.rs | 68 ++++ apps/labrinth/src/validate/project/summary.rs | 95 +++++ apps/labrinth/src/validate/project/tags.rs | 55 +++ apps/labrinth/src/validate/project/text.rs | 342 ++++++++++++++++++ .../labrinth/src/validate/project/versions.rs | 16 + 16 files changed, 1336 insertions(+) create mode 100644 apps/labrinth/src/validate/project/description.rs create mode 100644 apps/labrinth/src/validate/project/disclosures.rs create mode 100644 apps/labrinth/src/validate/project/gallery.rs create mode 100644 apps/labrinth/src/validate/project/icon.rs create mode 100644 apps/labrinth/src/validate/project/license.rs create mode 100644 apps/labrinth/src/validate/project/links.rs create mode 100644 apps/labrinth/src/validate/project/mod.rs create mode 100644 apps/labrinth/src/validate/project/moderation.rs create mode 100644 apps/labrinth/src/validate/project/name.rs create mode 100644 apps/labrinth/src/validate/project/permissions.rs create mode 100644 apps/labrinth/src/validate/project/server_settings.rs create mode 100644 apps/labrinth/src/validate/project/summary.rs create mode 100644 apps/labrinth/src/validate/project/tags.rs create mode 100644 apps/labrinth/src/validate/project/text.rs create mode 100644 apps/labrinth/src/validate/project/versions.rs diff --git a/apps/labrinth/src/validate/mod.rs b/apps/labrinth/src/validate/mod.rs index 3a7aca84e2..fed2a8d077 100644 --- a/apps/labrinth/src/validate/mod.rs +++ b/apps/labrinth/src/validate/mod.rs @@ -34,6 +34,7 @@ mod liteloader; mod modpack; mod neoforge; pub mod plugin; +pub mod project; mod quilt; mod resourcepack; mod rift; diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs new file mode 100644 index 0000000000..adbe9e678c --- /dev/null +++ b/apps/labrinth/src/validate/project/description.rs @@ -0,0 +1,92 @@ +use super::text::{ + contains_banned_description_link, contains_spam, + extract_description_blocks, extract_description_text, + has_image_without_alt_text, has_long_header, is_likely_english, + non_standard_text_ratio, profanity_count, +}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::exp::minecraft::Language; +use crate::models::projects::Project; + +const MIN_DESCRIPTION_CHARS: usize = 125; +const MAX_PROFANITY_COUNT: usize = 2; +const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let description = project.description.trim(); + let text = extract_description_text(description); + + if profanity_count(description) > MAX_PROFANITY_COUNT { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionProfanity, + ProjectNagSeverity::Required, + )); + } + if non_standard_text_ratio(description) + >= NON_STANDARD_TEXT_FAILURE_THRESHOLD + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && text.chars().count() >= MIN_DESCRIPTION_CHARS + && !contains_spam(&text) + && extract_description_blocks(description) + .iter() + .any(|block| !is_likely_english(block)) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if description.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::AddDescription, + ProjectNagSeverity::Required, + )); + } else if text.chars().count() < MIN_DESCRIPTION_CHARS { + nags.push(ProjectNag::new( + ProjectNagKind::DescriptionTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(&text) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionSpam, + ProjectNagSeverity::Required, + )); + } + if contains_banned_description_link(description) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionBannedLink, + ProjectNagSeverity::Required, + )); + } + if has_long_header(description) { + nags.push(ProjectNag::new( + ProjectNagKind::LongHeaders, + ProjectNagSeverity::Required, + )); + } + if has_image_without_alt_text(description) { + nags.push(ProjectNag::new( + ProjectNagKind::MissingAltText, + ProjectNagSeverity::Warning, + )); + } + + nags +} + +fn project_requires_english(project: &Project) -> bool { + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) +} diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs new file mode 100644 index 0000000000..c4785b2b0d --- /dev/null +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -0,0 +1,12 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + let _ = project; + + vec![ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + )] +} diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs new file mode 100644 index 0000000000..2e3eccc63c --- /dev/null +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -0,0 +1,63 @@ +use super::text::{contains_profanity, has_non_standard_text}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +fn validate_text(text: Option<&str>) -> Vec { + let text = text.unwrap_or_default(); + let mut nags = Vec::new(); + + if contains_profanity(text) { + nags.push(ProjectNag::new( + ProjectNagKind::GalleryTextProfanity, + ProjectNagSeverity::Required, + )); + } + + if has_non_standard_text(text) { + nags.push(ProjectNag::new( + ProjectNagKind::GalleryTextNonStandard, + ProjectNagSeverity::Required, + )); + } + + nags +} + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + let mut nags = Vec::new(); + let gallery_is_empty = project.gallery.is_empty(); + let is_shader = project.project_types.iter().any(|ty| ty == "shader"); + let is_resource_pack = + project.project_types.iter().any(|ty| ty == "resourcepack"); + let has_gallery_exemption = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "audio" || category == "locale"); + + if (is_shader && project.gallery.len() < 3) + || (is_resource_pack && gallery_is_empty && !has_gallery_exemption) + { + nags.push(ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + )); + } + + let is_minecraft_server = project.components.minecraft_server.is_some(); + if !is_minecraft_server && !project.gallery.iter().any(|item| item.featured) + { + nags.push(ProjectNag::new( + ProjectNagKind::FeatureGalleryImage, + ProjectNagSeverity::Suggestion, + )); + } + + for item in &project.gallery { + nags.extend(validate_text(item.name.as_deref())); + nags.extend(validate_text(item.description.as_deref())); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/icon.rs b/apps/labrinth/src/validate/project/icon.rs new file mode 100644 index 0000000000..2ec211c477 --- /dev/null +++ b/apps/labrinth/src/validate/project/icon.rs @@ -0,0 +1,14 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + if project.icon_url.as_deref().is_none_or(str::is_empty) { + vec![ProjectNag::new( + ProjectNagKind::AddIcon, + ProjectNagSeverity::Suggestion, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/license.rs b/apps/labrinth/src/validate/project/license.rs new file mode 100644 index 0000000000..7e0d70cb5f --- /dev/null +++ b/apps/labrinth/src/validate/project/license.rs @@ -0,0 +1,92 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const INAPPROPRIATE_LICENSE_DOMAINS: &[&str] = &[ + "youtube.com", + "youtu.be", + "modrinth.com", + "curseforge.com", + "twitter.com", + "x.com", + "discord.gg", + "discord.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "reddit.com", + "twitch.tv", + "patreon.com", + "ko-fi.com", + "paypal.com", + "buymeacoffee.com", + "google.com", + "example.com", + "t.me", +]; + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) +} + +fn has_invalid_license_url(url: &str) -> bool { + if url.is_empty() { + return false; + } + + let Ok(url) = url::Url::parse(url) else { + return true; + }; + let Some(hostname) = url.host_str() else { + return true; + }; + let hostname = hostname.to_ascii_lowercase(); + let hostname = hostname.trim_end_matches('.'); + + INAPPROPRIATE_LICENSE_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) +} + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + let license = &project.license; + let has_unknown_license = matches!( + license.id.as_str(), + "LicenseRef-Unknown" | "NOASSERTION" | "LicenseRef-NOASSERTION" + ); + + if has_unknown_license && !is_minecraft_server { + nags.push(ProjectNag::new( + ProjectNagKind::SelectLicense, + ProjectNagSeverity::Required, + )); + } + + let has_license_url = + license.url.as_deref().is_some_and(|url| !url.is_empty()); + let missing_custom_license_details = license.id == "LicenseRef-" + || (license.id.starts_with("LicenseRef-") + && !has_license_url + && license.id != "LicenseRef-Unknown" + && license.id != "LicenseRef-All-Rights-Reserved"); + if missing_custom_license_details && !is_minecraft_server { + nags.push(ProjectNag::new( + ProjectNagKind::AddCustomLicenseDetails, + ProjectNagSeverity::Required, + )); + } + + if license.url.as_deref().is_some_and(has_invalid_license_url) { + nags.push(ProjectNag::new( + ProjectNagKind::InvalidLicenseUrl, + ProjectNagSeverity::Required, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/links.rs b/apps/labrinth/src/validate/project/links.rs new file mode 100644 index 0000000000..0aa3b7d841 --- /dev/null +++ b/apps/labrinth/src/validate/project/links.rs @@ -0,0 +1,243 @@ +use std::net::IpAddr; + +use url::Url; + +use super::{ProjectNagKind, ProjectNagSeverity}; +use crate::models::projects::Version; + +const SOURCE_DOMAINS: &[&str] = &[ + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "git.sr.ht", + "tangled.org", + "git.gay", +]; +const ISSUES_DOMAINS: &[&str] = &[ + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "docs.google.com", + "tangled.org", + "git.gay", +]; +const DISCORD_DOMAINS: &[&str] = &["discord.gg", "discord.com", "dsc.gg"]; +const URL_SHORTENERS: &[&str] = + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; +const BLOCKED_EXTERNAL_LINK_DOMAINS: &[&str] = &[ + "twitter.com", + "x.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "telegram.org", + "t.me", + "bilibili.com", + "bsky.app", + "twitch.tv", + "reddit.com", + "redd.it", + "modrinth.com", + "minecraft.net", + "curseforge.com", + "planetminecraft.com", + "9minecraft.net", + "mcmod.cn", + "creativemode.net", + "orcaclient.com", + "autoforged.cn", +]; +const SOURCE_REQUIRING_LICENSES: &[&str] = &[ + "GPL-2.0", + "GPL-2.0+", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-3.0", + "GPL-3.0+", + "GPL-3.0-only", + "GPL-3.0-or-later", + "LGPL-2.1", + "LGPL-2.1+", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-3.0", + "LGPL-3.0+", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "AGPL-3.0", + "AGPL-3.0+", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "MPL-2.0", +]; + +pub(super) fn validate( + project: &crate::models::projects::Project, + versions: &[Version], +) -> Vec { + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + + if !is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinks, + ProjectNagSeverity::Suggestion, + )); + } + + if is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinksServer, + ProjectNagSeverity::Suggestion, + )); + } + + let link_count = project.link_urls.len(); + let unique_link_count = project + .link_urls + .values() + .map(|link| link.url.as_str()) + .collect::>() + .len(); + if unique_link_count != link_count { + nags.push(super::ProjectNag::new( + ProjectNagKind::IdenticalLinks, + ProjectNagSeverity::Required, + )); + } + + if named_link_is_uncommon(project, "source", SOURCE_DOMAINS) + || named_link_is_uncommon(project, "issues", ISSUES_DOMAINS) + || named_link_is_uncommon(project, "discord", DISCORD_DOMAINS) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::VerifyExternalLinks, + ProjectNagSeverity::Warning, + )); + } + + if ["source", "issues", "wiki", "site", "store"] + .into_iter() + .filter_map(|key| named_link(project, key)) + .any(is_discord_link) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::MisusedDiscordLink, + ProjectNagSeverity::Required, + )); + } + + if project + .link_urls + .values() + .any(|link| is_blocked_external_link(&link.url)) + || project + .license + .url + .as_deref() + .is_some_and(is_blocked_external_link) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::BannedLinkUsage, + ProjectNagSeverity::Required, + )); + } + + let is_source_project = project + .project_types + .iter() + .any(|project_type| project_type == "mod" || project_type == "plugin"); + let is_datapack = project + .project_types + .iter() + .any(|project_type| project_type == "datapack"); + let has_source_link = + named_link(project, "source").is_some_and(|url| !url.is_empty()); + let every_version_has_additional_files = + versions.iter().all(|version| version.files.len() >= 2); + + if !is_datapack + && is_source_project + && SOURCE_REQUIRING_LICENSES.contains(&project.license.id.as_str()) + && !has_source_link + && !every_version_has_additional_files + { + nags.push(super::ProjectNag::new( + ProjectNagKind::GplLicenseSourceRequired, + ProjectNagSeverity::Required, + )); + } + + nags +} + +fn named_link<'a>( + project: &'a crate::models::projects::Project, + key: &str, +) -> Option<&'a str> { + project.link_urls.get(key).map(|link| link.url.as_str()) +} + +fn named_link_is_uncommon( + project: &crate::models::projects::Project, + key: &str, + domains: &[&str], +) -> bool { + named_link(project, key) + .is_some_and(|url| !is_link_from_domains(url, domains)) +} + +fn is_discord_link(url: &str) -> bool { + is_link_from_domains(url, DISCORD_DOMAINS) +} + +fn is_blocked_external_link(url: &str) -> bool { + let Some(hostname) = get_link_hostname(url) else { + return false; + }; + + is_ip_address(&hostname) + || URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) + || BLOCKED_EXTERNAL_LINK_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) +} + +fn is_link_from_domains(url: &str, domains: &[&str]) -> bool { + let Some(hostname) = get_link_hostname(url) else { + return false; + }; + + domains + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) +} + +fn get_link_hostname(url: &str) -> Option { + let parsed = Url::parse(url).ok()?; + let hostname = parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(); + + (!hostname.is_empty()).then_some(hostname) +} + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) +} + +fn is_ip_address(hostname: &str) -> bool { + hostname + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok() +} diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs new file mode 100644 index 0000000000..2011c584f4 --- /dev/null +++ b/apps/labrinth/src/validate/project/mod.rs @@ -0,0 +1,157 @@ +use serde::{Deserialize, Serialize}; + +use crate::models::projects::{Project, Version}; + +mod description; +mod disclosures; +mod gallery; +mod icon; +mod license; +mod links; +mod moderation; +mod name; +mod permissions; +mod server_settings; +mod summary; +mod tags; +mod text; +mod versions; + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum ProjectNagKind { + // Project name + ProjectNameProfanity, + ProjectNameNonStandardText, + ProjectNameVersion, + MinecraftTitleClause, + + // Project summary + ProjectSummaryProfanity, + ProjectSummaryNonStandardText, + ProjectSummaryNonEnglish, + ProjectSummaryMatchesTitle, + SummaryTooShort, + ProjectSummarySpam, + SummarySpecialFormatting, + ProjectSummaryLinks, + + // Project icon + AddIcon, + + // Gallery + GalleryTextProfanity, + GalleryTextNonStandard, + UploadGalleryImage, + FeatureGalleryImage, + + // Project description + ProjectDescriptionProfanity, + ProjectDescriptionNonStandardText, + ProjectDescriptionNonEnglish, + AddDescription, + DescriptionTooShort, + ProjectDescriptionSpam, + ProjectDescriptionBannedLink, + LongHeaders, + MissingAltText, + + // License + SelectLicense, + AddCustomLicenseDetails, + InvalidLicenseUrl, + + // External links + AddLinks, + AddLinksServer, + IdenticalLinks, + VerifyExternalLinks, + MisusedDiscordLink, + BannedLinkUsage, + GplLicenseSourceRequired, + + // Permissions + ReviewPermissions, + + // Server settings + SelectCountry, + AllLanguages, + AddJavaAddress, + SelectCompatibility, + TooManyLanguages, + SelectLanguage, + + // Tags + SelectTags, + TooManyTags, + TooManyTagsServer, + MultipleResolutionTags, + AllTagsSelected, + + // Versions + UploadVersion, + + // Disclosures + CheckDisclosures, + + // Moderation + ModeratorFeedback, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "lowercase")] +pub enum ProjectNagSeverity { + Required, + Warning, + Suggestion, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, +)] +pub struct ProjectNag { + pub kind: ProjectNagKind, + pub severity: ProjectNagSeverity, +} + +impl ProjectNag { + pub(super) fn new( + kind: ProjectNagKind, + severity: ProjectNagSeverity, + ) -> Self { + Self { kind, severity } + } +} + +pub fn validate(project: &Project, versions: &[Version]) -> Vec { + let mut nags = [ + name::validate, + summary::validate, + icon::validate, + gallery::validate, + description::validate, + license::validate, + ] + .into_iter() + .flat_map(|validate| validate(project)) + .collect::>(); + + nags.extend(links::validate(project, versions)); + nags.extend(permissions::validate(versions)); + nags.extend( + [ + server_settings::validate, + tags::validate, + versions::validate, + disclosures::validate, + moderation::validate, + ] + .into_iter() + .flat_map(|validate| validate(project)), + ); + nags +} diff --git a/apps/labrinth/src/validate/project/moderation.rs b/apps/labrinth/src/validate/project/moderation.rs new file mode 100644 index 0000000000..1f68fbbe84 --- /dev/null +++ b/apps/labrinth/src/validate/project/moderation.rs @@ -0,0 +1,19 @@ +use crate::models::projects::ProjectStatus; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + if matches!( + project.status, + ProjectStatus::Rejected | ProjectStatus::Withheld + ) { + vec![ProjectNag::new( + ProjectNagKind::ModeratorFeedback, + ProjectNagSeverity::Warning, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/name.rs b/apps/labrinth/src/validate/project/name.rs new file mode 100644 index 0000000000..b592eefba6 --- /dev/null +++ b/apps/labrinth/src/validate/project/name.rs @@ -0,0 +1,51 @@ +use std::sync::LazyLock; + +use regex::Regex; + +use super::text::{contains_profanity, has_non_standard_text}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::projects::Project; + +static VERSION_NUMBER: LazyLock = + LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)+").unwrap()); +static PORT_OR_FORK: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b(?:port|fork)\b").unwrap()); + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let name = &project.name; + + if contains_profanity(name) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameProfanity, + ProjectNagSeverity::Required, + )); + } + if has_non_standard_text(name) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameNonStandardText, + ProjectNagSeverity::Required, + )); + } + + let normalized = name.to_lowercase(); + if VERSION_NUMBER + .find_iter(&normalized) + .any(|version| !PORT_OR_FORK.is_match(&normalized[version.end()..])) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameVersion, + ProjectNagSeverity::Required, + )); + } + if normalized.contains("minecraft") + && normalized.split_whitespace().count() <= 3 + { + nags.push(ProjectNag::new( + ProjectNagKind::MinecraftTitleClause, + ProjectNagSeverity::Required, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/permissions.rs b/apps/labrinth/src/validate/project/permissions.rs new file mode 100644 index 0000000000..6c5dce01c7 --- /dev/null +++ b/apps/labrinth/src/validate/project/permissions.rs @@ -0,0 +1,16 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::projects::Version; + +pub(super) fn validate(versions: &[Version]) -> Vec { + if versions + .iter() + .any(|version| !version.files_missing_attribution.is_empty()) + { + vec![ProjectNag::new( + ProjectNagKind::ReviewPermissions, + ProjectNagSeverity::Required, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/server_settings.rs b/apps/labrinth/src/validate/project/server_settings.rs new file mode 100644 index 0000000000..21793a40e4 --- /dev/null +++ b/apps/labrinth/src/validate/project/server_settings.rs @@ -0,0 +1,68 @@ +use crate::models::{exp::minecraft::ServerContentQuery, projects::Project}; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const MAX_LANGUAGE_COUNT: usize = 10; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let server = project.components.minecraft_server.as_ref(); + + if server.is_some_and(|server| server.region.is_none()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCountry, + ProjectNagSeverity::Required, + )); + } + + if server.is_some() + && project + .components + .minecraft_java_server + .as_ref() + .is_none_or(|server| server.address.is_empty()) + { + nags.push(ProjectNag::new( + ProjectNagKind::AddJavaAddress, + ProjectNagSeverity::Required, + )); + } + + if project + .components + .minecraft_java_server + .as_ref() + .is_some_and(|server| { + matches!( + &server.content, + ServerContentQuery::Vanilla { + recommended_game_version, + .. + } if recommended_game_version + .as_deref() + .is_none_or(str::is_empty) + ) + }) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCompatibility, + ProjectNagSeverity::Required, + )); + } + + if server.is_some_and(|server| server.languages.len() > MAX_LANGUAGE_COUNT) + { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyLanguages, + ProjectNagSeverity::Warning, + )); + } + + if server.is_some_and(|server| server.languages.is_empty()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectLanguage, + ProjectNagSeverity::Suggestion, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs new file mode 100644 index 0000000000..40ddd3b576 --- /dev/null +++ b/apps/labrinth/src/validate/project/summary.rs @@ -0,0 +1,95 @@ +use super::text::{ + contains_link_or_ip, contains_profanity, contains_spam, + has_non_standard_text, has_summary_formatting, is_likely_english, +}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::exp::minecraft::Language; +use crate::models::projects::Project; + +const MIN_SUMMARY_CHARS: usize = 25; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let summary = project.summary.trim(); + let contains_link = contains_link_or_ip(summary); + + if contains_profanity(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryProfanity, + ProjectNagSeverity::Required, + )); + } + if has_non_standard_text(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && summary.chars().count() >= MIN_SUMMARY_CHARS + && !contains_link + && !contains_spam(summary) + && !is_likely_english(summary) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if !summary.is_empty() + && !contains_link + && normalized_without_whitespace(summary) + == normalized_without_whitespace(&project.name) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryMatchesTitle, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() + && !contains_link + && summary.chars().count() < MIN_SUMMARY_CHARS + { + nags.push(ProjectNag::new( + ProjectNagKind::SummaryTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummarySpam, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && has_summary_formatting(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::SummarySpecialFormatting, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && contains_link { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryLinks, + ProjectNagSeverity::Required, + )); + } + + nags +} + +fn normalized_without_whitespace(text: &str) -> String { + text.trim() + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn project_requires_english(project: &Project) -> bool { + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) +} diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs new file mode 100644 index 0000000000..55135ed3b1 --- /dev/null +++ b/apps/labrinth/src/validate/project/tags.rs @@ -0,0 +1,55 @@ +use crate::models::projects::Project; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const MAX_TAG_COUNT: usize = 8; +const MAX_TAG_COUNT_SERVER: usize = 18; +const RESOLUTION_TAGS: [&str; 8] = + ["8x-", "16x", "32x", "48x", "64x", "128x", "256x", "512x+"]; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let tag_count = + project.categories.len() + project.additional_categories.len(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + + if !project.versions.is_empty() && project.categories.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::SelectTags, + ProjectNagSeverity::Suggestion, + )); + } + + if !is_minecraft_server && tag_count > MAX_TAG_COUNT { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyTags, + ProjectNagSeverity::Warning, + )); + } + + if is_minecraft_server && tag_count > MAX_TAG_COUNT_SERVER { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyTagsServer, + ProjectNagSeverity::Required, + )); + } + + if project + .project_types + .iter() + .any(|project_type| project_type == "resourcepack") + && project + .categories + .iter() + .chain(&project.additional_categories) + .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) + .count() > 1 + { + nags.push(ProjectNag::new( + ProjectNagKind::MultipleResolutionTags, + ProjectNagSeverity::Warning, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs new file mode 100644 index 0000000000..7f60a1067f --- /dev/null +++ b/apps/labrinth/src/validate/project/text.rs @@ -0,0 +1,342 @@ +use std::collections::HashMap; +use std::sync::LazyLock; + +use censor::Censor; +use regex::Regex; +use url::Url; + +static PROFANITY_CENSOR: LazyLock = + LazyLock::new(|| Censor::Standard + Censor::Sex); + +static WORD: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); +static LINK_OR_IP: LazyLock = LazyLock::new(|| { + Regex::new(concat!( + r"(?i)(?:https?://|www\.)[^\s<>()]+|", + r"\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+\b|", + r"\b(?:\d{1,3}\.){3}\d{1,3}\b", + )) + .unwrap() +}); +static MARKDOWN_LINK: LazyLock = + LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); +static HTML_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); +static CODE_BLOCK: LazyLock = + LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); +static INLINE_CODE: LazyLock = + LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); +static MARKDOWN_IMAGE: LazyLock = + LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); +static HTML_IMAGE: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); +static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { + Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() +}); +static DESCRIPTION_LINK: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() +}); +static HEADER: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); +static HTML_HEADER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?is)]*>(.*?)").unwrap() +}); +static INLINE_MARKDOWN: LazyLock = LazyLock::new(|| { + Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() +}); + +const URL_SHORTENERS: &[&str] = + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + +pub(super) fn contains_profanity(text: &str) -> bool { + PROFANITY_CENSOR.check(text) +} + +pub(super) fn profanity_count(text: &str) -> usize { + PROFANITY_CENSOR.count(text) +} + +pub(super) fn has_non_standard_text(text: &str) -> bool { + non_standard_character_count(text) > 0 +} + +pub(super) fn non_standard_text_ratio(text: &str) -> f64 { + let character_count = text.chars().count(); + if character_count == 0 { + return 0.0; + } + + non_standard_character_count(text) as f64 / character_count as f64 +} + +fn non_standard_character_count(text: &str) -> usize { + let mut count = 0; + let mut combining_marks = 0; + let mut has_base = false; + + for character in text.chars() { + let code = character as u32; + let is_combining = is_in_ranges( + code, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ); + if is_combining { + combining_marks += 1; + if !has_base || combining_marks > 2 { + count += 1; + } + continue; + } + + combining_marks = 0; + has_base = !character.is_whitespace(); + let allowed_control = matches!(character, '\n' | '\r' | '\t'); + let control = character.is_control() && !allowed_control; + let private_use = is_in_ranges( + code, + &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], + ); + let invisible = is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x061c, 0x061c), + (0x200e, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0xfeff, 0xfeff), + ], + ); + let fancy = is_in_ranges( + code, + &[ + (0x02b0, 0x02ff), + (0x1d400, 0x1d7ff), + (0x2460, 0x24ff), + (0x2070, 0x209f), + (0x2100, 0x214f), + (0xfb00, 0xfb06), + (0xff10, 0xff19), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0x1f100, 0x1f1ad), + ], + ) && !matches!( + code, + 0x02d6 + | 0x02d7 | 0x02d8 + | 0x02d9 | 0x02da + | 0x02db | 0x02dc + | 0x02dd | 0x207a + | 0x207b | 0x208a + | 0x208b | 0x2120 + | 0x2122 | 0x2139 + ); + + if control || private_use || invisible || fancy { + count += 1; + } + } + + count +} + +fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { + ranges + .iter() + .any(|(start, end)| code >= *start && code <= *end) +} + +pub(super) fn contains_spam(text: &str) -> bool { + let normalized = text.to_lowercase(); + let mut previous = None; + let mut repeated = 0; + for character in normalized.chars() { + if !character.is_whitespace() && previous == Some(character) { + repeated += 1; + } else { + repeated = usize::from(!character.is_whitespace()); + previous = Some(character); + } + if repeated >= 8 { + return true; + } + } + + let words = WORD + .find_iter(&normalized) + .map(|word| word.as_str()) + .collect::>(); + let mut repeated_words = 1; + for pair in words.windows(2) { + repeated_words = if pair[0] == pair[1] { + repeated_words + 1 + } else { + 1 + }; + if repeated_words >= 4 { + return true; + } + } + + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for phrases in words.windows(phrase_words * 3) { + if phrases[..phrase_words] + == phrases[phrase_words..phrase_words * 2] + && phrases[..phrase_words] + == phrases[phrase_words * 2..phrase_words * 3] + { + return true; + } + } + } + + false +} + +pub(super) fn contains_link_or_ip(text: &str) -> bool { + LINK_OR_IP.is_match(text) +} + +pub(super) fn has_summary_formatting(summary: &str) -> bool { + HTML_TAG.is_match(summary) + || MARKDOWN_LINK.is_match(summary) + || summary.lines().any(|line| { + let line = line.trim_start(); + line.starts_with('#') + || line.starts_with('>') + || line.starts_with("- ") + || line.starts_with("* ") + || line.starts_with("+ ") + || line.starts_with("```") + || line.contains("**") + || line.contains("__") + || line.contains("~~") + || INLINE_CODE.is_match(line) + }) +} + +pub(super) fn extract_description_text(markdown: &str) -> String { + let without_code = CODE_BLOCK.replace_all(markdown, " "); + let without_code = INLINE_CODE.replace_all(&without_code, " "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); + let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); + let without_html = HTML_TAG.replace_all(&without_links, " "); + without_html + .replace(['*', '_', '~', '`', '>', '-', '|'], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +pub(super) fn extract_description_blocks(markdown: &str) -> Vec { + CODE_BLOCK + .replace_all(markdown, "") + .split("\n\n") + .map(extract_description_text) + .filter(|block| !block.is_empty()) + .collect() +} + +pub(super) fn has_long_header(markdown: &str) -> bool { + HEADER.captures_iter(markdown).any(|captures| { + INLINE_MARKDOWN + .replace_all(&captures[1], "$1") + .trim() + .chars() + .count() > 80 + }) || HTML_HEADER.captures_iter(markdown).any(|captures| { + HTML_TAG + .replace_all(&captures[1], " ") + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .count() > 80 + }) +} + +pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { + let without_code = CODE_BLOCK.replace_all(markdown, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + MARKDOWN_IMAGE + .captures_iter(&without_code) + .any(|captures| captures[1].trim().is_empty()) + || HTML_IMAGE.find_iter(&without_code).any(|image| { + ALT_ATTRIBUTE + .captures(image.as_str()) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .is_none_or(|alt| alt.as_str().trim().is_empty()) + }) +} + +pub(super) fn contains_banned_description_link(markdown: &str) -> bool { + DESCRIPTION_LINK.find_iter(markdown).any(|link| { + let raw = link.as_str(); + let normalized = if raw.to_ascii_lowercase().starts_with("www.") { + format!("https://{raw}") + } else { + raw.to_owned() + }; + Url::parse(&normalized).is_ok_and(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) + }) + }) +} + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname.eq_ignore_ascii_case(domain) + || hostname + .to_ascii_lowercase() + .ends_with(&format!(".{domain}")) +} + +pub(super) fn is_likely_english(text: &str) -> bool { + let words = WORD.find_iter(text).count(); + if words < 8 || text.chars().count() < 35 { + return true; + } + + let mut scripts = HashMap::<&'static str, usize>::new(); + for character in text.chars().filter(|character| character.is_alphabetic()) + { + let code = character as u32; + let script = if code <= 0x024f { + "latin" + } else if is_in_ranges(code, &[(0x0400, 0x052f)]) { + "cyrillic" + } else if is_in_ranges(code, &[(0x0370, 0x03ff)]) { + "greek" + } else if is_in_ranges( + code, + &[ + (0x3040, 0x30ff), + (0x3400, 0x4dbf), + (0x4e00, 0x9fff), + (0xac00, 0xd7af), + ], + ) { + "cjk" + } else { + "other" + }; + *scripts.entry(script).or_default() += 1; + } + + let total = scripts.values().sum::(); + total == 0 + || scripts.get("latin").copied().unwrap_or_default() * 10 >= total * 7 +} diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs new file mode 100644 index 0000000000..2df1f43402 --- /dev/null +++ b/apps/labrinth/src/validate/project/versions.rs @@ -0,0 +1,16 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + if project.versions.is_empty() + && project.components.minecraft_server.is_none() + { + vec![ProjectNag::new( + ProjectNagKind::UploadVersion, + ProjectNagSeverity::Required, + )] + } else { + Vec::new() + } +} From 3ba6d0d8bcc0369705133af354078d8f87d68c57 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:32:40 +0100 Subject: [PATCH 04/28] validation CLI --- apps/labrinth/src/bin/check_project.rs | 74 ++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 apps/labrinth/src/bin/check_project.rs diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs new file mode 100644 index 0000000000..c8c4fcd511 --- /dev/null +++ b/apps/labrinth/src/bin/check_project.rs @@ -0,0 +1,74 @@ +use clap::Parser; +use eyre::{Result, WrapErr, eyre}; +use labrinth::models::projects::{Project, Version}; +use labrinth::validate::project::validate; +use serde::de::DeserializeOwned; +use url::Url; + +const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; + +#[derive(Parser)] +#[command(version, about = "Validate a project from the Modrinth API")] +struct Args { + /// The ID of the project to validate + project_id: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install().wrap_err("installing color-eyre")?; + let args = Args::parse(); + let client = reqwest::Client::builder() + .user_agent(concat!( + "labrinth-check-project/", + env!("CARGO_PKG_VERSION") + )) + .build() + .wrap_err("building HTTP client")?; + + let project_url = api_url(&["project", &args.project_id])?; + let mut versions_url = api_url(&["project", &args.project_id, "version"])?; + versions_url + .query_pairs_mut() + .append_pair("include_changelog", "false"); + + let (project, versions) = tokio::try_join!( + fetch::(&client, project_url), + fetch::>(&client, versions_url), + )?; + let nags = validate(&project, &versions); + + println!( + "{}", + serde_json::to_string_pretty(&nags) + .wrap_err("serializing project nags")? + ); + + Ok(()) +} + +fn api_url(path: &[&str]) -> Result { + let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; + url.path_segments_mut() + .map_err(|_| eyre!("API base URL cannot be a base URL"))? + .extend(path); + Ok(url) +} + +async fn fetch( + client: &reqwest::Client, + url: Url, +) -> Result { + let response = client + .get(url.clone()) + .send() + .await + .wrap_err_with(|| format!("requesting `{url}`"))? + .error_for_status() + .wrap_err_with(|| format!("requesting `{url}`"))?; + + response + .json() + .await + .wrap_err_with(|| format!("deserializing response from `{url}`")) +} From 20235f2d267e979b6625cd18b02355e9102c9bf6 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:38:44 +0100 Subject: [PATCH 05/28] add --token option --- apps/labrinth/src/bin/check_project.rs | 100 ++-- .../src/validate/project/description.rs | 150 +++--- .../src/validate/project/disclosures.rs | 12 +- apps/labrinth/src/validate/project/gallery.rs | 108 ++-- apps/labrinth/src/validate/project/icon.rs | 18 +- apps/labrinth/src/validate/project/license.rs | 144 ++--- apps/labrinth/src/validate/project/links.rs | 388 ++++++------- .../src/validate/project/moderation.rs | 24 +- apps/labrinth/src/validate/project/name.rs | 70 +-- .../src/validate/project/permissions.rs | 22 +- .../src/validate/project/server_settings.rs | 109 ++-- apps/labrinth/src/validate/project/summary.rs | 154 +++--- apps/labrinth/src/validate/project/tags.rs | 91 ++-- apps/labrinth/src/validate/project/text.rs | 509 +++++++++--------- .../labrinth/src/validate/project/versions.rs | 22 +- 15 files changed, 971 insertions(+), 950 deletions(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index c8c4fcd511..5e6778716b 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -10,65 +10,75 @@ const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; #[derive(Parser)] #[command(version, about = "Validate a project from the Modrinth API")] struct Args { - /// The ID of the project to validate - project_id: String, + /// The ID of the project to validate + project_id: String, + + /// A Modrinth token to send as bearer authentication + #[arg(long, value_name = "TOKEN")] + token: Option, } #[tokio::main] async fn main() -> Result<()> { - color_eyre::install().wrap_err("installing color-eyre")?; - let args = Args::parse(); - let client = reqwest::Client::builder() - .user_agent(concat!( - "labrinth-check-project/", - env!("CARGO_PKG_VERSION") - )) - .build() - .wrap_err("building HTTP client")?; + color_eyre::install().wrap_err("installing color-eyre")?; + let args = Args::parse(); + let client = reqwest::Client::builder() + .user_agent(concat!( + "labrinth-check-project/", + env!("CARGO_PKG_VERSION") + )) + .build() + .wrap_err("building HTTP client")?; - let project_url = api_url(&["project", &args.project_id])?; - let mut versions_url = api_url(&["project", &args.project_id, "version"])?; - versions_url - .query_pairs_mut() - .append_pair("include_changelog", "false"); + let project_url = api_url(&["project", &args.project_id])?; + let mut versions_url = api_url(&["project", &args.project_id, "version"])?; + versions_url + .query_pairs_mut() + .append_pair("include_changelog", "false"); - let (project, versions) = tokio::try_join!( - fetch::(&client, project_url), - fetch::>(&client, versions_url), - )?; - let nags = validate(&project, &versions); + let token = args.token.as_deref(); + let (project, versions) = tokio::try_join!( + fetch::(&client, project_url, token), + fetch::>(&client, versions_url, token), + )?; + let nags = validate(&project, &versions); - println!( - "{}", - serde_json::to_string_pretty(&nags) - .wrap_err("serializing project nags")? - ); + println!( + "{}", + serde_json::to_string_pretty(&nags) + .wrap_err("serializing project nags")? + ); - Ok(()) + Ok(()) } fn api_url(path: &[&str]) -> Result { - let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; - url.path_segments_mut() - .map_err(|_| eyre!("API base URL cannot be a base URL"))? - .extend(path); - Ok(url) + let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; + url.path_segments_mut() + .map_err(|_| eyre!("API base URL cannot be a base URL"))? + .extend(path); + Ok(url) } async fn fetch( - client: &reqwest::Client, - url: Url, + client: &reqwest::Client, + url: Url, + token: Option<&str>, ) -> Result { - let response = client - .get(url.clone()) - .send() - .await - .wrap_err_with(|| format!("requesting `{url}`"))? - .error_for_status() - .wrap_err_with(|| format!("requesting `{url}`"))?; + let mut request = client.get(url.clone()); + if let Some(token) = token { + request = request.bearer_auth(token); + } + + let response = request + .send() + .await + .wrap_err_with(|| format!("requesting `{url}`"))? + .error_for_status() + .wrap_err_with(|| format!("requesting `{url}`"))?; - response - .json() - .await - .wrap_err_with(|| format!("deserializing response from `{url}`")) + response + .json() + .await + .wrap_err_with(|| format!("deserializing response from `{url}`")) } diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index adbe9e678c..ee5e6f49ee 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -1,8 +1,8 @@ use super::text::{ - contains_banned_description_link, contains_spam, - extract_description_blocks, extract_description_text, - has_image_without_alt_text, has_long_header, is_likely_english, - non_standard_text_ratio, profanity_count, + contains_banned_description_link, contains_spam, + extract_description_blocks, extract_description_text, + has_image_without_alt_text, has_long_header, is_likely_english, + non_standard_text_ratio, profanity_count, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::exp::minecraft::Language; @@ -13,80 +13,80 @@ const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let description = project.description.trim(); - let text = extract_description_text(description); + let mut nags = Vec::new(); + let description = project.description.trim(); + let text = extract_description_text(description); - if profanity_count(description) > MAX_PROFANITY_COUNT { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionProfanity, - ProjectNagSeverity::Required, - )); - } - if non_standard_text_ratio(description) - >= NON_STANDARD_TEXT_FAILURE_THRESHOLD - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && text.chars().count() >= MIN_DESCRIPTION_CHARS - && !contains_spam(&text) - && extract_description_blocks(description) - .iter() - .any(|block| !is_likely_english(block)) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if description.is_empty() { - nags.push(ProjectNag::new( - ProjectNagKind::AddDescription, - ProjectNagSeverity::Required, - )); - } else if text.chars().count() < MIN_DESCRIPTION_CHARS { - nags.push(ProjectNag::new( - ProjectNagKind::DescriptionTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(&text) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionSpam, - ProjectNagSeverity::Required, - )); - } - if contains_banned_description_link(description) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionBannedLink, - ProjectNagSeverity::Required, - )); - } - if has_long_header(description) { - nags.push(ProjectNag::new( - ProjectNagKind::LongHeaders, - ProjectNagSeverity::Required, - )); - } - if has_image_without_alt_text(description) { - nags.push(ProjectNag::new( - ProjectNagKind::MissingAltText, - ProjectNagSeverity::Warning, - )); - } + if profanity_count(description) > MAX_PROFANITY_COUNT { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionProfanity, + ProjectNagSeverity::Required, + )); + } + if non_standard_text_ratio(description) + >= NON_STANDARD_TEXT_FAILURE_THRESHOLD + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && text.chars().count() >= MIN_DESCRIPTION_CHARS + && !contains_spam(&text) + && extract_description_blocks(description) + .iter() + .any(|block| !is_likely_english(block)) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if description.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::AddDescription, + ProjectNagSeverity::Required, + )); + } else if text.chars().count() < MIN_DESCRIPTION_CHARS { + nags.push(ProjectNag::new( + ProjectNagKind::DescriptionTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(&text) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionSpam, + ProjectNagSeverity::Required, + )); + } + if contains_banned_description_link(description) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionBannedLink, + ProjectNagSeverity::Required, + )); + } + if has_long_header(description) { + nags.push(ProjectNag::new( + ProjectNagKind::LongHeaders, + ProjectNagSeverity::Required, + )); + } + if has_image_without_alt_text(description) { + nags.push(ProjectNag::new( + ProjectNagKind::MissingAltText, + ProjectNagSeverity::Warning, + )); + } - nags + nags } fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) } diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index c4785b2b0d..f7b0f165ee 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -1,12 +1,12 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - let _ = project; + let _ = project; - vec![ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - )] + vec![ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + )] } diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs index 2e3eccc63c..251862fd17 100644 --- a/apps/labrinth/src/validate/project/gallery.rs +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -2,62 +2,62 @@ use super::text::{contains_profanity, has_non_standard_text}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; fn validate_text(text: Option<&str>) -> Vec { - let text = text.unwrap_or_default(); - let mut nags = Vec::new(); - - if contains_profanity(text) { - nags.push(ProjectNag::new( - ProjectNagKind::GalleryTextProfanity, - ProjectNagSeverity::Required, - )); - } - - if has_non_standard_text(text) { - nags.push(ProjectNag::new( - ProjectNagKind::GalleryTextNonStandard, - ProjectNagSeverity::Required, - )); - } - - nags + let text = text.unwrap_or_default(); + let mut nags = Vec::new(); + + if contains_profanity(text) { + nags.push(ProjectNag::new( + ProjectNagKind::GalleryTextProfanity, + ProjectNagSeverity::Required, + )); + } + + if has_non_standard_text(text) { + nags.push(ProjectNag::new( + ProjectNagKind::GalleryTextNonStandard, + ProjectNagSeverity::Required, + )); + } + + nags } pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - let mut nags = Vec::new(); - let gallery_is_empty = project.gallery.is_empty(); - let is_shader = project.project_types.iter().any(|ty| ty == "shader"); - let is_resource_pack = - project.project_types.iter().any(|ty| ty == "resourcepack"); - let has_gallery_exemption = project - .categories - .iter() - .chain(&project.additional_categories) - .any(|category| category == "audio" || category == "locale"); - - if (is_shader && project.gallery.len() < 3) - || (is_resource_pack && gallery_is_empty && !has_gallery_exemption) - { - nags.push(ProjectNag::new( - ProjectNagKind::UploadGalleryImage, - ProjectNagSeverity::Required, - )); - } - - let is_minecraft_server = project.components.minecraft_server.is_some(); - if !is_minecraft_server && !project.gallery.iter().any(|item| item.featured) - { - nags.push(ProjectNag::new( - ProjectNagKind::FeatureGalleryImage, - ProjectNagSeverity::Suggestion, - )); - } - - for item in &project.gallery { - nags.extend(validate_text(item.name.as_deref())); - nags.extend(validate_text(item.description.as_deref())); - } - - nags + let mut nags = Vec::new(); + let gallery_is_empty = project.gallery.is_empty(); + let is_shader = project.project_types.iter().any(|ty| ty == "shader"); + let is_resource_pack = + project.project_types.iter().any(|ty| ty == "resourcepack"); + let has_gallery_exemption = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "audio" || category == "locale"); + + if (is_shader && project.gallery.len() < 3) + || (is_resource_pack && gallery_is_empty && !has_gallery_exemption) + { + nags.push(ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + )); + } + + let is_minecraft_server = project.components.minecraft_server.is_some(); + if !is_minecraft_server && !project.gallery.iter().any(|item| item.featured) + { + nags.push(ProjectNag::new( + ProjectNagKind::FeatureGalleryImage, + ProjectNagSeverity::Suggestion, + )); + } + + for item in &project.gallery { + nags.extend(validate_text(item.name.as_deref())); + nags.extend(validate_text(item.description.as_deref())); + } + + nags } diff --git a/apps/labrinth/src/validate/project/icon.rs b/apps/labrinth/src/validate/project/icon.rs index 2ec211c477..566015b8fd 100644 --- a/apps/labrinth/src/validate/project/icon.rs +++ b/apps/labrinth/src/validate/project/icon.rs @@ -1,14 +1,14 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - if project.icon_url.as_deref().is_none_or(str::is_empty) { - vec![ProjectNag::new( - ProjectNagKind::AddIcon, - ProjectNagSeverity::Suggestion, - )] - } else { - Vec::new() - } + if project.icon_url.as_deref().is_none_or(str::is_empty) { + vec![ProjectNag::new( + ProjectNagKind::AddIcon, + ProjectNagSeverity::Suggestion, + )] + } else { + Vec::new() + } } diff --git a/apps/labrinth/src/validate/project/license.rs b/apps/labrinth/src/validate/project/license.rs index 7e0d70cb5f..8bef9ed25f 100644 --- a/apps/labrinth/src/validate/project/license.rs +++ b/apps/labrinth/src/validate/project/license.rs @@ -1,92 +1,92 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; const INAPPROPRIATE_LICENSE_DOMAINS: &[&str] = &[ - "youtube.com", - "youtu.be", - "modrinth.com", - "curseforge.com", - "twitter.com", - "x.com", - "discord.gg", - "discord.com", - "instagram.com", - "facebook.com", - "tiktok.com", - "reddit.com", - "twitch.tv", - "patreon.com", - "ko-fi.com", - "paypal.com", - "buymeacoffee.com", - "google.com", - "example.com", - "t.me", + "youtube.com", + "youtu.be", + "modrinth.com", + "curseforge.com", + "twitter.com", + "x.com", + "discord.gg", + "discord.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "reddit.com", + "twitch.tv", + "patreon.com", + "ko-fi.com", + "paypal.com", + "buymeacoffee.com", + "google.com", + "example.com", + "t.me", ]; fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { - hostname == domain - || hostname - .strip_suffix(domain) - .is_some_and(|prefix| prefix.ends_with('.')) + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) } fn has_invalid_license_url(url: &str) -> bool { - if url.is_empty() { - return false; - } + if url.is_empty() { + return false; + } - let Ok(url) = url::Url::parse(url) else { - return true; - }; - let Some(hostname) = url.host_str() else { - return true; - }; - let hostname = hostname.to_ascii_lowercase(); - let hostname = hostname.trim_end_matches('.'); + let Ok(url) = url::Url::parse(url) else { + return true; + }; + let Some(hostname) = url.host_str() else { + return true; + }; + let hostname = hostname.to_ascii_lowercase(); + let hostname = hostname.trim_end_matches('.'); - INAPPROPRIATE_LICENSE_DOMAINS - .iter() - .any(|domain| hostname_matches_domain(hostname, domain)) + INAPPROPRIATE_LICENSE_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) } pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - let mut nags = Vec::new(); - let is_minecraft_server = project.components.minecraft_server.is_some(); - let license = &project.license; - let has_unknown_license = matches!( - license.id.as_str(), - "LicenseRef-Unknown" | "NOASSERTION" | "LicenseRef-NOASSERTION" - ); + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + let license = &project.license; + let has_unknown_license = matches!( + license.id.as_str(), + "LicenseRef-Unknown" | "NOASSERTION" | "LicenseRef-NOASSERTION" + ); - if has_unknown_license && !is_minecraft_server { - nags.push(ProjectNag::new( - ProjectNagKind::SelectLicense, - ProjectNagSeverity::Required, - )); - } + if has_unknown_license && !is_minecraft_server { + nags.push(ProjectNag::new( + ProjectNagKind::SelectLicense, + ProjectNagSeverity::Required, + )); + } - let has_license_url = - license.url.as_deref().is_some_and(|url| !url.is_empty()); - let missing_custom_license_details = license.id == "LicenseRef-" - || (license.id.starts_with("LicenseRef-") - && !has_license_url - && license.id != "LicenseRef-Unknown" - && license.id != "LicenseRef-All-Rights-Reserved"); - if missing_custom_license_details && !is_minecraft_server { - nags.push(ProjectNag::new( - ProjectNagKind::AddCustomLicenseDetails, - ProjectNagSeverity::Required, - )); - } + let has_license_url = + license.url.as_deref().is_some_and(|url| !url.is_empty()); + let missing_custom_license_details = license.id == "LicenseRef-" + || (license.id.starts_with("LicenseRef-") + && !has_license_url + && license.id != "LicenseRef-Unknown" + && license.id != "LicenseRef-All-Rights-Reserved"); + if missing_custom_license_details && !is_minecraft_server { + nags.push(ProjectNag::new( + ProjectNagKind::AddCustomLicenseDetails, + ProjectNagSeverity::Required, + )); + } - if license.url.as_deref().is_some_and(has_invalid_license_url) { - nags.push(ProjectNag::new( - ProjectNagKind::InvalidLicenseUrl, - ProjectNagSeverity::Required, - )); - } + if license.url.as_deref().is_some_and(has_invalid_license_url) { + nags.push(ProjectNag::new( + ProjectNagKind::InvalidLicenseUrl, + ProjectNagSeverity::Required, + )); + } - nags + nags } diff --git a/apps/labrinth/src/validate/project/links.rs b/apps/labrinth/src/validate/project/links.rs index 0aa3b7d841..4505a711ce 100644 --- a/apps/labrinth/src/validate/project/links.rs +++ b/apps/labrinth/src/validate/project/links.rs @@ -6,238 +6,238 @@ use super::{ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Version; const SOURCE_DOMAINS: &[&str] = &[ - "github.com", - "gitlab.com", - "bitbucket.org", - "codeberg.org", - "git.sr.ht", - "tangled.org", - "git.gay", + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "git.sr.ht", + "tangled.org", + "git.gay", ]; const ISSUES_DOMAINS: &[&str] = &[ - "github.com", - "gitlab.com", - "bitbucket.org", - "codeberg.org", - "docs.google.com", - "tangled.org", - "git.gay", + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "docs.google.com", + "tangled.org", + "git.gay", ]; const DISCORD_DOMAINS: &[&str] = &["discord.gg", "discord.com", "dsc.gg"]; const URL_SHORTENERS: &[&str] = - &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; const BLOCKED_EXTERNAL_LINK_DOMAINS: &[&str] = &[ - "twitter.com", - "x.com", - "instagram.com", - "facebook.com", - "tiktok.com", - "telegram.org", - "t.me", - "bilibili.com", - "bsky.app", - "twitch.tv", - "reddit.com", - "redd.it", - "modrinth.com", - "minecraft.net", - "curseforge.com", - "planetminecraft.com", - "9minecraft.net", - "mcmod.cn", - "creativemode.net", - "orcaclient.com", - "autoforged.cn", + "twitter.com", + "x.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "telegram.org", + "t.me", + "bilibili.com", + "bsky.app", + "twitch.tv", + "reddit.com", + "redd.it", + "modrinth.com", + "minecraft.net", + "curseforge.com", + "planetminecraft.com", + "9minecraft.net", + "mcmod.cn", + "creativemode.net", + "orcaclient.com", + "autoforged.cn", ]; const SOURCE_REQUIRING_LICENSES: &[&str] = &[ - "GPL-2.0", - "GPL-2.0+", - "GPL-2.0-only", - "GPL-2.0-or-later", - "GPL-3.0", - "GPL-3.0+", - "GPL-3.0-only", - "GPL-3.0-or-later", - "LGPL-2.1", - "LGPL-2.1+", - "LGPL-2.1-only", - "LGPL-2.1-or-later", - "LGPL-3.0", - "LGPL-3.0+", - "LGPL-3.0-only", - "LGPL-3.0-or-later", - "AGPL-3.0", - "AGPL-3.0+", - "AGPL-3.0-only", - "AGPL-3.0-or-later", - "MPL-2.0", + "GPL-2.0", + "GPL-2.0+", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-3.0", + "GPL-3.0+", + "GPL-3.0-only", + "GPL-3.0-or-later", + "LGPL-2.1", + "LGPL-2.1+", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-3.0", + "LGPL-3.0+", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "AGPL-3.0", + "AGPL-3.0+", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "MPL-2.0", ]; pub(super) fn validate( - project: &crate::models::projects::Project, - versions: &[Version], + project: &crate::models::projects::Project, + versions: &[Version], ) -> Vec { - let mut nags = Vec::new(); - let is_minecraft_server = project.components.minecraft_server.is_some(); - - if !is_minecraft_server && project.link_urls.is_empty() { - nags.push(super::ProjectNag::new( - ProjectNagKind::AddLinks, - ProjectNagSeverity::Suggestion, - )); - } - - if is_minecraft_server && project.link_urls.is_empty() { - nags.push(super::ProjectNag::new( - ProjectNagKind::AddLinksServer, - ProjectNagSeverity::Suggestion, - )); - } - - let link_count = project.link_urls.len(); - let unique_link_count = project - .link_urls - .values() - .map(|link| link.url.as_str()) - .collect::>() - .len(); - if unique_link_count != link_count { - nags.push(super::ProjectNag::new( - ProjectNagKind::IdenticalLinks, - ProjectNagSeverity::Required, - )); - } - - if named_link_is_uncommon(project, "source", SOURCE_DOMAINS) - || named_link_is_uncommon(project, "issues", ISSUES_DOMAINS) - || named_link_is_uncommon(project, "discord", DISCORD_DOMAINS) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::VerifyExternalLinks, - ProjectNagSeverity::Warning, - )); - } - - if ["source", "issues", "wiki", "site", "store"] - .into_iter() - .filter_map(|key| named_link(project, key)) - .any(is_discord_link) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::MisusedDiscordLink, - ProjectNagSeverity::Required, - )); - } - - if project - .link_urls - .values() - .any(|link| is_blocked_external_link(&link.url)) - || project - .license - .url - .as_deref() - .is_some_and(is_blocked_external_link) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::BannedLinkUsage, - ProjectNagSeverity::Required, - )); - } - - let is_source_project = project - .project_types - .iter() - .any(|project_type| project_type == "mod" || project_type == "plugin"); - let is_datapack = project - .project_types - .iter() - .any(|project_type| project_type == "datapack"); - let has_source_link = - named_link(project, "source").is_some_and(|url| !url.is_empty()); - let every_version_has_additional_files = - versions.iter().all(|version| version.files.len() >= 2); - - if !is_datapack - && is_source_project - && SOURCE_REQUIRING_LICENSES.contains(&project.license.id.as_str()) - && !has_source_link - && !every_version_has_additional_files - { - nags.push(super::ProjectNag::new( - ProjectNagKind::GplLicenseSourceRequired, - ProjectNagSeverity::Required, - )); - } - - nags + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + + if !is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinks, + ProjectNagSeverity::Suggestion, + )); + } + + if is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinksServer, + ProjectNagSeverity::Suggestion, + )); + } + + let link_count = project.link_urls.len(); + let unique_link_count = project + .link_urls + .values() + .map(|link| link.url.as_str()) + .collect::>() + .len(); + if unique_link_count != link_count { + nags.push(super::ProjectNag::new( + ProjectNagKind::IdenticalLinks, + ProjectNagSeverity::Required, + )); + } + + if named_link_is_uncommon(project, "source", SOURCE_DOMAINS) + || named_link_is_uncommon(project, "issues", ISSUES_DOMAINS) + || named_link_is_uncommon(project, "discord", DISCORD_DOMAINS) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::VerifyExternalLinks, + ProjectNagSeverity::Warning, + )); + } + + if ["source", "issues", "wiki", "site", "store"] + .into_iter() + .filter_map(|key| named_link(project, key)) + .any(is_discord_link) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::MisusedDiscordLink, + ProjectNagSeverity::Required, + )); + } + + if project + .link_urls + .values() + .any(|link| is_blocked_external_link(&link.url)) + || project + .license + .url + .as_deref() + .is_some_and(is_blocked_external_link) + { + nags.push(super::ProjectNag::new( + ProjectNagKind::BannedLinkUsage, + ProjectNagSeverity::Required, + )); + } + + let is_source_project = project + .project_types + .iter() + .any(|project_type| project_type == "mod" || project_type == "plugin"); + let is_datapack = project + .project_types + .iter() + .any(|project_type| project_type == "datapack"); + let has_source_link = + named_link(project, "source").is_some_and(|url| !url.is_empty()); + let every_version_has_additional_files = + versions.iter().all(|version| version.files.len() >= 2); + + if !is_datapack + && is_source_project + && SOURCE_REQUIRING_LICENSES.contains(&project.license.id.as_str()) + && !has_source_link + && !every_version_has_additional_files + { + nags.push(super::ProjectNag::new( + ProjectNagKind::GplLicenseSourceRequired, + ProjectNagSeverity::Required, + )); + } + + nags } fn named_link<'a>( - project: &'a crate::models::projects::Project, - key: &str, + project: &'a crate::models::projects::Project, + key: &str, ) -> Option<&'a str> { - project.link_urls.get(key).map(|link| link.url.as_str()) + project.link_urls.get(key).map(|link| link.url.as_str()) } fn named_link_is_uncommon( - project: &crate::models::projects::Project, - key: &str, - domains: &[&str], + project: &crate::models::projects::Project, + key: &str, + domains: &[&str], ) -> bool { - named_link(project, key) - .is_some_and(|url| !is_link_from_domains(url, domains)) + named_link(project, key) + .is_some_and(|url| !is_link_from_domains(url, domains)) } fn is_discord_link(url: &str) -> bool { - is_link_from_domains(url, DISCORD_DOMAINS) + is_link_from_domains(url, DISCORD_DOMAINS) } fn is_blocked_external_link(url: &str) -> bool { - let Some(hostname) = get_link_hostname(url) else { - return false; - }; - - is_ip_address(&hostname) - || URL_SHORTENERS - .iter() - .any(|domain| hostname_matches_domain(&hostname, domain)) - || BLOCKED_EXTERNAL_LINK_DOMAINS - .iter() - .any(|domain| hostname_matches_domain(&hostname, domain)) + let Some(hostname) = get_link_hostname(url) else { + return false; + }; + + is_ip_address(&hostname) + || URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) + || BLOCKED_EXTERNAL_LINK_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) } fn is_link_from_domains(url: &str, domains: &[&str]) -> bool { - let Some(hostname) = get_link_hostname(url) else { - return false; - }; + let Some(hostname) = get_link_hostname(url) else { + return false; + }; - domains - .iter() - .any(|domain| hostname_matches_domain(&hostname, domain)) + domains + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) } fn get_link_hostname(url: &str) -> Option { - let parsed = Url::parse(url).ok()?; - let hostname = parsed - .host_str()? - .trim_end_matches('.') - .to_ascii_lowercase(); + let parsed = Url::parse(url).ok()?; + let hostname = parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(); - (!hostname.is_empty()).then_some(hostname) + (!hostname.is_empty()).then_some(hostname) } fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { - hostname == domain - || hostname - .strip_suffix(domain) - .is_some_and(|prefix| prefix.ends_with('.')) + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) } fn is_ip_address(hostname: &str) -> bool { - hostname - .trim_start_matches('[') - .trim_end_matches(']') - .parse::() - .is_ok() + hostname + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok() } diff --git a/apps/labrinth/src/validate/project/moderation.rs b/apps/labrinth/src/validate/project/moderation.rs index 1f68fbbe84..8f4774a61a 100644 --- a/apps/labrinth/src/validate/project/moderation.rs +++ b/apps/labrinth/src/validate/project/moderation.rs @@ -3,17 +3,17 @@ use crate::models::projects::ProjectStatus; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - if matches!( - project.status, - ProjectStatus::Rejected | ProjectStatus::Withheld - ) { - vec![ProjectNag::new( - ProjectNagKind::ModeratorFeedback, - ProjectNagSeverity::Warning, - )] - } else { - Vec::new() - } + if matches!( + project.status, + ProjectStatus::Rejected | ProjectStatus::Withheld + ) { + vec![ProjectNag::new( + ProjectNagKind::ModeratorFeedback, + ProjectNagSeverity::Warning, + )] + } else { + Vec::new() + } } diff --git a/apps/labrinth/src/validate/project/name.rs b/apps/labrinth/src/validate/project/name.rs index b592eefba6..bef840d7ca 100644 --- a/apps/labrinth/src/validate/project/name.rs +++ b/apps/labrinth/src/validate/project/name.rs @@ -7,45 +7,45 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; static VERSION_NUMBER: LazyLock = - LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)+").unwrap()); + LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)+").unwrap()); static PORT_OR_FORK: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\b(?:port|fork)\b").unwrap()); + LazyLock::new(|| Regex::new(r"(?i)\b(?:port|fork)\b").unwrap()); pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let name = &project.name; + let mut nags = Vec::new(); + let name = &project.name; - if contains_profanity(name) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectNameProfanity, - ProjectNagSeverity::Required, - )); - } - if has_non_standard_text(name) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectNameNonStandardText, - ProjectNagSeverity::Required, - )); - } + if contains_profanity(name) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameProfanity, + ProjectNagSeverity::Required, + )); + } + if has_non_standard_text(name) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameNonStandardText, + ProjectNagSeverity::Required, + )); + } - let normalized = name.to_lowercase(); - if VERSION_NUMBER - .find_iter(&normalized) - .any(|version| !PORT_OR_FORK.is_match(&normalized[version.end()..])) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectNameVersion, - ProjectNagSeverity::Required, - )); - } - if normalized.contains("minecraft") - && normalized.split_whitespace().count() <= 3 - { - nags.push(ProjectNag::new( - ProjectNagKind::MinecraftTitleClause, - ProjectNagSeverity::Required, - )); - } + let normalized = name.to_lowercase(); + if VERSION_NUMBER + .find_iter(&normalized) + .any(|version| !PORT_OR_FORK.is_match(&normalized[version.end()..])) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameVersion, + ProjectNagSeverity::Required, + )); + } + if normalized.contains("minecraft") + && normalized.split_whitespace().count() <= 3 + { + nags.push(ProjectNag::new( + ProjectNagKind::MinecraftTitleClause, + ProjectNagSeverity::Required, + )); + } - nags + nags } diff --git a/apps/labrinth/src/validate/project/permissions.rs b/apps/labrinth/src/validate/project/permissions.rs index 6c5dce01c7..8bef6a0836 100644 --- a/apps/labrinth/src/validate/project/permissions.rs +++ b/apps/labrinth/src/validate/project/permissions.rs @@ -2,15 +2,15 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Version; pub(super) fn validate(versions: &[Version]) -> Vec { - if versions - .iter() - .any(|version| !version.files_missing_attribution.is_empty()) - { - vec![ProjectNag::new( - ProjectNagKind::ReviewPermissions, - ProjectNagSeverity::Required, - )] - } else { - Vec::new() - } + if versions + .iter() + .any(|version| !version.files_missing_attribution.is_empty()) + { + vec![ProjectNag::new( + ProjectNagKind::ReviewPermissions, + ProjectNagSeverity::Required, + )] + } else { + Vec::new() + } } diff --git a/apps/labrinth/src/validate/project/server_settings.rs b/apps/labrinth/src/validate/project/server_settings.rs index 21793a40e4..eb73c730a6 100644 --- a/apps/labrinth/src/validate/project/server_settings.rs +++ b/apps/labrinth/src/validate/project/server_settings.rs @@ -5,64 +5,65 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; const MAX_LANGUAGE_COUNT: usize = 10; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let server = project.components.minecraft_server.as_ref(); + let mut nags = Vec::new(); + let server = project.components.minecraft_server.as_ref(); - if server.is_some_and(|server| server.region.is_none()) { - nags.push(ProjectNag::new( - ProjectNagKind::SelectCountry, - ProjectNagSeverity::Required, - )); - } + if server.is_some_and(|server| server.region.is_none()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCountry, + ProjectNagSeverity::Required, + )); + } - if server.is_some() - && project - .components - .minecraft_java_server - .as_ref() - .is_none_or(|server| server.address.is_empty()) - { - nags.push(ProjectNag::new( - ProjectNagKind::AddJavaAddress, - ProjectNagSeverity::Required, - )); - } + if server.is_some() + && project + .components + .minecraft_java_server + .as_ref() + .is_none_or(|server| server.address.is_empty()) + { + nags.push(ProjectNag::new( + ProjectNagKind::AddJavaAddress, + ProjectNagSeverity::Required, + )); + } - if project - .components - .minecraft_java_server - .as_ref() - .is_some_and(|server| { - matches!( - &server.content, - ServerContentQuery::Vanilla { - recommended_game_version, - .. - } if recommended_game_version - .as_deref() - .is_none_or(str::is_empty) - ) - }) { - nags.push(ProjectNag::new( - ProjectNagKind::SelectCompatibility, - ProjectNagSeverity::Required, - )); - } + if project + .components + .minecraft_java_server + .as_ref() + .is_some_and(|server| { + matches!( + &server.content, + ServerContentQuery::Vanilla { + recommended_game_version, + .. + } if recommended_game_version + .as_deref() + .is_none_or(str::is_empty) + ) + }) + { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCompatibility, + ProjectNagSeverity::Required, + )); + } - if server.is_some_and(|server| server.languages.len() > MAX_LANGUAGE_COUNT) - { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyLanguages, - ProjectNagSeverity::Warning, - )); - } + if server.is_some_and(|server| server.languages.len() > MAX_LANGUAGE_COUNT) + { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyLanguages, + ProjectNagSeverity::Warning, + )); + } - if server.is_some_and(|server| server.languages.is_empty()) { - nags.push(ProjectNag::new( - ProjectNagKind::SelectLanguage, - ProjectNagSeverity::Suggestion, - )); - } + if server.is_some_and(|server| server.languages.is_empty()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectLanguage, + ProjectNagSeverity::Suggestion, + )); + } - nags + nags } diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 40ddd3b576..22da89221b 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,6 +1,6 @@ use super::text::{ - contains_link_or_ip, contains_profanity, contains_spam, - has_non_standard_text, has_summary_formatting, is_likely_english, + contains_link_or_ip, contains_profanity, contains_spam, + has_non_standard_text, has_summary_formatting, is_likely_english, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::exp::minecraft::Language; @@ -9,87 +9,87 @@ use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let summary = project.summary.trim(); - let contains_link = contains_link_or_ip(summary); + let mut nags = Vec::new(); + let summary = project.summary.trim(); + let contains_link = contains_link_or_ip(summary); - if contains_profanity(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryProfanity, - ProjectNagSeverity::Required, - )); - } - if has_non_standard_text(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && summary.chars().count() >= MIN_SUMMARY_CHARS - && !contains_link - && !contains_spam(summary) - && !is_likely_english(summary) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if !summary.is_empty() - && !contains_link - && normalized_without_whitespace(summary) - == normalized_without_whitespace(&project.name) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryMatchesTitle, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() - && !contains_link - && summary.chars().count() < MIN_SUMMARY_CHARS - { - nags.push(ProjectNag::new( - ProjectNagKind::SummaryTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummarySpam, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && has_summary_formatting(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::SummarySpecialFormatting, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && contains_link { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryLinks, - ProjectNagSeverity::Required, - )); - } + if contains_profanity(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryProfanity, + ProjectNagSeverity::Required, + )); + } + if has_non_standard_text(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && summary.chars().count() >= MIN_SUMMARY_CHARS + && !contains_link + && !contains_spam(summary) + && !is_likely_english(summary) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if !summary.is_empty() + && !contains_link + && normalized_without_whitespace(summary) + == normalized_without_whitespace(&project.name) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryMatchesTitle, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() + && !contains_link + && summary.chars().count() < MIN_SUMMARY_CHARS + { + nags.push(ProjectNag::new( + ProjectNagKind::SummaryTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummarySpam, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && has_summary_formatting(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::SummarySpecialFormatting, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && contains_link { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryLinks, + ProjectNagSeverity::Required, + )); + } - nags + nags } fn normalized_without_whitespace(text: &str) -> String { - text.trim() - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() + text.trim() + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() } fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) } diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs index 55135ed3b1..6d6404d9e6 100644 --- a/apps/labrinth/src/validate/project/tags.rs +++ b/apps/labrinth/src/validate/project/tags.rs @@ -5,51 +5,52 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; const MAX_TAG_COUNT: usize = 8; const MAX_TAG_COUNT_SERVER: usize = 18; const RESOLUTION_TAGS: [&str; 8] = - ["8x-", "16x", "32x", "48x", "64x", "128x", "256x", "512x+"]; + ["8x-", "16x", "32x", "48x", "64x", "128x", "256x", "512x+"]; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let tag_count = - project.categories.len() + project.additional_categories.len(); - let is_minecraft_server = project.components.minecraft_server.is_some(); - - if !project.versions.is_empty() && project.categories.is_empty() { - nags.push(ProjectNag::new( - ProjectNagKind::SelectTags, - ProjectNagSeverity::Suggestion, - )); - } - - if !is_minecraft_server && tag_count > MAX_TAG_COUNT { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyTags, - ProjectNagSeverity::Warning, - )); - } - - if is_minecraft_server && tag_count > MAX_TAG_COUNT_SERVER { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyTagsServer, - ProjectNagSeverity::Required, - )); - } - - if project - .project_types - .iter() - .any(|project_type| project_type == "resourcepack") - && project - .categories - .iter() - .chain(&project.additional_categories) - .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) - .count() > 1 - { - nags.push(ProjectNag::new( - ProjectNagKind::MultipleResolutionTags, - ProjectNagSeverity::Warning, - )); - } - - nags + let mut nags = Vec::new(); + let tag_count = + project.categories.len() + project.additional_categories.len(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + + if !project.versions.is_empty() && project.categories.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::SelectTags, + ProjectNagSeverity::Suggestion, + )); + } + + if !is_minecraft_server && tag_count > MAX_TAG_COUNT { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyTags, + ProjectNagSeverity::Warning, + )); + } + + if is_minecraft_server && tag_count > MAX_TAG_COUNT_SERVER { + nags.push(ProjectNag::new( + ProjectNagKind::TooManyTagsServer, + ProjectNagSeverity::Required, + )); + } + + if project + .project_types + .iter() + .any(|project_type| project_type == "resourcepack") + && project + .categories + .iter() + .chain(&project.additional_categories) + .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) + .count() + > 1 + { + nags.push(ProjectNag::new( + ProjectNagKind::MultipleResolutionTags, + ProjectNagSeverity::Warning, + )); + } + + nags } diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 7f60a1067f..4776037011 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -6,12 +6,12 @@ use regex::Regex; use url::Url; static PROFANITY_CENSOR: LazyLock = - LazyLock::new(|| Censor::Standard + Censor::Sex); + LazyLock::new(|| Censor::Standard + Censor::Sex); static WORD: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static LINK_OR_IP: LazyLock = LazyLock::new(|| { - Regex::new(concat!( + Regex::new(concat!( r"(?i)(?:https?://|www\.)[^\s<>()]+|", r"\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+\b|", r"\b(?:\d{1,3}\.){3}\d{1,3}\b", @@ -19,324 +19,333 @@ static LINK_OR_IP: LazyLock = LazyLock::new(|| { .unwrap() }); static MARKDOWN_LINK: LazyLock = - LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); static CODE_BLOCK: LazyLock = - LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); + LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); static INLINE_CODE: LazyLock = - LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); + LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); static MARKDOWN_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { - Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() + Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() }); static DESCRIPTION_LINK: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() + Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() }); static HEADER: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); + LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static INLINE_MARKDOWN: LazyLock = LazyLock::new(|| { - Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() + Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() }); const URL_SHORTENERS: &[&str] = - &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; pub(super) fn contains_profanity(text: &str) -> bool { - PROFANITY_CENSOR.check(text) + PROFANITY_CENSOR.check(text) } pub(super) fn profanity_count(text: &str) -> usize { - PROFANITY_CENSOR.count(text) + PROFANITY_CENSOR.count(text) } pub(super) fn has_non_standard_text(text: &str) -> bool { - non_standard_character_count(text) > 0 + non_standard_character_count(text) > 0 } pub(super) fn non_standard_text_ratio(text: &str) -> f64 { - let character_count = text.chars().count(); - if character_count == 0 { - return 0.0; - } + let character_count = text.chars().count(); + if character_count == 0 { + return 0.0; + } - non_standard_character_count(text) as f64 / character_count as f64 + non_standard_character_count(text) as f64 / character_count as f64 } fn non_standard_character_count(text: &str) -> usize { - let mut count = 0; - let mut combining_marks = 0; - let mut has_base = false; + let mut count = 0; + let mut combining_marks = 0; + let mut has_base = false; - for character in text.chars() { - let code = character as u32; - let is_combining = is_in_ranges( - code, - &[ - (0x0300, 0x036f), - (0x1ab0, 0x1aff), - (0x1dc0, 0x1dff), - (0x20d0, 0x20ff), - (0xfe20, 0xfe2f), - ], - ); - if is_combining { - combining_marks += 1; - if !has_base || combining_marks > 2 { - count += 1; - } - continue; - } + for character in text.chars() { + let code = character as u32; + let is_combining = is_in_ranges( + code, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ); + if is_combining { + combining_marks += 1; + if !has_base || combining_marks > 2 { + count += 1; + } + continue; + } - combining_marks = 0; - has_base = !character.is_whitespace(); - let allowed_control = matches!(character, '\n' | '\r' | '\t'); - let control = character.is_control() && !allowed_control; - let private_use = is_in_ranges( - code, - &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], - ); - let invisible = is_in_ranges( - code, - &[ - (0x00ad, 0x00ad), - (0x061c, 0x061c), - (0x200e, 0x200f), - (0x202a, 0x202e), - (0x2060, 0x206f), - (0xfeff, 0xfeff), - ], - ); - let fancy = is_in_ranges( - code, - &[ - (0x02b0, 0x02ff), - (0x1d400, 0x1d7ff), - (0x2460, 0x24ff), - (0x2070, 0x209f), - (0x2100, 0x214f), - (0xfb00, 0xfb06), - (0xff10, 0xff19), - (0xff21, 0xff3a), - (0xff41, 0xff5a), - (0x1f100, 0x1f1ad), - ], - ) && !matches!( - code, - 0x02d6 - | 0x02d7 | 0x02d8 - | 0x02d9 | 0x02da - | 0x02db | 0x02dc - | 0x02dd | 0x207a - | 0x207b | 0x208a - | 0x208b | 0x2120 - | 0x2122 | 0x2139 - ); + combining_marks = 0; + has_base = !character.is_whitespace(); + let allowed_control = matches!(character, '\n' | '\r' | '\t'); + let control = character.is_control() && !allowed_control; + let private_use = is_in_ranges( + code, + &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], + ); + let invisible = is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x061c, 0x061c), + (0x200e, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0xfeff, 0xfeff), + ], + ); + let fancy = is_in_ranges( + code, + &[ + (0x02b0, 0x02ff), + (0x1d400, 0x1d7ff), + (0x2460, 0x24ff), + (0x2070, 0x209f), + (0x2100, 0x214f), + (0xfb00, 0xfb06), + (0xff10, 0xff19), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0x1f100, 0x1f1ad), + ], + ) && !matches!( + code, + 0x02d6 + | 0x02d7 + | 0x02d8 + | 0x02d9 + | 0x02da + | 0x02db + | 0x02dc + | 0x02dd + | 0x207a + | 0x207b + | 0x208a + | 0x208b + | 0x2120 + | 0x2122 + | 0x2139 + ); - if control || private_use || invisible || fancy { - count += 1; - } - } + if control || private_use || invisible || fancy { + count += 1; + } + } - count + count } fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { - ranges - .iter() - .any(|(start, end)| code >= *start && code <= *end) + ranges + .iter() + .any(|(start, end)| code >= *start && code <= *end) } pub(super) fn contains_spam(text: &str) -> bool { - let normalized = text.to_lowercase(); - let mut previous = None; - let mut repeated = 0; - for character in normalized.chars() { - if !character.is_whitespace() && previous == Some(character) { - repeated += 1; - } else { - repeated = usize::from(!character.is_whitespace()); - previous = Some(character); - } - if repeated >= 8 { - return true; - } - } + let normalized = text.to_lowercase(); + let mut previous = None; + let mut repeated = 0; + for character in normalized.chars() { + if !character.is_whitespace() && previous == Some(character) { + repeated += 1; + } else { + repeated = usize::from(!character.is_whitespace()); + previous = Some(character); + } + if repeated >= 8 { + return true; + } + } - let words = WORD - .find_iter(&normalized) - .map(|word| word.as_str()) - .collect::>(); - let mut repeated_words = 1; - for pair in words.windows(2) { - repeated_words = if pair[0] == pair[1] { - repeated_words + 1 - } else { - 1 - }; - if repeated_words >= 4 { - return true; - } - } + let words = WORD + .find_iter(&normalized) + .map(|word| word.as_str()) + .collect::>(); + let mut repeated_words = 1; + for pair in words.windows(2) { + repeated_words = if pair[0] == pair[1] { + repeated_words + 1 + } else { + 1 + }; + if repeated_words >= 4 { + return true; + } + } - let max_phrase_words = 8.min(words.len() / 3); - for phrase_words in (2..=max_phrase_words).rev() { - for phrases in words.windows(phrase_words * 3) { - if phrases[..phrase_words] - == phrases[phrase_words..phrase_words * 2] - && phrases[..phrase_words] - == phrases[phrase_words * 2..phrase_words * 3] - { - return true; - } - } - } + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for phrases in words.windows(phrase_words * 3) { + if phrases[..phrase_words] + == phrases[phrase_words..phrase_words * 2] + && phrases[..phrase_words] + == phrases[phrase_words * 2..phrase_words * 3] + { + return true; + } + } + } - false + false } pub(super) fn contains_link_or_ip(text: &str) -> bool { - LINK_OR_IP.is_match(text) + LINK_OR_IP.is_match(text) } pub(super) fn has_summary_formatting(summary: &str) -> bool { - HTML_TAG.is_match(summary) - || MARKDOWN_LINK.is_match(summary) - || summary.lines().any(|line| { - let line = line.trim_start(); - line.starts_with('#') - || line.starts_with('>') - || line.starts_with("- ") - || line.starts_with("* ") - || line.starts_with("+ ") - || line.starts_with("```") - || line.contains("**") - || line.contains("__") - || line.contains("~~") - || INLINE_CODE.is_match(line) - }) + HTML_TAG.is_match(summary) + || MARKDOWN_LINK.is_match(summary) + || summary.lines().any(|line| { + let line = line.trim_start(); + line.starts_with('#') + || line.starts_with('>') + || line.starts_with("- ") + || line.starts_with("* ") + || line.starts_with("+ ") + || line.starts_with("```") + || line.contains("**") + || line.contains("__") + || line.contains("~~") + || INLINE_CODE.is_match(line) + }) } pub(super) fn extract_description_text(markdown: &str) -> String { - let without_code = CODE_BLOCK.replace_all(markdown, " "); - let without_code = INLINE_CODE.replace_all(&without_code, " "); - let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); - let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); - let without_html = HTML_TAG.replace_all(&without_links, " "); - without_html - .replace(['*', '_', '~', '`', '>', '-', '|'], " ") - .split_whitespace() - .collect::>() - .join(" ") + let without_code = CODE_BLOCK.replace_all(markdown, " "); + let without_code = INLINE_CODE.replace_all(&without_code, " "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); + let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); + let without_html = HTML_TAG.replace_all(&without_links, " "); + without_html + .replace(['*', '_', '~', '`', '>', '-', '|'], " ") + .split_whitespace() + .collect::>() + .join(" ") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - CODE_BLOCK - .replace_all(markdown, "") - .split("\n\n") - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + CODE_BLOCK + .replace_all(markdown, "") + .split("\n\n") + .map(extract_description_text) + .filter(|block| !block.is_empty()) + .collect() } pub(super) fn has_long_header(markdown: &str) -> bool { - HEADER.captures_iter(markdown).any(|captures| { - INLINE_MARKDOWN - .replace_all(&captures[1], "$1") - .trim() - .chars() - .count() > 80 - }) || HTML_HEADER.captures_iter(markdown).any(|captures| { - HTML_TAG - .replace_all(&captures[1], " ") - .split_whitespace() - .collect::>() - .join(" ") - .chars() - .count() > 80 - }) + HEADER.captures_iter(markdown).any(|captures| { + INLINE_MARKDOWN + .replace_all(&captures[1], "$1") + .trim() + .chars() + .count() + > 80 + }) || HTML_HEADER.captures_iter(markdown).any(|captures| { + HTML_TAG + .replace_all(&captures[1], " ") + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .count() + > 80 + }) } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { - let without_code = CODE_BLOCK.replace_all(markdown, ""); - let without_code = INLINE_CODE.replace_all(&without_code, ""); - MARKDOWN_IMAGE - .captures_iter(&without_code) - .any(|captures| captures[1].trim().is_empty()) - || HTML_IMAGE.find_iter(&without_code).any(|image| { - ALT_ATTRIBUTE - .captures(image.as_str()) - .and_then(|captures| { - captures.get(1).or_else(|| captures.get(2)) - }) - .is_none_or(|alt| alt.as_str().trim().is_empty()) - }) + let without_code = CODE_BLOCK.replace_all(markdown, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + MARKDOWN_IMAGE + .captures_iter(&without_code) + .any(|captures| captures[1].trim().is_empty()) + || HTML_IMAGE.find_iter(&without_code).any(|image| { + ALT_ATTRIBUTE + .captures(image.as_str()) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .is_none_or(|alt| alt.as_str().trim().is_empty()) + }) } pub(super) fn contains_banned_description_link(markdown: &str) -> bool { - DESCRIPTION_LINK.find_iter(markdown).any(|link| { - let raw = link.as_str(); - let normalized = if raw.to_ascii_lowercase().starts_with("www.") { - format!("https://{raw}") - } else { - raw.to_owned() - }; - Url::parse(&normalized).is_ok_and(|url| { - url.host_str().is_some_and(|hostname| { - URL_SHORTENERS - .iter() - .any(|domain| hostname_matches_domain(hostname, domain)) - }) - }) - }) + DESCRIPTION_LINK.find_iter(markdown).any(|link| { + let raw = link.as_str(); + let normalized = if raw.to_ascii_lowercase().starts_with("www.") { + format!("https://{raw}") + } else { + raw.to_owned() + }; + Url::parse(&normalized).is_ok_and(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) + }) + }) } fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { - hostname.eq_ignore_ascii_case(domain) - || hostname - .to_ascii_lowercase() - .ends_with(&format!(".{domain}")) + hostname.eq_ignore_ascii_case(domain) + || hostname + .to_ascii_lowercase() + .ends_with(&format!(".{domain}")) } pub(super) fn is_likely_english(text: &str) -> bool { - let words = WORD.find_iter(text).count(); - if words < 8 || text.chars().count() < 35 { - return true; - } + let words = WORD.find_iter(text).count(); + if words < 8 || text.chars().count() < 35 { + return true; + } - let mut scripts = HashMap::<&'static str, usize>::new(); - for character in text.chars().filter(|character| character.is_alphabetic()) - { - let code = character as u32; - let script = if code <= 0x024f { - "latin" - } else if is_in_ranges(code, &[(0x0400, 0x052f)]) { - "cyrillic" - } else if is_in_ranges(code, &[(0x0370, 0x03ff)]) { - "greek" - } else if is_in_ranges( - code, - &[ - (0x3040, 0x30ff), - (0x3400, 0x4dbf), - (0x4e00, 0x9fff), - (0xac00, 0xd7af), - ], - ) { - "cjk" - } else { - "other" - }; - *scripts.entry(script).or_default() += 1; - } + let mut scripts = HashMap::<&'static str, usize>::new(); + for character in text.chars().filter(|character| character.is_alphabetic()) + { + let code = character as u32; + let script = if code <= 0x024f { + "latin" + } else if is_in_ranges(code, &[(0x0400, 0x052f)]) { + "cyrillic" + } else if is_in_ranges(code, &[(0x0370, 0x03ff)]) { + "greek" + } else if is_in_ranges( + code, + &[ + (0x3040, 0x30ff), + (0x3400, 0x4dbf), + (0x4e00, 0x9fff), + (0xac00, 0xd7af), + ], + ) { + "cjk" + } else { + "other" + }; + *scripts.entry(script).or_default() += 1; + } - let total = scripts.values().sum::(); - total == 0 - || scripts.get("latin").copied().unwrap_or_default() * 10 >= total * 7 + let total = scripts.values().sum::(); + total == 0 + || scripts.get("latin").copied().unwrap_or_default() * 10 >= total * 7 } diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs index 2df1f43402..94c65372a9 100644 --- a/apps/labrinth/src/validate/project/versions.rs +++ b/apps/labrinth/src/validate/project/versions.rs @@ -1,16 +1,16 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &crate::models::projects::Project, + project: &crate::models::projects::Project, ) -> Vec { - if project.versions.is_empty() - && project.components.minecraft_server.is_none() - { - vec![ProjectNag::new( - ProjectNagKind::UploadVersion, - ProjectNagSeverity::Required, - )] - } else { - Vec::new() - } + if project.versions.is_empty() + && project.components.minecraft_server.is_none() + { + vec![ProjectNag::new( + ProjectNagKind::UploadVersion, + ProjectNagSeverity::Required, + )] + } else { + Vec::new() + } } From 79f7a078ac91dd38fe26ca04c31224b6420a87e1 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:25:03 +0100 Subject: [PATCH 06/28] bulk check json, adjust profanity --- Cargo.lock | 30 + Cargo.toml | 3 + apps/labrinth/Cargo.toml | 3 + apps/labrinth/src/bin/check_project.rs | 289 +++++++-- apps/labrinth/src/routes/v3/mod.rs | 1 + .../src/routes/v3/projects/validate.rs | 89 ++- .../src/validate/project/description.rs | 149 +++-- apps/labrinth/src/validate/project/mod.rs | 12 +- apps/labrinth/src/validate/project/summary.rs | 154 ++--- apps/labrinth/src/validate/project/text.rs | 590 ++++++++++-------- 10 files changed, 816 insertions(+), 504 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c93f4e9c9f..e65a6d0cc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5601,6 +5601,7 @@ dependencies = [ "json-patch 4.1.0", "labrinth", "lettre", + "linkify", "modrinth-content-management", "modrinth-util", "muralpay", @@ -5610,6 +5611,7 @@ dependencies = [ "path-util", "postcard", "prometheus", + "psl", "quick-xml 0.38.3", "rand 0.8.5", "rand_chacha 0.3.1", @@ -5651,6 +5653,7 @@ dependencies = [ "webauthn-rs", "webauthn-rs-proto", "webp", + "whatlang", "woothee", "xredis", "yaserde", @@ -5845,6 +5848,15 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linkify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92f9ae2f3d31697697311f016320fe984127e636b45cf5eb5c175b7eb103dadb" +dependencies = [ + "memchr", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -7995,6 +8007,15 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "psl" +version = "2.1.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc88482eea924ca3a2f56a547454169af58deef35567965eb4fc2392a834841" +dependencies = [ + "psl-types", +] + [[package]] name = "psl-types" version = "2.0.11" @@ -12931,6 +12952,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "whatlang" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index cfcb0e8c2b..96da8f469d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,6 +113,7 @@ itertools = "0.14.0" jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" +linkify = "0.11.0" lettre = { version = "0.11.19", default-features = false, features = [ "aws-lc-rs", "builder", @@ -151,6 +152,7 @@ postcard = { version = "1.1.3", default-features = false, features = ["alloc"] } postcard-bindgen = "0.8.0" proc-macro2 = { version = "1.0" } prometheus = "0.14.0" +psl = "2.1.226" quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } @@ -237,6 +239,7 @@ uuid = "1.18.1" validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" +whatlang = "0.18.0" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry whoami = "1.6.1" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index d0fedde2ec..6369ea7629 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -71,6 +71,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +linkify = { workspace = true } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } muralpay = { workspace = true, features = ["client", "mock", "utoipa"] } @@ -80,6 +81,7 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } +psl = { workspace = true } quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } @@ -126,6 +128,7 @@ uuid = { workspace = true, features = ["fast-rng", "serde", "v4", "v7"] } validator = { workspace = true, features = ["derive"] } webauthn-rs = { workspace = true, features = ["conditional-ui", "danger-allow-state-serialisation"] } webauthn-rs-proto = { workspace = true } +whatlang = { workspace = true } webp = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index 5e6778716b..2ad4ea3eea 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -1,84 +1,249 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, PathBuf}; + use clap::Parser; use eyre::{Result, WrapErr, eyre}; use labrinth::models::projects::{Project, Version}; -use labrinth::validate::project::validate; -use serde::de::DeserializeOwned; +use labrinth::validate::project::{ + ProjectNagKind, ProjectNagSeverity, validate, +}; +use serde::Serialize; +use serde::de::{ + DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, + Visitor, +}; +use strum::IntoEnumIterator; use url::Url; const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; #[derive(Parser)] -#[command(version, about = "Validate a project from the Modrinth API")] +#[command( + version, + about = "Validate a project from the Modrinth API or a project dataset" +)] struct Args { - /// The ID of the project to validate - project_id: String, + /// The ID of the project to validate + #[arg( + value_name = "PROJECT_ID", + required_unless_present = "file", + conflicts_with = "file" + )] + project_id: Option, + + /// Validate all projects in a moderation project dataset + #[arg(long, value_name = "PATH", conflicts_with = "token")] + file: Option, + + /// A Modrinth token to send as bearer authentication + #[arg(long, value_name = "TOKEN")] + token: Option, +} + +#[derive(Serialize)] +struct BatchSummary { + projects: usize, + projects_with_at_least_one_required_nag: usize, + version_details_available: bool, + nag_counts: BTreeMap, +} + +impl BatchSummary { + fn new() -> Self { + Self { + projects: 0, + projects_with_at_least_one_required_nag: 0, + version_details_available: false, + nag_counts: ProjectNagKind::iter().map(|kind| (kind, 0)).collect(), + } + } - /// A Modrinth token to send as bearer authentication - #[arg(long, value_name = "TOKEN")] - token: Option, + fn add_project(&mut self, project: &Project) { + let nags = validate(project, &[]); + self.projects += 1; + if nags + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) + { + self.projects_with_at_least_one_required_nag += 1; + } + for nag in nags { + *self.nag_counts.entry(nag.kind).or_default() += 1; + } + } } #[tokio::main] async fn main() -> Result<()> { - color_eyre::install().wrap_err("installing color-eyre")?; - let args = Args::parse(); - let client = reqwest::Client::builder() - .user_agent(concat!( - "labrinth-check-project/", - env!("CARGO_PKG_VERSION") - )) - .build() - .wrap_err("building HTTP client")?; - - let project_url = api_url(&["project", &args.project_id])?; - let mut versions_url = api_url(&["project", &args.project_id, "version"])?; - versions_url - .query_pairs_mut() - .append_pair("include_changelog", "false"); - - let token = args.token.as_deref(); - let (project, versions) = tokio::try_join!( - fetch::(&client, project_url, token), - fetch::>(&client, versions_url, token), - )?; - let nags = validate(&project, &versions); - - println!( - "{}", - serde_json::to_string_pretty(&nags) - .wrap_err("serializing project nags")? - ); - - Ok(()) + color_eyre::install().wrap_err("installing color-eyre")?; + let args = Args::parse(); + + if let Some(path) = args.file { + let summary = summarize_file(&path)?; + print_json(&summary)?; + return Ok(()); + } + + let project_id = args + .project_id + .as_deref() + .ok_or_else(|| eyre!("a project ID or `--file` is required"))?; + check_api_project(project_id, args.token.as_deref()).await +} + +async fn check_api_project( + project_id: &str, + token: Option<&str>, +) -> Result<()> { + let client = reqwest::Client::builder() + .user_agent(concat!( + "labrinth-check-project/", + env!("CARGO_PKG_VERSION") + )) + .build() + .wrap_err("building HTTP client")?; + + let project_url = api_url(&["project", project_id])?; + let mut versions_url = api_url(&["project", project_id, "version"])?; + versions_url + .query_pairs_mut() + .append_pair("include_changelog", "false"); + + let (project, versions) = tokio::try_join!( + fetch::(&client, project_url, token), + fetch::>(&client, versions_url, token), + )?; + print_json(&validate(&project, &versions)) +} + +fn summarize_file(path: &Path) -> Result { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(file)); + let summary = DatasetSeed + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(summary) +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!( + "{}", + serde_json::to_string_pretty(value).wrap_err("serializing output")? + ); + Ok(()) } fn api_url(path: &[&str]) -> Result { - let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; - url.path_segments_mut() - .map_err(|_| eyre!("API base URL cannot be a base URL"))? - .extend(path); - Ok(url) + let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; + url.path_segments_mut() + .map_err(|_| eyre!("API base URL cannot be a base URL"))? + .extend(path); + Ok(url) } async fn fetch( - client: &reqwest::Client, - url: Url, - token: Option<&str>, + client: &reqwest::Client, + url: Url, + token: Option<&str>, ) -> Result { - let mut request = client.get(url.clone()); - if let Some(token) = token { - request = request.bearer_auth(token); - } - - let response = request - .send() - .await - .wrap_err_with(|| format!("requesting `{url}`"))? - .error_for_status() - .wrap_err_with(|| format!("requesting `{url}`"))?; - - response - .json() - .await - .wrap_err_with(|| format!("deserializing response from `{url}`")) + let mut request = client.get(url.clone()); + if let Some(token) = token { + request = request.bearer_auth(token); + } + + let response = request + .send() + .await + .wrap_err_with(|| format!("requesting `{url}`"))? + .error_for_status() + .wrap_err_with(|| format!("requesting `{url}`"))?; + + response + .json() + .await + .wrap_err_with(|| format!("deserializing response from `{url}`")) +} + +struct DatasetSeed; + +impl<'de> DeserializeSeed<'de> for DatasetSeed { + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(DatasetVisitor) + } +} + +struct DatasetVisitor; + +impl<'de> Visitor<'de> for DatasetVisitor { + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut summary = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if summary.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + summary = Some(map.next_value_seed(ProjectsSeed)?); + } else { + map.next_value::()?; + } + } + + summary.ok_or_else(|| serde::de::Error::missing_field("projects")) + } +} + +struct ProjectsSeed; + +impl<'de> DeserializeSeed<'de> for ProjectsSeed { + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProjectsVisitor) + } +} + +struct ProjectsVisitor; + +impl<'de> Visitor<'de> for ProjectsVisitor { + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut summary = BatchSummary::new(); + while let Some(project) = sequence.next_element::()? { + summary.add_project(&project); + } + Ok(summary) + } } diff --git a/apps/labrinth/src/routes/v3/mod.rs b/apps/labrinth/src/routes/v3/mod.rs index 1b3d952881..e3bc46858d 100644 --- a/apps/labrinth/src/routes/v3/mod.rs +++ b/apps/labrinth/src/routes/v3/mod.rs @@ -102,6 +102,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { payouts::platform_revenue, projects::project_get, projects::project_get_check, + projects::validate::validate, projects::project_delete, projects::project_edit, projects::project_icon_edit, diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 88a5608630..d92ec20b70 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -1,23 +1,86 @@ use actix_web::{HttpRequest, get, web}; +use eyre::eyre; +use serde::Serialize; use xredis::RedisPool; -use crate::{database::PgPool, routes::ApiError}; +use crate::auth::get_user_from_headers; +use crate::database::{PgPool, ReadOnlyPgPool, models as db_models}; +use crate::models::pats::Scopes; +use crate::models::projects::{Project, Version}; +use crate::models::teams::ProjectPermissions; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; +use crate::util::error::Context as _; +use crate::validate::project::{ProjectNag, validate as validate_project}; -pub fn config(cfg: &mut actix_web::web::ServiceConfig) { - cfg.service(validate); +#[derive(Serialize, utoipa::ToSchema)] +pub struct ProjectValidationResponse { + pub nags: Vec, } -#[derive(Debug)] -pub enum ValidateError {} - /// Validate that a project is ready to be submitted for review. -#[utoipa::path(tag = "projects")] +#[utoipa::path( + context_path = "/project", + tag = "projects", + responses((status = OK, body = ProjectValidationResponse)) +)] #[get("/{id}/validate")] pub async fn validate( - req: HttpRequest, - info: web::Path<(String,)>, - pool: web::Data, - redis: web::Data, -) -> Result { - Ok(subsecond::call(|| String::from("foobar world"))) + req: HttpRequest, + info: web::Path<(String,)>, + pool: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PROJECT_READ, + ) + .await + .wrap_auth_err("authenticating API request")? + .1; + + let project = + db_models::DBProject::get(&info.into_inner().0, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project from database")? + .wrap_not_found_err("resource not found")?; + + let (team_member, organization_team_member) = + db_models::DBTeamMember::get_for_project_permissions( + &project.inner, + user.id.into(), + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project permissions")?; + + if ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member, + &organization_team_member, + ) + .is_none() + { + return Err(ApiError::Auth(eyre!( + "you do not have permission to validate this project" + ))); + } + + let versions = + db_models::DBVersion::get_many(&project.versions, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project versions from database")? + .into_iter() + .map(Version::from) + .collect::>(); + let project = Project::from(project); + + Ok(web::Json(ProjectValidationResponse { + nags: validate_project(&project, &versions), + })) } diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index ee5e6f49ee..6a8d63eb22 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -1,8 +1,8 @@ use super::text::{ - contains_banned_description_link, contains_spam, - extract_description_blocks, extract_description_text, - has_image_without_alt_text, has_long_header, is_likely_english, - non_standard_text_ratio, profanity_count, + contains_banned_description_link, contains_spam, + extract_description_blocks, extract_description_text, + has_image_without_alt_text, has_long_header, has_sufficient_english_blocks, + non_standard_text_ratio, profanity_count, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::exp::minecraft::Language; @@ -13,80 +13,79 @@ const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let description = project.description.trim(); - let text = extract_description_text(description); + let mut nags = Vec::new(); + let description = project.description.trim(); + let text = extract_description_text(description); + let blocks = extract_description_blocks(description); - if profanity_count(description) > MAX_PROFANITY_COUNT { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionProfanity, - ProjectNagSeverity::Required, - )); - } - if non_standard_text_ratio(description) - >= NON_STANDARD_TEXT_FAILURE_THRESHOLD - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && text.chars().count() >= MIN_DESCRIPTION_CHARS - && !contains_spam(&text) - && extract_description_blocks(description) - .iter() - .any(|block| !is_likely_english(block)) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if description.is_empty() { - nags.push(ProjectNag::new( - ProjectNagKind::AddDescription, - ProjectNagSeverity::Required, - )); - } else if text.chars().count() < MIN_DESCRIPTION_CHARS { - nags.push(ProjectNag::new( - ProjectNagKind::DescriptionTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(&text) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionSpam, - ProjectNagSeverity::Required, - )); - } - if contains_banned_description_link(description) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionBannedLink, - ProjectNagSeverity::Required, - )); - } - if has_long_header(description) { - nags.push(ProjectNag::new( - ProjectNagKind::LongHeaders, - ProjectNagSeverity::Required, - )); - } - if has_image_without_alt_text(description) { - nags.push(ProjectNag::new( - ProjectNagKind::MissingAltText, - ProjectNagSeverity::Warning, - )); - } + if profanity_count(description) > MAX_PROFANITY_COUNT { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionProfanity, + ProjectNagSeverity::Required, + )); + } + if non_standard_text_ratio(description) + >= NON_STANDARD_TEXT_FAILURE_THRESHOLD + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && text.chars().count() >= MIN_DESCRIPTION_CHARS + && !contains_spam(&text) + && !has_sufficient_english_blocks(&blocks) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if description.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::AddDescription, + ProjectNagSeverity::Required, + )); + } else if text.chars().count() < MIN_DESCRIPTION_CHARS { + nags.push(ProjectNag::new( + ProjectNagKind::DescriptionTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(&text) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionSpam, + ProjectNagSeverity::Required, + )); + } + if contains_banned_description_link(description) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionBannedLink, + ProjectNagSeverity::Required, + )); + } + if has_long_header(description) { + nags.push(ProjectNag::new( + ProjectNagKind::LongHeaders, + ProjectNagSeverity::Required, + )); + } + if has_image_without_alt_text(description) { + nags.push(ProjectNag::new( + ProjectNagKind::MissingAltText, + ProjectNagSeverity::Warning, + )); + } - nags + nags } fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) } diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 2011c584f4..769b7f9ca7 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -18,7 +18,17 @@ mod text; mod versions; #[derive( - Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + strum::EnumIter, + utoipa::ToSchema, )] #[serde(rename_all = "snake_case")] pub enum ProjectNagKind { diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 22da89221b..41ca169f07 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,6 +1,6 @@ use super::text::{ - contains_link_or_ip, contains_profanity, contains_spam, - has_non_standard_text, has_summary_formatting, is_likely_english, + contains_link_or_ip, contains_profanity, contains_spam, + has_non_standard_text, has_summary_formatting, is_likely_english_summary, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::exp::minecraft::Language; @@ -9,87 +9,87 @@ use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let summary = project.summary.trim(); - let contains_link = contains_link_or_ip(summary); + let mut nags = Vec::new(); + let summary = project.summary.trim(); + let contains_link = contains_link_or_ip(summary); - if contains_profanity(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryProfanity, - ProjectNagSeverity::Required, - )); - } - if has_non_standard_text(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && summary.chars().count() >= MIN_SUMMARY_CHARS - && !contains_link - && !contains_spam(summary) - && !is_likely_english(summary) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if !summary.is_empty() - && !contains_link - && normalized_without_whitespace(summary) - == normalized_without_whitespace(&project.name) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryMatchesTitle, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() - && !contains_link - && summary.chars().count() < MIN_SUMMARY_CHARS - { - nags.push(ProjectNag::new( - ProjectNagKind::SummaryTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummarySpam, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && has_summary_formatting(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::SummarySpecialFormatting, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && contains_link { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryLinks, - ProjectNagSeverity::Required, - )); - } + if contains_profanity(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryProfanity, + ProjectNagSeverity::Required, + )); + } + if has_non_standard_text(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && summary.chars().count() >= MIN_SUMMARY_CHARS + && !contains_link + && !contains_spam(summary) + && !is_likely_english_summary(summary) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonEnglish, + ProjectNagSeverity::Warning, + )); + } + if !summary.is_empty() + && !contains_link + && normalized_without_whitespace(summary) + == normalized_without_whitespace(&project.name) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryMatchesTitle, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() + && !contains_link + && summary.chars().count() < MIN_SUMMARY_CHARS + { + nags.push(ProjectNag::new( + ProjectNagKind::SummaryTooShort, + ProjectNagSeverity::Required, + )); + } + if contains_spam(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummarySpam, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && has_summary_formatting(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::SummarySpecialFormatting, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && contains_link { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryLinks, + ProjectNagSeverity::Required, + )); + } - nags + nags } fn normalized_without_whitespace(text: &str) -> String { - text.trim() - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() + text.trim() + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() } fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) + project.components.minecraft_java_server.is_none() + || project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)) } diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 4776037011..76fa2b9547 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,351 +1,389 @@ -use std::collections::HashMap; +use std::collections::BTreeSet; use std::sync::LazyLock; use censor::Censor; +use linkify::{LinkFinder, LinkKind}; use regex::Regex; use url::Url; +use whatlang::{Detector, Lang}; static PROFANITY_CENSOR: LazyLock = - LazyLock::new(|| Censor::Standard + Censor::Sex); + LazyLock::new(|| Censor::Standard + Censor::Sex); static WORD: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); -static LINK_OR_IP: LazyLock = LazyLock::new(|| { - Regex::new(concat!( - r"(?i)(?:https?://|www\.)[^\s<>()]+|", - r"\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+\b|", - r"\b(?:\d{1,3}\.){3}\d{1,3}\b", - )) - .unwrap() + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); +static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder }); +static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = - LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); static CODE_BLOCK: LazyLock = - LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); + LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); static INLINE_CODE: LazyLock = - LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); + LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); static MARKDOWN_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { - Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() + Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() }); static DESCRIPTION_LINK: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() + Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() }); static HEADER: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); + LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static INLINE_MARKDOWN: LazyLock = LazyLock::new(|| { - Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() + Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() }); const URL_SHORTENERS: &[&str] = - &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; pub(super) fn contains_profanity(text: &str) -> bool { - PROFANITY_CENSOR.check(text) + PROFANITY_CENSOR.check(text) } pub(super) fn profanity_count(text: &str) -> usize { - PROFANITY_CENSOR.count(text) + PROFANITY_CENSOR.count(text) } pub(super) fn has_non_standard_text(text: &str) -> bool { - non_standard_character_count(text) > 0 + non_standard_character_count(text) > 0 } pub(super) fn non_standard_text_ratio(text: &str) -> f64 { - let character_count = text.chars().count(); - if character_count == 0 { - return 0.0; - } + let character_count = text.chars().count(); + if character_count == 0 { + return 0.0; + } - non_standard_character_count(text) as f64 / character_count as f64 + non_standard_character_count(text) as f64 / character_count as f64 } fn non_standard_character_count(text: &str) -> usize { - let mut count = 0; - let mut combining_marks = 0; - let mut has_base = false; - - for character in text.chars() { - let code = character as u32; - let is_combining = is_in_ranges( - code, - &[ - (0x0300, 0x036f), - (0x1ab0, 0x1aff), - (0x1dc0, 0x1dff), - (0x20d0, 0x20ff), - (0xfe20, 0xfe2f), - ], - ); - if is_combining { - combining_marks += 1; - if !has_base || combining_marks > 2 { - count += 1; - } - continue; - } - - combining_marks = 0; - has_base = !character.is_whitespace(); - let allowed_control = matches!(character, '\n' | '\r' | '\t'); - let control = character.is_control() && !allowed_control; - let private_use = is_in_ranges( - code, - &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], - ); - let invisible = is_in_ranges( - code, - &[ - (0x00ad, 0x00ad), - (0x061c, 0x061c), - (0x200e, 0x200f), - (0x202a, 0x202e), - (0x2060, 0x206f), - (0xfeff, 0xfeff), - ], - ); - let fancy = is_in_ranges( - code, - &[ - (0x02b0, 0x02ff), - (0x1d400, 0x1d7ff), - (0x2460, 0x24ff), - (0x2070, 0x209f), - (0x2100, 0x214f), - (0xfb00, 0xfb06), - (0xff10, 0xff19), - (0xff21, 0xff3a), - (0xff41, 0xff5a), - (0x1f100, 0x1f1ad), - ], - ) && !matches!( - code, - 0x02d6 - | 0x02d7 - | 0x02d8 - | 0x02d9 - | 0x02da - | 0x02db - | 0x02dc - | 0x02dd - | 0x207a - | 0x207b - | 0x208a - | 0x208b - | 0x2120 - | 0x2122 - | 0x2139 - ); - - if control || private_use || invisible || fancy { - count += 1; - } - } - - count + let mut count = 0; + let mut combining_marks = 0; + let mut has_base = false; + + for character in text.chars() { + let code = character as u32; + let is_combining = is_in_ranges( + code, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ); + if is_combining { + combining_marks += 1; + if !has_base || combining_marks > 2 { + count += 1; + } + continue; + } + + combining_marks = 0; + has_base = !character.is_whitespace(); + let allowed_control = matches!(character, '\n' | '\r' | '\t'); + let control = character.is_control() && !allowed_control; + let private_use = is_in_ranges( + code, + &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], + ); + let invisible = is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x061c, 0x061c), + (0x200e, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0xfeff, 0xfeff), + ], + ); + let fancy = is_in_ranges( + code, + &[ + (0x02b0, 0x02ff), + (0x1d400, 0x1d7ff), + (0x2460, 0x24ff), + (0x2070, 0x209f), + (0x2100, 0x214f), + (0xfb00, 0xfb06), + (0xff10, 0xff19), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0x1f100, 0x1f1ad), + ], + ) && !matches!( + code, + 0x02d6 + | 0x02d7 | 0x02d8 + | 0x02d9 | 0x02da + | 0x02db | 0x02dc + | 0x02dd | 0x207a + | 0x207b | 0x208a + | 0x208b | 0x2120 + | 0x2122 | 0x2139 + ); + + if control || private_use || invisible || fancy { + count += 1; + } + } + + count } fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { - ranges - .iter() - .any(|(start, end)| code >= *start && code <= *end) + ranges + .iter() + .any(|(start, end)| code >= *start && code <= *end) } pub(super) fn contains_spam(text: &str) -> bool { - let normalized = text.to_lowercase(); - let mut previous = None; - let mut repeated = 0; - for character in normalized.chars() { - if !character.is_whitespace() && previous == Some(character) { - repeated += 1; - } else { - repeated = usize::from(!character.is_whitespace()); - previous = Some(character); - } - if repeated >= 8 { - return true; - } - } - - let words = WORD - .find_iter(&normalized) - .map(|word| word.as_str()) - .collect::>(); - let mut repeated_words = 1; - for pair in words.windows(2) { - repeated_words = if pair[0] == pair[1] { - repeated_words + 1 - } else { - 1 - }; - if repeated_words >= 4 { - return true; - } - } - - let max_phrase_words = 8.min(words.len() / 3); - for phrase_words in (2..=max_phrase_words).rev() { - for phrases in words.windows(phrase_words * 3) { - if phrases[..phrase_words] - == phrases[phrase_words..phrase_words * 2] - && phrases[..phrase_words] - == phrases[phrase_words * 2..phrase_words * 3] - { - return true; - } - } - } - - false + let normalized = text.to_lowercase(); + let mut previous = None; + let mut repeated = 0; + for character in normalized.chars() { + if !character.is_whitespace() && previous == Some(character) { + repeated += 1; + } else { + repeated = usize::from(!character.is_whitespace()); + previous = Some(character); + } + if repeated >= 8 { + return true; + } + } + + let words = WORD + .find_iter(&normalized) + .map(|word| word.as_str()) + .collect::>(); + let mut repeated_words = 1; + for pair in words.windows(2) { + repeated_words = if pair[0] == pair[1] { + repeated_words + 1 + } else { + 1 + }; + if repeated_words >= 4 { + return true; + } + } + + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for phrases in words.windows(phrase_words * 3) { + if phrases[..phrase_words] + == phrases[phrase_words..phrase_words * 2] + && phrases[..phrase_words] + == phrases[phrase_words * 2..phrase_words * 3] + { + return true; + } + } + } + + false } pub(super) fn contains_link_or_ip(text: &str) -> bool { - LINK_OR_IP.is_match(text) + SUMMARY_LINK_FINDER.links(text).any(|link| { + let raw = link.as_str(); + if raw.contains("://") { + return true; + } + + Url::parse(&format!("https://{raw}")).is_ok_and(|url| { + url.host_str().is_some_and(|hostname| { + psl::domain(hostname.as_bytes()) + .is_some_and(|domain| domain.suffix().typ().is_some()) + }) + }) + }) } pub(super) fn has_summary_formatting(summary: &str) -> bool { - HTML_TAG.is_match(summary) - || MARKDOWN_LINK.is_match(summary) - || summary.lines().any(|line| { - let line = line.trim_start(); - line.starts_with('#') - || line.starts_with('>') - || line.starts_with("- ") - || line.starts_with("* ") - || line.starts_with("+ ") - || line.starts_with("```") - || line.contains("**") - || line.contains("__") - || line.contains("~~") - || INLINE_CODE.is_match(line) - }) + HTML_TAG.is_match(summary) + || MARKDOWN_LINK.is_match(summary) + || summary.lines().any(|line| { + let line = line.trim_start(); + line.starts_with('#') + || line.starts_with('>') + || line.starts_with("- ") + || line.starts_with("* ") + || line.starts_with("+ ") + || line.starts_with("```") + || line.contains("**") + || line.contains("__") + || line.contains("~~") + || INLINE_CODE.is_match(line) + }) } pub(super) fn extract_description_text(markdown: &str) -> String { - let without_code = CODE_BLOCK.replace_all(markdown, " "); - let without_code = INLINE_CODE.replace_all(&without_code, " "); - let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); - let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); - let without_html = HTML_TAG.replace_all(&without_links, " "); - without_html - .replace(['*', '_', '~', '`', '>', '-', '|'], " ") - .split_whitespace() - .collect::>() - .join(" ") + let without_code = CODE_BLOCK.replace_all(markdown, " "); + let without_code = INLINE_CODE.replace_all(&without_code, " "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); + let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); + let without_html = HTML_TAG.replace_all(&without_links, " "); + without_html + .replace(['*', '_', '~', '`', '>', '-', '|'], " ") + .split_whitespace() + .collect::>() + .join(" ") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - CODE_BLOCK - .replace_all(markdown, "") - .split("\n\n") - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + CODE_BLOCK + .replace_all(markdown, "") + .split("\n\n") + .map(extract_description_text) + .filter(|block| !block.is_empty()) + .collect() } pub(super) fn has_long_header(markdown: &str) -> bool { - HEADER.captures_iter(markdown).any(|captures| { - INLINE_MARKDOWN - .replace_all(&captures[1], "$1") - .trim() - .chars() - .count() - > 80 - }) || HTML_HEADER.captures_iter(markdown).any(|captures| { - HTML_TAG - .replace_all(&captures[1], " ") - .split_whitespace() - .collect::>() - .join(" ") - .chars() - .count() - > 80 - }) + HEADER.captures_iter(markdown).any(|captures| { + INLINE_MARKDOWN + .replace_all(&captures[1], "$1") + .trim() + .chars() + .count() > 80 + }) || HTML_HEADER.captures_iter(markdown).any(|captures| { + HTML_TAG + .replace_all(&captures[1], " ") + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .count() > 80 + }) } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { - let without_code = CODE_BLOCK.replace_all(markdown, ""); - let without_code = INLINE_CODE.replace_all(&without_code, ""); - MARKDOWN_IMAGE - .captures_iter(&without_code) - .any(|captures| captures[1].trim().is_empty()) - || HTML_IMAGE.find_iter(&without_code).any(|image| { - ALT_ATTRIBUTE - .captures(image.as_str()) - .and_then(|captures| { - captures.get(1).or_else(|| captures.get(2)) - }) - .is_none_or(|alt| alt.as_str().trim().is_empty()) - }) + let without_code = CODE_BLOCK.replace_all(markdown, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + MARKDOWN_IMAGE + .captures_iter(&without_code) + .any(|captures| captures[1].trim().is_empty()) + || HTML_IMAGE.find_iter(&without_code).any(|image| { + ALT_ATTRIBUTE + .captures(image.as_str()) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .is_none_or(|alt| alt.as_str().trim().is_empty()) + }) } pub(super) fn contains_banned_description_link(markdown: &str) -> bool { - DESCRIPTION_LINK.find_iter(markdown).any(|link| { - let raw = link.as_str(); - let normalized = if raw.to_ascii_lowercase().starts_with("www.") { - format!("https://{raw}") - } else { - raw.to_owned() - }; - Url::parse(&normalized).is_ok_and(|url| { - url.host_str().is_some_and(|hostname| { - URL_SHORTENERS - .iter() - .any(|domain| hostname_matches_domain(hostname, domain)) - }) - }) - }) + DESCRIPTION_LINK.find_iter(markdown).any(|link| { + let raw = link.as_str(); + let normalized = if raw.to_ascii_lowercase().starts_with("www.") { + format!("https://{raw}") + } else { + raw.to_owned() + }; + Url::parse(&normalized).is_ok_and(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) + }) + }) } fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { - hostname.eq_ignore_ascii_case(domain) - || hostname - .to_ascii_lowercase() - .ends_with(&format!(".{domain}")) + hostname.eq_ignore_ascii_case(domain) + || hostname + .to_ascii_lowercase() + .ends_with(&format!(".{domain}")) +} + +pub(super) fn is_likely_english_summary(text: &str) -> bool { + if !has_enough_language_content(text) { + return true; + } + + LANGUAGE_DETECTOR + .detect(text) + .is_none_or(|info| info.lang() == Lang::Eng || info.confidence() < 0.5) +} + +pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { + let mut english_chunks = 0; + let mut non_english_chunks = 0; + + for block in blocks { + for chunk in language_chunks(block) { + let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { + continue; + }; + + if info.lang() == Lang::Eng { + english_chunks += 1; + } else if info.confidence() >= 0.8 { + non_english_chunks += 1; + } + } + } + + let classified_chunks = english_chunks + non_english_chunks; + classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 +} + +fn language_chunks(block: &str) -> Vec { + const CHUNK_WORDS: usize = 24; + const CHUNK_STRIDE_WORDS: usize = 12; + + let words = WORD + .find_iter(block) + .map(|word| word.as_str()) + .collect::>(); + if words.len() < 8 { + return Vec::new(); + } + if words.len() <= CHUNK_WORDS { + let chunk = words.join(" "); + return has_enough_language_content(&chunk) + .then_some(chunk) + .into_iter() + .collect(); + } + + let mut starts = BTreeSet::new(); + let mut start = 0; + while start + 8 <= words.len() { + starts.insert(start); + start += CHUNK_STRIDE_WORDS; + } + starts.insert(words.len() - CHUNK_WORDS); + + starts + .into_iter() + .map(|start| { + words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") + }) + .filter(|chunk| has_enough_language_content(chunk)) + .collect() } -pub(super) fn is_likely_english(text: &str) -> bool { - let words = WORD.find_iter(text).count(); - if words < 8 || text.chars().count() < 35 { - return true; - } - - let mut scripts = HashMap::<&'static str, usize>::new(); - for character in text.chars().filter(|character| character.is_alphabetic()) - { - let code = character as u32; - let script = if code <= 0x024f { - "latin" - } else if is_in_ranges(code, &[(0x0400, 0x052f)]) { - "cyrillic" - } else if is_in_ranges(code, &[(0x0370, 0x03ff)]) { - "greek" - } else if is_in_ranges( - code, - &[ - (0x3040, 0x30ff), - (0x3400, 0x4dbf), - (0x4e00, 0x9fff), - (0xac00, 0xd7af), - ], - ) { - "cjk" - } else { - "other" - }; - *scripts.entry(script).or_default() += 1; - } - - let total = scripts.values().sum::(); - total == 0 - || scripts.get("latin").copied().unwrap_or_default() * 10 >= total * 7 +fn has_enough_language_content(text: &str) -> bool { + WORD.find_iter(text).count() >= 8 && text.trim().chars().count() >= 35 } From 71d95a3b7e570596e25393457ea7f65a26ba8507 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:42:07 +0100 Subject: [PATCH 07/28] use rustrict --- Cargo.lock | 56 +++++++++++++++++++--- Cargo.toml | 1 + apps/labrinth/Cargo.toml | 1 + apps/labrinth/src/validate/project/text.rs | 35 +++++++++++--- 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e65a6d0cc1..05000f2cf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -585,7 +585,7 @@ dependencies = [ "futures-core", "libc", "portable-atomic", - "rustc-hash", + "rustc-hash 2.1.1", "tokio", "tokio-stream", "xattr", @@ -1440,7 +1440,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex", "syn 2.0.106", ] @@ -3020,7 +3020,7 @@ dependencies = [ "futures-util", "generational-box", "longest-increasing-subsequence", - "rustc-hash", + "rustc-hash 2.1.1", "rustversion", "serde", "slab", @@ -3075,7 +3075,7 @@ dependencies = [ "futures-util", "generational-box", "parking_lot", - "rustc-hash", + "rustc-hash 2.1.1", "tracing", "warnings", ] @@ -3216,6 +3216,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + [[package]] name = "docker_credential" version = "1.3.2" @@ -3708,6 +3714,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + [[package]] name = "flate2" version = "1.1.4" @@ -5337,6 +5349,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -5622,6 +5643,7 @@ dependencies = [ "rust_decimal", "rust_iso3166", "rustls 0.23.32", + "rustrict", "rusty-money", "scalar_api_reference", "sentry", @@ -8156,7 +8178,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.32", "socket2 0.6.5", "thiserror 2.0.17", @@ -8176,7 +8198,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.32", "rustls-pki-types", "slab", @@ -8968,6 +8990,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -9147,6 +9175,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustrict" +version = "0.7.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa26cdd448930328cd86e35244d8c7c8e3368e8b8e2fb444befd65419f5b3771" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "doc-comment", + "finl_unicode", + "itertools 0.10.5", + "lazy_static", + "rustc-hash 1.1.0", + "unicode-normalization", +] + [[package]] name = "rustversion" version = "1.0.22" diff --git a/Cargo.toml b/Cargo.toml index 96da8f469d..2d45e48345 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,6 +165,7 @@ reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" +rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } rust-s3 = { version = "0.37.0", default-features = false, features = [ "fail-on-err", "tags", diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 6369ea7629..71b4f567b1 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -91,6 +91,7 @@ regex = { workspace = true } reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } +rustrict = { workspace = true } rustls.workspace = true rusty-money = { workspace = true } scalar_api_reference = { workspace = true, features = ["actix-web"] } diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 76fa2b9547..e9375de9f5 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,15 +1,12 @@ use std::collections::BTreeSet; use std::sync::LazyLock; -use censor::Censor; use linkify::{LinkFinder, LinkKind}; use regex::Regex; +use rustrict::{Censor, Type}; use url::Url; use whatlang::{Detector, Lang}; -static PROFANITY_CENSOR: LazyLock = - LazyLock::new(|| Censor::Standard + Censor::Sex); - static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { @@ -49,11 +46,37 @@ const URL_SHORTENERS: &[&str] = &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; pub(super) fn contains_profanity(text: &str) -> bool { - PROFANITY_CENSOR.check(text) + let mut censor = Censor::from_str(text); + censor.with_ignore_self_censoring(true); + censor.analyze().is(profanity_types()) } pub(super) fn profanity_count(text: &str) -> usize { - PROFANITY_CENSOR.count(text) + // Rustrict only exposes match counts through its tracing features. + const CENSORED: char = '\0'; + + let threshold = profanity_types(); + let mut censor = Censor::from_str(text); + censor + .with_ignore_self_censoring(true) + .with_censor_threshold(threshold) + .with_censor_first_character_threshold(threshold) + .with_censor_replacement(CENSORED); + + let mut count = 0; + let mut in_censored_text = false; + for character in censor.censor().chars() { + let is_censored = character == CENSORED; + if is_censored && !in_censored_text { + count += 1; + } + in_censored_text = is_censored; + } + count +} + +fn profanity_types() -> Type { + Type::PROFANE & Type::MODERATE_OR_HIGHER } pub(super) fn has_non_standard_text(text: &str) -> bool { From 2037c3425c7e1023fdeda206e17d7dd843994cb4 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:26:15 +0100 Subject: [PATCH 08/28] don't allow submitting a project if it fails validation --- apps/labrinth/src/bin/check_project.rs | 170 ++++++- .../src/bin/check_project/profanity.rs | 446 ++++++++++++++++++ .../src/database/models/project_item.rs | 34 +- .../src/database/models/version_item.rs | 26 +- apps/labrinth/src/routes/v3/projects/mod.rs | 198 +++++++- apps/labrinth/src/validate/project/mod.rs | 9 +- .../labrinth/src/validate/project/versions.rs | 47 +- apps/labrinth/tests/project.rs | 46 ++ packages/xredis/src/lib.rs | 76 +++ 9 files changed, 1008 insertions(+), 44 deletions(-) create mode 100644 apps/labrinth/src/bin/check_project/profanity.rs diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index 2ad4ea3eea..fe8cde8932 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use std::fmt; use std::fs::File; -use std::io::BufReader; +use std::io::{BufReader, IsTerminal}; use std::path::{Path, PathBuf}; use clap::Parser; @@ -18,6 +18,9 @@ use serde::de::{ use strum::IntoEnumIterator; use url::Url; +#[path = "check_project/profanity.rs"] +mod profanity; + const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; #[derive(Parser)] @@ -29,18 +32,52 @@ struct Args { /// The ID of the project to validate #[arg( value_name = "PROJECT_ID", - required_unless_present = "file", - conflicts_with = "file" + required_unless_present_any = ["file", "read_profanity_report"], + conflicts_with_all = ["file", "read_profanity_report"] )] project_id: Option, /// Validate all projects in a moderation project dataset - #[arg(long, value_name = "PATH", conflicts_with = "token")] + #[arg( + long, + value_name = "PATH", + conflicts_with_all = ["token", "read_profanity_report"] + )] file: Option, /// A Modrinth token to send as bearer authentication #[arg(long, value_name = "TOKEN")] token: Option, + + /// Show this many descriptions flagged for profanity + #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] + show_description_profanity: usize, + + /// Scan every profanity-bearing field into a reusable JSON report + #[arg( + long, + value_name = "PATH", + requires = "file", + conflicts_with = "show_description_profanity" + )] + write_profanity_report: Option, + + /// Render a previously generated profanity report + #[arg(long, value_name = "PATH", conflicts_with = "file")] + read_profanity_report: Option, + + /// Render at most this many projects from a profanity report (0 means all) + #[arg( + long, + value_name = "COUNT", + default_value_t = 0, + requires = "read_profanity_report" + )] + profanity_report_limit: usize, + + /// Use bracket markers instead of ANSI colors when rendering a report + #[arg(long, requires = "read_profanity_report")] + profanity_report_brackets: bool, } #[derive(Serialize)] @@ -49,15 +86,27 @@ struct BatchSummary { projects_with_at_least_one_required_nag: usize, version_details_available: bool, nag_counts: BTreeMap, + #[serde(skip)] + description_profanity_sample_limit: usize, + #[serde(skip)] + description_profanity_samples: Vec, +} + +struct DescriptionProfanitySample { + id: String, + name: String, + description: String, } impl BatchSummary { - fn new() -> Self { + fn new(description_profanity_sample_limit: usize) -> Self { Self { projects: 0, projects_with_at_least_one_required_nag: 0, version_details_available: false, nag_counts: ProjectNagKind::iter().map(|kind| (kind, 0)).collect(), + description_profanity_sample_limit, + description_profanity_samples: Vec::new(), } } @@ -70,6 +119,19 @@ impl BatchSummary { { self.projects_with_at_least_one_required_nag += 1; } + if self.description_profanity_samples.len() + < self.description_profanity_sample_limit + && nags.iter().any(|nag| { + nag.kind == ProjectNagKind::ProjectDescriptionProfanity + }) { + self.description_profanity_samples.push( + DescriptionProfanitySample { + id: project.id.to_string(), + name: project.name.clone(), + description: project.description.clone(), + }, + ); + } for nag in nags { *self.nag_counts.entry(nag.kind).or_default() += 1; } @@ -81,8 +143,26 @@ async fn main() -> Result<()> { color_eyre::install().wrap_err("installing color-eyre")?; let args = Args::parse(); - if let Some(path) = args.file { - let summary = summarize_file(&path)?; + if let Some(report_path) = args.read_profanity_report.as_deref() { + return profanity::render_report( + report_path, + args.profanity_report_limit, + args.profanity_report_brackets, + ); + } + + if let Some(report_path) = args.write_profanity_report.as_deref() { + let dataset_path = args.file.as_deref().ok_or_else(|| { + eyre!("`--write-profanity-report` requires `--file`") + })?; + return profanity::write_report(dataset_path, report_path); + } + + if let Some(path) = args.file.as_deref() { + let summary = summarize_file(path, args.show_description_profanity)?; + print_description_profanity_samples( + &summary.description_profanity_samples, + ); print_json(&summary)?; return Ok(()); } @@ -119,20 +199,54 @@ async fn check_api_project( print_json(&validate(&project, &versions)) } -fn summarize_file(path: &Path) -> Result { +fn summarize_file( + path: &Path, + description_profanity_sample_limit: usize, +) -> Result { let file = File::open(path) .wrap_err_with(|| format!("opening `{}`", path.display()))?; let mut deserializer = serde_json::Deserializer::from_reader(BufReader::new(file)); - let summary = DatasetSeed - .deserialize(&mut deserializer) - .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let summary = DatasetSeed { + description_profanity_sample_limit, + } + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; deserializer .end() .wrap_err_with(|| format!("reading `{}`", path.display()))?; Ok(summary) } +fn print_description_profanity_samples(samples: &[DescriptionProfanitySample]) { + if samples.is_empty() { + return; + } + + let use_color = std::io::stderr().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + eprintln!("\n=== Description profanity samples ==="); + if use_color { + eprintln!( + "Detected text is shown with a bold white-on-red background." + ); + } else { + eprintln!("Detected text is enclosed in ⟦double brackets⟧."); + } + + for (index, sample) in samples.iter().enumerate() { + eprintln!("\n--- Sample {} of {} ---", index + 1, samples.len()); + eprintln!("Project: {}", sample.name); + eprintln!("ID: {}", sample.id); + eprintln!("URL: https://modrinth.com/project/{}", sample.id); + eprintln!( + "\n{}", + profanity::highlight_text(&sample.description, use_color) + ); + } + eprintln!("\n=== End description profanity samples ===\n"); +} + fn print_json(value: &impl Serialize) -> Result<()> { println!( "{}", @@ -172,7 +286,9 @@ async fn fetch( .wrap_err_with(|| format!("deserializing response from `{url}`")) } -struct DatasetSeed; +struct DatasetSeed { + description_profanity_sample_limit: usize, +} impl<'de> DeserializeSeed<'de> for DatasetSeed { type Value = BatchSummary; @@ -181,11 +297,16 @@ impl<'de> DeserializeSeed<'de> for DatasetSeed { where D: serde::Deserializer<'de>, { - deserializer.deserialize_map(DatasetVisitor) + deserializer.deserialize_map(DatasetVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) } } -struct DatasetVisitor; +struct DatasetVisitor { + description_profanity_sample_limit: usize, +} impl<'de> Visitor<'de> for DatasetVisitor { type Value = BatchSummary; @@ -204,7 +325,10 @@ impl<'de> Visitor<'de> for DatasetVisitor { if summary.is_some() { return Err(serde::de::Error::duplicate_field("projects")); } - summary = Some(map.next_value_seed(ProjectsSeed)?); + summary = Some(map.next_value_seed(ProjectsSeed { + description_profanity_sample_limit: + self.description_profanity_sample_limit, + })?); } else { map.next_value::()?; } @@ -214,7 +338,9 @@ impl<'de> Visitor<'de> for DatasetVisitor { } } -struct ProjectsSeed; +struct ProjectsSeed { + description_profanity_sample_limit: usize, +} impl<'de> DeserializeSeed<'de> for ProjectsSeed { type Value = BatchSummary; @@ -223,11 +349,16 @@ impl<'de> DeserializeSeed<'de> for ProjectsSeed { where D: serde::Deserializer<'de>, { - deserializer.deserialize_seq(ProjectsVisitor) + deserializer.deserialize_seq(ProjectsVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) } } -struct ProjectsVisitor; +struct ProjectsVisitor { + description_profanity_sample_limit: usize, +} impl<'de> Visitor<'de> for ProjectsVisitor { type Value = BatchSummary; @@ -240,7 +371,8 @@ impl<'de> Visitor<'de> for ProjectsVisitor { where A: SeqAccess<'de>, { - let mut summary = BatchSummary::new(); + let mut summary = + BatchSummary::new(self.description_profanity_sample_limit); while let Some(project) = sequence.next_element::()? { summary.add_project(&project); } diff --git a/apps/labrinth/src/bin/check_project/profanity.rs b/apps/labrinth/src/bin/check_project/profanity.rs new file mode 100644 index 0000000000..8d71d0a2ec --- /dev/null +++ b/apps/labrinth/src/bin/check_project/profanity.rs @@ -0,0 +1,446 @@ +use std::fmt; +use std::fs::File; +use std::io::{BufReader, BufWriter, IsTerminal}; +use std::path::Path; + +use eyre::{Result, WrapErr, eyre}; +use labrinth::models::projects::Project; +use rustrict::{Censor, Type}; +use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; + +const REPORT_FORMAT_VERSION: u32 = 1; +const CENSORED: char = '\0'; +const ANSI_START: &str = "\x1b[1;97;41m"; +const ANSI_END: &str = "\x1b[0m"; + +#[derive(Serialize, Deserialize)] +struct ProfanityReport { + format_version: u32, + matcher: String, + projects_scanned: usize, + projects_with_profanity: usize, + fields_with_profanity: usize, + fields_triggering_nags: usize, + projects: Vec, +} + +impl ProfanityReport { + fn new() -> Self { + Self { + format_version: REPORT_FORMAT_VERSION, + matcher: "rustrict: profane and moderate-or-higher".to_owned(), + projects_scanned: 0, + projects_with_profanity: 0, + fields_with_profanity: 0, + fields_triggering_nags: 0, + projects: Vec::new(), + } + } + + fn add_project(&mut self, project: &Project) { + self.projects_scanned += 1; + let mut fields = Vec::new(); + push_field( + &mut fields, + ProfanityFieldKind::ProjectName, + None, + &project.name, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectSummary, + None, + &project.summary, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectDescription, + None, + &project.description, + ); + for (gallery_index, gallery_item) in project.gallery.iter().enumerate() + { + if let Some(name) = &gallery_item.name { + push_field( + &mut fields, + ProfanityFieldKind::GalleryName, + Some(gallery_index), + name, + ); + } + if let Some(description) = &gallery_item.description { + push_field( + &mut fields, + ProfanityFieldKind::GalleryDescription, + Some(gallery_index), + description, + ); + } + } + + if fields.is_empty() { + return; + } + + self.projects_with_profanity += 1; + self.fields_with_profanity += fields.len(); + self.fields_triggering_nags += + fields.iter().filter(|field| field.triggers_nag).count(); + self.projects.push(ProjectProfanityReport { + id: project.id.to_string(), + name: project.name.clone(), + fields, + }); + } +} + +#[derive(Serialize, Deserialize)] +struct ProjectProfanityReport { + id: String, + name: String, + fields: Vec, +} + +#[derive(Serialize, Deserialize)] +struct ProfanityFieldReport { + kind: ProfanityFieldKind, + #[serde(skip_serializing_if = "Option::is_none")] + gallery_index: Option, + text: String, + matches: Vec, + triggers_nag: bool, +} + +#[derive(Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum ProfanityFieldKind { + ProjectName, + ProjectSummary, + ProjectDescription, + GalleryName, + GalleryDescription, +} + +impl ProfanityFieldKind { + fn label(self, gallery_index: Option) -> String { + match (self, gallery_index) { + (Self::ProjectName, _) => "Project name".to_owned(), + (Self::ProjectSummary, _) => "Project summary".to_owned(), + (Self::ProjectDescription, _) => "Project description".to_owned(), + (Self::GalleryName, Some(index)) => { + format!("Gallery image {} name", index + 1) + } + (Self::GalleryDescription, Some(index)) => { + format!("Gallery image {} description", index + 1) + } + (Self::GalleryName, None) => "Gallery image name".to_owned(), + (Self::GalleryDescription, None) => { + "Gallery image description".to_owned() + } + } + } +} + +#[derive(Serialize, Deserialize)] +struct ProfanityMatch { + start_char: usize, + end_char: usize, +} + +fn push_field( + fields: &mut Vec, + kind: ProfanityFieldKind, + gallery_index: Option, + text: &str, +) { + let matches = profanity_matches(text); + if matches.is_empty() { + return; + } + + let triggers_nag = match kind { + ProfanityFieldKind::ProjectDescription => matches.len() > 2, + _ => true, + }; + fields.push(ProfanityFieldReport { + kind, + gallery_index, + text: text.to_owned(), + matches, + triggers_nag, + }); +} + +pub(super) fn write_report( + dataset_path: &Path, + report_path: &Path, +) -> Result<()> { + let report = scan_dataset(dataset_path)?; + let output = File::create(report_path) + .wrap_err_with(|| format!("creating `{}`", report_path.display()))?; + serde_json::to_writer_pretty(BufWriter::new(output), &report) + .wrap_err_with(|| format!("writing `{}`", report_path.display()))?; + + eprintln!( + "wrote {} projects with profanity across {} fields to `{}`", + report.projects_with_profanity, + report.fields_with_profanity, + report_path.display(), + ); + Ok(()) +} + +pub(super) fn render_report( + report_path: &Path, + project_limit: usize, + force_brackets: bool, +) -> Result<()> { + let input = File::open(report_path) + .wrap_err_with(|| format!("opening `{}`", report_path.display()))?; + let report: ProfanityReport = + serde_json::from_reader(BufReader::new(input)) + .wrap_err_with(|| format!("reading `{}`", report_path.display()))?; + if report.format_version != REPORT_FORMAT_VERSION { + return Err(eyre!( + "unsupported profanity report format version `{}`", + report.format_version + )); + } + + let use_color = !force_brackets + && std::io::stdout().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + println!("=== Project profanity report ==="); + println!("Matcher: {}", report.matcher); + println!("Projects scanned: {}", report.projects_scanned); + println!( + "Projects with profanity: {}", + report.projects_with_profanity + ); + println!("Fields with profanity: {}", report.fields_with_profanity); + println!( + "Fields triggering nags: {}", + report.fields_triggering_nags + ); + if use_color { + println!("Detected text uses a bold white-on-red background."); + } else { + println!("Detected text is enclosed in ⟦double brackets⟧."); + } + + let projects = if project_limit == 0 { + report.projects.as_slice() + } else { + &report.projects[..project_limit.min(report.projects.len())] + }; + for (project_index, project) in projects.iter().enumerate() { + println!( + "\n--- Project {} of {} ---", + project_index + 1, + projects.len() + ); + println!("Project: {}", sanitize_text(&project.name)); + println!("ID: {}", project.id); + println!("URL: https://modrinth.com/project/{}", project.id); + + for field in &project.fields { + println!( + "\n[{}; {} match{}; nag: {}]", + field.kind.label(field.gallery_index), + field.matches.len(), + if field.matches.len() == 1 { "" } else { "es" }, + if field.triggers_nag { "yes" } else { "no" }, + ); + println!( + "{}", + highlight_matches(&field.text, &field.matches, use_color) + ); + } + } + Ok(()) +} + +pub(super) fn highlight_text(text: &str, use_color: bool) -> String { + let matches = profanity_matches(text); + highlight_matches(text, &matches, use_color) +} + +fn scan_dataset(path: &Path) -> Result { + let input = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(input)); + let report = ProfanityDatasetSeed + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(report) +} + +fn profanity_matches(text: &str) -> Vec { + let threshold = Type::PROFANE & Type::MODERATE_OR_HIGHER; + let mut censor = Censor::from_str(text); + censor + .with_ignore_self_censoring(true) + .with_censor_threshold(threshold) + .with_censor_first_character_threshold(threshold) + .with_censor_replacement(CENSORED); + let censored = censor.censor().chars().collect::>(); + + let mut matches = Vec::new(); + let mut start = None; + let mut character_count = 0; + for (index, _) in text.chars().enumerate() { + character_count = index + 1; + let is_censored = censored.get(index) == Some(&CENSORED); + match (start, is_censored) { + (None, true) => start = Some(index), + (Some(start_char), false) => { + matches.push(ProfanityMatch { + start_char, + end_char: index, + }); + start = None; + } + _ => {} + } + } + if let Some(start_char) = start { + matches.push(ProfanityMatch { + start_char, + end_char: character_count, + }); + } + matches +} + +fn highlight_matches( + text: &str, + matches: &[ProfanityMatch], + use_color: bool, +) -> String { + let mut output = String::with_capacity(text.len()); + let mut matches = matches.iter().peekable(); + let mut highlighting = false; + for (index, character) in text.chars().enumerate() { + while matches + .peek() + .is_some_and(|matched| index >= matched.end_char) + { + matches.next(); + } + let is_censored = matches.peek().is_some_and(|matched| { + index >= matched.start_char && index < matched.end_char + }); + if is_censored != highlighting { + output.push_str(if is_censored { + if use_color { ANSI_START } else { "⟦" } + } else if use_color { + ANSI_END + } else { + "⟧" + }); + highlighting = is_censored; + } + push_safe_character(&mut output, character); + } + if highlighting { + output.push_str(if use_color { ANSI_END } else { "⟧" }); + } + output +} + +fn sanitize_text(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + for character in text.chars() { + push_safe_character(&mut output, character); + } + output +} + +fn push_safe_character(output: &mut String, character: char) { + match character { + '\n' | '\t' => output.push(character), + '\r' => {} + character if character.is_control() => output.push('�'), + character => output.push(character), + } +} + +struct ProfanityDatasetSeed; + +impl<'de> DeserializeSeed<'de> for ProfanityDatasetSeed { + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(ProfanityDatasetVisitor) + } +} + +struct ProfanityDatasetVisitor; + +impl<'de> Visitor<'de> for ProfanityDatasetVisitor { + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut report = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if report.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + report = Some(map.next_value_seed(ProfanityProjectsSeed)?); + } else { + map.next_value::()?; + } + } + report.ok_or_else(|| serde::de::Error::missing_field("projects")) + } +} + +struct ProfanityProjectsSeed; + +impl<'de> DeserializeSeed<'de> for ProfanityProjectsSeed { + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProfanityProjectsVisitor) + } +} + +struct ProfanityProjectsVisitor; + +impl<'de> Visitor<'de> for ProfanityProjectsVisitor { + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut report = ProfanityReport::new(); + while let Some(project) = sequence.next_element::()? { + report.add_project(&project); + } + Ok(report) + } +} diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index f4d0d3118a..7d6b46afa8 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -576,10 +576,42 @@ impl DBProject { where E: crate::database::Acquire<'a, Database = sqlx::Postgres>, { - let val = redis.get_cached_keys_with_slug( + Self::get_many_inner(project_strings, exec, redis, true).await + } + + pub async fn get_many_uncached< + 'a, + E, + T: Display + Hash + Eq + PartialEq + Clone + Debug, + >( + project_strings: &[T], + exec: E, + redis: &RedisPool, + ) -> Result, ApiError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + Self::get_many_inner(project_strings, exec, redis, false).await + } + + async fn get_many_inner< + 'a, + E, + T: Display + Hash + Eq + PartialEq + Clone + Debug, + >( + project_strings: &[T], + exec: E, + redis: &RedisPool, + use_cache: bool, + ) -> Result, ApiError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + let val = redis.get_keys_with_slug_cache( PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, false, + use_cache, project_strings, |ids| async move { let mut exec = exec diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index aff56773e5..a54a933ad5 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -595,8 +595,32 @@ impl DBVersion { where E: crate::database::Acquire<'a, Database = sqlx::Postgres>, { - let mut val = redis.get_cached_keys( + Self::get_many_inner(version_ids, exec, redis, true).await + } + + pub async fn get_many_uncached<'a, E>( + version_ids: &[DBVersionId], + exec: E, + redis: &RedisPool, + ) -> Result, DatabaseError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + Self::get_many_inner(version_ids, exec, redis, false).await + } + + async fn get_many_inner<'a, E>( + version_ids: &[DBVersionId], + exec: E, + redis: &RedisPool, + use_cache: bool, + ) -> Result, DatabaseError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + let mut val = redis.get_keys_with_cache( VERSIONS_NAMESPACE, + use_cache, &version_ids.iter().map(|x| x.0).collect::>(), |version_ids| async move { let mut exec = exec.acquire().await?; diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index c85ebae776..25c0e17275 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -6,7 +6,9 @@ use std::collections::HashMap; use crate::auth::checks::{filter_visible_versions, is_visible_project}; use crate::auth::{filter_visible_projects, get_user_from_headers}; use crate::database::models::notification_item::NotificationBuilder; -use crate::database::models::project_item::{DBGalleryItem, DBModCategory}; +use crate::database::models::project_item::{ + DBGalleryItem, DBModCategory, ProjectQueryResult, +}; use crate::database::models::thread_item::ThreadMessageBuilder; use crate::database::models::{ DBModerationLock, DBProjectId, DBTeamMember, ids as db_ids, image_item, @@ -24,10 +26,11 @@ use crate::models::notifications::NotificationBody; use crate::models::pats::Scopes; use crate::models::projects::{ MonetizationStatus, Project, ProjectStatus, SideTypesMigrationReviewStatus, + Version, }; use crate::models::teams::{DEFAULT_ROLE, ProjectPermissions}; use crate::models::threads::MessageBody; -use crate::models::users::DELETED_USER; +use crate::models::users::{DELETED_USER, User}; use crate::models::{self, exp}; use crate::queue::session::AuthQueue; use crate::routes::ApiError; @@ -39,6 +42,7 @@ use crate::util::error::Context; use crate::util::img; use crate::util::img::{delete_old_images, upload_image_optimized}; use crate::util::routes::read_limited_from_payload; +use crate::validate::project::has_required_nags; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web}; use chrono::Utc; use eyre::eyre; @@ -494,6 +498,22 @@ pub async fn project_edit_internal( ))); }; + let submit_for_review = new_project.status + == Some(ProjectStatus::Processing) + && !user.role.is_mod(); + if submit_for_review { + if !perms.contains(ProjectPermissions::EDIT_DETAILS) { + return Err(ApiError::Auth(eyre!( + "you do not have permission to submit this project for review" + ))); + } + if project_item.inner.status.is_approved() { + return Err(ApiError::Auth(eyre!( + "you do not have permission to submit this project for review" + ))); + } + } + let mut transaction = pool .begin() .await @@ -541,7 +561,9 @@ pub async fn project_edit_internal( .wrap_internal_err("querying database for `project_edit_internal`")?; } - if let Some(status) = &new_project.status { + if let Some(status) = &new_project.status + && !submit_for_review + { if !perms.contains(ProjectPermissions::EDIT_DETAILS) { return Err(ApiError::Auth(eyre::eyre!( "You do not have the permissions to edit the status of this project!", @@ -614,12 +636,6 @@ pub async fn project_edit_internal( } if status == &ProjectStatus::Processing { - if project_item.versions.is_empty() { - return Err(ApiError::Request(eyre::eyre!(String::from( - "Project submitted for review with no initial versions", - )))); - } - sqlx::query!( " UPDATE mods @@ -704,7 +720,7 @@ pub async fn project_edit_internal( .ok(); } - if team_member.is_none_or(|x| !x.accepted) { + if team_member.as_ref().is_none_or(|x| !x.accepted) { let notified_members = sqlx::query!( " SELECT tm.user_id id @@ -1342,6 +1358,47 @@ pub async fn project_edit_internal( .await .wrap_api_err("deleting unused images")?; + if submit_for_review { + let mut projects = db_models::DBProject::get_many_uncached( + &[ProjectId::from(id)], + &mut transaction, + &redis, + ) + .await + .wrap_internal_err("reloading project for submission validation")?; + let reloaded_project = + projects.pop().wrap_not_found_err("resource not found")?; + let versions = db_models::DBVersion::get_many_uncached( + &reloaded_project.versions, + &mut transaction, + &redis, + ) + .await + .wrap_internal_err( + "reloading project versions for submission validation", + )? + .into_iter() + .map(Version::from) + .collect::>(); + let project = Project::from(reloaded_project.clone()); + + if has_required_nags(&project, &versions) { + return Err(ApiError::Request(eyre!( + "project must have no required validation nags before being submitted for review" + ))); + } + + submit_project_for_review( + &reloaded_project, + &user, + team_member.as_ref().is_none_or(|member| !member.accepted), + sync_archival_disclosure, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await @@ -1388,6 +1445,127 @@ pub async fn project_edit_internal( Ok(HttpResponse::NoContent().body("")) } +async fn submit_project_for_review( + project: &ProjectQueryResult, + user: &User, + notify_team_members: bool, + sync_archival_disclosure: bool, + transaction: &mut PgTransaction<'_>, + redis: &RedisPool, +) -> Result<(), ApiError> { + let archival_disclosure = + db_models::DBProjectDisclosure::get_many_for_project( + project.inner.id, + false, + &mut *transaction, + ) + .await + .wrap_internal_err("failed to fetch project disclosures")? + .into_iter() + .find(|disclosure| { + matches!(disclosure.disclosure, ProjectDisclosure::Archived { .. }) + }); + + sqlx::query!( + " + UPDATE mods + SET moderation_message = NULL, moderation_message_body = NULL, queued = NOW() + WHERE (id = $1) + ", + project.inner.id as db_ids::DBProjectId, + ) + .execute(&mut *transaction) + .await + .wrap_internal_err("querying database for `project_edit_internal`")?; + + if notify_team_members { + let notified_members = sqlx::query!( + " + SELECT tm.user_id id + FROM team_members tm + WHERE tm.team_id = $1 AND tm.accepted + ", + project.inner.team_id as db_ids::DBTeamId + ) + .fetch(&mut *transaction) + .map_ok(|row| db_models::DBUserId(row.id)) + .try_collect::>() + .await + .wrap_internal_err("fetching notified members from database")?; + + NotificationBuilder { + body: NotificationBody::StatusChange { + project_id: project.inner.id.into(), + old_status: project.inner.status, + new_status: ProjectStatus::Processing, + }, + } + .insert_many(notified_members.clone(), &mut *transaction, redis) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + + NotificationBuilder { + body: NotificationBody::ProjectStatusNeutral { + project_id: project.inner.id.into(), + old_status: project.inner.status, + new_status: ProjectStatus::Processing, + }, + } + .insert_many(notified_members, &mut *transaction, redis) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + } + + ThreadMessageBuilder { + author_id: Some(user.id.into()), + body: MessageBody::StatusChange { + new_status: ProjectStatus::Processing, + old_status: project.inner.status, + }, + thread_id: project.thread_id, + hide_identity: false, + } + .insert(&mut *transaction) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + + sqlx::query!( + " + UPDATE mods + SET status = $1 + WHERE (id = $2) + ", + ProjectStatus::Processing.as_str(), + project.inner.id as db_ids::DBProjectId, + ) + .execute(&mut *transaction) + .await + .wrap_internal_err("querying database for `project_edit_internal`")?; + + if sync_archival_disclosure + && archival_disclosure + .is_some_and(|disclosure| disclosure.lock_status.allows_removal()) + { + db_models::DBProjectDisclosure::remove( + project.inner.id, + ProjectDisclosureType::Archived, + user.id.into(), + false, + &mut *transaction, + ) + .await + .wrap_internal_err("failed to remove archival disclosure")?; + } + + Ok(()) +} + pub async fn edit_project_categories( categories: &Vec, perms: &ProjectPermissions, diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 769b7f9ca7..665bd3684f 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -102,6 +102,7 @@ pub enum ProjectNagKind { // Versions UploadVersion, + SelectEnvironment, // Disclosures CheckDisclosures, @@ -152,11 +153,11 @@ pub fn validate(project: &Project, versions: &[Version]) -> Vec { nags.extend(links::validate(project, versions)); nags.extend(permissions::validate(versions)); + nags.extend(versions::validate(project, versions)); nags.extend( [ server_settings::validate, tags::validate, - versions::validate, disclosures::validate, moderation::validate, ] @@ -165,3 +166,9 @@ pub fn validate(project: &Project, versions: &[Version]) -> Vec { ); nags } + +pub fn has_required_nags(project: &Project, versions: &[Version]) -> bool { + validate(project, versions) + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) +} diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs index 94c65372a9..028bb38ec8 100644 --- a/apps/labrinth/src/validate/project/versions.rs +++ b/apps/labrinth/src/validate/project/versions.rs @@ -1,16 +1,39 @@ +use crate::models::projects::{Project, Version}; + use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &crate::models::projects::Project, -) -> Vec { - if project.versions.is_empty() - && project.components.minecraft_server.is_none() - { - vec![ProjectNag::new( - ProjectNagKind::UploadVersion, - ProjectNagSeverity::Required, - )] - } else { - Vec::new() - } + project: &Project, + versions: &[Version], +) -> Vec { + let mut nags = Vec::new(); + + if project.versions.is_empty() + && project.components.minecraft_server.is_none() + { + nags.push(ProjectNag::new( + ProjectNagKind::UploadVersion, + ProjectNagSeverity::Required, + )); + } + + let requires_environment = project + .project_types + .iter() + .any(|project_type| matches!(project_type.as_str(), "mod" | "modpack")); + if requires_environment + && versions.iter().any(|version| { + version + .fields + .get("environment") + .and_then(serde_json::Value::as_str) + .is_none_or(|environment| environment.trim().is_empty()) + }) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectEnvironment, + ProjectNagSeverity::Required, + )); + } + + nags } diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 239ce885f0..aaed59404d 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -23,6 +23,7 @@ use labrinth::database::models::project_item::{ PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, ProjectQueryResult, }; use labrinth::models::ids::ProjectId; +use labrinth::models::projects::ProjectStatus; use labrinth::models::teams::ProjectPermissions; use labrinth::util::actix::{MultipartSegment, MultipartSegmentData}; use serde_json::json; @@ -501,6 +502,51 @@ pub async fn test_patch_project() { .await; } +#[actix_rt::test] +async fn test_submit_invalid_project_for_review() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "draft" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_before.status, ProjectStatus::Draft); + let original_description = project_before.description; + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": "processing", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_after.status, ProjectStatus::Draft); + assert_eq!(project_after.description, original_description); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v3() { // Hits V3-specific patchable fields diff --git a/packages/xredis/src/lib.rs b/packages/xredis/src/lib.rs index 19d4afce1e..f1d08ba5ab 100644 --- a/packages/xredis/src/lib.rs +++ b/packages/xredis/src/lib.rs @@ -113,6 +113,39 @@ impl RedisPool { .wrap_err("registering Redis blocking pool metrics") } + pub async fn get_keys_with_cache( + &self, + namespace: &str, + use_cache: bool, + keys: &[K], + closure: F, + ) -> Result> + where + F: FnOnce(Vec) -> Fut, + Fut: Future, E>>, + E: std::error::Error + Send + Sync + 'static, + T: Serialize + DeserializeOwned, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize + + Debug, + { + if use_cache { + self.get_cached_keys(namespace, keys, closure).await + } else { + Ok(closure(keys.to_vec()) + .await + .wrap_err("fetching uncached values")? + .into_iter() + .map(|(_, value)| value) + .collect()) + } + } + pub async fn get_cached_keys( &self, namespace: &str, @@ -163,6 +196,49 @@ impl RedisPool { .await } + pub async fn get_keys_with_slug_cache( + &self, + namespace: &str, + slug_namespace: &str, + case_sensitive: bool, + use_cache: bool, + keys: &[I], + closure: F, + ) -> Result> + where + F: FnOnce(Vec) -> Fut, + Fut: Future, T)>, E>>, + E: std::error::Error + Send + Sync + 'static, + T: Serialize + DeserializeOwned, + I: Display + Hash + Eq + PartialEq + Clone + Debug, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize, + S: Display + Clone + DeserializeOwned + Serialize + Debug, + { + if use_cache { + self.get_cached_keys_with_slug( + namespace, + slug_namespace, + case_sensitive, + keys, + closure, + ) + .await + } else { + Ok(closure(keys.to_vec()) + .await + .wrap_err("fetching uncached values by slug")? + .into_iter() + .map(|(_, (_, value))| value) + .collect()) + } + } + pub async fn get_cached_keys_with_slug( &self, namespace: &str, From 9dd4395ae2f08f134d5e29f600ff8584df7195fb Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:40:37 +0100 Subject: [PATCH 09/28] update to latest frontend validators --- Cargo.lock | 2 + Cargo.toml | 2 + apps/labrinth/Cargo.toml | 2 + apps/labrinth/src/bin/check_project.rs | 6 +- apps/labrinth/src/routes/v3/projects/mod.rs | 23 +- .../src/routes/v3/projects/validate.rs | 25 +- .../src/validate/project/description.rs | 201 +++--- .../src/validate/project/disclosures.rs | 57 +- apps/labrinth/src/validate/project/gallery.rs | 95 ++- apps/labrinth/src/validate/project/license.rs | 54 +- apps/labrinth/src/validate/project/links.rs | 72 +- apps/labrinth/src/validate/project/mod.rs | 73 +- apps/labrinth/src/validate/project/name.rs | 46 +- .../src/validate/project/server_settings.rs | 17 +- apps/labrinth/src/validate/project/summary.rs | 222 ++++--- apps/labrinth/src/validate/project/tags.rs | 94 ++- apps/labrinth/src/validate/project/text.rs | 625 ++++++++++++++++-- .../labrinth/src/validate/project/versions.rs | 24 +- 18 files changed, 1293 insertions(+), 347 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0cb2fb7d4..37a9f69e62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5667,6 +5667,8 @@ dependencies = [ "totp-rs", "tracing", "tracing-actix-web", + "unicode-normalization", + "unicode-segmentation", "url", "urlencoding", "utoipa", diff --git a/Cargo.toml b/Cargo.toml index 2d45e48345..61e2156db9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -235,6 +235,8 @@ ts-rs = "12.0.1" typed-path = "0.12.0" url = "2.5.7" urlencoding = "2.1.3" +unicode-normalization = "0.1.24" +unicode-segmentation = "1.12.0" utoipa = { version = "5.4.0", features = ["actix_extras", "chrono", "decimal"] } uuid = "1.18.1" validator = "0.20.0" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 71b4f567b1..4305caf363 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -124,6 +124,8 @@ tracing = { workspace = true } tracing-actix-web = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } +unicode-normalization = { workspace = true } +unicode-segmentation = { workspace = true } utoipa = { workspace = true, features = ["url"] } uuid = { workspace = true, features = ["fast-rng", "serde", "v4", "v7"] } validator = { workspace = true, features = ["derive"] } diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index fe8cde8932..186b9fb5c5 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -122,7 +122,11 @@ impl BatchSummary { if self.description_profanity_samples.len() < self.description_profanity_sample_limit && nags.iter().any(|nag| { - nag.kind == ProjectNagKind::ProjectDescriptionProfanity + matches!( + nag.kind, + ProjectNagKind::ProjectDescriptionSlur + | ProjectNagKind::ProjectDescriptionProfanity + ) }) { self.description_profanity_samples.push( DescriptionProfanitySample { diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index 25c0e17275..819d78ad22 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -42,7 +42,7 @@ use crate::util::error::Context; use crate::util::img; use crate::util::img::{delete_old_images, upload_image_optimized}; use crate::util::routes::read_limited_from_payload; -use crate::validate::project::has_required_nags; +use crate::validate::project::has_required_nags_with_context; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web}; use chrono::Utc; use eyre::eyre; @@ -1380,9 +1380,28 @@ pub async fn project_edit_internal( .into_iter() .map(Version::from) .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, &redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + reloaded_project.inner.id, + false, + &mut transaction, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); let project = Project::from(reloaded_project.clone()); - if has_required_nags(&project, &versions) { + if has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) { return Err(ApiError::Request(eyre!( "project must have no required validation nags before being submitted for review" ))); diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index d92ec20b70..3af56dc7df 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -11,7 +11,9 @@ use crate::models::teams::ProjectPermissions; use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::util::error::Context as _; -use crate::validate::project::{ProjectNag, validate as validate_project}; +use crate::validate::project::{ + ProjectNag, validate_with_context as validate_project, +}; #[derive(Serialize, utoipa::ToSchema)] pub struct ProjectValidationResponse { @@ -78,9 +80,28 @@ pub async fn validate( .into_iter() .map(Version::from) .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, &redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + project.inner.id, + false, + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); let project = Project::from(project); Ok(web::Json(ProjectValidationResponse { - nags: validate_project(&project, &versions), + nags: validate_project( + &project, + &versions, + &available_categories, + &disclosures, + ), })) } diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 6a8d63eb22..b80a8e8666 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -1,11 +1,15 @@ +use serde_json::json; + use super::text::{ - contains_banned_description_link, contains_spam, - extract_description_blocks, extract_description_text, - has_image_without_alt_text, has_long_header, has_sufficient_english_blocks, - non_standard_text_ratio, profanity_count, + ProfanityKind, contains_spam, description_ends_with_header, + extract_description_blocks, extract_description_text, + find_banned_description_link, has_adjacent_same_level_headers, + has_image_without_alt_text, has_sufficient_english_blocks, + js_string_length, long_header_count, non_standard_text_ratio, + normalize_project_field_text, profanity_matches, project_requires_english, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; -use crate::models::exp::minecraft::Language; + use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; @@ -13,79 +17,120 @@ const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let description = project.description.trim(); - let text = extract_description_text(description); - let blocks = extract_description_blocks(description); + let mut nags = Vec::new(); + let description = project.description.as_str(); + let normalized_description = normalize_project_field_text(description); + let text = extract_description_text(description); + let normalized_text = extract_description_text(&normalized_description); + let blocks = extract_description_blocks(description); + let profanity = profanity_matches(description); - if profanity_count(description) > MAX_PROFANITY_COUNT { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionProfanity, - ProjectNagSeverity::Required, - )); - } - if non_standard_text_ratio(description) - >= NON_STANDARD_TEXT_FAILURE_THRESHOLD - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && text.chars().count() >= MIN_DESCRIPTION_CHARS - && !contains_spam(&text) - && !has_sufficient_english_blocks(&blocks) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if description.is_empty() { - nags.push(ProjectNag::new( - ProjectNagKind::AddDescription, - ProjectNagSeverity::Required, - )); - } else if text.chars().count() < MIN_DESCRIPTION_CHARS { - nags.push(ProjectNag::new( - ProjectNagKind::DescriptionTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(&text) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionSpam, - ProjectNagSeverity::Required, - )); - } - if contains_banned_description_link(description) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionBannedLink, - ProjectNagSeverity::Required, - )); - } - if has_long_header(description) { - nags.push(ProjectNag::new( - ProjectNagKind::LongHeaders, - ProjectNagSeverity::Required, - )); - } - if has_image_without_alt_text(description) { - nags.push(ProjectNag::new( - ProjectNagKind::MissingAltText, - ProjectNagSeverity::Warning, - )); - } - - nags -} + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .filter(|matched| matched.kind == ProfanityKind::Profanity) + .nth(MAX_PROFANITY_COUNT) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if non_standard_text_ratio(description) + >= NON_STANDARD_TEXT_FAILURE_THRESHOLD + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && js_string_length(&text) >= MIN_DESCRIPTION_CHARS + && !contains_spam(&text) + && !has_sufficient_english_blocks(&blocks) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonEnglish, + ProjectNagSeverity::Required, + )); + } + if normalized_description.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::AddDescription, + ProjectNagSeverity::Required, + )); + } else { + let description_length = js_string_length(&normalized_text); + if description_length < MIN_DESCRIPTION_CHARS { + nags.push( + ProjectNag::new( + ProjectNagKind::DescriptionTooShort, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "length": description_length, + "min_chars": MIN_DESCRIPTION_CHARS, + })), + ); + } + } + if contains_spam(&text) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionSpam, + ProjectNagSeverity::Required, + )); + } + if let Some(url) = find_banned_description_link(description) { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionBannedLink, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "full_url": url })), + ); + } + let long_headers = long_header_count(description); + if long_headers > 0 { + nags.push( + ProjectNag::new( + ProjectNagKind::LongHeaders, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "count": long_headers })), + ); + } + if description_ends_with_header(description) { + nags.push(ProjectNag::new( + ProjectNagKind::DescriptionEndsWithHeader, + ProjectNagSeverity::Required, + )); + } + if has_adjacent_same_level_headers(description) { + nags.push(ProjectNag::new( + ProjectNagKind::AdjacentHeaders, + ProjectNagSeverity::Required, + )); + } + if has_image_without_alt_text(description) { + nags.push(ProjectNag::new( + ProjectNagKind::MissingAltText, + ProjectNagSeverity::Warning, + )); + } -fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) + nags } diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index f7b0f165ee..3355c72099 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -1,12 +1,57 @@ +use super::text::has_paired_html_formatting; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::disclosures::ProjectDisclosure; +use crate::models::v2::projects::LegacyProject; pub(super) fn validate( project: &crate::models::projects::Project, -) -> Vec { - let _ = project; + disclosures: Option<&[ProjectDisclosure]>, +) -> Vec { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + let mut nags = vec![ + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ]; - vec![ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - )] + if disclosures.is_some_and(|disclosures| { + disclosures.iter().any(disclosure_has_paired_html) + }) { + nags.push(ProjectNag::new( + ProjectNagKind::DisclosuresSpecialFormatting, + ProjectNagSeverity::Required, + )); + } + + nags +} + +fn disclosure_has_paired_html(disclosure: &ProjectDisclosure) -> bool { + match disclosure { + ProjectDisclosure::AiContent { note, .. } + | ProjectDisclosure::Advertisements { note } + | ProjectDisclosure::EpilepsyTriggers { note } + | ProjectDisclosure::SystemInteractions { note, .. } + | ProjectDisclosure::Archived { note } => { + note.as_deref().is_some_and(has_paired_html_formatting) + } + ProjectDisclosure::Telemetry { data_collected, .. } => data_collected + .iter() + .any(|text| has_paired_html_formatting(text)), + ProjectDisclosure::DerivativeWork { sources } => { + sources.iter().any(|source| { + has_paired_html_formatting(&source.label) + || source + .note + .as_deref() + .is_some_and(has_paired_html_formatting) + }) + } + ProjectDisclosure::PaidFeatures { features } => { + features.iter().any(|text| has_paired_html_formatting(text)) + } + } } diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs index 251862fd17..11679c85ee 100644 --- a/apps/labrinth/src/validate/project/gallery.rs +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -1,22 +1,60 @@ -use super::text::{contains_profanity, has_non_standard_text}; +use serde_json::json; + +use super::text::{ProfanityKind, has_non_standard_text, profanity_matches}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; -fn validate_text(text: Option<&str>) -> Vec { +fn validate_text( + text: Option<&str>, + gallery_index: usize, + field: &'static str, +) -> Vec { let text = text.unwrap_or_default(); let mut nags = Vec::new(); + let profanity = profanity_matches(text); - if contains_profanity(text) { - nags.push(ProjectNag::new( - ProjectNagKind::GalleryTextProfanity, - ProjectNagSeverity::Required, - )); + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + "value": matched.raw_text, + })), + ); + } + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Profanity) + { + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + "value": matched.raw_text, + })), + ); } - if has_non_standard_text(text) { - nags.push(ProjectNag::new( - ProjectNagKind::GalleryTextNonStandard, - ProjectNagSeverity::Required, - )); + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextNonStandard, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + })), + ); } nags @@ -36,13 +74,22 @@ pub(super) fn validate( .chain(&project.additional_categories) .any(|category| category == "audio" || category == "locale"); - if (is_shader && project.gallery.len() < 3) - || (is_resource_pack && gallery_is_empty && !has_gallery_exemption) - { - nags.push(ProjectNag::new( - ProjectNagKind::UploadGalleryImage, - ProjectNagSeverity::Required, - )); + if is_shader && project.gallery.len() < 3 { + nags.push( + ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "project_type": "shader" })), + ); + } else if is_resource_pack && gallery_is_empty && !has_gallery_exemption { + nags.push( + ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "project_type": "resourcepack" })), + ); } let is_minecraft_server = project.components.minecraft_server.is_some(); @@ -54,9 +101,13 @@ pub(super) fn validate( )); } - for item in &project.gallery { - nags.extend(validate_text(item.name.as_deref())); - nags.extend(validate_text(item.description.as_deref())); + for (index, item) in project.gallery.iter().enumerate() { + nags.extend(validate_text(item.name.as_deref(), index, "name")); + nags.extend(validate_text( + item.description.as_deref(), + index, + "description", + )); } nags diff --git a/apps/labrinth/src/validate/project/license.rs b/apps/labrinth/src/validate/project/license.rs index 8bef9ed25f..d84d154573 100644 --- a/apps/labrinth/src/validate/project/license.rs +++ b/apps/labrinth/src/validate/project/license.rs @@ -1,4 +1,5 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::v2::projects::LegacyProject; const INAPPROPRIATE_LICENSE_DOMAINS: &[&str] = &[ "youtube.com", @@ -30,16 +31,21 @@ fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { .is_some_and(|prefix| prefix.ends_with('.')) } -fn has_invalid_license_url(url: &str) -> bool { +enum InvalidLicenseUrl { + InappropriateDomain(String), + Malformed, +} + +fn get_invalid_license_url(url: &str) -> Option { if url.is_empty() { - return false; + return None; } let Ok(url) = url::Url::parse(url) else { - return true; + return Some(InvalidLicenseUrl::Malformed); }; let Some(hostname) = url.host_str() else { - return true; + return Some(InvalidLicenseUrl::Malformed); }; let hostname = hostname.to_ascii_lowercase(); let hostname = hostname.trim_end_matches('.'); @@ -47,6 +53,7 @@ fn has_invalid_license_url(url: &str) -> bool { INAPPROPRIATE_LICENSE_DOMAINS .iter() .any(|domain| hostname_matches_domain(hostname, domain)) + .then(|| InvalidLicenseUrl::InappropriateDomain(hostname.to_string())) } pub(super) fn validate( @@ -54,6 +61,8 @@ pub(super) fn validate( ) -> Vec { let mut nags = Vec::new(); let is_minecraft_server = project.components.minecraft_server.is_some(); + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); let license = &project.license; let has_unknown_license = matches!( license.id.as_str(), @@ -61,10 +70,13 @@ pub(super) fn validate( ); if has_unknown_license && !is_minecraft_server { - nags.push(ProjectNag::new( - ProjectNagKind::SelectLicense, - ProjectNagSeverity::Required, - )); + nags.push( + ProjectNag::new( + ProjectNagKind::SelectLicense, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); } let has_license_url = @@ -81,11 +93,27 @@ pub(super) fn validate( )); } - if license.url.as_deref().is_some_and(has_invalid_license_url) { - nags.push(ProjectNag::new( - ProjectNagKind::InvalidLicenseUrl, - ProjectNagSeverity::Required, - )); + if let Some(invalid_url) = + license.url.as_deref().and_then(get_invalid_license_url) + { + let details = match invalid_url { + InvalidLicenseUrl::InappropriateDomain(domain) => { + serde_json::json!({ + "domain": domain, + "reason": "inappropriate_domain", + }) + } + InvalidLicenseUrl::Malformed => serde_json::json!({ + "reason": "malformed", + }), + }; + nags.push( + ProjectNag::new( + ProjectNagKind::InvalidLicenseUrl, + ProjectNagSeverity::Required, + ) + .with_details(details), + ); } nags diff --git a/apps/labrinth/src/validate/project/links.rs b/apps/labrinth/src/validate/project/links.rs index 4505a711ce..f563378b76 100644 --- a/apps/labrinth/src/validate/project/links.rs +++ b/apps/labrinth/src/validate/project/links.rs @@ -3,7 +3,7 @@ use std::net::IpAddr; use url::Url; use super::{ProjectNagKind, ProjectNagSeverity}; -use crate::models::projects::Version; +use crate::models::{projects::Version, v2::projects::LegacyProject}; const SOURCE_DOMAINS: &[&str] = &[ "github.com", @@ -37,6 +37,8 @@ const BLOCKED_EXTERNAL_LINK_DOMAINS: &[&str] = &[ "bilibili.com", "bsky.app", "twitch.tv", + "youtube.com", + "youtu.be", "reddit.com", "redd.it", "modrinth.com", @@ -129,20 +131,14 @@ pub(super) fn validate( )); } - if project - .link_urls - .values() - .any(|link| is_blocked_external_link(&link.url)) - || project - .license - .url - .as_deref() - .is_some_and(is_blocked_external_link) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::BannedLinkUsage, - ProjectNagSeverity::Required, - )); + if let Some(url) = find_blocked_external_link(project) { + nags.push( + super::ProjectNag::new( + ProjectNagKind::BannedLinkUsage, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "url": url })), + ); } let is_source_project = project @@ -164,10 +160,15 @@ pub(super) fn validate( && !has_source_link && !every_version_has_additional_files { - nags.push(super::ProjectNag::new( - ProjectNagKind::GplLicenseSourceRequired, - ProjectNagSeverity::Required, - )); + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + super::ProjectNag::new( + ProjectNagKind::GplLicenseSourceRequired, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); } nags @@ -193,6 +194,39 @@ fn is_discord_link(url: &str) -> bool { is_link_from_domains(url, DISCORD_DOMAINS) } +fn find_blocked_external_link( + project: &crate::models::projects::Project, +) -> Option<&str> { + const LEGACY_LINK_KEYS: &[&str] = &["source", "issues", "wiki", "discord"]; + + for key in LEGACY_LINK_KEYS { + if let Some(url) = + named_link(project, key).filter(|url| is_blocked_external_link(url)) + { + return Some(url); + } + } + if let Some(url) = project + .license + .url + .as_deref() + .filter(|url| is_blocked_external_link(url)) + { + return Some(url); + } + + let mut remaining_links = project + .link_urls + .iter() + .filter(|(key, _)| !LEGACY_LINK_KEYS.contains(&key.as_str())) + .collect::>(); + remaining_links.sort_unstable_by_key(|(key, _)| key.as_str()); + remaining_links + .into_iter() + .map(|(_, link)| link.url.as_str()) + .find(|url| is_blocked_external_link(url)) +} + fn is_blocked_external_link(url: &str) -> bool { let Some(hostname) = get_link_hostname(url) else { return false; diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 665bd3684f..6bc417f824 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -1,5 +1,8 @@ use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use crate::database::models::categories::Category; +use crate::models::disclosures::ProjectDisclosure; use crate::models::projects::{Project, Version}; mod description; @@ -33,12 +36,14 @@ mod versions; #[serde(rename_all = "snake_case")] pub enum ProjectNagKind { // Project name + ProjectNameSlur, ProjectNameProfanity, ProjectNameNonStandardText, ProjectNameVersion, MinecraftTitleClause, // Project summary + ProjectSummarySlur, ProjectSummaryProfanity, ProjectSummaryNonStandardText, ProjectSummaryNonEnglish, @@ -52,12 +57,14 @@ pub enum ProjectNagKind { AddIcon, // Gallery + GalleryTextSlur, GalleryTextProfanity, GalleryTextNonStandard, UploadGalleryImage, FeatureGalleryImage, // Project description + ProjectDescriptionSlur, ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, @@ -66,6 +73,8 @@ pub enum ProjectNagKind { ProjectDescriptionSpam, ProjectDescriptionBannedLink, LongHeaders, + DescriptionEndsWithHeader, + AdjacentHeaders, MissingAltText, // License @@ -106,6 +115,7 @@ pub enum ProjectNagKind { // Disclosures CheckDisclosures, + DisclosuresSpecialFormatting, // Moderation ModeratorFeedback, @@ -122,11 +132,13 @@ pub enum ProjectNagSeverity { } #[derive( - Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, )] pub struct ProjectNag { pub kind: ProjectNagKind, pub severity: ProjectNagSeverity, + #[schema(value_type = serde_json::Value)] + pub details: Value, } impl ProjectNag { @@ -134,11 +146,44 @@ impl ProjectNag { kind: ProjectNagKind, severity: ProjectNagSeverity, ) -> Self { - Self { kind, severity } + Self { + kind, + severity, + details: json!({}), + } + } + + pub(super) fn with_details(mut self, details: Value) -> Self { + debug_assert!(details.is_object()); + self.details = details; + self } } pub fn validate(project: &Project, versions: &[Version]) -> Vec { + validate_inner(project, versions, None, None) +} + +pub fn validate_with_context( + project: &Project, + versions: &[Version], + available_categories: &[Category], + disclosures: &[ProjectDisclosure], +) -> Vec { + validate_inner( + project, + versions, + Some(available_categories), + Some(disclosures), + ) +} + +fn validate_inner( + project: &Project, + versions: &[Version], + available_categories: Option<&[Category]>, + disclosures: Option<&[ProjectDisclosure]>, +) -> Vec { let mut nags = [ name::validate, summary::validate, @@ -155,15 +200,12 @@ pub fn validate(project: &Project, versions: &[Version]) -> Vec { nags.extend(permissions::validate(versions)); nags.extend(versions::validate(project, versions)); nags.extend( - [ - server_settings::validate, - tags::validate, - disclosures::validate, - moderation::validate, - ] - .into_iter() - .flat_map(|validate| validate(project)), + [server_settings::validate, moderation::validate] + .into_iter() + .flat_map(|validate| validate(project)), ); + nags.extend(tags::validate(project, available_categories)); + nags.extend(disclosures::validate(project, disclosures)); nags } @@ -172,3 +214,14 @@ pub fn has_required_nags(project: &Project, versions: &[Version]) -> bool { .iter() .any(|nag| nag.severity == ProjectNagSeverity::Required) } + +pub fn has_required_nags_with_context( + project: &Project, + versions: &[Version], + available_categories: &[Category], + disclosures: &[ProjectDisclosure], +) -> bool { + validate_with_context(project, versions, available_categories, disclosures) + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) +} diff --git a/apps/labrinth/src/validate/project/name.rs b/apps/labrinth/src/validate/project/name.rs index bef840d7ca..dae386ecc3 100644 --- a/apps/labrinth/src/validate/project/name.rs +++ b/apps/labrinth/src/validate/project/name.rs @@ -1,25 +1,46 @@ use std::sync::LazyLock; use regex::Regex; +use serde_json::json; -use super::text::{contains_profanity, has_non_standard_text}; +use super::text::{ + ProfanityKind, has_non_standard_text, normalize_project_field_text, + profanity_matches, +}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; static VERSION_NUMBER: LazyLock = LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)+").unwrap()); -static PORT_OR_FORK: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\b(?:port|fork)\b").unwrap()); pub(super) fn validate(project: &Project) -> Vec { let mut nags = Vec::new(); let name = &project.name; + let profanity = profanity_matches(name); - if contains_profanity(name) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectNameProfanity, - ProjectNagSeverity::Required, - )); + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectNameSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Profanity) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectNameProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); } if has_non_standard_text(name) { nags.push(ProjectNag::new( @@ -28,11 +49,10 @@ pub(super) fn validate(project: &Project) -> Vec { )); } - let normalized = name.to_lowercase(); - if VERSION_NUMBER - .find_iter(&normalized) - .any(|version| !PORT_OR_FORK.is_match(&normalized[version.end()..])) - { + let normalized = normalize_project_field_text(name).to_lowercase(); + let is_port_or_fork = + normalized.contains("port") || normalized.contains("fork"); + if VERSION_NUMBER.is_match(&normalized) && !is_port_or_fork { nags.push(ProjectNag::new( ProjectNagKind::ProjectNameVersion, ProjectNagSeverity::Required, diff --git a/apps/labrinth/src/validate/project/server_settings.rs b/apps/labrinth/src/validate/project/server_settings.rs index eb73c730a6..cd9a0841ed 100644 --- a/apps/labrinth/src/validate/project/server_settings.rs +++ b/apps/labrinth/src/validate/project/server_settings.rs @@ -50,12 +50,19 @@ pub(super) fn validate(project: &Project) -> Vec { )); } - if server.is_some_and(|server| server.languages.len() > MAX_LANGUAGE_COUNT) + if let Some(language_count) = server + .map(|server| server.languages.len()) + .filter(|language_count| *language_count > MAX_LANGUAGE_COUNT) { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyLanguages, - ProjectNagSeverity::Warning, - )); + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyLanguages, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "language_count": language_count, + })), + ); } if server.is_some_and(|server| server.languages.is_empty()) { diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 41ca169f07..6f768dba0a 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,95 +1,157 @@ +use serde_json::json; + use super::text::{ - contains_link_or_ip, contains_profanity, contains_spam, - has_non_standard_text, has_summary_formatting, is_likely_english_summary, + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_likely_english_summary, js_string_length, + normalize_project_field_text, profanity_matches, project_requires_english, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; -use crate::models::exp::minecraft::Language; + use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; +const MAX_SUMMARY_NAME_SIMILARITY: f64 = 0.8; pub(super) fn validate(project: &Project) -> Vec { - let mut nags = Vec::new(); - let summary = project.summary.trim(); - let contains_link = contains_link_or_ip(summary); + let mut nags = Vec::new(); + let summary = project.summary.as_str(); + let normalized_summary = normalize_project_field_text(summary); + let summary_link = find_link_or_ip(summary); + let contains_link = summary_link.is_some(); + let profanity = profanity_matches(summary); + + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummarySlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Profanity) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummaryProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if has_non_standard_text(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonStandardText, + ProjectNagSeverity::Required, + )); + } + if project_requires_english(project) + && js_string_length(&normalized_summary) >= MIN_SUMMARY_CHARS + && !contains_link + && !contains_spam(&normalized_summary) + && !is_likely_english_summary(&normalized_summary) + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonEnglish, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() + && !contains_link + && !project.name.is_empty() + && summary_name_similarity(summary, &project.name) + >= MAX_SUMMARY_NAME_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryMatchesTitle, + ProjectNagSeverity::Required, + )); + } + let summary_length = js_string_length(&normalized_summary); + if !summary.is_empty() + && !contains_link + && summary_length < MIN_SUMMARY_CHARS + { + nags.push( + ProjectNag::new( + ProjectNagKind::SummaryTooShort, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "length": summary_length, + "min_chars": MIN_SUMMARY_CHARS, + })), + ); + } + if contains_spam(&normalized_summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummarySpam, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && has_summary_formatting(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::SummarySpecialFormatting, + ProjectNagSeverity::Required, + )); + } + if let Some(value) = summary_link { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummaryLinks, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": value })), + ); + } - if contains_profanity(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryProfanity, - ProjectNagSeverity::Required, - )); - } - if has_non_standard_text(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonStandardText, - ProjectNagSeverity::Required, - )); - } - if project_requires_english(project) - && summary.chars().count() >= MIN_SUMMARY_CHARS - && !contains_link - && !contains_spam(summary) - && !is_likely_english_summary(summary) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryNonEnglish, - ProjectNagSeverity::Warning, - )); - } - if !summary.is_empty() - && !contains_link - && normalized_without_whitespace(summary) - == normalized_without_whitespace(&project.name) - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryMatchesTitle, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() - && !contains_link - && summary.chars().count() < MIN_SUMMARY_CHARS - { - nags.push(ProjectNag::new( - ProjectNagKind::SummaryTooShort, - ProjectNagSeverity::Required, - )); - } - if contains_spam(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummarySpam, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && has_summary_formatting(summary) { - nags.push(ProjectNag::new( - ProjectNagKind::SummarySpecialFormatting, - ProjectNagSeverity::Required, - )); - } - if !summary.is_empty() && contains_link { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectSummaryLinks, - ProjectNagSeverity::Required, - )); - } + nags +} + +fn summary_name_similarity(summary: &str, name: &str) -> f64 { + let summary = normalized_for_similarity(summary); + let name = normalized_for_similarity(name); + let longest_length = summary.len().max(name.len()); + if longest_length == 0 { + return 0.0; + } - nags + 1.0 - levenshtein_distance(&summary, &name) as f64 / longest_length as f64 } -fn normalized_without_whitespace(text: &str) -> String { - text.trim() - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() +fn normalized_for_similarity(text: &str) -> Vec { + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() } -fn project_requires_english(project: &Project) -> bool { - project.components.minecraft_java_server.is_none() - || project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)) +fn levenshtein_distance(left: &[char], right: &[char]) -> usize { + if left.len() > right.len() { + return levenshtein_distance(right, left); + } + + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] } diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs index 6d6404d9e6..53a8f40746 100644 --- a/apps/labrinth/src/validate/project/tags.rs +++ b/apps/labrinth/src/validate/project/tags.rs @@ -1,4 +1,7 @@ -use crate::models::projects::Project; +use crate::{ + database::models::categories::Category, + models::{projects::Project, v2::projects::LegacyProject}, +}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -7,11 +10,16 @@ const MAX_TAG_COUNT_SERVER: usize = 18; const RESOLUTION_TAGS: [&str; 8] = ["8x-", "16x", "32x", "48x", "64x", "128x", "256x", "512x+"]; -pub(super) fn validate(project: &Project) -> Vec { +pub(super) fn validate( + project: &Project, + available_categories: Option<&[Category]>, +) -> Vec { let mut nags = Vec::new(); let tag_count = project.categories.len() + project.additional_categories.len(); let is_minecraft_server = project.components.minecraft_server.is_some(); + let (project_type, actual_project_type) = + LegacyProject::get_project_type(&project.project_types); if !project.versions.is_empty() && project.categories.is_empty() { nags.push(ProjectNag::new( @@ -21,35 +29,73 @@ pub(super) fn validate(project: &Project) -> Vec { } if !is_minecraft_server && tag_count > MAX_TAG_COUNT { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyTags, - ProjectNagSeverity::Warning, - )); + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyTags, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "tag_count": tag_count, + "max_tag_count": MAX_TAG_COUNT, + })), + ); } if is_minecraft_server && tag_count > MAX_TAG_COUNT_SERVER { - nags.push(ProjectNag::new( - ProjectNagKind::TooManyTagsServer, - ProjectNagSeverity::Required, - )); + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyTagsServer, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "tag_count": tag_count, + "max_tag_count": MAX_TAG_COUNT_SERVER, + })), + ); } - if project - .project_types + let mut resolution_tags = project + .categories .iter() - .any(|project_type| project_type == "resourcepack") - && project - .categories + .chain(&project.additional_categories) + .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) + .map(String::as_str) + .collect::>(); + resolution_tags.sort_by_key(|tag| { + RESOLUTION_TAGS .iter() - .chain(&project.additional_categories) - .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) - .count() - > 1 - { - nags.push(ProjectNag::new( - ProjectNagKind::MultipleResolutionTags, - ProjectNagSeverity::Warning, - )); + .position(|resolution| resolution == tag) + .unwrap_or(RESOLUTION_TAGS.len()) + }); + if project_type == "resourcepack" && resolution_tags.len() > 1 { + nags.push( + ProjectNag::new( + ProjectNagKind::MultipleResolutionTags, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "count": resolution_tags.len(), + "tags": resolution_tags.join("|"), + })), + ); + } + + if let Some(available_categories) = available_categories { + let total_available_tags = available_categories + .iter() + .filter(|category| category.project_type == actual_project_type) + .count(); + if tag_count == total_available_tags && project_type != "project" { + nags.push( + ProjectNag::new( + ProjectNagKind::AllTagsSelected, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "total_available_tags": total_available_tags, + })), + ); + } } nags diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index e9375de9f5..150a7313d1 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -4,9 +4,14 @@ use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; use regex::Regex; use rustrict::{Censor, Type}; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; use url::Url; use whatlang::{Detector, Lang}; +use crate::models::exp::minecraft::Language; +use crate::models::projects::Project; + static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { @@ -16,11 +21,17 @@ static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { }); static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = - LazyLock::new(|| Regex::new(r"!?\[[^\]]*\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)<[a-z][^>]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)|]*>").unwrap()); +static HTML_OPEN_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)<([a-z][\w:-]*)\b[^>]*>").unwrap()); +static HTML_CLOSE_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)").unwrap()); static CODE_BLOCK: LazyLock = LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); +static DESCRIPTION_BLOCK_BREAK: LazyLock = + LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); static MARKDOWN_IMAGE: LazyLock = @@ -30,32 +41,117 @@ static HTML_IMAGE: LazyLock = static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() }); -static DESCRIPTION_LINK: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)(?:https?://|www\.)[^\s<>()\]]+").unwrap() +static DESCRIPTION_LINK_FINDER: LazyLock = LazyLock::new(|| { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder }); static HEADER: LazyLock = LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); +static HEADER_LINE: LazyLock = + LazyLock::new(|| Regex::new(r"^([#]{1,6})[\t ]+.+?\s*#*\s*$").unwrap()); +static SETEXT_HEADER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^([^\r\n]+)\r?\n[\t ]*(?:=+|-+)[\t ]*$").unwrap() +}); static HTML_HEADER: LazyLock = LazyLock::new(|| { Regex::new(r"(?is)]*>(.*?)").unwrap() }); -static INLINE_MARKDOWN: LazyLock = LazyLock::new(|| { - Regex::new(r"[*_~`]|!?\[([^\]]*)\]\([^)]+\)|<[^>]+>").unwrap() +static ADJACENT_HTML_HEADERS: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)\s* = LazyLock::new(|| { + Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); const URL_SHORTENERS: &[&str] = &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; -pub(super) fn contains_profanity(text: &str) -> bool { - let mut censor = Censor::from_str(text); - censor.with_ignore_self_censoring(true); - censor.analyze().is(profanity_types()) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ProfanityKind { + Profanity, + Slur, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ProfanityMatch { + pub(super) kind: ProfanityKind, + pub(super) raw_text: String, +} + +const SLUR_TERMS: &[&str] = &[ + "beaner", + "cameljockey", + "chankoro", + "chink", + "chingchong", + "coon", + "cottonpic", + "cottonpik", + "darkie", + "downie", + "dyke", + "fag", + "gook", + "jap", + "jigabo", + "junglebunny", + "kike", + "koon", + "niqa", + "nigga", + "niqqa", + "niggu", + "niqqu", + "niggr", + "nigger", + "niglet", + "nignog", + "paki", + "raghead", + "retard", + "trannie", + "tranny", + "wetback", +]; + +pub(super) fn normalize_project_field_text(text: &str) -> String { + text.trim().nfc().collect() +} + +pub(super) fn js_string_length(text: &str) -> usize { + text.encode_utf16().count() +} + +pub(super) fn profanity_matches(text: &str) -> Vec { + let (prepared, raw_ranges) = prepare_profanity_text(text); + let mut spans = censored_spans(&prepared, profanity_types()); + for span in censored_spans(&prepared, slur_types()) { + if spans.contains(&span) { + continue; + } + if raw_text_for_span(text, &raw_ranges, span).is_some_and(is_slur) { + spans.push(span); + } + } + spans.extend(frontend_slur_spans(&prepared)); + spans.sort_unstable_by(|left, right| { + left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1)) + }); + + let mut matches = Vec::new(); + let mut previous_end = 0; + for (start, end) in spans { + if start < previous_end { + continue; + } + push_profanity_match(text, &raw_ranges, start, end, &mut matches); + previous_end = end; + } + matches } -pub(super) fn profanity_count(text: &str) -> usize { - // Rustrict only exposes match counts through its tracing features. +fn censored_spans(text: &str, threshold: Type) -> Vec<(usize, usize)> { const CENSORED: char = '\0'; - let threshold = profanity_types(); let mut censor = Censor::from_str(text); censor .with_ignore_self_censoring(true) @@ -63,22 +159,272 @@ pub(super) fn profanity_count(text: &str) -> usize { .with_censor_first_character_threshold(threshold) .with_censor_replacement(CENSORED); - let mut count = 0; - let mut in_censored_text = false; - for character in censor.censor().chars() { - let is_censored = character == CENSORED; - if is_censored && !in_censored_text { - count += 1; + let mut spans = Vec::new(); + let mut start = None; + let mut character_count = 0; + for (index, character) in censor.censor().chars().enumerate() { + character_count = index + 1; + match (start, character == CENSORED) { + (None, true) => start = Some(index), + (Some(match_start), false) => { + spans.push((match_start, index)); + start = None; + } + _ => {} } - in_censored_text = is_censored; } - count + if let Some(start) = start { + spans.push((start, character_count)); + } + spans +} + +fn frontend_slur_spans(text: &str) -> Vec<(usize, usize)> { + let characters = text.chars().collect::>(); + let mut spans = Vec::new(); + + for start in 0..characters.len() { + if start > 0 && is_profanity_word_character(characters[start - 1]) { + continue; + } + + let end = SLUR_TERMS + .iter() + .flat_map(|term| { + [ + match_repeated_term(&characters, start, term), + match_separated_term(&characters, start, term), + ] + }) + .flatten() + .max(); + if let Some(end) = end { + spans.push((start, end)); + } + } + + spans +} + +fn match_repeated_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let term = term.as_bytes(); + let mut input_index = start; + let mut term_index = 0; + + while term_index < term.len() { + let expected = term[term_index] as char; + let mut required = 1; + while term.get(term_index + required) == Some(&term[term_index]) { + required += 1; + } + + let mut matched = 0; + while characters.get(input_index) == Some(&expected) { + matched += 1; + input_index += 1; + } + if matched < required { + return None; + } + term_index += required; + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + +fn match_separated_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let mut input_index = start; + let mut term = term.bytes().peekable(); + + while let Some(expected) = term.next() { + if characters.get(input_index) != Some(&(expected as char)) { + return None; + } + input_index += 1; + + if term.peek().is_some() { + let separator_start = input_index; + while characters.get(input_index).is_some_and(|character| { + !is_profanity_word_character(*character) + }) { + input_index += 1; + } + if input_index == separator_start { + return None; + } + } + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + +fn is_whole_word_end(characters: &[char], end: usize) -> bool { + characters + .get(end) + .is_none_or(|character| !is_profanity_word_character(*character)) +} + +fn is_profanity_word_character(character: char) -> bool { + character.is_alphanumeric() + || character == '_' + || is_in_ranges( + character as u32, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ) +} + +fn prepare_profanity_text(text: &str) -> (String, Vec<(usize, usize)>) { + let mut prepared = String::new(); + let mut raw_ranges = Vec::new(); + + for (start, grapheme) in text.grapheme_indices(true) { + let end = start + grapheme.len(); + for character in grapheme.nfkc().flat_map(char::to_lowercase) { + if is_invisible_separator(character) { + continue; + } + + prepared.push(normalize_obfuscated_character(character)); + raw_ranges.push((start, end)); + } + } + + (prepared, raw_ranges) +} + +fn push_profanity_match( + text: &str, + raw_ranges: &[(usize, usize)], + start: usize, + end: usize, + matches: &mut Vec, +) { + let Some(raw_text) = raw_text_for_span(text, raw_ranges, (start, end)) + else { + return; + }; + if normalize_project_field_text(raw_text).to_lowercase() == "кооп" { + return; + } + + matches.push(ProfanityMatch { + kind: if is_slur(raw_text) { + ProfanityKind::Slur + } else { + ProfanityKind::Profanity + }, + raw_text: raw_text.to_owned(), + }); +} + +fn raw_text_for_span<'a>( + text: &'a str, + raw_ranges: &[(usize, usize)], + (start, end): (usize, usize), +) -> Option<&'a str> { + let &(raw_start, _) = raw_ranges.get(start)?; + let &(_, raw_end) = raw_ranges.get(end.checked_sub(1)?)?; + text.get(raw_start..raw_end) +} + +fn is_slur(raw_text: &str) -> bool { + let normalized = normalized_profanity_term(raw_text); + SLUR_TERMS + .iter() + .any(|term| collapse_duplicate_letters(term) == normalized) +} + +fn normalized_profanity_term(text: &str) -> String { + let (prepared, _) = prepare_profanity_text(text); + collapse_duplicate_letters( + &prepared + .chars() + .filter(char::is_ascii_alphabetic) + .collect::(), + ) +} + +fn collapse_duplicate_letters(text: &str) -> String { + let mut collapsed = String::with_capacity(text.len()); + let mut previous = None; + for character in text.chars() { + if previous != Some(character) { + collapsed.push(character); + previous = Some(character); + } + } + collapsed +} + +fn normalize_obfuscated_character(character: char) -> char { + match character { + '@' | '4' => 'a', + '8' => 'b', + '(' | '[' | '{' | 'с' | 'ϲ' => 'c', + '3' | 'е' | 'ε' => 'e', + '6' | '9' => 'g', + '!' | '1' | '/' | '|' | 'і' | 'ι' | 'ı' => 'i', + 'ј' => 'j', + 'к' | 'κ' => 'k', + 'м' | 'μ' => 'm', + 'п' => 'n', + '0' | 'о' | 'ο' => 'o', + 'р' | 'ρ' => 'p', + '$' | '5' | 'ѕ' => 's', + '+' | '7' | 'т' | 'τ' => 't', + 'υ' | 'ս' => 'u', + 'х' | 'χ' => 'x', + 'у' | 'γ' => 'y', + '2' => 'z', + _ => character, + } +} + +fn is_invisible_separator(character: char) -> bool { + let code = character as u32; + is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x034f, 0x034f), + (0x061c, 0x061c), + (0x115f, 0x1160), + (0x17b4, 0x17b5), + (0x180b, 0x180f), + (0x200b, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0x3164, 0x3164), + (0xfe00, 0xfe0f), + (0xfeff, 0xfeff), + (0xffa0, 0xffa0), + (0xe0100, 0xe01ef), + ], + ) } fn profanity_types() -> Type { Type::PROFANE & Type::MODERATE_OR_HIGHER } +fn slur_types() -> Type { + Type::OFFENSIVE & Type::MILD_OR_HIGHER +} + pub(super) fn has_non_standard_text(text: &str) -> bool { non_standard_character_count(text) > 0 } @@ -177,7 +523,8 @@ fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { } pub(super) fn contains_spam(text: &str) -> bool { - let normalized = text.to_lowercase(); + let normalized = + text.nfc().flat_map(char::to_lowercase).collect::(); let mut previous = None; let mut repeated = 0; for character in normalized.chars() { @@ -224,24 +571,29 @@ pub(super) fn contains_spam(text: &str) -> bool { false } -pub(super) fn contains_link_or_ip(text: &str) -> bool { - SUMMARY_LINK_FINDER.links(text).any(|link| { +pub(super) fn find_link_or_ip(text: &str) -> Option { + SUMMARY_LINK_FINDER.links(text).find_map(|link| { let raw = link.as_str(); if raw.contains("://") { - return true; + return Some(raw.to_owned()); } - Url::parse(&format!("https://{raw}")).is_ok_and(|url| { - url.host_str().is_some_and(|hostname| { - psl::domain(hostname.as_bytes()) - .is_some_and(|domain| domain.suffix().typ().is_some()) + Url::parse(&format!("https://{raw}")) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + hostname.parse::().is_ok() + || psl::domain(hostname.as_bytes()).is_some_and( + |domain| domain.suffix().typ().is_some(), + ) + }) }) - }) + .map(|_| raw.to_owned()) }) } pub(super) fn has_summary_formatting(summary: &str) -> bool { - HTML_TAG.is_match(summary) + has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) || summary.lines().any(|line| { let line = line.trim_start(); @@ -258,13 +610,50 @@ pub(super) fn has_summary_formatting(summary: &str) -> bool { }) } +pub(super) fn has_paired_html_formatting(text: &str) -> bool { + let without_code = CODE_BLOCK.replace_all(text, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + HTML_OPEN_TAG.captures_iter(&without_code).any(|opening| { + let Some(tag) = opening.get(1) else { + return false; + }; + let Some(opening_match) = opening.get(0) else { + return false; + }; + HTML_CLOSE_TAG + .captures_iter(&without_code[opening_match.end()..]) + .any(|closing| { + closing.get(1).is_some_and(|closing_tag| { + closing_tag.as_str().eq_ignore_ascii_case(tag.as_str()) + }) + }) + }) +} + pub(super) fn extract_description_text(markdown: &str) -> String { let without_code = CODE_BLOCK.replace_all(markdown, " "); let without_code = INLINE_CODE.replace_all(&without_code, " "); let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); - let without_html = HTML_TAG.replace_all(&without_links, " "); + let with_html_image_alt = HTML_IMAGE.replace_all( + &without_links, + |captures: ®ex::Captures<'_>| { + ALT_ATTRIBUTE + .captures(&captures[0]) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) + }, + ); + let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); without_html + .lines() + .map(|line| { + line.trim_start_matches(|character| matches!(character, '>' | '#')) + }) + .collect::>() + .join("\n") .replace(['*', '_', '~', '`', '>', '-', '|'], " ") .split_whitespace() .collect::>() @@ -272,30 +661,120 @@ pub(super) fn extract_description_text(markdown: &str) -> String { } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - CODE_BLOCK - .replace_all(markdown, "") - .split("\n\n") + let without_code = CODE_BLOCK.replace_all(markdown, ""); + DESCRIPTION_BLOCK_BREAK + .split(&without_code) .map(extract_description_text) .filter(|block| !block.is_empty()) .collect() } -pub(super) fn has_long_header(markdown: &str) -> bool { - HEADER.captures_iter(markdown).any(|captures| { - INLINE_MARKDOWN - .replace_all(&captures[1], "$1") - .trim() - .chars() - .count() > 80 - }) || HTML_HEADER.captures_iter(markdown).any(|captures| { - HTML_TAG - .replace_all(&captures[1], " ") - .split_whitespace() - .collect::>() - .join(" ") - .chars() - .count() > 80 - }) +pub(super) fn long_header_count(markdown: &str) -> usize { + let markdown_headers = HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let setext_headers = SETEXT_HEADER + .captures_iter(markdown) + .filter(|captures| !captures[1].trim_start().starts_with('#')) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let html_headers = HTML_HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + + markdown_headers + setext_headers + html_headers +} + +fn header_is_long(header: &str) -> bool { + let with_image_alt = MARKDOWN_IMAGE.replace_all(header, "$1"); + let with_link_text = MARKDOWN_LINK.replace_all(&with_image_alt, "$1"); + let without_html = HTML_TAG.replace_all(&with_link_text, " "); + let rendered = without_html + .replace(['*', '_', '~', '`'], "") + .split_whitespace() + .collect::>() + .join(" "); + + rendered.graphemes(true).count() > 80 +} + +pub(super) fn description_ends_with_header(markdown: &str) -> bool { + let trimmed = markdown.trim_end(); + if trimmed.is_empty() { + return false; + } + + let lines = trimmed.lines().collect::>(); + let last_line = lines.last().map_or("", |line| line.trim()); + if HEADER_LINE.is_match(last_line) { + return true; + } + if lines.len() >= 2 + && is_setext_underline(last_line) + && !lines[lines.len() - 2].trim().is_empty() + { + return true; + } + + TRAILING_HTML_HEADER.is_match(trimmed) +} + +pub(super) fn has_adjacent_same_level_headers(markdown: &str) -> bool { + let lines = markdown.lines().collect::>(); + let mut previous_header = None; + let mut index = 0; + while index < lines.len() { + let line = lines[index].trim(); + if line.is_empty() { + index += 1; + continue; + } + + let mut header_level = HEADER_LINE + .captures(line) + .and_then(|captures| captures.get(1)) + .map(|hashes| hashes.as_str().len()); + if header_level.is_none() + && lines + .get(index + 1) + .is_some_and(|underline| is_setext_underline(underline.trim())) + { + header_level = + Some(if lines[index + 1].trim_start().starts_with('=') { + 1 + } else { + 2 + }); + index += 1; + } + + if let Some(level) = header_level { + if level <= 3 && previous_header == Some(level) { + return true; + } + previous_header = Some(level); + } else { + previous_header = None; + } + index += 1; + } + + ADJACENT_HTML_HEADERS + .captures_iter(markdown) + .any(|captures| { + captures.get(1).map(|level| level.as_str()) + == captures.get(2).map(|level| level.as_str()) + }) +} + +fn is_setext_underline(line: &str) -> bool { + let mut characters = line.chars(); + let Some(marker @ ('=' | '-')) = characters.next() else { + return false; + }; + characters.all(|character| character == marker) } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -314,21 +793,24 @@ pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { }) } -pub(super) fn contains_banned_description_link(markdown: &str) -> bool { - DESCRIPTION_LINK.find_iter(markdown).any(|link| { +pub(super) fn find_banned_description_link(markdown: &str) -> Option { + DESCRIPTION_LINK_FINDER.links(markdown).find_map(|link| { let raw = link.as_str(); - let normalized = if raw.to_ascii_lowercase().starts_with("www.") { - format!("https://{raw}") - } else { + let normalized = if raw.contains("://") { raw.to_owned() + } else { + format!("http://{raw}") }; - Url::parse(&normalized).is_ok_and(|url| { - url.host_str().is_some_and(|hostname| { - URL_SHORTENERS - .iter() - .any(|domain| hostname_matches_domain(hostname, domain)) + Url::parse(&normalized) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) }) - }) + .map(|_| normalized) }) } @@ -339,6 +821,22 @@ fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { .ends_with(&format!(".{domain}")) } +pub(super) fn project_requires_english(project: &Project) -> bool { + let has_locale_tag = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "locale"); + let is_english_server = project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)); + + (project.components.minecraft_java_server.is_none() && !has_locale_tag) + || is_english_server +} + pub(super) fn is_likely_english_summary(text: &str) -> bool { if !has_enough_language_content(text) { return true; @@ -408,5 +906,6 @@ fn language_chunks(block: &str) -> Vec { } fn has_enough_language_content(text: &str) -> bool { - WORD.find_iter(text).count() >= 8 && text.trim().chars().count() >= 35 + WORD.find_iter(text).count() >= 8 + && text.trim().graphemes(true).count() >= 35 } diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs index 028bb38ec8..3ebc3d80a9 100644 --- a/apps/labrinth/src/validate/project/versions.rs +++ b/apps/labrinth/src/validate/project/versions.rs @@ -4,7 +4,7 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( project: &Project, - versions: &[Version], + _versions: &[Version], ) -> Vec { let mut nags = Vec::new(); @@ -21,14 +21,20 @@ pub(super) fn validate( .project_types .iter() .any(|project_type| matches!(project_type.as_str(), "mod" | "modpack")); - if requires_environment - && versions.iter().any(|version| { - version - .fields - .get("environment") - .and_then(serde_json::Value::as_str) - .is_none_or(|environment| environment.trim().is_empty()) - }) { + let has_valid_environment = + project + .fields + .get("environment") + .is_some_and(|environments| { + !environments.is_empty() + && environments.iter().all(|environment| { + environment.as_str().is_some_and(|environment| { + !environment.trim().is_empty() + && environment != "unknown" + }) + }) + }); + if requires_environment && !has_valid_environment { nags.push(ProjectNag::new( ProjectNagKind::SelectEnvironment, ProjectNagSeverity::Required, From be2eff839fb5e5da421df94bc1d5ed3b60b274d0 Mon Sep 17 00:00:00 2001 From: tdgao Date: Wed, 2 Sep 2026 22:33:05 -0600 Subject: [PATCH 10/28] feat: remove links from text before spam check --- .../src/validate/project/description.rs | 6 +- apps/labrinth/src/validate/project/text.rs | 1460 +++++++++-------- 2 files changed, 746 insertions(+), 720 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index b80a8e8666..a77f1891e2 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -7,6 +7,7 @@ use super::text::{ has_image_without_alt_text, has_sufficient_english_blocks, js_string_length, long_header_count, non_standard_text_ratio, normalize_project_field_text, profanity_matches, project_requires_english, + text_without_links, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -21,6 +22,7 @@ pub(super) fn validate(project: &Project) -> Vec { let description = project.description.as_str(); let normalized_description = normalize_project_field_text(description); let text = extract_description_text(description); + let spam_text = text_without_links(&text); let normalized_text = extract_description_text(&normalized_description); let blocks = extract_description_blocks(description); let profanity = profanity_matches(description); @@ -60,7 +62,7 @@ pub(super) fn validate(project: &Project) -> Vec { } if project_requires_english(project) && js_string_length(&text) >= MIN_DESCRIPTION_CHARS - && !contains_spam(&text) + && !contains_spam(&spam_text) && !has_sufficient_english_blocks(&blocks) { nags.push(ProjectNag::new( @@ -88,7 +90,7 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } - if contains_spam(&text) { + if contains_spam(&spam_text) { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, ProjectNagSeverity::Required, diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 150a7313d1..ce5cd489da 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -13,899 +13,923 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; static WORD: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { - let mut finder = LinkFinder::new(); - finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); - finder + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder }); static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = - LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)|]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)|]*>").unwrap()); static HTML_OPEN_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)<([a-z][\w:-]*)\b[^>]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)<([a-z][\w:-]*)\b[^>]*>").unwrap()); static HTML_CLOSE_TAG: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)").unwrap()); static CODE_BLOCK: LazyLock = - LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); + LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); static DESCRIPTION_BLOCK_BREAK: LazyLock = - LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); + LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = - LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); + LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); static MARKDOWN_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); + LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_IMAGE: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); + LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { - Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() + Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() }); static DESCRIPTION_LINK_FINDER: LazyLock = LazyLock::new(|| { - let mut finder = LinkFinder::new(); - finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); - finder + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder }); static HEADER: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); + LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); static HEADER_LINE: LazyLock = - LazyLock::new(|| Regex::new(r"^([#]{1,6})[\t ]+.+?\s*#*\s*$").unwrap()); + LazyLock::new(|| Regex::new(r"^([#]{1,6})[\t ]+.+?\s*#*\s*$").unwrap()); static SETEXT_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?m)^([^\r\n]+)\r?\n[\t ]*(?:=+|-+)[\t ]*$").unwrap() + Regex::new(r"(?m)^([^\r\n]+)\r?\n[\t ]*(?:=+|-+)[\t ]*$").unwrap() }); static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { - Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() + Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); const URL_SHORTENERS: &[&str] = - &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum ProfanityKind { - Profanity, - Slur, + Profanity, + Slur, } #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct ProfanityMatch { - pub(super) kind: ProfanityKind, - pub(super) raw_text: String, + pub(super) kind: ProfanityKind, + pub(super) raw_text: String, } const SLUR_TERMS: &[&str] = &[ - "beaner", - "cameljockey", - "chankoro", - "chink", - "chingchong", - "coon", - "cottonpic", - "cottonpik", - "darkie", - "downie", - "dyke", - "fag", - "gook", - "jap", - "jigabo", - "junglebunny", - "kike", - "koon", - "niqa", - "nigga", - "niqqa", - "niggu", - "niqqu", - "niggr", - "nigger", - "niglet", - "nignog", - "paki", - "raghead", - "retard", - "trannie", - "tranny", - "wetback", + "beaner", + "cameljockey", + "chankoro", + "chink", + "chingchong", + "coon", + "cottonpic", + "cottonpik", + "darkie", + "downie", + "dyke", + "fag", + "gook", + "jap", + "jigabo", + "junglebunny", + "kike", + "koon", + "niqa", + "nigga", + "niqqa", + "niggu", + "niqqu", + "niggr", + "nigger", + "niglet", + "nignog", + "paki", + "raghead", + "retard", + "trannie", + "tranny", + "wetback", ]; pub(super) fn normalize_project_field_text(text: &str) -> String { - text.trim().nfc().collect() + text.trim().nfc().collect() } pub(super) fn js_string_length(text: &str) -> usize { - text.encode_utf16().count() + text.encode_utf16().count() } pub(super) fn profanity_matches(text: &str) -> Vec { - let (prepared, raw_ranges) = prepare_profanity_text(text); - let mut spans = censored_spans(&prepared, profanity_types()); - for span in censored_spans(&prepared, slur_types()) { - if spans.contains(&span) { - continue; - } - if raw_text_for_span(text, &raw_ranges, span).is_some_and(is_slur) { - spans.push(span); - } - } - spans.extend(frontend_slur_spans(&prepared)); - spans.sort_unstable_by(|left, right| { - left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1)) - }); - - let mut matches = Vec::new(); - let mut previous_end = 0; - for (start, end) in spans { - if start < previous_end { - continue; - } - push_profanity_match(text, &raw_ranges, start, end, &mut matches); - previous_end = end; - } - matches + let (prepared, raw_ranges) = prepare_profanity_text(text); + let mut spans = censored_spans(&prepared, profanity_types()); + for span in censored_spans(&prepared, slur_types()) { + if spans.contains(&span) { + continue; + } + if raw_text_for_span(text, &raw_ranges, span).is_some_and(is_slur) { + spans.push(span); + } + } + spans.extend(frontend_slur_spans(&prepared)); + spans.sort_unstable_by(|left, right| { + left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1)) + }); + + let mut matches = Vec::new(); + let mut previous_end = 0; + for (start, end) in spans { + if start < previous_end { + continue; + } + push_profanity_match(text, &raw_ranges, start, end, &mut matches); + previous_end = end; + } + matches } fn censored_spans(text: &str, threshold: Type) -> Vec<(usize, usize)> { - const CENSORED: char = '\0'; - - let mut censor = Censor::from_str(text); - censor - .with_ignore_self_censoring(true) - .with_censor_threshold(threshold) - .with_censor_first_character_threshold(threshold) - .with_censor_replacement(CENSORED); - - let mut spans = Vec::new(); - let mut start = None; - let mut character_count = 0; - for (index, character) in censor.censor().chars().enumerate() { - character_count = index + 1; - match (start, character == CENSORED) { - (None, true) => start = Some(index), - (Some(match_start), false) => { - spans.push((match_start, index)); - start = None; - } - _ => {} - } - } - if let Some(start) = start { - spans.push((start, character_count)); - } - spans + const CENSORED: char = '\0'; + + let mut censor = Censor::from_str(text); + censor + .with_ignore_self_censoring(true) + .with_censor_threshold(threshold) + .with_censor_first_character_threshold(threshold) + .with_censor_replacement(CENSORED); + + let mut spans = Vec::new(); + let mut start = None; + let mut character_count = 0; + for (index, character) in censor.censor().chars().enumerate() { + character_count = index + 1; + match (start, character == CENSORED) { + (None, true) => start = Some(index), + (Some(match_start), false) => { + spans.push((match_start, index)); + start = None; + } + _ => {} + } + } + if let Some(start) = start { + spans.push((start, character_count)); + } + spans } fn frontend_slur_spans(text: &str) -> Vec<(usize, usize)> { - let characters = text.chars().collect::>(); - let mut spans = Vec::new(); - - for start in 0..characters.len() { - if start > 0 && is_profanity_word_character(characters[start - 1]) { - continue; - } - - let end = SLUR_TERMS - .iter() - .flat_map(|term| { - [ - match_repeated_term(&characters, start, term), - match_separated_term(&characters, start, term), - ] - }) - .flatten() - .max(); - if let Some(end) = end { - spans.push((start, end)); - } - } - - spans + let characters = text.chars().collect::>(); + let mut spans = Vec::new(); + + for start in 0..characters.len() { + if start > 0 && is_profanity_word_character(characters[start - 1]) { + continue; + } + + let end = SLUR_TERMS + .iter() + .flat_map(|term| { + [ + match_repeated_term(&characters, start, term), + match_separated_term(&characters, start, term), + ] + }) + .flatten() + .max(); + if let Some(end) = end { + spans.push((start, end)); + } + } + + spans } fn match_repeated_term( - characters: &[char], - start: usize, - term: &str, + characters: &[char], + start: usize, + term: &str, ) -> Option { - let term = term.as_bytes(); - let mut input_index = start; - let mut term_index = 0; - - while term_index < term.len() { - let expected = term[term_index] as char; - let mut required = 1; - while term.get(term_index + required) == Some(&term[term_index]) { - required += 1; - } - - let mut matched = 0; - while characters.get(input_index) == Some(&expected) { - matched += 1; - input_index += 1; - } - if matched < required { - return None; - } - term_index += required; - } - - is_whole_word_end(characters, input_index).then_some(input_index) + let term = term.as_bytes(); + let mut input_index = start; + let mut term_index = 0; + + while term_index < term.len() { + let expected = term[term_index] as char; + let mut required = 1; + while term.get(term_index + required) == Some(&term[term_index]) { + required += 1; + } + + let mut matched = 0; + while characters.get(input_index) == Some(&expected) { + matched += 1; + input_index += 1; + } + if matched < required { + return None; + } + term_index += required; + } + + is_whole_word_end(characters, input_index).then_some(input_index) } fn match_separated_term( - characters: &[char], - start: usize, - term: &str, + characters: &[char], + start: usize, + term: &str, ) -> Option { - let mut input_index = start; - let mut term = term.bytes().peekable(); - - while let Some(expected) = term.next() { - if characters.get(input_index) != Some(&(expected as char)) { - return None; - } - input_index += 1; - - if term.peek().is_some() { - let separator_start = input_index; - while characters.get(input_index).is_some_and(|character| { - !is_profanity_word_character(*character) - }) { - input_index += 1; - } - if input_index == separator_start { - return None; - } - } - } - - is_whole_word_end(characters, input_index).then_some(input_index) + let mut input_index = start; + let mut term = term.bytes().peekable(); + + while let Some(expected) = term.next() { + if characters.get(input_index) != Some(&(expected as char)) { + return None; + } + input_index += 1; + + if term.peek().is_some() { + let separator_start = input_index; + while characters.get(input_index).is_some_and(|character| { + !is_profanity_word_character(*character) + }) { + input_index += 1; + } + if input_index == separator_start { + return None; + } + } + } + + is_whole_word_end(characters, input_index).then_some(input_index) } fn is_whole_word_end(characters: &[char], end: usize) -> bool { - characters - .get(end) - .is_none_or(|character| !is_profanity_word_character(*character)) + characters + .get(end) + .is_none_or(|character| !is_profanity_word_character(*character)) } fn is_profanity_word_character(character: char) -> bool { - character.is_alphanumeric() - || character == '_' - || is_in_ranges( - character as u32, - &[ - (0x0300, 0x036f), - (0x1ab0, 0x1aff), - (0x1dc0, 0x1dff), - (0x20d0, 0x20ff), - (0xfe20, 0xfe2f), - ], - ) + character.is_alphanumeric() + || character == '_' + || is_in_ranges( + character as u32, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ) } fn prepare_profanity_text(text: &str) -> (String, Vec<(usize, usize)>) { - let mut prepared = String::new(); - let mut raw_ranges = Vec::new(); + let mut prepared = String::new(); + let mut raw_ranges = Vec::new(); - for (start, grapheme) in text.grapheme_indices(true) { - let end = start + grapheme.len(); - for character in grapheme.nfkc().flat_map(char::to_lowercase) { - if is_invisible_separator(character) { - continue; - } + for (start, grapheme) in text.grapheme_indices(true) { + let end = start + grapheme.len(); + for character in grapheme.nfkc().flat_map(char::to_lowercase) { + if is_invisible_separator(character) { + continue; + } - prepared.push(normalize_obfuscated_character(character)); - raw_ranges.push((start, end)); - } - } + prepared.push(normalize_obfuscated_character(character)); + raw_ranges.push((start, end)); + } + } - (prepared, raw_ranges) + (prepared, raw_ranges) } fn push_profanity_match( - text: &str, - raw_ranges: &[(usize, usize)], - start: usize, - end: usize, - matches: &mut Vec, + text: &str, + raw_ranges: &[(usize, usize)], + start: usize, + end: usize, + matches: &mut Vec, ) { - let Some(raw_text) = raw_text_for_span(text, raw_ranges, (start, end)) - else { - return; - }; - if normalize_project_field_text(raw_text).to_lowercase() == "кооп" { - return; - } - - matches.push(ProfanityMatch { - kind: if is_slur(raw_text) { - ProfanityKind::Slur - } else { - ProfanityKind::Profanity - }, - raw_text: raw_text.to_owned(), - }); + let Some(raw_text) = raw_text_for_span(text, raw_ranges, (start, end)) + else { + return; + }; + if normalize_project_field_text(raw_text).to_lowercase() == "кооп" { + return; + } + + matches.push(ProfanityMatch { + kind: if is_slur(raw_text) { + ProfanityKind::Slur + } else { + ProfanityKind::Profanity + }, + raw_text: raw_text.to_owned(), + }); } fn raw_text_for_span<'a>( - text: &'a str, - raw_ranges: &[(usize, usize)], - (start, end): (usize, usize), + text: &'a str, + raw_ranges: &[(usize, usize)], + (start, end): (usize, usize), ) -> Option<&'a str> { - let &(raw_start, _) = raw_ranges.get(start)?; - let &(_, raw_end) = raw_ranges.get(end.checked_sub(1)?)?; - text.get(raw_start..raw_end) + let &(raw_start, _) = raw_ranges.get(start)?; + let &(_, raw_end) = raw_ranges.get(end.checked_sub(1)?)?; + text.get(raw_start..raw_end) } fn is_slur(raw_text: &str) -> bool { - let normalized = normalized_profanity_term(raw_text); - SLUR_TERMS - .iter() - .any(|term| collapse_duplicate_letters(term) == normalized) + let normalized = normalized_profanity_term(raw_text); + SLUR_TERMS + .iter() + .any(|term| collapse_duplicate_letters(term) == normalized) } fn normalized_profanity_term(text: &str) -> String { - let (prepared, _) = prepare_profanity_text(text); - collapse_duplicate_letters( - &prepared - .chars() - .filter(char::is_ascii_alphabetic) - .collect::(), - ) + let (prepared, _) = prepare_profanity_text(text); + collapse_duplicate_letters( + &prepared + .chars() + .filter(char::is_ascii_alphabetic) + .collect::(), + ) } fn collapse_duplicate_letters(text: &str) -> String { - let mut collapsed = String::with_capacity(text.len()); - let mut previous = None; - for character in text.chars() { - if previous != Some(character) { - collapsed.push(character); - previous = Some(character); - } - } - collapsed + let mut collapsed = String::with_capacity(text.len()); + let mut previous = None; + for character in text.chars() { + if previous != Some(character) { + collapsed.push(character); + previous = Some(character); + } + } + collapsed } fn normalize_obfuscated_character(character: char) -> char { - match character { - '@' | '4' => 'a', - '8' => 'b', - '(' | '[' | '{' | 'с' | 'ϲ' => 'c', - '3' | 'е' | 'ε' => 'e', - '6' | '9' => 'g', - '!' | '1' | '/' | '|' | 'і' | 'ι' | 'ı' => 'i', - 'ј' => 'j', - 'к' | 'κ' => 'k', - 'м' | 'μ' => 'm', - 'п' => 'n', - '0' | 'о' | 'ο' => 'o', - 'р' | 'ρ' => 'p', - '$' | '5' | 'ѕ' => 's', - '+' | '7' | 'т' | 'τ' => 't', - 'υ' | 'ս' => 'u', - 'х' | 'χ' => 'x', - 'у' | 'γ' => 'y', - '2' => 'z', - _ => character, - } + match character { + '@' | '4' => 'a', + '8' => 'b', + '(' | '[' | '{' | 'с' | 'ϲ' => 'c', + '3' | 'е' | 'ε' => 'e', + '6' | '9' => 'g', + '!' | '1' | '/' | '|' | 'і' | 'ι' | 'ı' => 'i', + 'ј' => 'j', + 'к' | 'κ' => 'k', + 'м' | 'μ' => 'm', + 'п' => 'n', + '0' | 'о' | 'ο' => 'o', + 'р' | 'ρ' => 'p', + '$' | '5' | 'ѕ' => 's', + '+' | '7' | 'т' | 'τ' => 't', + 'υ' | 'ս' => 'u', + 'х' | 'χ' => 'x', + 'у' | 'γ' => 'y', + '2' => 'z', + _ => character, + } } fn is_invisible_separator(character: char) -> bool { - let code = character as u32; - is_in_ranges( - code, - &[ - (0x00ad, 0x00ad), - (0x034f, 0x034f), - (0x061c, 0x061c), - (0x115f, 0x1160), - (0x17b4, 0x17b5), - (0x180b, 0x180f), - (0x200b, 0x200f), - (0x202a, 0x202e), - (0x2060, 0x206f), - (0x3164, 0x3164), - (0xfe00, 0xfe0f), - (0xfeff, 0xfeff), - (0xffa0, 0xffa0), - (0xe0100, 0xe01ef), - ], - ) + let code = character as u32; + is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x034f, 0x034f), + (0x061c, 0x061c), + (0x115f, 0x1160), + (0x17b4, 0x17b5), + (0x180b, 0x180f), + (0x200b, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0x3164, 0x3164), + (0xfe00, 0xfe0f), + (0xfeff, 0xfeff), + (0xffa0, 0xffa0), + (0xe0100, 0xe01ef), + ], + ) } fn profanity_types() -> Type { - Type::PROFANE & Type::MODERATE_OR_HIGHER + Type::PROFANE & Type::MODERATE_OR_HIGHER } fn slur_types() -> Type { - Type::OFFENSIVE & Type::MILD_OR_HIGHER + Type::OFFENSIVE & Type::MILD_OR_HIGHER } pub(super) fn has_non_standard_text(text: &str) -> bool { - non_standard_character_count(text) > 0 + non_standard_character_count(text) > 0 } pub(super) fn non_standard_text_ratio(text: &str) -> f64 { - let character_count = text.chars().count(); - if character_count == 0 { - return 0.0; - } + let character_count = text.chars().count(); + if character_count == 0 { + return 0.0; + } - non_standard_character_count(text) as f64 / character_count as f64 + non_standard_character_count(text) as f64 / character_count as f64 } fn non_standard_character_count(text: &str) -> usize { - let mut count = 0; - let mut combining_marks = 0; - let mut has_base = false; - - for character in text.chars() { - let code = character as u32; - let is_combining = is_in_ranges( - code, - &[ - (0x0300, 0x036f), - (0x1ab0, 0x1aff), - (0x1dc0, 0x1dff), - (0x20d0, 0x20ff), - (0xfe20, 0xfe2f), - ], - ); - if is_combining { - combining_marks += 1; - if !has_base || combining_marks > 2 { - count += 1; - } - continue; - } - - combining_marks = 0; - has_base = !character.is_whitespace(); - let allowed_control = matches!(character, '\n' | '\r' | '\t'); - let control = character.is_control() && !allowed_control; - let private_use = is_in_ranges( - code, - &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], - ); - let invisible = is_in_ranges( - code, - &[ - (0x00ad, 0x00ad), - (0x061c, 0x061c), - (0x200e, 0x200f), - (0x202a, 0x202e), - (0x2060, 0x206f), - (0xfeff, 0xfeff), - ], - ); - let fancy = is_in_ranges( - code, - &[ - (0x02b0, 0x02ff), - (0x1d400, 0x1d7ff), - (0x2460, 0x24ff), - (0x2070, 0x209f), - (0x2100, 0x214f), - (0xfb00, 0xfb06), - (0xff10, 0xff19), - (0xff21, 0xff3a), - (0xff41, 0xff5a), - (0x1f100, 0x1f1ad), - ], - ) && !matches!( - code, - 0x02d6 - | 0x02d7 | 0x02d8 - | 0x02d9 | 0x02da - | 0x02db | 0x02dc - | 0x02dd | 0x207a - | 0x207b | 0x208a - | 0x208b | 0x2120 - | 0x2122 | 0x2139 - ); - - if control || private_use || invisible || fancy { - count += 1; - } - } - - count + let mut count = 0; + let mut combining_marks = 0; + let mut has_base = false; + + for character in text.chars() { + let code = character as u32; + let is_combining = is_in_ranges( + code, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ); + if is_combining { + combining_marks += 1; + if !has_base || combining_marks > 2 { + count += 1; + } + continue; + } + + combining_marks = 0; + has_base = !character.is_whitespace(); + let allowed_control = matches!(character, '\n' | '\r' | '\t'); + let control = character.is_control() && !allowed_control; + let private_use = is_in_ranges( + code, + &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], + ); + let invisible = is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x061c, 0x061c), + (0x200e, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0xfeff, 0xfeff), + ], + ); + let fancy = is_in_ranges( + code, + &[ + (0x02b0, 0x02ff), + (0x1d400, 0x1d7ff), + (0x2460, 0x24ff), + (0x2070, 0x209f), + (0x2100, 0x214f), + (0xfb00, 0xfb06), + (0xff10, 0xff19), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0x1f100, 0x1f1ad), + ], + ) && !matches!( + code, + 0x02d6 + | 0x02d7 + | 0x02d8 + | 0x02d9 + | 0x02da + | 0x02db + | 0x02dc + | 0x02dd + | 0x207a + | 0x207b + | 0x208a + | 0x208b + | 0x2120 + | 0x2122 + | 0x2139 + ); + + if control || private_use || invisible || fancy { + count += 1; + } + } + + count } fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { - ranges - .iter() - .any(|(start, end)| code >= *start && code <= *end) + ranges + .iter() + .any(|(start, end)| code >= *start && code <= *end) } pub(super) fn contains_spam(text: &str) -> bool { - let normalized = - text.nfc().flat_map(char::to_lowercase).collect::(); - let mut previous = None; - let mut repeated = 0; - for character in normalized.chars() { - if !character.is_whitespace() && previous == Some(character) { - repeated += 1; - } else { - repeated = usize::from(!character.is_whitespace()); - previous = Some(character); - } - if repeated >= 8 { - return true; - } - } - - let words = WORD - .find_iter(&normalized) - .map(|word| word.as_str()) - .collect::>(); - let mut repeated_words = 1; - for pair in words.windows(2) { - repeated_words = if pair[0] == pair[1] { - repeated_words + 1 - } else { - 1 - }; - if repeated_words >= 4 { - return true; - } - } - - let max_phrase_words = 8.min(words.len() / 3); - for phrase_words in (2..=max_phrase_words).rev() { - for phrases in words.windows(phrase_words * 3) { - if phrases[..phrase_words] - == phrases[phrase_words..phrase_words * 2] - && phrases[..phrase_words] - == phrases[phrase_words * 2..phrase_words * 3] - { - return true; - } - } - } - - false + let normalized = + text.nfc().flat_map(char::to_lowercase).collect::(); + let mut previous = None; + let mut repeated = 0; + for character in normalized.chars() { + if !character.is_whitespace() && previous == Some(character) { + repeated += 1; + } else { + repeated = usize::from(!character.is_whitespace()); + previous = Some(character); + } + if repeated >= 8 { + return true; + } + } + + let words = WORD + .find_iter(&normalized) + .map(|word| word.as_str()) + .collect::>(); + let mut repeated_words = 1; + for pair in words.windows(2) { + repeated_words = if pair[0] == pair[1] { + repeated_words + 1 + } else { + 1 + }; + if repeated_words >= 4 { + return true; + } + } + + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for phrases in words.windows(phrase_words * 3) { + if phrases[..phrase_words] + == phrases[phrase_words..phrase_words * 2] + && phrases[..phrase_words] + == phrases[phrase_words * 2..phrase_words * 3] + { + return true; + } + } + } + + false } pub(super) fn find_link_or_ip(text: &str) -> Option { - SUMMARY_LINK_FINDER.links(text).find_map(|link| { - let raw = link.as_str(); - if raw.contains("://") { - return Some(raw.to_owned()); - } - - Url::parse(&format!("https://{raw}")) - .ok() - .filter(|url| { - url.host_str().is_some_and(|hostname| { - hostname.parse::().is_ok() - || psl::domain(hostname.as_bytes()).is_some_and( - |domain| domain.suffix().typ().is_some(), - ) - }) - }) - .map(|_| raw.to_owned()) - }) + SUMMARY_LINK_FINDER.links(text).find_map(|link| { + let raw = link.as_str(); + if raw.contains("://") { + return Some(raw.to_owned()); + } + + Url::parse(&format!("https://{raw}")) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + hostname.parse::().is_ok() + || psl::domain(hostname.as_bytes()).is_some_and( + |domain| domain.suffix().typ().is_some(), + ) + }) + }) + .map(|_| raw.to_owned()) + }) } pub(super) fn has_summary_formatting(summary: &str) -> bool { - has_paired_html_formatting(summary) - || MARKDOWN_LINK.is_match(summary) - || summary.lines().any(|line| { - let line = line.trim_start(); - line.starts_with('#') - || line.starts_with('>') - || line.starts_with("- ") - || line.starts_with("* ") - || line.starts_with("+ ") - || line.starts_with("```") - || line.contains("**") - || line.contains("__") - || line.contains("~~") - || INLINE_CODE.is_match(line) - }) + has_paired_html_formatting(summary) + || MARKDOWN_LINK.is_match(summary) + || summary.lines().any(|line| { + let line = line.trim_start(); + line.starts_with('#') + || line.starts_with('>') + || line.starts_with("- ") + || line.starts_with("* ") + || line.starts_with("+ ") + || line.starts_with("```") + || line.contains("**") + || line.contains("__") + || line.contains("~~") + || INLINE_CODE.is_match(line) + }) } pub(super) fn has_paired_html_formatting(text: &str) -> bool { - let without_code = CODE_BLOCK.replace_all(text, ""); - let without_code = INLINE_CODE.replace_all(&without_code, ""); - HTML_OPEN_TAG.captures_iter(&without_code).any(|opening| { - let Some(tag) = opening.get(1) else { - return false; - }; - let Some(opening_match) = opening.get(0) else { - return false; - }; - HTML_CLOSE_TAG - .captures_iter(&without_code[opening_match.end()..]) - .any(|closing| { - closing.get(1).is_some_and(|closing_tag| { - closing_tag.as_str().eq_ignore_ascii_case(tag.as_str()) - }) - }) - }) + let without_code = CODE_BLOCK.replace_all(text, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + HTML_OPEN_TAG.captures_iter(&without_code).any(|opening| { + let Some(tag) = opening.get(1) else { + return false; + }; + let Some(opening_match) = opening.get(0) else { + return false; + }; + HTML_CLOSE_TAG + .captures_iter(&without_code[opening_match.end()..]) + .any(|closing| { + closing.get(1).is_some_and(|closing_tag| { + closing_tag.as_str().eq_ignore_ascii_case(tag.as_str()) + }) + }) + }) } pub(super) fn extract_description_text(markdown: &str) -> String { - let without_code = CODE_BLOCK.replace_all(markdown, " "); - let without_code = INLINE_CODE.replace_all(&without_code, " "); - let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); - let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); - let with_html_image_alt = HTML_IMAGE.replace_all( - &without_links, - |captures: ®ex::Captures<'_>| { - ALT_ATTRIBUTE - .captures(&captures[0]) - .and_then(|captures| { - captures.get(1).or_else(|| captures.get(2)) - }) - .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) - }, - ); - let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); - without_html - .lines() - .map(|line| { - line.trim_start_matches(|character| matches!(character, '>' | '#')) - }) - .collect::>() - .join("\n") - .replace(['*', '_', '~', '`', '>', '-', '|'], " ") - .split_whitespace() - .collect::>() - .join(" ") + let without_code = CODE_BLOCK.replace_all(markdown, " "); + let without_code = INLINE_CODE.replace_all(&without_code, " "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); + let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); + let with_html_image_alt = HTML_IMAGE.replace_all( + &without_links, + |captures: ®ex::Captures<'_>| { + ALT_ATTRIBUTE + .captures(&captures[0]) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) + }, + ); + let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); + without_html + .lines() + .map(|line| { + line.trim_start_matches(|character| matches!(character, '>' | '#')) + }) + .collect::>() + .join("\n") + .replace(['*', '_', '~', '`', '>', '-', '|'], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +pub(super) fn text_without_links(text: &str) -> String { + let mut without_links = String::with_capacity(text.len()); + let mut previous_end = 0; + + for link in DESCRIPTION_LINK_FINDER.links(&text) { + without_links.push_str(&text[previous_end..link.start()]); + without_links.push(' '); + previous_end = link.end(); + } + without_links.push_str(&text[previous_end..]); + + without_links + .split_whitespace() + .collect::>() + .join(" ") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - let without_code = CODE_BLOCK.replace_all(markdown, ""); - DESCRIPTION_BLOCK_BREAK - .split(&without_code) - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + let without_code = CODE_BLOCK.replace_all(markdown, ""); + DESCRIPTION_BLOCK_BREAK + .split(&without_code) + .map(extract_description_text) + .filter(|block| !block.is_empty()) + .collect() } pub(super) fn long_header_count(markdown: &str) -> usize { - let markdown_headers = HEADER - .captures_iter(markdown) - .filter(|captures| header_is_long(&captures[1])) - .count(); - let setext_headers = SETEXT_HEADER - .captures_iter(markdown) - .filter(|captures| !captures[1].trim_start().starts_with('#')) - .filter(|captures| header_is_long(&captures[1])) - .count(); - let html_headers = HTML_HEADER - .captures_iter(markdown) - .filter(|captures| header_is_long(&captures[1])) - .count(); - - markdown_headers + setext_headers + html_headers + let markdown_headers = HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let setext_headers = SETEXT_HEADER + .captures_iter(markdown) + .filter(|captures| !captures[1].trim_start().starts_with('#')) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let html_headers = HTML_HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + + markdown_headers + setext_headers + html_headers } fn header_is_long(header: &str) -> bool { - let with_image_alt = MARKDOWN_IMAGE.replace_all(header, "$1"); - let with_link_text = MARKDOWN_LINK.replace_all(&with_image_alt, "$1"); - let without_html = HTML_TAG.replace_all(&with_link_text, " "); - let rendered = without_html - .replace(['*', '_', '~', '`'], "") - .split_whitespace() - .collect::>() - .join(" "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(header, "$1"); + let with_link_text = MARKDOWN_LINK.replace_all(&with_image_alt, "$1"); + let without_html = HTML_TAG.replace_all(&with_link_text, " "); + let rendered = without_html + .replace(['*', '_', '~', '`'], "") + .split_whitespace() + .collect::>() + .join(" "); - rendered.graphemes(true).count() > 80 + rendered.graphemes(true).count() > 80 } pub(super) fn description_ends_with_header(markdown: &str) -> bool { - let trimmed = markdown.trim_end(); - if trimmed.is_empty() { - return false; - } - - let lines = trimmed.lines().collect::>(); - let last_line = lines.last().map_or("", |line| line.trim()); - if HEADER_LINE.is_match(last_line) { - return true; - } - if lines.len() >= 2 - && is_setext_underline(last_line) - && !lines[lines.len() - 2].trim().is_empty() - { - return true; - } - - TRAILING_HTML_HEADER.is_match(trimmed) + let trimmed = markdown.trim_end(); + if trimmed.is_empty() { + return false; + } + + let lines = trimmed.lines().collect::>(); + let last_line = lines.last().map_or("", |line| line.trim()); + if HEADER_LINE.is_match(last_line) { + return true; + } + if lines.len() >= 2 + && is_setext_underline(last_line) + && !lines[lines.len() - 2].trim().is_empty() + { + return true; + } + + TRAILING_HTML_HEADER.is_match(trimmed) } pub(super) fn has_adjacent_same_level_headers(markdown: &str) -> bool { - let lines = markdown.lines().collect::>(); - let mut previous_header = None; - let mut index = 0; - while index < lines.len() { - let line = lines[index].trim(); - if line.is_empty() { - index += 1; - continue; - } - - let mut header_level = HEADER_LINE - .captures(line) - .and_then(|captures| captures.get(1)) - .map(|hashes| hashes.as_str().len()); - if header_level.is_none() - && lines - .get(index + 1) - .is_some_and(|underline| is_setext_underline(underline.trim())) - { - header_level = - Some(if lines[index + 1].trim_start().starts_with('=') { - 1 - } else { - 2 - }); - index += 1; - } - - if let Some(level) = header_level { - if level <= 3 && previous_header == Some(level) { - return true; - } - previous_header = Some(level); - } else { - previous_header = None; - } - index += 1; - } - - ADJACENT_HTML_HEADERS - .captures_iter(markdown) - .any(|captures| { - captures.get(1).map(|level| level.as_str()) - == captures.get(2).map(|level| level.as_str()) - }) + let lines = markdown.lines().collect::>(); + let mut previous_header = None; + let mut index = 0; + while index < lines.len() { + let line = lines[index].trim(); + if line.is_empty() { + index += 1; + continue; + } + + let mut header_level = HEADER_LINE + .captures(line) + .and_then(|captures| captures.get(1)) + .map(|hashes| hashes.as_str().len()); + if header_level.is_none() + && lines + .get(index + 1) + .is_some_and(|underline| is_setext_underline(underline.trim())) + { + header_level = + Some(if lines[index + 1].trim_start().starts_with('=') { + 1 + } else { + 2 + }); + index += 1; + } + + if let Some(level) = header_level { + if level <= 3 && previous_header == Some(level) { + return true; + } + previous_header = Some(level); + } else { + previous_header = None; + } + index += 1; + } + + ADJACENT_HTML_HEADERS + .captures_iter(markdown) + .any(|captures| { + captures.get(1).map(|level| level.as_str()) + == captures.get(2).map(|level| level.as_str()) + }) } fn is_setext_underline(line: &str) -> bool { - let mut characters = line.chars(); - let Some(marker @ ('=' | '-')) = characters.next() else { - return false; - }; - characters.all(|character| character == marker) + let mut characters = line.chars(); + let Some(marker @ ('=' | '-')) = characters.next() else { + return false; + }; + characters.all(|character| character == marker) } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { - let without_code = CODE_BLOCK.replace_all(markdown, ""); - let without_code = INLINE_CODE.replace_all(&without_code, ""); - MARKDOWN_IMAGE - .captures_iter(&without_code) - .any(|captures| captures[1].trim().is_empty()) - || HTML_IMAGE.find_iter(&without_code).any(|image| { - ALT_ATTRIBUTE - .captures(image.as_str()) - .and_then(|captures| { - captures.get(1).or_else(|| captures.get(2)) - }) - .is_none_or(|alt| alt.as_str().trim().is_empty()) - }) + let without_code = CODE_BLOCK.replace_all(markdown, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + MARKDOWN_IMAGE + .captures_iter(&without_code) + .any(|captures| captures[1].trim().is_empty()) + || HTML_IMAGE.find_iter(&without_code).any(|image| { + ALT_ATTRIBUTE + .captures(image.as_str()) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .is_none_or(|alt| alt.as_str().trim().is_empty()) + }) } pub(super) fn find_banned_description_link(markdown: &str) -> Option { - DESCRIPTION_LINK_FINDER.links(markdown).find_map(|link| { - let raw = link.as_str(); - let normalized = if raw.contains("://") { - raw.to_owned() - } else { - format!("http://{raw}") - }; - Url::parse(&normalized) - .ok() - .filter(|url| { - url.host_str().is_some_and(|hostname| { - URL_SHORTENERS - .iter() - .any(|domain| hostname_matches_domain(hostname, domain)) - }) - }) - .map(|_| normalized) - }) + DESCRIPTION_LINK_FINDER.links(markdown).find_map(|link| { + let raw = link.as_str(); + let normalized = if raw.contains("://") { + raw.to_owned() + } else { + format!("http://{raw}") + }; + Url::parse(&normalized) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) + }) + .map(|_| normalized) + }) } fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { - hostname.eq_ignore_ascii_case(domain) - || hostname - .to_ascii_lowercase() - .ends_with(&format!(".{domain}")) + hostname.eq_ignore_ascii_case(domain) + || hostname + .to_ascii_lowercase() + .ends_with(&format!(".{domain}")) } pub(super) fn project_requires_english(project: &Project) -> bool { - let has_locale_tag = project - .categories - .iter() - .chain(&project.additional_categories) - .any(|category| category == "locale"); - let is_english_server = project - .components - .minecraft_server - .as_ref() - .is_some_and(|server| server.languages.contains(&Language::En)); + let has_locale_tag = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "locale"); + let is_english_server = project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)); - (project.components.minecraft_java_server.is_none() && !has_locale_tag) - || is_english_server + (project.components.minecraft_java_server.is_none() && !has_locale_tag) + || is_english_server } pub(super) fn is_likely_english_summary(text: &str) -> bool { - if !has_enough_language_content(text) { - return true; - } + if !has_enough_language_content(text) { + return true; + } - LANGUAGE_DETECTOR - .detect(text) - .is_none_or(|info| info.lang() == Lang::Eng || info.confidence() < 0.5) + LANGUAGE_DETECTOR + .detect(text) + .is_none_or(|info| info.lang() == Lang::Eng || info.confidence() < 0.5) } pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let mut english_chunks = 0; - let mut non_english_chunks = 0; + let mut english_chunks = 0; + let mut non_english_chunks = 0; - for block in blocks { - for chunk in language_chunks(block) { - let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { - continue; - }; + for block in blocks { + for chunk in language_chunks(block) { + let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { + continue; + }; - if info.lang() == Lang::Eng { - english_chunks += 1; - } else if info.confidence() >= 0.8 { - non_english_chunks += 1; - } - } - } + if info.lang() == Lang::Eng { + english_chunks += 1; + } else if info.confidence() >= 0.8 { + non_english_chunks += 1; + } + } + } - let classified_chunks = english_chunks + non_english_chunks; - classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 + let classified_chunks = english_chunks + non_english_chunks; + classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 } fn language_chunks(block: &str) -> Vec { - const CHUNK_WORDS: usize = 24; - const CHUNK_STRIDE_WORDS: usize = 12; - - let words = WORD - .find_iter(block) - .map(|word| word.as_str()) - .collect::>(); - if words.len() < 8 { - return Vec::new(); - } - if words.len() <= CHUNK_WORDS { - let chunk = words.join(" "); - return has_enough_language_content(&chunk) - .then_some(chunk) - .into_iter() - .collect(); - } - - let mut starts = BTreeSet::new(); - let mut start = 0; - while start + 8 <= words.len() { - starts.insert(start); - start += CHUNK_STRIDE_WORDS; - } - starts.insert(words.len() - CHUNK_WORDS); - - starts - .into_iter() - .map(|start| { - words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") - }) - .filter(|chunk| has_enough_language_content(chunk)) - .collect() + const CHUNK_WORDS: usize = 24; + const CHUNK_STRIDE_WORDS: usize = 12; + + let words = WORD + .find_iter(block) + .map(|word| word.as_str()) + .collect::>(); + if words.len() < 8 { + return Vec::new(); + } + if words.len() <= CHUNK_WORDS { + let chunk = words.join(" "); + return has_enough_language_content(&chunk) + .then_some(chunk) + .into_iter() + .collect(); + } + + let mut starts = BTreeSet::new(); + let mut start = 0; + while start + 8 <= words.len() { + starts.insert(start); + start += CHUNK_STRIDE_WORDS; + } + starts.insert(words.len() - CHUNK_WORDS); + + starts + .into_iter() + .map(|start| { + words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") + }) + .filter(|chunk| has_enough_language_content(chunk)) + .collect() } fn has_enough_language_content(text: &str) -> bool { - WORD.find_iter(text).count() >= 8 - && text.trim().graphemes(true).count() >= 35 + WORD.find_iter(text).count() >= 8 + && text.trim().graphemes(true).count() >= 35 } From 68b9b40297c6df679f1ab58104e8341b54f57544 Mon Sep 17 00:00:00 2001 From: tdgao Date: Wed, 2 Sep 2026 22:35:12 -0600 Subject: [PATCH 11/28] feat: allow gallery description to have 2 fucks --- apps/labrinth/src/validate/project/gallery.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs index 11679c85ee..38a811f0bc 100644 --- a/apps/labrinth/src/validate/project/gallery.rs +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -3,10 +3,13 @@ use serde_json::json; use super::text::{ProfanityKind, has_non_standard_text, profanity_matches}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +const MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT: usize = 2; + fn validate_text( text: Option<&str>, gallery_index: usize, field: &'static str, + max_profanity_count: usize, ) -> Vec { let text = text.unwrap_or_default(); let mut nags = Vec::new(); @@ -30,7 +33,8 @@ fn validate_text( } if let Some(matched) = profanity .iter() - .find(|matched| matched.kind == ProfanityKind::Profanity) + .filter(|matched| matched.kind == ProfanityKind::Profanity) + .nth(max_profanity_count) { nags.push( ProjectNag::new( @@ -102,11 +106,12 @@ pub(super) fn validate( } for (index, item) in project.gallery.iter().enumerate() { - nags.extend(validate_text(item.name.as_deref(), index, "name")); + nags.extend(validate_text(item.name.as_deref(), index, "name", 0)); nags.extend(validate_text( item.description.as_deref(), index, "description", + MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT, )); } From 942f8a9942d53546bf5d2a47e134e5f72033f11d Mon Sep 17 00:00:00 2001 From: tdgao Date: Wed, 2 Sep 2026 23:31:12 -0600 Subject: [PATCH 12/28] feat: improve description spam detection to reduce false positives --- .../src/validate/project/description.rs | 32 ++- apps/labrinth/src/validate/project/text.rs | 241 ++++++++++++++++-- 2 files changed, 249 insertions(+), 24 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index a77f1891e2..e18a549d01 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -1,13 +1,12 @@ use serde_json::json; use super::text::{ - ProfanityKind, contains_spam, description_ends_with_header, + ProfanityKind, contains_description_spam, description_ends_with_header, extract_description_blocks, extract_description_text, find_banned_description_link, has_adjacent_same_level_headers, has_image_without_alt_text, has_sufficient_english_blocks, js_string_length, long_header_count, non_standard_text_ratio, normalize_project_field_text, profanity_matches, project_requires_english, - text_without_links, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -22,7 +21,7 @@ pub(super) fn validate(project: &Project) -> Vec { let description = project.description.as_str(); let normalized_description = normalize_project_field_text(description); let text = extract_description_text(description); - let spam_text = text_without_links(&text); + let has_spam = has_description_spam(description); let normalized_text = extract_description_text(&normalized_description); let blocks = extract_description_blocks(description); let profanity = profanity_matches(description); @@ -62,7 +61,7 @@ pub(super) fn validate(project: &Project) -> Vec { } if project_requires_english(project) && js_string_length(&text) >= MIN_DESCRIPTION_CHARS - && !contains_spam(&spam_text) + && !has_spam && !has_sufficient_english_blocks(&blocks) { nags.push(ProjectNag::new( @@ -90,7 +89,7 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } - if contains_spam(&spam_text) { + if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, ProjectNagSeverity::Required, @@ -136,3 +135,26 @@ pub(super) fn validate(project: &Project) -> Vec { nags } + +fn has_description_spam(markdown: &str) -> bool { + contains_description_spam(markdown) +} + +#[cfg(test)] +mod tests { + use super::has_description_spam; + + #[test] + fn description_spam_ignores_formatting() { + assert!(!has_description_spam("======== ━━━━━━━━ !!!!!!!!")); + assert!(has_description_spam(&"a".repeat(64))); + } + + #[test] + fn description_spam_allows_isolated_repetition() { + assert!(!has_description_spam(&format!( + "aaaaaaaa {}", + "text ".repeat(140) + ))); + } +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index ce5cd489da..b8932fd8c6 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; @@ -14,6 +14,10 @@ use crate::models::projects::Project; static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); +static SPAM_TOKEN: LazyLock = LazyLock::new(|| { + Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) + .unwrap() +}); static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { let mut finder = LinkFinder::new(); finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); @@ -24,6 +28,9 @@ static MARKDOWN_LINK: LazyLock = LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?is)|]*>").unwrap()); +static HTML_ENTITY: LazyLock = LazyLock::new(|| { + Regex::new(r"&(?:#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]+);").unwrap() +}); static HTML_OPEN_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?is)<([a-z][\w:-]*)\b[^>]*>").unwrap()); static HTML_CLOSE_TAG: LazyLock = @@ -33,7 +40,7 @@ static CODE_BLOCK: LazyLock = static DESCRIPTION_BLOCK_BREAK: LazyLock = LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = - LazyLock::new(|| Regex::new(r"`[^`]*`").unwrap()); + LazyLock::new(|| Regex::new(r"`([^`]*)`").unwrap()); static MARKDOWN_IMAGE: LazyLock = LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_IMAGE: LazyLock = @@ -578,6 +585,219 @@ pub(super) fn contains_spam(text: &str) -> bool { false } +pub(super) fn contains_description_spam(markdown: &str) -> bool { + const MIN_CHARACTER_RUN: usize = 16; + const MIN_CHARACTER_EXCESS: usize = 20; + const MIN_CHARACTER_EXCESS_PERCENT: usize = 10; + const EXTREME_CHARACTER_RUN: usize = 64; + const MIN_REPEATED_WORDS: usize = 6; + const MIN_REPEATED_PHRASE_WORDS: usize = 12; + const MIN_DUPLICATE_BLOCK_PERCENT: usize = 20; + + let blocks = extract_description_spam_blocks(markdown); + let mut block_counts = BTreeMap::, usize>::new(); + let mut visible_characters = 0; + let mut repeated_character_excess = 0; + let mut longest_character_run = 0; + let mut total_words = 0; + + for block in blocks { + let words = spam_words(&block); + if words.is_empty() { + continue; + } + total_words += words.len(); + + update_character_repetition( + &words, + &mut visible_characters, + &mut repeated_character_excess, + &mut longest_character_run, + ); + + if has_repeated_words(&words, MIN_REPEATED_WORDS) + || has_repeated_phrase(&words, MIN_REPEATED_PHRASE_WORDS) + { + return true; + } + + if is_duplicate_spam_block_candidate(&block) { + *block_counts.entry(words).or_default() += 1; + } + } + + let duplicate_block_words = block_counts + .iter() + .filter(|(words, count)| { + **count >= 3 && words.len() * **count >= MIN_REPEATED_PHRASE_WORDS + }) + .map(|(words, count)| words.len() * (count - 1)) + .sum::(); + if duplicate_block_words > 0 + && duplicate_block_words * 100 + >= total_words * MIN_DUPLICATE_BLOCK_PERCENT + { + return true; + } + + longest_character_run >= EXTREME_CHARACTER_RUN + || (longest_character_run >= MIN_CHARACTER_RUN + && repeated_character_excess >= MIN_CHARACTER_EXCESS + && repeated_character_excess * 100 + >= visible_characters * MIN_CHARACTER_EXCESS_PERCENT) +} + +fn extract_description_spam_blocks(markdown: &str) -> Vec { + let without_code = CODE_BLOCK.replace_all(markdown, "\n\n"); + let with_inline_code = INLINE_CODE.replace_all(&without_code, "$1"); + let without_images = MARKDOWN_IMAGE.replace_all(&with_inline_code, " "); + let with_link_labels = MARKDOWN_LINK.replace_all(&without_images, "$1"); + let without_html_images = HTML_IMAGE.replace_all(&with_link_labels, " "); + let without_html = HTML_TAG.replace_all(&without_html_images, " "); + let without_entities = HTML_ENTITY.replace_all(&without_html, " "); + let without_links = text_without_explicit_links(&without_entities); + + let mut blocks = Vec::new(); + let mut paragraph = String::new(); + for line in without_links.lines() { + let line = line.trim(); + if line.is_empty() { + push_spam_block(&mut blocks, &mut paragraph); + continue; + } + + if is_standalone_spam_block(line) { + push_spam_block(&mut blocks, &mut paragraph); + blocks.push(line.to_owned()); + } else { + if !paragraph.is_empty() { + paragraph.push(' '); + } + paragraph.push_str(line); + } + } + push_spam_block(&mut blocks, &mut paragraph); + + blocks +} + +fn text_without_explicit_links(text: &str) -> String { + let mut without_links = String::with_capacity(text.len()); + let mut previous_end = 0; + + for link in DESCRIPTION_LINK_FINDER.links(text).filter(|link| { + let link = link.as_str(); + link.contains("://") || link.starts_with("www.") + }) { + without_links.push_str(&text[previous_end..link.start()]); + without_links.push(' '); + previous_end = link.end(); + } + without_links.push_str(&text[previous_end..]); + without_links +} + +fn push_spam_block(blocks: &mut Vec, paragraph: &mut String) { + if !paragraph.is_empty() { + blocks.push(std::mem::take(paragraph)); + } +} + +fn is_standalone_spam_block(line: &str) -> bool { + let line = line.trim_start_matches('>'); + let trimmed = line.trim_start(); + let is_heading = trimmed.starts_with('#'); + let is_list_item = ["- ", "* ", "+ ", "•"] + .iter() + .any(|prefix| trimmed.starts_with(prefix)) + || trimmed.split_once(". ").is_some_and(|(prefix, _)| { + prefix.chars().all(|char| char.is_ascii_digit()) + }); + let is_table_row = trimmed.contains('|'); + + is_heading || is_list_item || is_table_row +} + +fn is_duplicate_spam_block_candidate(block: &str) -> bool { + let trimmed = block.trim_start_matches('>').trim_start(); + !trimmed.starts_with('#') && !trimmed.contains('|') +} + +fn spam_words(text: &str) -> Vec { + SPAM_TOKEN + .find_iter(text) + .map(|word| word.as_str().nfc().flat_map(char::to_lowercase).collect()) + .collect() +} + +fn update_character_repetition( + words: &[String], + visible_characters: &mut usize, + repeated_character_excess: &mut usize, + longest_character_run: &mut usize, +) { + for word in words { + *visible_characters += word.chars().count(); + let mut previous = None; + let mut run: usize = 0; + for character in word.chars() { + if previous == Some(character) { + run += 1; + } else { + *repeated_character_excess += run.saturating_sub(3); + *longest_character_run = (*longest_character_run).max(run); + previous = Some(character); + run = 1; + } + } + *repeated_character_excess += run.saturating_sub(3); + *longest_character_run = (*longest_character_run).max(run); + } +} + +fn has_repeated_words(words: &[String], minimum_repetitions: usize) -> bool { + let mut repetitions = 1; + for pair in words.windows(2) { + repetitions = if pair[0] == pair[1] { + repetitions + 1 + } else { + 1 + }; + if repetitions >= minimum_repetitions { + return true; + } + } + + false +} + +fn has_repeated_phrase( + words: &[String], + minimum_repeated_words: usize, +) -> bool { + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for start in 0..=words.len() - phrase_words * 3 { + let phrase = &words[start..start + phrase_words]; + let mut repetitions = 1; + while start + phrase_words * (repetitions + 1) <= words.len() + && phrase + == &words[start + phrase_words * repetitions + ..start + phrase_words * (repetitions + 1)] + { + repetitions += 1; + } + if repetitions >= 3 + && phrase_words * repetitions >= minimum_repeated_words + { + return true; + } + } + } + + false +} + pub(super) fn find_link_or_ip(text: &str) -> Option { SUMMARY_LINK_FINDER.links(text).find_map(|link| { let raw = link.as_str(); @@ -667,23 +887,6 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } -pub(super) fn text_without_links(text: &str) -> String { - let mut without_links = String::with_capacity(text.len()); - let mut previous_end = 0; - - for link in DESCRIPTION_LINK_FINDER.links(&text) { - without_links.push_str(&text[previous_end..link.start()]); - without_links.push(' '); - previous_end = link.end(); - } - without_links.push_str(&text[previous_end..]); - - without_links - .split_whitespace() - .collect::>() - .join(" ") -} - pub(super) fn extract_description_blocks(markdown: &str) -> Vec { let without_code = CODE_BLOCK.replace_all(markdown, ""); DESCRIPTION_BLOCK_BREAK From 0594618e302e11a4d813c47adfbc93f8b14d9ffd Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 3 Sep 2026 00:25:00 -0600 Subject: [PATCH 13/28] feat: add find nag project ids to check project script --- apps/labrinth/src/bin/check_project.rs | 41 +++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index 186b9fb5c5..b93bb5b920 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -10,7 +10,7 @@ use labrinth::models::projects::{Project, Version}; use labrinth::validate::project::{ ProjectNagKind, ProjectNagSeverity, validate, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde::de::{ DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor, @@ -53,6 +53,10 @@ struct Args { #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] show_description_profanity: usize, + /// Print project IDs grouped by every triggered nag kind + #[arg(long, requires = "file")] + nag_project_ids: bool, + /// Scan every profanity-bearing field into a reusable JSON report #[arg( long, @@ -163,6 +167,9 @@ async fn main() -> Result<()> { } if let Some(path) = args.file.as_deref() { + if args.nag_project_ids { + return print_json(&find_nag_project_ids(path)?); + } let summary = summarize_file(path, args.show_description_profanity)?; print_description_profanity_samples( &summary.description_profanity_samples, @@ -178,6 +185,38 @@ async fn main() -> Result<()> { check_api_project(project_id, args.token.as_deref()).await } +#[derive(Deserialize)] +struct ProjectDataset { + projects: Vec, +} + +fn find_nag_project_ids( + path: &Path, +) -> Result>> { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let mut project_ids = ProjectNagKind::iter() + .map(|kind| (kind, Vec::new())) + .collect::>(); + + for project in dataset.projects { + let kinds = validate(&project, &[]) + .into_iter() + .map(|nag| nag.kind) + .collect::>(); + for kind in kinds { + project_ids + .entry(kind) + .or_default() + .push(project.id.to_string()); + } + } + + Ok(project_ids) +} + async fn check_api_project( project_id: &str, token: Option<&str>, From 6b912a5b0ef1f78c7bcd90468a7f4cb5b797c515 Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 3 Sep 2026 10:21:01 -0600 Subject: [PATCH 14/28] feat: add ngram density for spam detection --- apps/labrinth/src/validate/project/text.rs | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index b8932fd8c6..5e101f948f 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -617,6 +617,7 @@ pub(super) fn contains_description_spam(markdown: &str) -> bool { if has_repeated_words(&words, MIN_REPEATED_WORDS) || has_repeated_phrase(&words, MIN_REPEATED_PHRASE_WORDS) + || has_repeated_ngram_density(&words) { return true; } @@ -798,6 +799,42 @@ fn has_repeated_phrase( false } +fn has_repeated_ngram_density(words: &[String]) -> bool { + const NGRAM_WORDS: usize = 4; + const MIN_WORDS: usize = 30; + const MIN_DUPLICATE_NGRAM_PERCENT: usize = 50; + const MIN_COVERED_WORD_PERCENT: usize = 70; + + if words.len() < MIN_WORDS { + return false; + } + + let mut ngram_counts = BTreeMap::<&[String], usize>::new(); + for ngram in words.windows(NGRAM_WORDS) { + *ngram_counts.entry(ngram).or_default() += 1; + } + + let total_ngrams = words.len() - NGRAM_WORDS + 1; + let duplicate_ngrams = ngram_counts + .values() + .map(|count| count.saturating_sub(1)) + .sum::(); + if duplicate_ngrams * 100 + < total_ngrams * MIN_DUPLICATE_NGRAM_PERCENT + { + return false; + } + + let mut covered_words = vec![false; words.len()]; + for (start, ngram) in words.windows(NGRAM_WORDS).enumerate() { + if ngram_counts.get(ngram).is_some_and(|count| *count > 1) { + covered_words[start..start + NGRAM_WORDS].fill(true); + } + } + covered_words.iter().filter(|covered| **covered).count() * 100 + >= words.len() * MIN_COVERED_WORD_PERCENT +} + pub(super) fn find_link_or_ip(text: &str) -> Option { SUMMARY_LINK_FINDER.links(text).find_map(|link| { let raw = link.as_str(); From 410ae64e50b442928dfefbcfe9abc4dbced9f05e Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 3 Sep 2026 10:57:09 -0600 Subject: [PATCH 15/28] feat: add affected link fields to external link validation --- apps/labrinth/src/validate/project/links.rs | 52 ++++++++++++++------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/apps/labrinth/src/validate/project/links.rs b/apps/labrinth/src/validate/project/links.rs index f563378b76..16c2819e57 100644 --- a/apps/labrinth/src/validate/project/links.rs +++ b/apps/labrinth/src/validate/project/links.rs @@ -110,25 +110,43 @@ pub(super) fn validate( )); } - if named_link_is_uncommon(project, "source", SOURCE_DOMAINS) - || named_link_is_uncommon(project, "issues", ISSUES_DOMAINS) - || named_link_is_uncommon(project, "discord", DISCORD_DOMAINS) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::VerifyExternalLinks, - ProjectNagSeverity::Warning, - )); + let uncommon_link_fields = [ + ("source", SOURCE_DOMAINS), + ("issues", ISSUES_DOMAINS), + ("discord", DISCORD_DOMAINS), + ] + .into_iter() + .filter_map(|(key, domains)| { + named_link_is_uncommon(project, key, domains).then_some(key) + }) + .collect::>(); + if !uncommon_link_fields.is_empty() { + nags.push( + super::ProjectNag::new( + ProjectNagKind::VerifyExternalLinks, + ProjectNagSeverity::Warning, + ) + .with_details( + serde_json::json!({ "fields": uncommon_link_fields }), + ), + ); } - if ["source", "issues", "wiki", "site", "store"] - .into_iter() - .filter_map(|key| named_link(project, key)) - .any(is_discord_link) - { - nags.push(super::ProjectNag::new( - ProjectNagKind::MisusedDiscordLink, - ProjectNagSeverity::Required, - )); + let misused_discord_link_fields = + ["source", "issues", "wiki", "site", "store"] + .into_iter() + .filter(|key| named_link(project, key).is_some_and(is_discord_link)) + .collect::>(); + if !misused_discord_link_fields.is_empty() { + nags.push( + super::ProjectNag::new( + ProjectNagKind::MisusedDiscordLink, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "fields": misused_discord_link_fields, + })), + ); } if let Some(url) = find_blocked_external_link(project) { From 58299763ce7dad8e4dfa7d3e552bd82cde8626f6 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:33:40 +0100 Subject: [PATCH 16/28] fail to patch project if fails validation while in review --- apps/labrinth/Cargo.toml | 1 + apps/labrinth/src/routes/v3/projects/mod.rs | 26 +++++++------ apps/labrinth/tests/project.rs | 41 +++++++++++++++++++++ 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 4305caf363..1f0b064e9c 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "labrinth" version = "2.7.0" +default-run = "labrinth" edition.workspace = true license = "AGPL-3.0-only" diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index 44ccde7aef..72e4cab9f4 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -501,6 +501,8 @@ pub async fn project_edit_internal( let submit_for_review = new_project.status == Some(ProjectStatus::Processing) && !user.role.is_mod(); + let validate_for_review = submit_for_review + || project_item.inner.status == ProjectStatus::Processing; if submit_for_review { if !perms.contains(ProjectPermissions::EDIT_DETAILS) { return Err(ApiError::Auth(eyre!( @@ -1358,7 +1360,7 @@ pub async fn project_edit_internal( .await .wrap_api_err("deleting unused images")?; - if submit_for_review { + if validate_for_review { let mut projects = db_models::DBProject::get_many_uncached( &[ProjectId::from(id)], &mut transaction, @@ -1403,19 +1405,21 @@ pub async fn project_edit_internal( &disclosures, ) { return Err(ApiError::Request(eyre!( - "project must have no required validation nags before being submitted for review" + "project must have no required validation nags before or while under review" ))); } - submit_project_for_review( - &reloaded_project, - &user, - team_member.as_ref().is_none_or(|member| !member.accepted), - sync_archival_disclosure, - &mut transaction, - &redis, - ) - .await?; + if submit_for_review { + submit_project_for_review( + &reloaded_project, + &user, + team_member.as_ref().is_none_or(|member| !member.accepted), + sync_archival_disclosure, + &mut transaction, + &redis, + ) + .await?; + } } transaction diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index aaed59404d..83e5b7db0a 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -547,6 +547,47 @@ async fn test_submit_invalid_project_for_review() { .await; } +#[actix_rt::test] +async fn test_edit_invalid_project_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_before.status, ProjectStatus::Processing); + + let response = api + .edit_project( + project_slug, + json!({ "description": "" }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_after.status, ProjectStatus::Processing); + assert_eq!(project_after.description, project_before.description); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v3() { // Hits V3-specific patchable fields From c25c7a791f37b6e198dee4ae582b11d968e84b96 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 3 Sep 2026 20:10:19 +0100 Subject: [PATCH 17/28] add validation to more proj edit routes --- apps/labrinth/src/routes/v3/disclosures.rs | 11 + apps/labrinth/src/routes/v3/projects/mod.rs | 117 ++++----- .../src/routes/v3/projects/validate.rs | 65 ++++- apps/labrinth/src/routes/v3/versions.rs | 25 +- apps/labrinth/tests/project.rs | 228 +++++++++++++++++- apps/labrinth/tests/version.rs | 31 ++- 6 files changed, 413 insertions(+), 64 deletions(-) diff --git a/apps/labrinth/src/routes/v3/disclosures.rs b/apps/labrinth/src/routes/v3/disclosures.rs index b81fefdd32..9923e7b36e 100644 --- a/apps/labrinth/src/routes/v3/disclosures.rs +++ b/apps/labrinth/src/routes/v3/disclosures.rs @@ -14,6 +14,7 @@ use crate::models::disclosures::{ ProjectDisclosureType, }; use crate::models::pats::Scopes; +use crate::models::projects::ProjectStatus; use crate::models::teams::ProjectPermissions; use crate::queue::session::AuthQueue; use crate::routes::ApiError; @@ -279,6 +280,16 @@ pub async fn modify_project_disclosures( .wrap_internal_err("failed to remove project disclosure")?; } + if project.inner.status == ProjectStatus::Processing { + super::projects::validate::ensure_project_is_valid_for_review( + project.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index b354bf7ac2..f533a78d62 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -26,7 +26,6 @@ use crate::models::notifications::NotificationBody; use crate::models::pats::Scopes; use crate::models::projects::{ MonetizationStatus, Project, ProjectStatus, SideTypesMigrationReviewStatus, - Version, }; use crate::models::teams::{DEFAULT_ROLE, ProjectPermissions}; use crate::models::threads::MessageBody; @@ -42,7 +41,6 @@ use crate::util::error::Context; use crate::util::img; use crate::util::img::{delete_old_images, upload_image_optimized}; use crate::util::routes::read_limited_from_payload; -use crate::validate::project::has_required_nags_with_context; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web}; use chrono::Utc; use eyre::eyre; @@ -1361,53 +1359,13 @@ pub async fn project_edit_internal( .wrap_api_err("deleting unused images")?; if validate_for_review { - let mut projects = db_models::DBProject::get_many_uncached( - &[ProjectId::from(id)], - &mut transaction, - &redis, - ) - .await - .wrap_internal_err("reloading project for submission validation")?; - let reloaded_project = - projects.pop().wrap_not_found_err("resource not found")?; - let versions = db_models::DBVersion::get_many_uncached( - &reloaded_project.versions, + let reloaded_project = validate::ensure_project_is_valid_for_review( + id, + &pool, &mut transaction, &redis, ) - .await - .wrap_internal_err( - "reloading project versions for submission validation", - )? - .into_iter() - .map(Version::from) - .collect::>(); - let available_categories = - db_models::categories::Category::list(&**pool, &redis) - .await - .wrap_internal_err("fetching project categories")?; - let disclosures = db_models::DBProjectDisclosure::get_many_for_project( - reloaded_project.inner.id, - false, - &mut transaction, - ) - .await - .wrap_internal_err("fetching project disclosures")? - .into_iter() - .map(|disclosure| disclosure.disclosure) - .collect::>(); - let project = Project::from(reloaded_project.clone()); - - if has_required_nags_with_context( - &project, - &versions, - &available_categories, - &disclosures, - ) { - return Err(ApiError::Request(eyre!( - "project must have no required validation nags before or while under review" - ))); - } + .await?; if submit_for_review { submit_project_for_review( @@ -2740,8 +2698,8 @@ pub async fn add_gallery_item_internal( } let gallery_item = vec![db_models::project_item::DBGalleryItem { - image_url: upload_result.url, - raw_image_url: upload_result.raw_url, + image_url: upload_result.url.clone(), + raw_image_url: upload_result.raw_url.clone(), featured: item.featured, name: item.name, description: item.description, @@ -2756,6 +2714,31 @@ pub async fn add_gallery_item_internal( .await .wrap_internal_err("inserting galleries into database")?; + let validation_error = match project_item.inner.status { + ProjectStatus::Processing => { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await + .err() + } + _ => None, + }; + if let Some(error) = validation_error { + delete_old_images( + Some(upload_result.url), + Some(upload_result.raw_url), + FileHostPublicity::Public, + &**file_host, + ) + .await + .wrap_api_err("deleting rejected gallery image upload")?; + return Err(error); + } + transaction .commit() .await @@ -2994,6 +2977,16 @@ pub async fn edit_gallery_item_internal( )?; } + if project_item.inner.status == ProjectStatus::Processing { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await @@ -3127,15 +3120,6 @@ pub async fn delete_gallery_item_internal( } } - delete_old_images( - Some(item.image_url), - Some(item.raw_image_url), - FileHostPublicity::Public, - &**file_host, - ) - .await - .wrap_api_err("deleting old images")?; - let mut transaction = pool .begin() .await @@ -3154,11 +3138,30 @@ pub async fn delete_gallery_item_internal( "querying database for `delete_gallery_item_internal`", )?; + if project_item.inner.status == ProjectStatus::Processing { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await .wrap_internal_err("committing database transaction")?; + delete_old_images( + Some(item.image_url), + Some(item.raw_image_url), + FileHostPublicity::Public, + &**file_host, + ) + .await + .wrap_api_err("deleting old images")?; + clear_project_cache_and_queue_search( &redis, &search_state, diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 3af56dc7df..0a2529f999 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -4,7 +4,12 @@ use serde::Serialize; use xredis::RedisPool; use crate::auth::get_user_from_headers; -use crate::database::{PgPool, ReadOnlyPgPool, models as db_models}; +use crate::database::models::DBProjectId; +use crate::database::models::project_item::ProjectQueryResult; +use crate::database::{ + PgPool, PgTransaction, ReadOnlyPgPool, models as db_models, +}; +use crate::models::ids::ProjectId; use crate::models::pats::Scopes; use crate::models::projects::{Project, Version}; use crate::models::teams::ProjectPermissions; @@ -12,7 +17,8 @@ use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::util::error::Context as _; use crate::validate::project::{ - ProjectNag, validate_with_context as validate_project, + ProjectNag, has_required_nags_with_context, + validate_with_context as validate_project, }; #[derive(Serialize, utoipa::ToSchema)] @@ -20,6 +26,61 @@ pub struct ProjectValidationResponse { pub nags: Vec, } +pub(crate) async fn ensure_project_is_valid_for_review( + project_id: DBProjectId, + pool: &PgPool, + transaction: &mut PgTransaction<'_>, + redis: &RedisPool, +) -> Result { + let mut projects = db_models::DBProject::get_many_uncached( + &[ProjectId::from(project_id)], + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project for review validation")?; + let reloaded_project = + projects.pop().wrap_not_found_err("resource not found")?; + let versions = db_models::DBVersion::get_many_uncached( + &reloaded_project.versions, + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project versions for review validation")? + .into_iter() + .map(Version::from) + .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + reloaded_project.inner.id, + false, + &mut *transaction, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); + let project = Project::from(reloaded_project.clone()); + + if has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) { + return Err(ApiError::Request(eyre!( + "project must have no required validation nags before or while under review" + ))); + } + + Ok(reloaded_project) +} + /// Validate that a project is ready to be submitted for review. #[utoipa::path( context_path = "/project", diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 99eb84f19b..9a969bc531 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -20,7 +20,7 @@ use crate::models::ids::VersionId; use crate::models::images::ImageContext; use crate::models::pats::Scopes; use crate::models::projects::{ - Dependency, FileType, VersionStatus, VersionType, + Dependency, FileType, ProjectStatus, VersionStatus, VersionType, }; use crate::models::projects::{Loader, skip_nulls}; use crate::models::teams::ProjectPermissions; @@ -47,7 +47,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) { } // Given a project ID/slug and a version slug -/// Get a project version. +/// Get a project version. #[utoipa::path( context_path = "/project", tag = "versions", @@ -969,7 +969,7 @@ pub struct VersionListFilters { pub include_changelog: bool, } -/// List project versions. +/// List project versions. #[utoipa::path( context_path = "/project", tag = "versions", @@ -1243,6 +1243,15 @@ pub async fn version_delete( .wrap_request_err_with(|| { "the specified version does not exist!".to_string() })?; + let project = database::models::DBProject::get_id( + version.inner.project_id, + &**pool, + &redis, + ) + .await + .wrap_internal_err("fetching project from database")? + .wrap_not_found_err("resource not found")?; + let validate_for_review = project.inner.status == ProjectStatus::Processing; if !user.role.is_admin() { let team_member = @@ -1327,6 +1336,16 @@ pub async fn version_delete( "executing `tech_review_sync::sync_project_tech_review_state`", )?; + if validate_for_review { + super::projects::validate::ensure_project_is_valid_for_review( + version.inner.project_id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 83e5b7db0a..2847db5433 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -6,7 +6,9 @@ use common::dummy_data::DUMMY_CATEGORIES; use crate::common::api_common::models::CommonProject; use crate::common::api_common::request_data::ProjectCreationRequestData; -use crate::common::api_common::{ApiProject, ApiTeams, ApiVersion}; +use crate::common::api_common::{ + Api, ApiProject, ApiTeams, ApiVersion, AppendsOptionalPat, +}; use crate::common::dummy_data::{ DummyImage, DummyOrganizationZeta, DummyProjectAlpha, DummyProjectBeta, TestFile, @@ -588,6 +590,230 @@ async fn test_edit_invalid_project_in_review_rolls_back() { .await; } +#[actix_rt::test] +async fn test_add_invalid_gallery_item_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + false, + None, + Some("%EF%BC%A1".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_edit_invalid_gallery_item_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + true, + None, + Some("valid-gallery-description".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + let gallery_url = + project_before["gallery"][0]["url"].as_str().unwrap(); + + let response = api + .edit_gallery_item( + project_slug, + gallery_url, + vec![("name".to_string(), "Ａ".to_string())] + .into_iter() + .collect(), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_delete_gallery_item_from_invalid_project_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + true, + None, + Some("valid-gallery-description".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": "draft", + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + let gallery_url = + project_before["gallery"][0]["url"].as_str().unwrap(); + + let response = api + .remove_gallery_item(project_slug, gallery_url, USER_USER_PAT) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_edit_invalid_disclosures_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let disclosures_uri = + format!("/v3/project/{project_slug}/disclosures"); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let request = test::TestRequest::get() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let disclosures_before: serde_json::Value = + test::read_body_json(response).await; + + let request = test::TestRequest::patch() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .set_json(json!({ + "set": [{ + "type": "advertisements", + "note": "sponsored", + }], + "remove": [], + })) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let request = test::TestRequest::get() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let disclosures_after: serde_json::Value = + test::read_body_json(response).await; + assert_eq!(disclosures_after, disclosures_before); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v3() { // Hits V3-specific patchable fields diff --git a/apps/labrinth/tests/version.rs b/apps/labrinth/tests/version.rs index 0fd00c9475..19ab8fab79 100644 --- a/apps/labrinth/tests/version.rs +++ b/apps/labrinth/tests/version.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use crate::common::api_common::ApiVersion; +use crate::common::api_common::{ApiProject, ApiVersion}; use crate::common::database::*; use crate::common::dummy_data::{ DummyProjectAlpha, DummyProjectBeta, TestFile, @@ -91,6 +91,35 @@ async fn test_get_version() { .await; } +#[actix_rt::test] +async fn deleting_version_in_review_that_fails_validation_rolls_back() { + with_test_environment( + None, + |test_env: common::environment::TestEnvironment| async move { + let api = &test_env.api; + let project = &test_env.dummy.project_alpha; + + let response = api + .edit_project( + &project.project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = + api.remove_version(&project.version_id, USER_USER_PAT).await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let response = + api.get_version(&project.version_id, USER_USER_PAT).await; + assert_status!(&response, StatusCode::OK); + }, + ) + .await; +} + #[actix_rt::test] async fn rejects_duplicate_filename_when_uploading_file_to_version() { with_test_environment( From b5d7e979c53a942cddf6d31292c3187f5a8436b2 Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 3 Sep 2026 14:04:15 -0600 Subject: [PATCH 18/28] feat: improve description and summary language detection --- apps/labrinth/src/bin/check_project.rs | 38 ++++++++++----- .../src/validate/project/description.rs | 22 +++++++-- apps/labrinth/src/validate/project/mod.rs | 8 ++++ apps/labrinth/src/validate/project/summary.rs | 37 +++++++++++--- apps/labrinth/src/validate/project/text.rs | 48 +++++++++++++++++-- 5 files changed, 123 insertions(+), 30 deletions(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index b93bb5b920..0d5560a3e5 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -8,13 +8,14 @@ use clap::Parser; use eyre::{Result, WrapErr, eyre}; use labrinth::models::projects::{Project, Version}; use labrinth::validate::project::{ - ProjectNagKind, ProjectNagSeverity, validate, + ProjectNagKind, ProjectNagSeverity, is_project_description_non_english, + is_project_summary_non_english, validate, }; -use serde::{Deserialize, Serialize}; use serde::de::{ DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor, }; +use serde::{Deserialize, Serialize}; use strum::IntoEnumIterator; use url::Url; @@ -53,10 +54,14 @@ struct Args { #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] show_description_profanity: usize, - /// Print project IDs grouped by every triggered nag kind + /// Print project IDs triggering the summary language nag #[arg(long, requires = "file")] nag_project_ids: bool, + /// Print project IDs triggering the description language nag + #[arg(long, requires = "file", conflicts_with = "nag_project_ids")] + description_language_project_ids: bool, + /// Scan every profanity-bearing field into a reusable JSON report #[arg( long, @@ -168,7 +173,18 @@ async fn main() -> Result<()> { if let Some(path) = args.file.as_deref() { if args.nag_project_ids { - return print_json(&find_nag_project_ids(path)?); + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectSummaryNonEnglish, + is_project_summary_non_english, + )?); + } + if args.description_language_project_ids { + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectDescriptionNonEnglish, + is_project_description_non_english, + )?); } let summary = summarize_file(path, args.show_description_profanity)?; print_description_profanity_samples( @@ -190,23 +206,19 @@ struct ProjectDataset { projects: Vec, } -fn find_nag_project_ids( +fn find_language_project_ids( path: &Path, + kind: ProjectNagKind, + is_non_english: fn(&Project) -> bool, ) -> Result>> { let file = File::open(path) .wrap_err_with(|| format!("opening `{}`", path.display()))?; let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) .wrap_err_with(|| format!("reading `{}`", path.display()))?; - let mut project_ids = ProjectNagKind::iter() - .map(|kind| (kind, Vec::new())) - .collect::>(); + let mut project_ids = BTreeMap::from([(kind, Vec::new())]); for project in dataset.projects { - let kinds = validate(&project, &[]) - .into_iter() - .map(|nag| nag.kind) - .collect::>(); - for kind in kinds { + if is_non_english(&project) { project_ids .entry(kind) .or_default() diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index e18a549d01..e137812f87 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -59,11 +59,7 @@ pub(super) fn validate(project: &Project) -> Vec { ProjectNagSeverity::Required, )); } - if project_requires_english(project) - && js_string_length(&text) >= MIN_DESCRIPTION_CHARS - && !has_spam - && !has_sufficient_english_blocks(&blocks) - { + if !has_spam && is_non_english_text(project, &text, &blocks) { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionNonEnglish, ProjectNagSeverity::Required, @@ -136,6 +132,22 @@ pub(super) fn validate(project: &Project) -> Vec { nags } +pub(super) fn is_non_english(project: &Project) -> bool { + let text = extract_description_text(&project.description); + let blocks = extract_description_blocks(&project.description); + is_non_english_text(project, &text, &blocks) +} + +fn is_non_english_text( + project: &Project, + text: &str, + blocks: &[String], +) -> bool { + project_requires_english(project) + && js_string_length(text) >= MIN_DESCRIPTION_CHARS + && !has_sufficient_english_blocks(blocks) +} + fn has_description_spam(markdown: &str) -> bool { contains_description_spam(markdown) } diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 6bc417f824..308870b7a7 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -164,6 +164,14 @@ pub fn validate(project: &Project, versions: &[Version]) -> Vec { validate_inner(project, versions, None, None) } +pub fn is_project_summary_non_english(project: &Project) -> bool { + summary::is_non_english(project) +} + +pub fn is_project_description_non_english(project: &Project) -> bool { + description::is_non_english(project) +} + pub fn validate_with_context( project: &Project, versions: &[Version], diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 6f768dba0a..518bed6dfc 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -18,6 +18,7 @@ pub(super) fn validate(project: &Project) -> Vec { let normalized_summary = normalize_project_field_text(summary); let summary_link = find_link_or_ip(summary); let contains_link = summary_link.is_some(); + let has_spam = contains_spam(&normalized_summary); let profanity = profanity_matches(summary); if let Some(matched) = profanity @@ -50,12 +51,12 @@ pub(super) fn validate(project: &Project) -> Vec { ProjectNagSeverity::Required, )); } - if project_requires_english(project) - && js_string_length(&normalized_summary) >= MIN_SUMMARY_CHARS - && !contains_link - && !contains_spam(&normalized_summary) - && !is_likely_english_summary(&normalized_summary) - { + if requires_language_nag( + project, + &normalized_summary, + contains_link, + has_spam, + ) { nags.push(ProjectNag::new( ProjectNagKind::ProjectSummaryNonEnglish, ProjectNagSeverity::Required, @@ -88,7 +89,7 @@ pub(super) fn validate(project: &Project) -> Vec { })), ); } - if contains_spam(&normalized_summary) { + if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectSummarySpam, ProjectNagSeverity::Required, @@ -113,6 +114,28 @@ pub(super) fn validate(project: &Project) -> Vec { nags } +pub(super) fn is_non_english(project: &Project) -> bool { + let normalized_summary = normalize_project_field_text(&project.summary); + is_non_english_text(project, &normalized_summary) +} + +fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { + project_requires_english(project) + && js_string_length(normalized_summary) >= MIN_SUMMARY_CHARS + && !is_likely_english_summary(normalized_summary) +} + +fn requires_language_nag( + project: &Project, + normalized_summary: &str, + contains_link: bool, + has_spam: bool, +) -> bool { + is_non_english_text(project, normalized_summary) + && !contains_link + && !has_spam +} + fn summary_name_similarity(summary: &str, name: &str) -> f64 { let summary = normalized_for_similarity(summary); let name = normalized_for_similarity(name); diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 5e101f948f..437a5d5ef7 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -14,6 +14,8 @@ use crate::models::projects::Project; static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); +static NON_LATIN_LETTER: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); static SPAM_TOKEN: LazyLock = LazyLock::new(|| { Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) .unwrap() @@ -1102,13 +1104,18 @@ pub(super) fn project_requires_english(project: &Project) -> bool { } pub(super) fn is_likely_english_summary(text: &str) -> bool { - if !has_enough_language_content(text) { + let detection_text = normalize_language_text(text); + if has_dominant_non_latin_script(&detection_text) { + return false; + } + + if !has_enough_language_content(&detection_text) { return true; } LANGUAGE_DETECTOR - .detect(text) - .is_none_or(|info| info.lang() == Lang::Eng || info.confidence() < 0.5) + .detect(&detection_text) + .is_none_or(|info| info.lang() == Lang::Eng || !info.is_reliable()) } pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { @@ -1116,14 +1123,29 @@ pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { let mut non_english_chunks = 0; for block in blocks { - for chunk in language_chunks(block) { + let detection_text = normalize_language_text(block); + if has_dominant_non_latin_script(&detection_text) { + non_english_chunks += 1; + + let latin_text = NON_LATIN_LETTER.replace_all(&detection_text, " "); + if has_enough_language_content(&latin_text) + && LANGUAGE_DETECTOR + .detect(&latin_text) + .is_some_and(|info| info.lang() == Lang::Eng) + { + english_chunks += 1; + } + continue; + } + + for chunk in language_chunks(&detection_text) { let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { continue; }; if info.lang() == Lang::Eng { english_chunks += 1; - } else if info.confidence() >= 0.8 { + } else if info.is_reliable() { non_english_chunks += 1; } } @@ -1133,6 +1155,22 @@ pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 } +fn normalize_language_text(text: &str) -> String { + text.nfkc().collect() +} + +fn has_dominant_non_latin_script(text: &str) -> bool { + const MIN_NON_LATIN_LETTERS: usize = 5; + + let non_latin_letters = NON_LATIN_LETTER.find_iter(text).count(); + let alphabetic_letters = text + .chars() + .filter(|character| character.is_alphabetic()) + .count(); + non_latin_letters >= MIN_NON_LATIN_LETTERS + && non_latin_letters * 2 >= alphabetic_letters +} + fn language_chunks(block: &str) -> Vec { const CHUNK_WORDS: usize = 24; const CHUNK_STRIDE_WORDS: usize = 12; From 68ade7c95f39ab382b5bed2d16e9fc9faddd95b8 Mon Sep 17 00:00:00 2001 From: tdgao Date: Thu, 3 Sep 2026 15:05:21 -0600 Subject: [PATCH 19/28] feat: reduce false positives on slurs and profanity --- apps/labrinth/src/bin/check_project.rs | 114 ++++++++++++++++-- apps/labrinth/src/validate/project/gallery.rs | 2 +- apps/labrinth/src/validate/project/text.rs | 101 ++++++++-------- 3 files changed, 152 insertions(+), 65 deletions(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index 0d5560a3e5..dbca82c5d0 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::fs::File; use std::io::{BufReader, IsTerminal}; @@ -46,6 +46,19 @@ struct Args { )] file: Option, + /// Print every validator kind with the IDs of projects that trigger it + #[arg( + long, + requires = "file", + conflicts_with_all = [ + "nag_project_ids", + "description_language_project_ids", + "show_description_profanity", + "write_profanity_report" + ] + )] + all_nag_project_ids: bool, + /// A Modrinth token to send as bearer authentication #[arg(long, value_name = "TOKEN")] token: Option, @@ -186,6 +199,9 @@ async fn main() -> Result<()> { is_project_description_non_english, )?); } + if args.all_nag_project_ids { + return print_json(&find_all_nag_project_ids(path)?); + } let summary = summarize_file(path, args.show_description_profanity)?; print_description_profanity_samples( &summary.description_profanity_samples, @@ -206,6 +222,63 @@ struct ProjectDataset { projects: Vec, } +fn find_all_nag_project_ids( + path: &Path, +) -> Result>> { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let worker_results = std::thread::scope(|scope| { + let workers = dataset + .projects + .chunks(chunk_size) + .map(|projects| { + scope.spawn(move || { + let mut project_ids = + BTreeMap::>::new(); + for project in projects { + let kinds = validate(project, &[]) + .into_iter() + .map(|nag| nag.kind) + .collect::>(); + for kind in kinds { + project_ids + .entry(kind) + .or_default() + .push(project.id.to_string()); + } + } + project_ids + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("project validation worker panicked")) + }) + .collect::>>() + })?; + + let mut project_ids = ProjectNagKind::iter() + .map(|kind| (kind, Vec::new())) + .collect::>(); + for worker_result in worker_results { + for (kind, ids) in worker_result { + project_ids.entry(kind).or_default().extend(ids); + } + } + + Ok(project_ids) +} + fn find_language_project_ids( path: &Path, kind: ProjectNagKind, @@ -215,16 +288,35 @@ fn find_language_project_ids( .wrap_err_with(|| format!("opening `{}`", path.display()))?; let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) .wrap_err_with(|| format!("reading `{}`", path.display()))?; - let mut project_ids = BTreeMap::from([(kind, Vec::new())]); - - for project in dataset.projects { - if is_non_english(&project) { - project_ids - .entry(kind) - .or_default() - .push(project.id.to_string()); - } - } + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let ids = std::thread::scope(|scope| -> Result> { + let workers = dataset + .projects + .chunks(chunk_size) + .map(|projects| { + scope.spawn(move || { + projects + .iter() + .filter(|project| is_non_english(project)) + .map(|project| project.id.to_string()) + .collect::>() + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("language detection worker panicked")) + }) + .collect::>>() + .map(|ids| ids.into_iter().flatten().collect()) + })?; + let project_ids = BTreeMap::from([(kind, ids)]); Ok(project_ids) } diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs index 38a811f0bc..c40e82c04f 100644 --- a/apps/labrinth/src/validate/project/gallery.rs +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -3,7 +3,7 @@ use serde_json::json; use super::text::{ProfanityKind, has_non_standard_text, profanity_matches}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; -const MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT: usize = 2; +const MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT: usize = 0; fn validate_text( text: Option<&str>, diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 437a5d5ef7..972e32647f 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -3,7 +3,6 @@ use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; use regex::Regex; -use rustrict::{Censor, Type}; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; use url::Url; @@ -122,6 +121,25 @@ const SLUR_TERMS: &[&str] = &[ "wetback", ]; +const PROFANITY_TERMS: &[&str] = &[ + "asshole", + "bastard", + "bitch", + "bullshit", + "cum", + "cunt", + "douchebag", + "fck", + "fuck", + "incest", + "motherfucker", + "pussy", + "shit", + "slut", + "twat", + "whore", +]; + pub(super) fn normalize_project_field_text(text: &str) -> String { text.trim().nfc().collect() } @@ -132,16 +150,8 @@ pub(super) fn js_string_length(text: &str) -> usize { pub(super) fn profanity_matches(text: &str) -> Vec { let (prepared, raw_ranges) = prepare_profanity_text(text); - let mut spans = censored_spans(&prepared, profanity_types()); - for span in censored_spans(&prepared, slur_types()) { - if spans.contains(&span) { - continue; - } - if raw_text_for_span(text, &raw_ranges, span).is_some_and(is_slur) { - spans.push(span); - } - } - spans.extend(frontend_slur_spans(&prepared)); + let mut spans = term_spans(&prepared, PROFANITY_TERMS, false); + spans.extend(term_spans(&prepared, SLUR_TERMS, true)); spans.sort_unstable_by(|left, right| { left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1)) }); @@ -158,37 +168,11 @@ pub(super) fn profanity_matches(text: &str) -> Vec { matches } -fn censored_spans(text: &str, threshold: Type) -> Vec<(usize, usize)> { - const CENSORED: char = '\0'; - - let mut censor = Censor::from_str(text); - censor - .with_ignore_self_censoring(true) - .with_censor_threshold(threshold) - .with_censor_first_character_threshold(threshold) - .with_censor_replacement(CENSORED); - - let mut spans = Vec::new(); - let mut start = None; - let mut character_count = 0; - for (index, character) in censor.censor().chars().enumerate() { - character_count = index + 1; - match (start, character == CENSORED) { - (None, true) => start = Some(index), - (Some(match_start), false) => { - spans.push((match_start, index)); - start = None; - } - _ => {} - } - } - if let Some(start) = start { - spans.push((start, character_count)); - } - spans -} - -fn frontend_slur_spans(text: &str) -> Vec<(usize, usize)> { +fn term_spans( + text: &str, + terms: &[&str], + match_repeated_letters: bool, +) -> Vec<(usize, usize)> { let characters = text.chars().collect::>(); let mut spans = Vec::new(); @@ -197,11 +181,15 @@ fn frontend_slur_spans(text: &str) -> Vec<(usize, usize)> { continue; } - let end = SLUR_TERMS + let end = terms .iter() .flat_map(|term| { [ - match_repeated_term(&characters, start, term), + if match_repeated_letters { + match_repeated_term(&characters, start, term) + } else { + match_exact_term(&characters, start, term) + }, match_separated_term(&characters, start, term), ] }) @@ -215,6 +203,22 @@ fn frontend_slur_spans(text: &str) -> Vec<(usize, usize)> { spans } +fn match_exact_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let mut input_index = start; + for expected in term.chars() { + if characters.get(input_index) != Some(&expected) { + return None; + } + input_index += 1; + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + fn match_repeated_term( characters: &[char], start: usize, @@ -329,7 +333,6 @@ fn push_profanity_match( if normalize_project_field_text(raw_text).to_lowercase() == "кооп" { return; } - matches.push(ProfanityMatch { kind: if is_slur(raw_text) { ProfanityKind::Slur @@ -426,14 +429,6 @@ fn is_invisible_separator(character: char) -> bool { ) } -fn profanity_types() -> Type { - Type::PROFANE & Type::MODERATE_OR_HIGHER -} - -fn slur_types() -> Type { - Type::OFFENSIVE & Type::MILD_OR_HIGHER -} - pub(super) fn has_non_standard_text(text: &str) -> bool { non_standard_character_count(text) > 0 } From 40e54b8e4093a27f2a7303f1586b2a5f08053cc5 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:49:44 +0100 Subject: [PATCH 20/28] fix --- apps/labrinth/src/routes/v2/projects.rs | 3 +- apps/labrinth/tests/v2/project.rs | 39 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/apps/labrinth/src/routes/v2/projects.rs b/apps/labrinth/src/routes/v2/projects.rs index 679f740ea0..23691141bd 100644 --- a/apps/labrinth/src/routes/v2/projects.rs +++ b/apps/labrinth/src/routes/v2/projects.rs @@ -691,8 +691,7 @@ pub async fn project_edit( true, ) .await - .or_else(v2_reroute::flatten_404_error) - .wrap_api_err("flattening v2 not-found response")?; + .or_else(v2_reroute::flatten_404_error)?; // If client and server side were set, we will call // the version setting route for each version to set the side types for each of them. diff --git a/apps/labrinth/tests/v2/project.rs b/apps/labrinth/tests/v2/project.rs index 36bb380323..91ce1f789d 100644 --- a/apps/labrinth/tests/v2/project.rs +++ b/apps/labrinth/tests/v2/project.rs @@ -405,6 +405,45 @@ async fn permissions_upload_version() { .await; } +#[actix_rt::test] +async fn invalid_review_submission_returns_validation_error() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "draft" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "body": "", + "status": "processing", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let error: serde_json::Value = test::read_body_json(response).await; + assert_eq!( + error["description"], + "project must have no required validation nags before or while under review" + ); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v2() { // Hits V3-specific patchable fields From cb34b1834b1f7ba9c7f69ffa6ec3dff218c81142 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:03:38 +0100 Subject: [PATCH 21/28] skip check when rejecting --- apps/labrinth/src/routes/v3/projects/mod.rs | 7 ++- apps/labrinth/tests/project.rs | 50 +++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index f533a78d62..aafcd1af07 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -499,8 +499,13 @@ pub async fn project_edit_internal( let submit_for_review = new_project.status == Some(ProjectStatus::Processing) && !user.role.is_mod(); + let leave_review = matches!( + new_project.status, + Some(ProjectStatus::Draft | ProjectStatus::Rejected) + ); let validate_for_review = submit_for_review - || project_item.inner.status == ProjectStatus::Processing; + || (project_item.inner.status == ProjectStatus::Processing + && !leave_review); if submit_for_review { if !perms.contains(ProjectPermissions::EDIT_DETAILS) { return Err(ApiError::Auth(eyre!( diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 2847db5433..6ed1eb7d28 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -590,6 +590,56 @@ async fn test_edit_invalid_project_in_review_rolls_back() { .await; } +#[actix_rt::test] +async fn test_leaving_review_skips_validation() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let original_project = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + + for (status, expected_status) in [ + ("draft", ProjectStatus::Draft), + ("rejected", ProjectStatus::Rejected), + ] { + let response = api + .edit_project( + project_slug, + json!({ + "description": original_project.description.clone(), + "status": "processing", + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": status, + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project.status, expected_status); + assert!(project.description.is_empty()); + } + }, + ) + .await; +} + #[actix_rt::test] async fn test_add_invalid_gallery_item_in_review_rolls_back() { with_test_environment( From 5db296a49474fbdfa14aa766ccc28d2bf812a3b0 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:43:46 +0100 Subject: [PATCH 22/28] fix ci --- Cargo.lock | 1 - Cargo.toml | 13 ++++++------- apps/labrinth/Cargo.toml | 13 ++++++------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6d04f35f4..acce146a94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5714,7 +5714,6 @@ dependencies = [ "sqlx", "sqlx-tracing", "strum", - "subsecond", "thiserror 2.0.17", "tikv-jemalloc-ctl", "tikv-jemallocator", diff --git a/Cargo.toml b/Cargo.toml index 6c309e30b9..a4a779c2dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ dashmap = "6.1.0" data-url = "0.3.2" deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "c7209b086829572f368f1194aba54bdea8dc540b", version = "0.23.0" } derive_more = "2.1.1" +dioxus-devtools = "0.7.10" directories = "6.0.0" dirs = "6.0.0" discord-rich-presence = "1.0.0" @@ -96,8 +97,6 @@ hex = "0.4.3" hickory-resolver = "0.25.2" hmac = "0.12.1" httpdate = "1.0.3" -subsecond = "0.7.10" -dioxus-devtools = "0.7.10" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ "aws-lc-rs", @@ -114,7 +113,6 @@ itertools = "0.14.0" jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" -linkify = "0.11.0" lettre = { version = "0.11.19", default-features = false, features = [ "aws-lc-rs", "builder", @@ -126,6 +124,7 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ "checked-decode", "safe-decode", @@ -166,13 +165,13 @@ reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" -rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } rust-s3 = { version = "0.37.0", default-features = false, features = [ "fail-on-err", "tags", "tokio-rustls-tls", ] } rustls = "0.23.32" +rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } rusty-money = "0.4.1" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" @@ -234,18 +233,18 @@ tracing-error = "0.2.1" tracing-subscriber = "0.3.20" ts-rs = "12.0.1" typed-path = "0.12.0" -url = "2.5.7" -urlencoding = "2.1.3" unicode-normalization = "0.1.24" unicode-segmentation = "1.12.0" +url = "2.5.7" +urlencoding = "2.1.3" utoipa = { version = "5.4.0", features = ["actix_extras", "chrono", "decimal"] } uuid = "1.18.1" validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" -whatlang = "0.18.0" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry +whatlang = "0.18.0" whoami = "1.6.1" windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index cf032dc1a3..e813bd2c3d 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "labrinth" version = "2.7.0" -default-run = "labrinth" edition.workspace = true license = "AGPL-3.0-only" +default-run = "labrinth" # This seems redundant, but it's necessary for Docker to work [[bin]] @@ -31,10 +31,8 @@ bitflags = { workspace = true } bytes = { workspace = true } cel = { workspace = true } censor = { workspace = true } -dioxus-devtools = { workspace = true } chrono = { workspace = true, features = ["serde"] } chumsky = { workspace = true } -subsecond = { workspace = true } clap = { workspace = true, features = ["derive"] } clickhouse = { workspace = true, features = ["time", "uuid"] } color-eyre = { workspace = true } @@ -43,6 +41,7 @@ component-derive = { workspace = true } const_format = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true, features = ["deref", "deref_mut"] } +dioxus-devtools = { workspace = true } dotenvy = { workspace = true } either = { workspace = true } eyre = { workspace = true } @@ -93,8 +92,8 @@ regex = { workspace = true } reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } -rustrict = { workspace = true } rustls.workspace = true +rustrict = { workspace = true } rusty-money = { workspace = true } scalar_api_reference = { workspace = true, features = ["actix-web"] } sentry = { workspace = true } @@ -124,17 +123,17 @@ tokio-stream = { workspace = true } totp-rs = { workspace = true, features = ["gen_secret"] } tracing = { workspace = true } tracing-actix-web = { workspace = true } -url = { workspace = true } -urlencoding = { workspace = true } unicode-normalization = { workspace = true } unicode-segmentation = { workspace = true } +url = { workspace = true } +urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } uuid = { workspace = true, features = ["fast-rng", "serde", "v4", "v7"] } validator = { workspace = true, features = ["derive"] } webauthn-rs = { workspace = true, features = ["conditional-ui", "danger-allow-state-serialisation"] } webauthn-rs-proto = { workspace = true } -whatlang = { workspace = true } webp = { workspace = true } +whatlang = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } yaserde = { workspace = true, features = ["derive"] } From f72420c82d57d0451bcd55b4e4bba8c2a10619d1 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:28:07 +0100 Subject: [PATCH 23/28] fix ci --- apps/labrinth/src/bin/check_project.rs | 823 +++++++++--------- .../src/bin/check_project/profanity.rs | 706 +++++++-------- .../src/routes/v3/projects/validate.rs | 246 +++--- apps/labrinth/src/test/api_v2/request_data.rs | 2 +- apps/labrinth/src/test/api_v3/request_data.rs | 2 +- apps/labrinth/src/validate/project/text.rs | 4 +- .../labrinth/src/validate/project/versions.rs | 70 +- apps/labrinth/tests/v2/search.rs | 4 + 8 files changed, 930 insertions(+), 927 deletions(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index dbca82c5d0..7c4c1119ee 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -8,12 +8,12 @@ use clap::Parser; use eyre::{Result, WrapErr, eyre}; use labrinth::models::projects::{Project, Version}; use labrinth::validate::project::{ - ProjectNagKind, ProjectNagSeverity, is_project_description_non_english, - is_project_summary_non_english, validate, + ProjectNagKind, ProjectNagSeverity, is_project_description_non_english, + is_project_summary_non_english, validate, }; use serde::de::{ - DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, - Visitor, + DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, + Visitor, }; use serde::{Deserialize, Serialize}; use strum::IntoEnumIterator; @@ -26,28 +26,28 @@ const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; #[derive(Parser)] #[command( - version, - about = "Validate a project from the Modrinth API or a project dataset" + version, + about = "Validate a project from the Modrinth API or a project dataset" )] struct Args { - /// The ID of the project to validate - #[arg( + /// The ID of the project to validate + #[arg( value_name = "PROJECT_ID", required_unless_present_any = ["file", "read_profanity_report"], conflicts_with_all = ["file", "read_profanity_report"] )] - project_id: Option, + project_id: Option, - /// Validate all projects in a moderation project dataset - #[arg( + /// Validate all projects in a moderation project dataset + #[arg( long, value_name = "PATH", conflicts_with_all = ["token", "read_profanity_report"] )] - file: Option, + file: Option, - /// Print every validator kind with the IDs of projects that trigger it - #[arg( + /// Print every validator kind with the IDs of projects that trigger it + #[arg( long, requires = "file", conflicts_with_all = [ @@ -57,472 +57,473 @@ struct Args { "write_profanity_report" ] )] - all_nag_project_ids: bool, - - /// A Modrinth token to send as bearer authentication - #[arg(long, value_name = "TOKEN")] - token: Option, - - /// Show this many descriptions flagged for profanity - #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] - show_description_profanity: usize, - - /// Print project IDs triggering the summary language nag - #[arg(long, requires = "file")] - nag_project_ids: bool, - - /// Print project IDs triggering the description language nag - #[arg(long, requires = "file", conflicts_with = "nag_project_ids")] - description_language_project_ids: bool, - - /// Scan every profanity-bearing field into a reusable JSON report - #[arg( - long, - value_name = "PATH", - requires = "file", - conflicts_with = "show_description_profanity" - )] - write_profanity_report: Option, - - /// Render a previously generated profanity report - #[arg(long, value_name = "PATH", conflicts_with = "file")] - read_profanity_report: Option, - - /// Render at most this many projects from a profanity report (0 means all) - #[arg( - long, - value_name = "COUNT", - default_value_t = 0, - requires = "read_profanity_report" - )] - profanity_report_limit: usize, - - /// Use bracket markers instead of ANSI colors when rendering a report - #[arg(long, requires = "read_profanity_report")] - profanity_report_brackets: bool, + all_nag_project_ids: bool, + + /// A Modrinth token to send as bearer authentication + #[arg(long, value_name = "TOKEN")] + token: Option, + + /// Show this many descriptions flagged for profanity + #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] + show_description_profanity: usize, + + /// Print project IDs triggering the summary language nag + #[arg(long, requires = "file")] + nag_project_ids: bool, + + /// Print project IDs triggering the description language nag + #[arg(long, requires = "file", conflicts_with = "nag_project_ids")] + description_language_project_ids: bool, + + /// Scan every profanity-bearing field into a reusable JSON report + #[arg( + long, + value_name = "PATH", + requires = "file", + conflicts_with = "show_description_profanity" + )] + write_profanity_report: Option, + + /// Render a previously generated profanity report + #[arg(long, value_name = "PATH", conflicts_with = "file")] + read_profanity_report: Option, + + /// Render at most this many projects from a profanity report (0 means all) + #[arg( + long, + value_name = "COUNT", + default_value_t = 0, + requires = "read_profanity_report" + )] + profanity_report_limit: usize, + + /// Use bracket markers instead of ANSI colors when rendering a report + #[arg(long, requires = "read_profanity_report")] + profanity_report_brackets: bool, } #[derive(Serialize)] struct BatchSummary { - projects: usize, - projects_with_at_least_one_required_nag: usize, - version_details_available: bool, - nag_counts: BTreeMap, - #[serde(skip)] - description_profanity_sample_limit: usize, - #[serde(skip)] - description_profanity_samples: Vec, + projects: usize, + projects_with_at_least_one_required_nag: usize, + version_details_available: bool, + nag_counts: BTreeMap, + #[serde(skip)] + description_profanity_sample_limit: usize, + #[serde(skip)] + description_profanity_samples: Vec, } struct DescriptionProfanitySample { - id: String, - name: String, - description: String, + id: String, + name: String, + description: String, } impl BatchSummary { - fn new(description_profanity_sample_limit: usize) -> Self { - Self { - projects: 0, - projects_with_at_least_one_required_nag: 0, - version_details_available: false, - nag_counts: ProjectNagKind::iter().map(|kind| (kind, 0)).collect(), - description_profanity_sample_limit, - description_profanity_samples: Vec::new(), - } - } - - fn add_project(&mut self, project: &Project) { - let nags = validate(project, &[]); - self.projects += 1; - if nags - .iter() - .any(|nag| nag.severity == ProjectNagSeverity::Required) - { - self.projects_with_at_least_one_required_nag += 1; - } - if self.description_profanity_samples.len() - < self.description_profanity_sample_limit - && nags.iter().any(|nag| { - matches!( - nag.kind, - ProjectNagKind::ProjectDescriptionSlur - | ProjectNagKind::ProjectDescriptionProfanity - ) - }) { - self.description_profanity_samples.push( - DescriptionProfanitySample { - id: project.id.to_string(), - name: project.name.clone(), - description: project.description.clone(), - }, - ); - } - for nag in nags { - *self.nag_counts.entry(nag.kind).or_default() += 1; - } - } + fn new(description_profanity_sample_limit: usize) -> Self { + Self { + projects: 0, + projects_with_at_least_one_required_nag: 0, + version_details_available: false, + nag_counts: ProjectNagKind::iter().map(|kind| (kind, 0)).collect(), + description_profanity_sample_limit, + description_profanity_samples: Vec::new(), + } + } + + fn add_project(&mut self, project: &Project) { + let nags = validate(project, &[]); + self.projects += 1; + if nags + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) + { + self.projects_with_at_least_one_required_nag += 1; + } + if self.description_profanity_samples.len() + < self.description_profanity_sample_limit + && nags.iter().any(|nag| { + matches!( + nag.kind, + ProjectNagKind::ProjectDescriptionSlur + | ProjectNagKind::ProjectDescriptionProfanity + ) + }) + { + self.description_profanity_samples.push( + DescriptionProfanitySample { + id: project.id.to_string(), + name: project.name.clone(), + description: project.description.clone(), + }, + ); + } + for nag in nags { + *self.nag_counts.entry(nag.kind).or_default() += 1; + } + } } #[tokio::main] async fn main() -> Result<()> { - color_eyre::install().wrap_err("installing color-eyre")?; - let args = Args::parse(); - - if let Some(report_path) = args.read_profanity_report.as_deref() { - return profanity::render_report( - report_path, - args.profanity_report_limit, - args.profanity_report_brackets, - ); - } - - if let Some(report_path) = args.write_profanity_report.as_deref() { - let dataset_path = args.file.as_deref().ok_or_else(|| { - eyre!("`--write-profanity-report` requires `--file`") - })?; - return profanity::write_report(dataset_path, report_path); - } - - if let Some(path) = args.file.as_deref() { - if args.nag_project_ids { - return print_json(&find_language_project_ids( - path, - ProjectNagKind::ProjectSummaryNonEnglish, - is_project_summary_non_english, - )?); - } - if args.description_language_project_ids { - return print_json(&find_language_project_ids( - path, - ProjectNagKind::ProjectDescriptionNonEnglish, - is_project_description_non_english, - )?); - } - if args.all_nag_project_ids { - return print_json(&find_all_nag_project_ids(path)?); - } - let summary = summarize_file(path, args.show_description_profanity)?; - print_description_profanity_samples( - &summary.description_profanity_samples, - ); - print_json(&summary)?; - return Ok(()); - } - - let project_id = args - .project_id - .as_deref() - .ok_or_else(|| eyre!("a project ID or `--file` is required"))?; - check_api_project(project_id, args.token.as_deref()).await + color_eyre::install().wrap_err("installing color-eyre")?; + let args = Args::parse(); + + if let Some(report_path) = args.read_profanity_report.as_deref() { + return profanity::render_report( + report_path, + args.profanity_report_limit, + args.profanity_report_brackets, + ); + } + + if let Some(report_path) = args.write_profanity_report.as_deref() { + let dataset_path = args.file.as_deref().ok_or_else(|| { + eyre!("`--write-profanity-report` requires `--file`") + })?; + return profanity::write_report(dataset_path, report_path); + } + + if let Some(path) = args.file.as_deref() { + if args.nag_project_ids { + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectSummaryNonEnglish, + is_project_summary_non_english, + )?); + } + if args.description_language_project_ids { + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectDescriptionNonEnglish, + is_project_description_non_english, + )?); + } + if args.all_nag_project_ids { + return print_json(&find_all_nag_project_ids(path)?); + } + let summary = summarize_file(path, args.show_description_profanity)?; + print_description_profanity_samples( + &summary.description_profanity_samples, + ); + print_json(&summary)?; + return Ok(()); + } + + let project_id = args + .project_id + .as_deref() + .ok_or_else(|| eyre!("a project ID or `--file` is required"))?; + check_api_project(project_id, args.token.as_deref()).await } #[derive(Deserialize)] struct ProjectDataset { - projects: Vec, + projects: Vec, } fn find_all_nag_project_ids( - path: &Path, + path: &Path, ) -> Result>> { - let file = File::open(path) - .wrap_err_with(|| format!("opening `{}`", path.display()))?; - let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - let worker_count = std::thread::available_parallelism() - .map(usize::from) - .unwrap_or(1); - let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); - let worker_results = std::thread::scope(|scope| { - let workers = dataset - .projects - .chunks(chunk_size) - .map(|projects| { - scope.spawn(move || { - let mut project_ids = - BTreeMap::>::new(); - for project in projects { - let kinds = validate(project, &[]) - .into_iter() - .map(|nag| nag.kind) - .collect::>(); - for kind in kinds { - project_ids - .entry(kind) - .or_default() - .push(project.id.to_string()); - } - } - project_ids - }) - }) - .collect::>(); - workers - .into_iter() - .map(|worker| { - worker - .join() - .map_err(|_| eyre!("project validation worker panicked")) - }) - .collect::>>() - })?; - - let mut project_ids = ProjectNagKind::iter() - .map(|kind| (kind, Vec::new())) - .collect::>(); - for worker_result in worker_results { - for (kind, ids) in worker_result { - project_ids.entry(kind).or_default().extend(ids); - } - } - - Ok(project_ids) + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let worker_results = std::thread::scope(|scope| { + let workers = dataset + .projects + .chunks(chunk_size) + .map(|projects| { + scope.spawn(move || { + let mut project_ids = + BTreeMap::>::new(); + for project in projects { + let kinds = validate(project, &[]) + .into_iter() + .map(|nag| nag.kind) + .collect::>(); + for kind in kinds { + project_ids + .entry(kind) + .or_default() + .push(project.id.to_string()); + } + } + project_ids + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("project validation worker panicked")) + }) + .collect::>>() + })?; + + let mut project_ids = ProjectNagKind::iter() + .map(|kind| (kind, Vec::new())) + .collect::>(); + for worker_result in worker_results { + for (kind, ids) in worker_result { + project_ids.entry(kind).or_default().extend(ids); + } + } + + Ok(project_ids) } fn find_language_project_ids( - path: &Path, - kind: ProjectNagKind, - is_non_english: fn(&Project) -> bool, + path: &Path, + kind: ProjectNagKind, + is_non_english: fn(&Project) -> bool, ) -> Result>> { - let file = File::open(path) - .wrap_err_with(|| format!("opening `{}`", path.display()))?; - let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - let worker_count = std::thread::available_parallelism() - .map(usize::from) - .unwrap_or(1); - let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); - let ids = std::thread::scope(|scope| -> Result> { - let workers = dataset - .projects - .chunks(chunk_size) - .map(|projects| { - scope.spawn(move || { - projects - .iter() - .filter(|project| is_non_english(project)) - .map(|project| project.id.to_string()) - .collect::>() - }) - }) - .collect::>(); - workers - .into_iter() - .map(|worker| { - worker - .join() - .map_err(|_| eyre!("language detection worker panicked")) - }) - .collect::>>() - .map(|ids| ids.into_iter().flatten().collect()) - })?; - let project_ids = BTreeMap::from([(kind, ids)]); - - Ok(project_ids) + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let ids = std::thread::scope(|scope| -> Result> { + let workers = dataset + .projects + .chunks(chunk_size) + .map(|projects| { + scope.spawn(move || { + projects + .iter() + .filter(|project| is_non_english(project)) + .map(|project| project.id.to_string()) + .collect::>() + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("language detection worker panicked")) + }) + .collect::>>() + .map(|ids| ids.into_iter().flatten().collect()) + })?; + let project_ids = BTreeMap::from([(kind, ids)]); + + Ok(project_ids) } async fn check_api_project( - project_id: &str, - token: Option<&str>, + project_id: &str, + token: Option<&str>, ) -> Result<()> { - let client = reqwest::Client::builder() - .user_agent(concat!( - "labrinth-check-project/", - env!("CARGO_PKG_VERSION") - )) - .build() - .wrap_err("building HTTP client")?; - - let project_url = api_url(&["project", project_id])?; - let mut versions_url = api_url(&["project", project_id, "version"])?; - versions_url - .query_pairs_mut() - .append_pair("include_changelog", "false"); - - let (project, versions) = tokio::try_join!( - fetch::(&client, project_url, token), - fetch::>(&client, versions_url, token), - )?; - print_json(&validate(&project, &versions)) + let client = reqwest::Client::builder() + .user_agent(concat!( + "labrinth-check-project/", + env!("CARGO_PKG_VERSION") + )) + .build() + .wrap_err("building HTTP client")?; + + let project_url = api_url(&["project", project_id])?; + let mut versions_url = api_url(&["project", project_id, "version"])?; + versions_url + .query_pairs_mut() + .append_pair("include_changelog", "false"); + + let (project, versions) = tokio::try_join!( + fetch::(&client, project_url, token), + fetch::>(&client, versions_url, token), + )?; + print_json(&validate(&project, &versions)) } fn summarize_file( - path: &Path, - description_profanity_sample_limit: usize, + path: &Path, + description_profanity_sample_limit: usize, ) -> Result { - let file = File::open(path) - .wrap_err_with(|| format!("opening `{}`", path.display()))?; - let mut deserializer = - serde_json::Deserializer::from_reader(BufReader::new(file)); - let summary = DatasetSeed { - description_profanity_sample_limit, - } - .deserialize(&mut deserializer) - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - deserializer - .end() - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - Ok(summary) + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(file)); + let summary = DatasetSeed { + description_profanity_sample_limit, + } + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(summary) } fn print_description_profanity_samples(samples: &[DescriptionProfanitySample]) { - if samples.is_empty() { - return; - } - - let use_color = std::io::stderr().is_terminal() - && std::env::var_os("NO_COLOR").is_none(); - eprintln!("\n=== Description profanity samples ==="); - if use_color { - eprintln!( - "Detected text is shown with a bold white-on-red background." - ); - } else { - eprintln!("Detected text is enclosed in ⟦double brackets⟧."); - } - - for (index, sample) in samples.iter().enumerate() { - eprintln!("\n--- Sample {} of {} ---", index + 1, samples.len()); - eprintln!("Project: {}", sample.name); - eprintln!("ID: {}", sample.id); - eprintln!("URL: https://modrinth.com/project/{}", sample.id); - eprintln!( - "\n{}", - profanity::highlight_text(&sample.description, use_color) - ); - } - eprintln!("\n=== End description profanity samples ===\n"); + if samples.is_empty() { + return; + } + + let use_color = std::io::stderr().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + eprintln!("\n=== Description profanity samples ==="); + if use_color { + eprintln!( + "Detected text is shown with a bold white-on-red background." + ); + } else { + eprintln!("Detected text is enclosed in ⟦double brackets⟧."); + } + + for (index, sample) in samples.iter().enumerate() { + eprintln!("\n--- Sample {} of {} ---", index + 1, samples.len()); + eprintln!("Project: {}", sample.name); + eprintln!("ID: {}", sample.id); + eprintln!("URL: https://modrinth.com/project/{}", sample.id); + eprintln!( + "\n{}", + profanity::highlight_text(&sample.description, use_color) + ); + } + eprintln!("\n=== End description profanity samples ===\n"); } fn print_json(value: &impl Serialize) -> Result<()> { - println!( - "{}", - serde_json::to_string_pretty(value).wrap_err("serializing output")? - ); - Ok(()) + println!( + "{}", + serde_json::to_string_pretty(value).wrap_err("serializing output")? + ); + Ok(()) } fn api_url(path: &[&str]) -> Result { - let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; - url.path_segments_mut() - .map_err(|_| eyre!("API base URL cannot be a base URL"))? - .extend(path); - Ok(url) + let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; + url.path_segments_mut() + .map_err(|_| eyre!("API base URL cannot be a base URL"))? + .extend(path); + Ok(url) } async fn fetch( - client: &reqwest::Client, - url: Url, - token: Option<&str>, + client: &reqwest::Client, + url: Url, + token: Option<&str>, ) -> Result { - let mut request = client.get(url.clone()); - if let Some(token) = token { - request = request.bearer_auth(token); - } - - let response = request - .send() - .await - .wrap_err_with(|| format!("requesting `{url}`"))? - .error_for_status() - .wrap_err_with(|| format!("requesting `{url}`"))?; - - response - .json() - .await - .wrap_err_with(|| format!("deserializing response from `{url}`")) + let mut request = client.get(url.clone()); + if let Some(token) = token { + request = request.bearer_auth(token); + } + + let response = request + .send() + .await + .wrap_err_with(|| format!("requesting `{url}`"))? + .error_for_status() + .wrap_err_with(|| format!("requesting `{url}`"))?; + + response + .json() + .await + .wrap_err_with(|| format!("deserializing response from `{url}`")) } struct DatasetSeed { - description_profanity_sample_limit: usize, + description_profanity_sample_limit: usize, } impl<'de> DeserializeSeed<'de> for DatasetSeed { - type Value = BatchSummary; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_map(DatasetVisitor { - description_profanity_sample_limit: self - .description_profanity_sample_limit, - }) - } + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(DatasetVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) + } } struct DatasetVisitor { - description_profanity_sample_limit: usize, + description_profanity_sample_limit: usize, } impl<'de> Visitor<'de> for DatasetVisitor { - type Value = BatchSummary; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a moderation project dataset") - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let mut summary = None; - while let Some(key) = map.next_key::()? { - if key == "projects" { - if summary.is_some() { - return Err(serde::de::Error::duplicate_field("projects")); - } - summary = Some(map.next_value_seed(ProjectsSeed { - description_profanity_sample_limit: - self.description_profanity_sample_limit, - })?); - } else { - map.next_value::()?; - } - } - - summary.ok_or_else(|| serde::de::Error::missing_field("projects")) - } + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut summary = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if summary.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + summary = Some(map.next_value_seed(ProjectsSeed { + description_profanity_sample_limit: + self.description_profanity_sample_limit, + })?); + } else { + map.next_value::()?; + } + } + + summary.ok_or_else(|| serde::de::Error::missing_field("projects")) + } } struct ProjectsSeed { - description_profanity_sample_limit: usize, + description_profanity_sample_limit: usize, } impl<'de> DeserializeSeed<'de> for ProjectsSeed { - type Value = BatchSummary; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_seq(ProjectsVisitor { - description_profanity_sample_limit: self - .description_profanity_sample_limit, - }) - } + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProjectsVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) + } } struct ProjectsVisitor { - description_profanity_sample_limit: usize, + description_profanity_sample_limit: usize, } impl<'de> Visitor<'de> for ProjectsVisitor { - type Value = BatchSummary; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("an array of projects") - } - - fn visit_seq(self, mut sequence: A) -> Result - where - A: SeqAccess<'de>, - { - let mut summary = - BatchSummary::new(self.description_profanity_sample_limit); - while let Some(project) = sequence.next_element::()? { - summary.add_project(&project); - } - Ok(summary) - } + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut summary = + BatchSummary::new(self.description_profanity_sample_limit); + while let Some(project) = sequence.next_element::()? { + summary.add_project(&project); + } + Ok(summary) + } } diff --git a/apps/labrinth/src/bin/check_project/profanity.rs b/apps/labrinth/src/bin/check_project/profanity.rs index 8d71d0a2ec..58a6af8de8 100644 --- a/apps/labrinth/src/bin/check_project/profanity.rs +++ b/apps/labrinth/src/bin/check_project/profanity.rs @@ -16,431 +16,431 @@ const ANSI_END: &str = "\x1b[0m"; #[derive(Serialize, Deserialize)] struct ProfanityReport { - format_version: u32, - matcher: String, - projects_scanned: usize, - projects_with_profanity: usize, - fields_with_profanity: usize, - fields_triggering_nags: usize, - projects: Vec, + format_version: u32, + matcher: String, + projects_scanned: usize, + projects_with_profanity: usize, + fields_with_profanity: usize, + fields_triggering_nags: usize, + projects: Vec, } impl ProfanityReport { - fn new() -> Self { - Self { - format_version: REPORT_FORMAT_VERSION, - matcher: "rustrict: profane and moderate-or-higher".to_owned(), - projects_scanned: 0, - projects_with_profanity: 0, - fields_with_profanity: 0, - fields_triggering_nags: 0, - projects: Vec::new(), - } - } - - fn add_project(&mut self, project: &Project) { - self.projects_scanned += 1; - let mut fields = Vec::new(); - push_field( - &mut fields, - ProfanityFieldKind::ProjectName, - None, - &project.name, - ); - push_field( - &mut fields, - ProfanityFieldKind::ProjectSummary, - None, - &project.summary, - ); - push_field( - &mut fields, - ProfanityFieldKind::ProjectDescription, - None, - &project.description, - ); - for (gallery_index, gallery_item) in project.gallery.iter().enumerate() - { - if let Some(name) = &gallery_item.name { - push_field( - &mut fields, - ProfanityFieldKind::GalleryName, - Some(gallery_index), - name, - ); - } - if let Some(description) = &gallery_item.description { - push_field( - &mut fields, - ProfanityFieldKind::GalleryDescription, - Some(gallery_index), - description, - ); - } - } - - if fields.is_empty() { - return; - } - - self.projects_with_profanity += 1; - self.fields_with_profanity += fields.len(); - self.fields_triggering_nags += - fields.iter().filter(|field| field.triggers_nag).count(); - self.projects.push(ProjectProfanityReport { - id: project.id.to_string(), - name: project.name.clone(), - fields, - }); - } + fn new() -> Self { + Self { + format_version: REPORT_FORMAT_VERSION, + matcher: "rustrict: profane and moderate-or-higher".to_owned(), + projects_scanned: 0, + projects_with_profanity: 0, + fields_with_profanity: 0, + fields_triggering_nags: 0, + projects: Vec::new(), + } + } + + fn add_project(&mut self, project: &Project) { + self.projects_scanned += 1; + let mut fields = Vec::new(); + push_field( + &mut fields, + ProfanityFieldKind::ProjectName, + None, + &project.name, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectSummary, + None, + &project.summary, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectDescription, + None, + &project.description, + ); + for (gallery_index, gallery_item) in project.gallery.iter().enumerate() + { + if let Some(name) = &gallery_item.name { + push_field( + &mut fields, + ProfanityFieldKind::GalleryName, + Some(gallery_index), + name, + ); + } + if let Some(description) = &gallery_item.description { + push_field( + &mut fields, + ProfanityFieldKind::GalleryDescription, + Some(gallery_index), + description, + ); + } + } + + if fields.is_empty() { + return; + } + + self.projects_with_profanity += 1; + self.fields_with_profanity += fields.len(); + self.fields_triggering_nags += + fields.iter().filter(|field| field.triggers_nag).count(); + self.projects.push(ProjectProfanityReport { + id: project.id.to_string(), + name: project.name.clone(), + fields, + }); + } } #[derive(Serialize, Deserialize)] struct ProjectProfanityReport { - id: String, - name: String, - fields: Vec, + id: String, + name: String, + fields: Vec, } #[derive(Serialize, Deserialize)] struct ProfanityFieldReport { - kind: ProfanityFieldKind, - #[serde(skip_serializing_if = "Option::is_none")] - gallery_index: Option, - text: String, - matches: Vec, - triggers_nag: bool, + kind: ProfanityFieldKind, + #[serde(skip_serializing_if = "Option::is_none")] + gallery_index: Option, + text: String, + matches: Vec, + triggers_nag: bool, } #[derive(Clone, Copy, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] enum ProfanityFieldKind { - ProjectName, - ProjectSummary, - ProjectDescription, - GalleryName, - GalleryDescription, + ProjectName, + ProjectSummary, + ProjectDescription, + GalleryName, + GalleryDescription, } impl ProfanityFieldKind { - fn label(self, gallery_index: Option) -> String { - match (self, gallery_index) { - (Self::ProjectName, _) => "Project name".to_owned(), - (Self::ProjectSummary, _) => "Project summary".to_owned(), - (Self::ProjectDescription, _) => "Project description".to_owned(), - (Self::GalleryName, Some(index)) => { - format!("Gallery image {} name", index + 1) - } - (Self::GalleryDescription, Some(index)) => { - format!("Gallery image {} description", index + 1) - } - (Self::GalleryName, None) => "Gallery image name".to_owned(), - (Self::GalleryDescription, None) => { - "Gallery image description".to_owned() - } - } - } + fn label(self, gallery_index: Option) -> String { + match (self, gallery_index) { + (Self::ProjectName, _) => "Project name".to_owned(), + (Self::ProjectSummary, _) => "Project summary".to_owned(), + (Self::ProjectDescription, _) => "Project description".to_owned(), + (Self::GalleryName, Some(index)) => { + format!("Gallery image {} name", index + 1) + } + (Self::GalleryDescription, Some(index)) => { + format!("Gallery image {} description", index + 1) + } + (Self::GalleryName, None) => "Gallery image name".to_owned(), + (Self::GalleryDescription, None) => { + "Gallery image description".to_owned() + } + } + } } #[derive(Serialize, Deserialize)] struct ProfanityMatch { - start_char: usize, - end_char: usize, + start_char: usize, + end_char: usize, } fn push_field( - fields: &mut Vec, - kind: ProfanityFieldKind, - gallery_index: Option, - text: &str, + fields: &mut Vec, + kind: ProfanityFieldKind, + gallery_index: Option, + text: &str, ) { - let matches = profanity_matches(text); - if matches.is_empty() { - return; - } - - let triggers_nag = match kind { - ProfanityFieldKind::ProjectDescription => matches.len() > 2, - _ => true, - }; - fields.push(ProfanityFieldReport { - kind, - gallery_index, - text: text.to_owned(), - matches, - triggers_nag, - }); + let matches = profanity_matches(text); + if matches.is_empty() { + return; + } + + let triggers_nag = match kind { + ProfanityFieldKind::ProjectDescription => matches.len() > 2, + _ => true, + }; + fields.push(ProfanityFieldReport { + kind, + gallery_index, + text: text.to_owned(), + matches, + triggers_nag, + }); } pub(super) fn write_report( - dataset_path: &Path, - report_path: &Path, + dataset_path: &Path, + report_path: &Path, ) -> Result<()> { - let report = scan_dataset(dataset_path)?; - let output = File::create(report_path) - .wrap_err_with(|| format!("creating `{}`", report_path.display()))?; - serde_json::to_writer_pretty(BufWriter::new(output), &report) - .wrap_err_with(|| format!("writing `{}`", report_path.display()))?; - - eprintln!( - "wrote {} projects with profanity across {} fields to `{}`", - report.projects_with_profanity, - report.fields_with_profanity, - report_path.display(), - ); - Ok(()) + let report = scan_dataset(dataset_path)?; + let output = File::create(report_path) + .wrap_err_with(|| format!("creating `{}`", report_path.display()))?; + serde_json::to_writer_pretty(BufWriter::new(output), &report) + .wrap_err_with(|| format!("writing `{}`", report_path.display()))?; + + eprintln!( + "wrote {} projects with profanity across {} fields to `{}`", + report.projects_with_profanity, + report.fields_with_profanity, + report_path.display(), + ); + Ok(()) } pub(super) fn render_report( - report_path: &Path, - project_limit: usize, - force_brackets: bool, + report_path: &Path, + project_limit: usize, + force_brackets: bool, ) -> Result<()> { - let input = File::open(report_path) - .wrap_err_with(|| format!("opening `{}`", report_path.display()))?; - let report: ProfanityReport = - serde_json::from_reader(BufReader::new(input)) - .wrap_err_with(|| format!("reading `{}`", report_path.display()))?; - if report.format_version != REPORT_FORMAT_VERSION { - return Err(eyre!( - "unsupported profanity report format version `{}`", - report.format_version - )); - } - - let use_color = !force_brackets - && std::io::stdout().is_terminal() - && std::env::var_os("NO_COLOR").is_none(); - println!("=== Project profanity report ==="); - println!("Matcher: {}", report.matcher); - println!("Projects scanned: {}", report.projects_scanned); - println!( - "Projects with profanity: {}", - report.projects_with_profanity - ); - println!("Fields with profanity: {}", report.fields_with_profanity); - println!( - "Fields triggering nags: {}", - report.fields_triggering_nags - ); - if use_color { - println!("Detected text uses a bold white-on-red background."); - } else { - println!("Detected text is enclosed in ⟦double brackets⟧."); - } - - let projects = if project_limit == 0 { - report.projects.as_slice() - } else { - &report.projects[..project_limit.min(report.projects.len())] - }; - for (project_index, project) in projects.iter().enumerate() { - println!( - "\n--- Project {} of {} ---", - project_index + 1, - projects.len() - ); - println!("Project: {}", sanitize_text(&project.name)); - println!("ID: {}", project.id); - println!("URL: https://modrinth.com/project/{}", project.id); - - for field in &project.fields { - println!( - "\n[{}; {} match{}; nag: {}]", - field.kind.label(field.gallery_index), - field.matches.len(), - if field.matches.len() == 1 { "" } else { "es" }, - if field.triggers_nag { "yes" } else { "no" }, - ); - println!( - "{}", - highlight_matches(&field.text, &field.matches, use_color) - ); - } - } - Ok(()) + let input = File::open(report_path) + .wrap_err_with(|| format!("opening `{}`", report_path.display()))?; + let report: ProfanityReport = + serde_json::from_reader(BufReader::new(input)) + .wrap_err_with(|| format!("reading `{}`", report_path.display()))?; + if report.format_version != REPORT_FORMAT_VERSION { + return Err(eyre!( + "unsupported profanity report format version `{}`", + report.format_version + )); + } + + let use_color = !force_brackets + && std::io::stdout().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + println!("=== Project profanity report ==="); + println!("Matcher: {}", report.matcher); + println!("Projects scanned: {}", report.projects_scanned); + println!( + "Projects with profanity: {}", + report.projects_with_profanity + ); + println!("Fields with profanity: {}", report.fields_with_profanity); + println!( + "Fields triggering nags: {}", + report.fields_triggering_nags + ); + if use_color { + println!("Detected text uses a bold white-on-red background."); + } else { + println!("Detected text is enclosed in ⟦double brackets⟧."); + } + + let projects = if project_limit == 0 { + report.projects.as_slice() + } else { + &report.projects[..project_limit.min(report.projects.len())] + }; + for (project_index, project) in projects.iter().enumerate() { + println!( + "\n--- Project {} of {} ---", + project_index + 1, + projects.len() + ); + println!("Project: {}", sanitize_text(&project.name)); + println!("ID: {}", project.id); + println!("URL: https://modrinth.com/project/{}", project.id); + + for field in &project.fields { + println!( + "\n[{}; {} match{}; nag: {}]", + field.kind.label(field.gallery_index), + field.matches.len(), + if field.matches.len() == 1 { "" } else { "es" }, + if field.triggers_nag { "yes" } else { "no" }, + ); + println!( + "{}", + highlight_matches(&field.text, &field.matches, use_color) + ); + } + } + Ok(()) } pub(super) fn highlight_text(text: &str, use_color: bool) -> String { - let matches = profanity_matches(text); - highlight_matches(text, &matches, use_color) + let matches = profanity_matches(text); + highlight_matches(text, &matches, use_color) } fn scan_dataset(path: &Path) -> Result { - let input = File::open(path) - .wrap_err_with(|| format!("opening `{}`", path.display()))?; - let mut deserializer = - serde_json::Deserializer::from_reader(BufReader::new(input)); - let report = ProfanityDatasetSeed - .deserialize(&mut deserializer) - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - deserializer - .end() - .wrap_err_with(|| format!("reading `{}`", path.display()))?; - Ok(report) + let input = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(input)); + let report = ProfanityDatasetSeed + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(report) } fn profanity_matches(text: &str) -> Vec { - let threshold = Type::PROFANE & Type::MODERATE_OR_HIGHER; - let mut censor = Censor::from_str(text); - censor - .with_ignore_self_censoring(true) - .with_censor_threshold(threshold) - .with_censor_first_character_threshold(threshold) - .with_censor_replacement(CENSORED); - let censored = censor.censor().chars().collect::>(); - - let mut matches = Vec::new(); - let mut start = None; - let mut character_count = 0; - for (index, _) in text.chars().enumerate() { - character_count = index + 1; - let is_censored = censored.get(index) == Some(&CENSORED); - match (start, is_censored) { - (None, true) => start = Some(index), - (Some(start_char), false) => { - matches.push(ProfanityMatch { - start_char, - end_char: index, - }); - start = None; - } - _ => {} - } - } - if let Some(start_char) = start { - matches.push(ProfanityMatch { - start_char, - end_char: character_count, - }); - } - matches + let threshold = Type::PROFANE & Type::MODERATE_OR_HIGHER; + let mut censor = Censor::from_str(text); + censor + .with_ignore_self_censoring(true) + .with_censor_threshold(threshold) + .with_censor_first_character_threshold(threshold) + .with_censor_replacement(CENSORED); + let censored = censor.censor().chars().collect::>(); + + let mut matches = Vec::new(); + let mut start = None; + let mut character_count = 0; + for (index, _) in text.chars().enumerate() { + character_count = index + 1; + let is_censored = censored.get(index) == Some(&CENSORED); + match (start, is_censored) { + (None, true) => start = Some(index), + (Some(start_char), false) => { + matches.push(ProfanityMatch { + start_char, + end_char: index, + }); + start = None; + } + _ => {} + } + } + if let Some(start_char) = start { + matches.push(ProfanityMatch { + start_char, + end_char: character_count, + }); + } + matches } fn highlight_matches( - text: &str, - matches: &[ProfanityMatch], - use_color: bool, + text: &str, + matches: &[ProfanityMatch], + use_color: bool, ) -> String { - let mut output = String::with_capacity(text.len()); - let mut matches = matches.iter().peekable(); - let mut highlighting = false; - for (index, character) in text.chars().enumerate() { - while matches - .peek() - .is_some_and(|matched| index >= matched.end_char) - { - matches.next(); - } - let is_censored = matches.peek().is_some_and(|matched| { - index >= matched.start_char && index < matched.end_char - }); - if is_censored != highlighting { - output.push_str(if is_censored { - if use_color { ANSI_START } else { "⟦" } - } else if use_color { - ANSI_END - } else { - "⟧" - }); - highlighting = is_censored; - } - push_safe_character(&mut output, character); - } - if highlighting { - output.push_str(if use_color { ANSI_END } else { "⟧" }); - } - output + let mut output = String::with_capacity(text.len()); + let mut matches = matches.iter().peekable(); + let mut highlighting = false; + for (index, character) in text.chars().enumerate() { + while matches + .peek() + .is_some_and(|matched| index >= matched.end_char) + { + matches.next(); + } + let is_censored = matches.peek().is_some_and(|matched| { + index >= matched.start_char && index < matched.end_char + }); + if is_censored != highlighting { + output.push_str(if is_censored { + if use_color { ANSI_START } else { "⟦" } + } else if use_color { + ANSI_END + } else { + "⟧" + }); + highlighting = is_censored; + } + push_safe_character(&mut output, character); + } + if highlighting { + output.push_str(if use_color { ANSI_END } else { "⟧" }); + } + output } fn sanitize_text(text: &str) -> String { - let mut output = String::with_capacity(text.len()); - for character in text.chars() { - push_safe_character(&mut output, character); - } - output + let mut output = String::with_capacity(text.len()); + for character in text.chars() { + push_safe_character(&mut output, character); + } + output } fn push_safe_character(output: &mut String, character: char) { - match character { - '\n' | '\t' => output.push(character), - '\r' => {} - character if character.is_control() => output.push('�'), - character => output.push(character), - } + match character { + '\n' | '\t' => output.push(character), + '\r' => {} + character if character.is_control() => output.push('�'), + character => output.push(character), + } } struct ProfanityDatasetSeed; impl<'de> DeserializeSeed<'de> for ProfanityDatasetSeed { - type Value = ProfanityReport; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_map(ProfanityDatasetVisitor) - } + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(ProfanityDatasetVisitor) + } } struct ProfanityDatasetVisitor; impl<'de> Visitor<'de> for ProfanityDatasetVisitor { - type Value = ProfanityReport; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("a moderation project dataset") - } - - fn visit_map(self, mut map: A) -> Result - where - A: MapAccess<'de>, - { - let mut report = None; - while let Some(key) = map.next_key::()? { - if key == "projects" { - if report.is_some() { - return Err(serde::de::Error::duplicate_field("projects")); - } - report = Some(map.next_value_seed(ProfanityProjectsSeed)?); - } else { - map.next_value::()?; - } - } - report.ok_or_else(|| serde::de::Error::missing_field("projects")) - } + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut report = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if report.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + report = Some(map.next_value_seed(ProfanityProjectsSeed)?); + } else { + map.next_value::()?; + } + } + report.ok_or_else(|| serde::de::Error::missing_field("projects")) + } } struct ProfanityProjectsSeed; impl<'de> DeserializeSeed<'de> for ProfanityProjectsSeed { - type Value = ProfanityReport; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_seq(ProfanityProjectsVisitor) - } + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProfanityProjectsVisitor) + } } struct ProfanityProjectsVisitor; impl<'de> Visitor<'de> for ProfanityProjectsVisitor { - type Value = ProfanityReport; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("an array of projects") - } - - fn visit_seq(self, mut sequence: A) -> Result - where - A: SeqAccess<'de>, - { - let mut report = ProfanityReport::new(); - while let Some(project) = sequence.next_element::()? { - report.add_project(&project); - } - Ok(report) - } + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut report = ProfanityReport::new(); + while let Some(project) = sequence.next_element::()? { + report.add_project(&project); + } + Ok(report) + } } diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 0a2529f999..877242e7b2 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -7,7 +7,7 @@ use crate::auth::get_user_from_headers; use crate::database::models::DBProjectId; use crate::database::models::project_item::ProjectQueryResult; use crate::database::{ - PgPool, PgTransaction, ReadOnlyPgPool, models as db_models, + PgPool, PgTransaction, ReadOnlyPgPool, models as db_models, }; use crate::models::ids::ProjectId; use crate::models::pats::Scopes; @@ -17,68 +17,68 @@ use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::util::error::Context as _; use crate::validate::project::{ - ProjectNag, has_required_nags_with_context, - validate_with_context as validate_project, + ProjectNag, has_required_nags_with_context, + validate_with_context as validate_project, }; #[derive(Serialize, utoipa::ToSchema)] pub struct ProjectValidationResponse { - pub nags: Vec, + pub nags: Vec, } pub(crate) async fn ensure_project_is_valid_for_review( - project_id: DBProjectId, - pool: &PgPool, - transaction: &mut PgTransaction<'_>, - redis: &RedisPool, + project_id: DBProjectId, + pool: &PgPool, + transaction: &mut PgTransaction<'_>, + redis: &RedisPool, ) -> Result { - let mut projects = db_models::DBProject::get_many_uncached( - &[ProjectId::from(project_id)], - &mut *transaction, - redis, - ) - .await - .wrap_internal_err("reloading project for review validation")?; - let reloaded_project = - projects.pop().wrap_not_found_err("resource not found")?; - let versions = db_models::DBVersion::get_many_uncached( - &reloaded_project.versions, - &mut *transaction, - redis, - ) - .await - .wrap_internal_err("reloading project versions for review validation")? - .into_iter() - .map(Version::from) - .collect::>(); - let available_categories = - db_models::categories::Category::list(&**pool, redis) - .await - .wrap_internal_err("fetching project categories")?; - let disclosures = db_models::DBProjectDisclosure::get_many_for_project( - reloaded_project.inner.id, - false, - &mut *transaction, - ) - .await - .wrap_internal_err("fetching project disclosures")? - .into_iter() - .map(|disclosure| disclosure.disclosure) - .collect::>(); - let project = Project::from(reloaded_project.clone()); + let mut projects = db_models::DBProject::get_many_uncached( + &[ProjectId::from(project_id)], + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project for review validation")?; + let reloaded_project = + projects.pop().wrap_not_found_err("resource not found")?; + let versions = db_models::DBVersion::get_many_uncached( + &reloaded_project.versions, + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project versions for review validation")? + .into_iter() + .map(Version::from) + .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + reloaded_project.inner.id, + false, + &mut *transaction, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); + let project = Project::from(reloaded_project.clone()); - if has_required_nags_with_context( - &project, - &versions, - &available_categories, - &disclosures, - ) { - return Err(ApiError::Request(eyre!( - "project must have no required validation nags before or while under review" - ))); - } + if has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) { + return Err(ApiError::Request(eyre!( + "project must have no required validation nags before or while under review" + ))); + } - Ok(reloaded_project) + Ok(reloaded_project) } /// Validate that a project is ready to be submitted for review. @@ -89,80 +89,80 @@ pub(crate) async fn ensure_project_is_valid_for_review( )] #[get("/{id}/validate")] pub async fn validate( - req: HttpRequest, - info: web::Path<(String,)>, - pool: web::Data, - ro_pool: web::Data, - redis: web::Data, - session_queue: web::Data, + req: HttpRequest, + info: web::Path<(String,)>, + pool: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, ) -> Result, ApiError> { - let user = get_user_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Scopes::PROJECT_READ, - ) - .await - .wrap_auth_err("authenticating API request")? - .1; + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PROJECT_READ, + ) + .await + .wrap_auth_err("authenticating API request")? + .1; - let project = - db_models::DBProject::get(&info.into_inner().0, &***ro_pool, &redis) - .await - .wrap_internal_err("fetching project from database")? - .wrap_not_found_err("resource not found")?; + let project = + db_models::DBProject::get(&info.into_inner().0, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project from database")? + .wrap_not_found_err("resource not found")?; - let (team_member, organization_team_member) = - db_models::DBTeamMember::get_for_project_permissions( - &project.inner, - user.id.into(), - &***ro_pool, - ) - .await - .wrap_internal_err("fetching project permissions")?; + let (team_member, organization_team_member) = + db_models::DBTeamMember::get_for_project_permissions( + &project.inner, + user.id.into(), + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project permissions")?; - if ProjectPermissions::get_permissions_by_role( - &user.role, - &team_member, - &organization_team_member, - ) - .is_none() - { - return Err(ApiError::Auth(eyre!( - "you do not have permission to validate this project" - ))); - } + if ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member, + &organization_team_member, + ) + .is_none() + { + return Err(ApiError::Auth(eyre!( + "you do not have permission to validate this project" + ))); + } - let versions = - db_models::DBVersion::get_many(&project.versions, &***ro_pool, &redis) - .await - .wrap_internal_err("fetching project versions from database")? - .into_iter() - .map(Version::from) - .collect::>(); - let available_categories = - db_models::categories::Category::list(&**pool, &redis) - .await - .wrap_internal_err("fetching project categories")?; - let disclosures = db_models::DBProjectDisclosure::get_many_for_project( - project.inner.id, - false, - &***ro_pool, - ) - .await - .wrap_internal_err("fetching project disclosures")? - .into_iter() - .map(|disclosure| disclosure.disclosure) - .collect::>(); - let project = Project::from(project); + let versions = + db_models::DBVersion::get_many(&project.versions, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project versions from database")? + .into_iter() + .map(Version::from) + .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, &redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + project.inner.id, + false, + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); + let project = Project::from(project); - Ok(web::Json(ProjectValidationResponse { - nags: validate_project( - &project, - &versions, - &available_categories, - &disclosures, - ), - })) + Ok(web::Json(ProjectValidationResponse { + nags: validate_project( + &project, + &versions, + &available_categories, + &disclosures, + ), + })) } diff --git a/apps/labrinth/src/test/api_v2/request_data.rs b/apps/labrinth/src/test/api_v2/request_data.rs index e2e7156c41..4d98e9c572 100644 --- a/apps/labrinth/src/test/api_v2/request_data.rs +++ b/apps/labrinth/src/test/api_v2/request_data.rs @@ -91,7 +91,7 @@ pub fn get_public_project_creation_data_json( "slug": slug, "project_type": version_jar.as_ref().map_or("mod".to_string(), |f| f.project_type()), "description": "A dummy project for testing with.", - "body": "This project is approved, and versions are listed.", + "body": "This project is approved and its versions are listed. It contains enough descriptive information to satisfy project review validation requirements and explain what the project provides to its users.", "client_side": "required", "server_side": "optional", "initial_versions": initial_versions, diff --git a/apps/labrinth/src/test/api_v3/request_data.rs b/apps/labrinth/src/test/api_v3/request_data.rs index a3bc0eb4f7..6865f50add 100644 --- a/apps/labrinth/src/test/api_v3/request_data.rs +++ b/apps/labrinth/src/test/api_v3/request_data.rs @@ -100,7 +100,7 @@ pub fn get_public_project_creation_data_json( "name": format!("Test Project {slug}"), "slug": slug, "summary": "A dummy project for testing with.", - "description": "This project is approved, and versions are listed.", + "description": "This project is approved and its versions are listed. It contains enough descriptive information to satisfy project review validation requirements and explain what the project provides to its users.", "initial_versions": initial_versions, "is_draft": is_draft, "categories": [], diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 972e32647f..4d14ea7cb4 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -816,9 +816,7 @@ fn has_repeated_ngram_density(words: &[String]) -> bool { .values() .map(|count| count.saturating_sub(1)) .sum::(); - if duplicate_ngrams * 100 - < total_ngrams * MIN_DUPLICATE_NGRAM_PERCENT - { + if duplicate_ngrams * 100 < total_ngrams * MIN_DUPLICATE_NGRAM_PERCENT { return false; } diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs index 3ebc3d80a9..a9feb68642 100644 --- a/apps/labrinth/src/validate/project/versions.rs +++ b/apps/labrinth/src/validate/project/versions.rs @@ -3,43 +3,43 @@ use crate::models::projects::{Project, Version}; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; pub(super) fn validate( - project: &Project, - _versions: &[Version], + project: &Project, + _versions: &[Version], ) -> Vec { - let mut nags = Vec::new(); + let mut nags = Vec::new(); - if project.versions.is_empty() - && project.components.minecraft_server.is_none() - { - nags.push(ProjectNag::new( - ProjectNagKind::UploadVersion, - ProjectNagSeverity::Required, - )); - } + if project.versions.is_empty() + && project.components.minecraft_server.is_none() + { + nags.push(ProjectNag::new( + ProjectNagKind::UploadVersion, + ProjectNagSeverity::Required, + )); + } - let requires_environment = project - .project_types - .iter() - .any(|project_type| matches!(project_type.as_str(), "mod" | "modpack")); - let has_valid_environment = - project - .fields - .get("environment") - .is_some_and(|environments| { - !environments.is_empty() - && environments.iter().all(|environment| { - environment.as_str().is_some_and(|environment| { - !environment.trim().is_empty() - && environment != "unknown" - }) - }) - }); - if requires_environment && !has_valid_environment { - nags.push(ProjectNag::new( - ProjectNagKind::SelectEnvironment, - ProjectNagSeverity::Required, - )); - } + let requires_environment = project + .project_types + .iter() + .any(|project_type| matches!(project_type.as_str(), "mod" | "modpack")); + let has_valid_environment = + project + .fields + .get("environment") + .is_some_and(|environments| { + !environments.is_empty() + && environments.iter().all(|environment| { + environment.as_str().is_some_and(|environment| { + !environment.trim().is_empty() + && environment != "unknown" + }) + }) + }); + if requires_environment && !has_valid_environment { + nags.push(ProjectNag::new( + ProjectNagKind::SelectEnvironment, + ProjectNagSeverity::Required, + )); + } - nags + nags } diff --git a/apps/labrinth/tests/v2/search.rs b/apps/labrinth/tests/v2/search.rs index 510a21fde0..f16b0ee20a 100644 --- a/apps/labrinth/tests/v2/search.rs +++ b/apps/labrinth/tests/v2/search.rs @@ -64,6 +64,7 @@ async fn search_projects() { { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -141,6 +142,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -157,6 +159,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -176,6 +179,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, { "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] }, ])) From f7c9a5ee78972999025f728b98a6cd422b06d978 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:07:10 +0100 Subject: [PATCH 24/28] make sure test data has enough words in description --- apps/labrinth/src/test/dummy_data.rs | 2 +- apps/labrinth/src/test/search.rs | 4 ++++ apps/labrinth/tests/scopes.rs | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/labrinth/src/test/dummy_data.rs b/apps/labrinth/src/test/dummy_data.rs index 73000cfa77..4f26c5b9aa 100644 --- a/apps/labrinth/src/test/dummy_data.rs +++ b/apps/labrinth/src/test/dummy_data.rs @@ -363,7 +363,7 @@ pub async fn add_project_beta(api: &ApiV3) -> (Project, Version) { let modify_json = serde_json::from_value(json!([ { "op": "add", "path": "/summary", "value": "A dummy project for testing with." }, - { "op": "add", "path": "/description", "value": "This project is not-yet-approved, and versions are draft." }, + { "op": "add", "path": "/description", "value": "This project is not yet approved, but it contains enough descriptive information to satisfy project review validation requirements while remaining available for testing private project behavior." }, { "op": "add", "path": "/initial_versions/0/status", "value": "unlisted" }, { "op": "add", "path": "/status", "value": "private" }, { "op": "add", "path": "/requested_status", "value": "private" }, diff --git a/apps/labrinth/src/test/search.rs b/apps/labrinth/src/test/search.rs index 9159a5a14b..0f9d0a3026 100644 --- a/apps/labrinth/src/test/search.rs +++ b/apps/labrinth/src/test/search.rs @@ -63,6 +63,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "server_only" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -140,6 +141,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server" }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -155,6 +157,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server_prefers_both" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -172,6 +175,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server_prefers_both" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, { "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] }, ])) diff --git a/apps/labrinth/tests/scopes.rs b/apps/labrinth/tests/scopes.rs index 97264946a2..aa609285f5 100644 --- a/apps/labrinth/tests/scopes.rs +++ b/apps/labrinth/tests/scopes.rs @@ -547,7 +547,7 @@ pub async fn project_write_scopes() { &[beta_project_id.as_str()], json!( { - "description": "test_project_version_write_scopes Description" + "description": "This description verifies project write scopes while containing enough information to satisfy project review validation requirements for a project that is currently being processed." }), pat.as_deref(), ) From e5b3e9a6ab5b75b6d8892079cead81ba314e432e Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:03:48 +0100 Subject: [PATCH 25/28] fix --- apps/labrinth/src/validate/project/description.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index e137812f87..d38e8774d9 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -164,9 +164,8 @@ mod tests { #[test] fn description_spam_allows_isolated_repetition() { - assert!(!has_description_spam(&format!( - "aaaaaaaa {}", - "text ".repeat(140) - ))); + assert!(!has_description_spam( + "aaaaaaaa This project provides configurable tools for exploring worlds, organizing resources, managing settings, sharing creations, and helping players understand every available feature through clear documentation." + )); } } From 1bd70a4f4b458c08ddaa276d5fd2bfeab24a4b72 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:18:05 +0100 Subject: [PATCH 26/28] fix --- apps/labrinth/src/validate/project/text.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 4d14ea7cb4..0b04a3654f 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -908,9 +908,7 @@ pub(super) fn extract_description_text(markdown: &str) -> String { let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); without_html .lines() - .map(|line| { - line.trim_start_matches(|character| matches!(character, '>' | '#')) - }) + .map(|line| line.trim_start_matches(['>', '#'])) .collect::>() .join("\n") .replace(['*', '_', '~', '`', '>', '-', '|'], " ") From f544ea3040f52cf7a06fef509914fcd0ee83f1ae Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:00:11 +0100 Subject: [PATCH 27/28] improve error --- apps/labrinth/src/routes/v3/projects/mod.rs | 10 ++++++---- apps/labrinth/tests/project.rs | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/apps/labrinth/src/routes/v3/projects/mod.rs b/apps/labrinth/src/routes/v3/projects/mod.rs index aafcd1af07..5acae44de2 100644 --- a/apps/labrinth/src/routes/v3/projects/mod.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -462,10 +462,12 @@ pub async fn project_edit_internal( .wrap_auth_err("authenticating API request")? .1; - new_project - .validate() - .map_err(|err| eyre::eyre!(err)) - .wrap_request_err("validating request")?; + new_project.validate().map_err(|err| { + let message = + crate::util::validate::validation_errors_to_string(err, None) + .replacen("Field ", "field ", 1); + ApiError::Request(eyre!(message)) + })?; let Some(mut project_item) = db_models::DBProject::get(&info.into_inner().0, &**pool, &redis) diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 6ed1eb7d28..02f3551fa1 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -357,6 +357,23 @@ pub async fn test_patch_project() { assert_status!(&resp, StatusCode::BAD_REQUEST); } + let resp = api + .edit_project( + alpha_project_slug, + json!({ + "summary": "", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&resp, StatusCode::BAD_REQUEST); + let body: serde_json::Value = test::read_body_json(resp).await; + assert_eq!( + body["description"], + "field summary failed validation with error: length" + ); + assert!(body.get("details").is_none()); + // Failure because these are illegal requested statuses for a normal user. for req in ["unknown", "processing", "withheld", "scheduled"] { let resp = api From c23ac75b8703d3e77a0ecb3911402fcce7d2d5a4 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:07:06 +0100 Subject: [PATCH 28/28] fix --- apps/labrinth/src/bin/check_project.rs | 64 ++++++++++++-------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs index 7c4c1119ee..9de86106d1 100644 --- a/apps/labrinth/src/bin/check_project.rs +++ b/apps/labrinth/src/bin/check_project.rs @@ -235,29 +235,26 @@ fn find_all_nag_project_ids( .unwrap_or(1); let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); let worker_results = std::thread::scope(|scope| { - let workers = dataset - .projects - .chunks(chunk_size) - .map(|projects| { - scope.spawn(move || { - let mut project_ids = - BTreeMap::>::new(); - for project in projects { - let kinds = validate(project, &[]) - .into_iter() - .map(|nag| nag.kind) - .collect::>(); - for kind in kinds { - project_ids - .entry(kind) - .or_default() - .push(project.id.to_string()); - } + let mut workers = Vec::new(); + for projects in dataset.projects.chunks(chunk_size) { + workers.push(scope.spawn(move || { + let mut project_ids = + BTreeMap::>::new(); + for project in projects { + let kinds = validate(project, &[]) + .into_iter() + .map(|nag| nag.kind) + .collect::>(); + for kind in kinds { + project_ids + .entry(kind) + .or_default() + .push(project.id.to_string()); } - project_ids - }) - }) - .collect::>(); + } + project_ids + })); + } workers .into_iter() .map(|worker| { @@ -294,19 +291,16 @@ fn find_language_project_ids( .unwrap_or(1); let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); let ids = std::thread::scope(|scope| -> Result> { - let workers = dataset - .projects - .chunks(chunk_size) - .map(|projects| { - scope.spawn(move || { - projects - .iter() - .filter(|project| is_non_english(project)) - .map(|project| project.id.to_string()) - .collect::>() - }) - }) - .collect::>(); + let mut workers = Vec::new(); + for projects in dataset.projects.chunks(chunk_size) { + workers.push(scope.spawn(move || { + projects + .iter() + .filter(|project| is_non_english(project)) + .map(|project| project.id.to_string()) + .collect::>() + })); + } workers .into_iter() .map(|worker| {