diff --git a/Cargo.lock b/Cargo.lock index c03c8d0a29..acce146a94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,7 +602,7 @@ dependencies = [ "futures-core", "libc", "portable-atomic", - "rustc-hash", + "rustc-hash 2.1.1", "tokio", "tokio-stream", "xattr", @@ -851,7 +851,7 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", - "tungstenite", + "tungstenite 0.27.0", "webpki-roots 1.0.3", ] @@ -1463,7 +1463,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.1", "shlex 1.3.0", "syn 2.0.106", ] @@ -3056,6 +3056,85 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dioxus-cli-config" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322d10f47effebd85ee0662a2cb083ff920d40dd6295a4afd8cfa4f9bc2e05c6" + +[[package]] +name = "dioxus-core" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b232228ada232c0adc151e41ebd9ef70cc04683db097fa0dc6015979fa01722" +dependencies = [ + "anyhow", + "const_format", + "dioxus-core-types", + "futures-channel", + "futures-util", + "generational-box", + "longest-increasing-subsequence", + "rustc-hash 2.1.1", + "rustversion", + "serde", + "slab", + "slotmap", + "subsecond", + "tracing", +] + +[[package]] +name = "dioxus-core-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96007ed6cbad951dfed82d83c581aabd95bc591f351f5666d454ddbe7845b324" + +[[package]] +name = "dioxus-devtools" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434cebb282b3f820a341582ebfd37dab907d5a25c3eeafd87802468afcd2a959" +dependencies = [ + "dioxus-cli-config", + "dioxus-core", + "dioxus-devtools-types", + "dioxus-signals", + "serde", + "serde_json", + "subsecond", + "thiserror 2.0.17", + "tracing", + "tungstenite 0.28.0", +] + +[[package]] +name = "dioxus-devtools-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfd6f3475e38c93be245a664a9ce11679a8f2b99832620705cf7837f42aa39ce" +dependencies = [ + "dioxus-core", + "serde", + "subsecond-types", +] + +[[package]] +name = "dioxus-signals" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95d85e36fcaf7abdc986836801bf248776da7d5961f5fac65fdbf5ed491afb4f" +dependencies = [ + "dioxus-core", + "futures-channel", + "futures-util", + "generational-box", + "parking_lot", + "rustc-hash 2.1.1", + "tracing", + "warnings", +] + [[package]] name = "directories" version = "6.0.0" @@ -3192,6 +3271,12 @@ dependencies = [ "const-random", ] +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + [[package]] name = "docker_credential" version = "1.3.2" @@ -3684,6 +3769,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + [[package]] name = "flate2" version = "1.1.4" @@ -3840,9 +3931,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -3850,9 +3941,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" @@ -3878,9 +3969,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -3912,26 +4003,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -3941,9 +4032,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -3953,7 +4044,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -4087,6 +4177,16 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "generational-box" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b2dc9b873cd1a8adb80aa5dcef9a8205f781a8a313d5ffb867248cb3bcb764" +dependencies = [ + "parking_lot", + "tracing", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -5304,6 +5404,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -5550,6 +5659,7 @@ dependencies = [ "const_format", "dashmap", "derive_more 2.1.1", + "dioxus-devtools", "dotenv-build", "dotenvy", "either", @@ -5568,6 +5678,7 @@ dependencies = [ "json-patch 4.1.0", "labrinth", "lettre", + "linkify", "modrinth-content-management", "modrinth-util", "muralpay", @@ -5577,6 +5688,7 @@ dependencies = [ "path-util", "postcard", "prometheus", + "psl", "quick-xml 0.38.3", "rand 0.8.5", "rand_chacha 0.3.1", @@ -5587,6 +5699,7 @@ dependencies = [ "rust_decimal", "rust_iso3166", "rustls 0.23.32", + "rustrict", "rusty-money", "scalar_api_reference", "sentry", @@ -5609,6 +5722,8 @@ dependencies = [ "totp-rs", "tracing", "tracing-actix-web", + "unicode-normalization", + "unicode-segmentation", "url", "urlencoding", "utoipa", @@ -5617,6 +5732,7 @@ dependencies = [ "webauthn-rs", "webauthn-rs-proto", "webp", + "whatlang", "woothee", "xredis", "yaserde", @@ -5811,6 +5927,15 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linkify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92f9ae2f3d31697697311f016320fe984127e636b45cf5eb5c175b7eb103dadb" +dependencies = [ + "memchr", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -5867,6 +5992,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "longest-increasing-subsequence" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" + [[package]] name = "loop9" version = "0.1.5" @@ -6027,6 +6158,24 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix 1.1.2", +] + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -7952,6 +8101,15 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "psl" +version = "2.1.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc88482eea924ca3a2f56a547454169af58deef35567965eb4fc2392a834841" +dependencies = [ + "psl-types", +] + [[package]] name = "psl-types" version = "2.0.11" @@ -8092,7 +8250,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.32", "socket2 0.6.5", "thiserror 2.0.17", @@ -8112,7 +8270,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls 0.23.32", "rustls-pki-types", "slab", @@ -8904,6 +9062,12 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -9083,6 +9247,22 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustrict" +version = "0.7.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa26cdd448930328cd86e35244d8c7c8e3368e8b8e2fb444befd65419f5b3771" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "doc-comment", + "finl_unicode", + "itertools 0.10.5", + "lazy_static", + "rustc-hash 1.1.0", + "unicode-normalization", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -9902,6 +10082,16 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "slotmap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" +dependencies = [ + "serde", + "version_check", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -10413,6 +10603,34 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "subsecond" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d350d5788fa94d560d92269266a50efc77b036bfca6675e420b64df7b3211f37" +dependencies = [ + "js-sys", + "libc", + "libloading 0.8.8", + "memfd", + "memmap2", + "serde", + "subsecond-types", + "thiserror 2.0.17", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "subsecond-types" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf32d66269b5fbb8558334e8d22b657b2f7af0dd2ef56c231541099a862e464" +dependencies = [ + "serde", +] + [[package]] name = "subtle" version = "2.6.1" @@ -11939,6 +12157,23 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +dependencies = [ + "bytes", + "data-encoding", + "http 1.3.1", + "httparse", + "log", + "rand 0.9.2", + "sha1 0.10.6", + "thiserror 2.0.17", + "utf-8", +] + [[package]] name = "typed-arena" version = "2.0.2" @@ -12378,6 +12613,28 @@ dependencies = [ "try-lock", ] +[[package]] +name = "warnings" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f68998838dab65727c9b30465595c6f7c953313559371ca8bf31759b3680ad" +dependencies = [ + "pin-project", + "tracing", + "warnings-macro", +] + +[[package]] +name = "warnings-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59195a1db0e95b920366d949ba5e0d3fc0e70b67c09be15ce5abb790106b0571" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -12822,6 +13079,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "whatlang" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index 99b0c562a5..a4a779c2dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ dashmap = "6.1.0" data-url = "0.3.2" deadpool-redis = { git = "https://github.com/modrinth/deadpool", rev = "c7209b086829572f368f1194aba54bdea8dc540b", version = "0.23.0" } derive_more = "2.1.1" +dioxus-devtools = "0.7.10" directories = "6.0.0" dirs = "6.0.0" discord-rich-presence = "1.0.0" @@ -123,6 +124,7 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ "checked-decode", "safe-decode", @@ -150,6 +152,7 @@ postcard = { version = "1.1.3", default-features = false, features = ["alloc"] } postcard-bindgen = "0.8.0" proc-macro2 = { version = "1.0" } prometheus = "0.14.0" +psl = "2.1.226" quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } @@ -168,6 +171,7 @@ rust-s3 = { version = "0.37.0", default-features = false, features = [ "tokio-rustls-tls", ] } rustls = "0.23.32" +rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } rusty-money = "0.4.1" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" @@ -229,6 +233,8 @@ tracing-error = "0.2.1" tracing-subscriber = "0.3.20" ts-rs = "12.0.1" typed-path = "0.12.0" +unicode-normalization = "0.1.24" +unicode-segmentation = "1.12.0" url = "2.5.7" urlencoding = "2.1.3" utoipa = { version = "5.4.0", features = ["actix_extras", "chrono", "decimal"] } @@ -238,6 +244,7 @@ webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry +whatlang = "0.18.0" whoami = "1.6.1" windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index d94e808ed4..e813bd2c3d 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -3,6 +3,7 @@ name = "labrinth" version = "2.7.0" edition.workspace = true license = "AGPL-3.0-only" +default-run = "labrinth" # This seems redundant, but it's necessary for Docker to work [[bin]] @@ -40,6 +41,7 @@ component-derive = { workspace = true } const_format = { workspace = true } dashmap = { workspace = true } derive_more = { workspace = true, features = ["deref", "deref_mut"] } +dioxus-devtools = { workspace = true } dotenvy = { workspace = true } either = { workspace = true } eyre = { workspace = true } @@ -70,6 +72,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +linkify = { workspace = true } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } muralpay = { workspace = true, features = ["client", "mock", "utoipa"] } @@ -79,6 +82,7 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } +psl = { workspace = true } quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } @@ -89,6 +93,7 @@ reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls- rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } rustls.workspace = true +rustrict = { workspace = true } rusty-money = { workspace = true } scalar_api_reference = { workspace = true, features = ["actix-web"] } sentry = { workspace = true } @@ -118,6 +123,8 @@ tokio-stream = { workspace = true } totp-rs = { workspace = true, features = ["gen_secret"] } tracing = { workspace = true } tracing-actix-web = { workspace = true } +unicode-normalization = { workspace = true } +unicode-segmentation = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } utoipa = { workspace = true, features = ["url"] } @@ -126,6 +133,7 @@ validator = { workspace = true, features = ["derive"] } webauthn-rs = { workspace = true, features = ["conditional-ui", "danger-allow-state-serialisation"] } webauthn-rs-proto = { workspace = true } webp = { workspace = true } +whatlang = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } yaserde = { workspace = true, features = ["derive"] } diff --git a/apps/labrinth/src/bin/check_project.rs b/apps/labrinth/src/bin/check_project.rs new file mode 100644 index 0000000000..9de86106d1 --- /dev/null +++ b/apps/labrinth/src/bin/check_project.rs @@ -0,0 +1,523 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::fs::File; +use std::io::{BufReader, IsTerminal}; +use std::path::{Path, PathBuf}; + +use clap::Parser; +use eyre::{Result, WrapErr, eyre}; +use labrinth::models::projects::{Project, Version}; +use labrinth::validate::project::{ + ProjectNagKind, ProjectNagSeverity, is_project_description_non_english, + is_project_summary_non_english, validate, +}; +use serde::de::{ + DeserializeOwned, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, + Visitor, +}; +use serde::{Deserialize, Serialize}; +use strum::IntoEnumIterator; +use url::Url; + +#[path = "check_project/profanity.rs"] +mod profanity; + +const API_BASE_URL: &str = "https://api.modrinth.com/v3/"; + +#[derive(Parser)] +#[command( + version, + about = "Validate a project from the Modrinth API or a project dataset" +)] +struct Args { + /// The ID of the project to validate + #[arg( + value_name = "PROJECT_ID", + required_unless_present_any = ["file", "read_profanity_report"], + conflicts_with_all = ["file", "read_profanity_report"] + )] + project_id: Option, + + /// Validate all projects in a moderation project dataset + #[arg( + long, + value_name = "PATH", + conflicts_with_all = ["token", "read_profanity_report"] + )] + file: Option, + + /// Print every validator kind with the IDs of projects that trigger it + #[arg( + long, + requires = "file", + conflicts_with_all = [ + "nag_project_ids", + "description_language_project_ids", + "show_description_profanity", + "write_profanity_report" + ] + )] + all_nag_project_ids: bool, + + /// A Modrinth token to send as bearer authentication + #[arg(long, value_name = "TOKEN")] + token: Option, + + /// Show this many descriptions flagged for profanity + #[arg(long, value_name = "COUNT", default_value_t = 0, requires = "file")] + show_description_profanity: usize, + + /// Print project IDs triggering the summary language nag + #[arg(long, requires = "file")] + nag_project_ids: bool, + + /// Print project IDs triggering the description language nag + #[arg(long, requires = "file", conflicts_with = "nag_project_ids")] + description_language_project_ids: bool, + + /// Scan every profanity-bearing field into a reusable JSON report + #[arg( + long, + value_name = "PATH", + requires = "file", + conflicts_with = "show_description_profanity" + )] + write_profanity_report: Option, + + /// Render a previously generated profanity report + #[arg(long, value_name = "PATH", conflicts_with = "file")] + read_profanity_report: Option, + + /// Render at most this many projects from a profanity report (0 means all) + #[arg( + long, + value_name = "COUNT", + default_value_t = 0, + requires = "read_profanity_report" + )] + profanity_report_limit: usize, + + /// Use bracket markers instead of ANSI colors when rendering a report + #[arg(long, requires = "read_profanity_report")] + profanity_report_brackets: bool, +} + +#[derive(Serialize)] +struct BatchSummary { + projects: usize, + projects_with_at_least_one_required_nag: usize, + version_details_available: bool, + nag_counts: BTreeMap, + #[serde(skip)] + description_profanity_sample_limit: usize, + #[serde(skip)] + description_profanity_samples: Vec, +} + +struct DescriptionProfanitySample { + id: String, + name: String, + description: String, +} + +impl BatchSummary { + fn new(description_profanity_sample_limit: usize) -> Self { + Self { + projects: 0, + projects_with_at_least_one_required_nag: 0, + version_details_available: false, + nag_counts: ProjectNagKind::iter().map(|kind| (kind, 0)).collect(), + description_profanity_sample_limit, + description_profanity_samples: Vec::new(), + } + } + + fn add_project(&mut self, project: &Project) { + let nags = validate(project, &[]); + self.projects += 1; + if nags + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) + { + self.projects_with_at_least_one_required_nag += 1; + } + if self.description_profanity_samples.len() + < self.description_profanity_sample_limit + && nags.iter().any(|nag| { + matches!( + nag.kind, + ProjectNagKind::ProjectDescriptionSlur + | ProjectNagKind::ProjectDescriptionProfanity + ) + }) + { + self.description_profanity_samples.push( + DescriptionProfanitySample { + id: project.id.to_string(), + name: project.name.clone(), + description: project.description.clone(), + }, + ); + } + for nag in nags { + *self.nag_counts.entry(nag.kind).or_default() += 1; + } + } +} + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install().wrap_err("installing color-eyre")?; + let args = Args::parse(); + + if let Some(report_path) = args.read_profanity_report.as_deref() { + return profanity::render_report( + report_path, + args.profanity_report_limit, + args.profanity_report_brackets, + ); + } + + if let Some(report_path) = args.write_profanity_report.as_deref() { + let dataset_path = args.file.as_deref().ok_or_else(|| { + eyre!("`--write-profanity-report` requires `--file`") + })?; + return profanity::write_report(dataset_path, report_path); + } + + if let Some(path) = args.file.as_deref() { + if args.nag_project_ids { + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectSummaryNonEnglish, + is_project_summary_non_english, + )?); + } + if args.description_language_project_ids { + return print_json(&find_language_project_ids( + path, + ProjectNagKind::ProjectDescriptionNonEnglish, + is_project_description_non_english, + )?); + } + if args.all_nag_project_ids { + return print_json(&find_all_nag_project_ids(path)?); + } + let summary = summarize_file(path, args.show_description_profanity)?; + print_description_profanity_samples( + &summary.description_profanity_samples, + ); + print_json(&summary)?; + return Ok(()); + } + + let project_id = args + .project_id + .as_deref() + .ok_or_else(|| eyre!("a project ID or `--file` is required"))?; + check_api_project(project_id, args.token.as_deref()).await +} + +#[derive(Deserialize)] +struct ProjectDataset { + projects: Vec, +} + +fn find_all_nag_project_ids( + path: &Path, +) -> Result>> { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let worker_results = std::thread::scope(|scope| { + let mut workers = Vec::new(); + for projects in dataset.projects.chunks(chunk_size) { + workers.push(scope.spawn(move || { + let mut project_ids = + BTreeMap::>::new(); + for project in projects { + let kinds = validate(project, &[]) + .into_iter() + .map(|nag| nag.kind) + .collect::>(); + for kind in kinds { + project_ids + .entry(kind) + .or_default() + .push(project.id.to_string()); + } + } + project_ids + })); + } + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("project validation worker panicked")) + }) + .collect::>>() + })?; + + let mut project_ids = ProjectNagKind::iter() + .map(|kind| (kind, Vec::new())) + .collect::>(); + for worker_result in worker_results { + for (kind, ids) in worker_result { + project_ids.entry(kind).or_default().extend(ids); + } + } + + Ok(project_ids) +} + +fn find_language_project_ids( + path: &Path, + kind: ProjectNagKind, + is_non_english: fn(&Project) -> bool, +) -> Result>> { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let dataset: ProjectDataset = serde_json::from_reader(BufReader::new(file)) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + let worker_count = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1); + let chunk_size = dataset.projects.len().div_ceil(worker_count).max(1); + let ids = std::thread::scope(|scope| -> Result> { + let mut workers = Vec::new(); + for projects in dataset.projects.chunks(chunk_size) { + workers.push(scope.spawn(move || { + projects + .iter() + .filter(|project| is_non_english(project)) + .map(|project| project.id.to_string()) + .collect::>() + })); + } + workers + .into_iter() + .map(|worker| { + worker + .join() + .map_err(|_| eyre!("language detection worker panicked")) + }) + .collect::>>() + .map(|ids| ids.into_iter().flatten().collect()) + })?; + let project_ids = BTreeMap::from([(kind, ids)]); + + Ok(project_ids) +} + +async fn check_api_project( + project_id: &str, + token: Option<&str>, +) -> Result<()> { + let client = reqwest::Client::builder() + .user_agent(concat!( + "labrinth-check-project/", + env!("CARGO_PKG_VERSION") + )) + .build() + .wrap_err("building HTTP client")?; + + let project_url = api_url(&["project", project_id])?; + let mut versions_url = api_url(&["project", project_id, "version"])?; + versions_url + .query_pairs_mut() + .append_pair("include_changelog", "false"); + + let (project, versions) = tokio::try_join!( + fetch::(&client, project_url, token), + fetch::>(&client, versions_url, token), + )?; + print_json(&validate(&project, &versions)) +} + +fn summarize_file( + path: &Path, + description_profanity_sample_limit: usize, +) -> Result { + let file = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(file)); + let summary = DatasetSeed { + description_profanity_sample_limit, + } + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(summary) +} + +fn print_description_profanity_samples(samples: &[DescriptionProfanitySample]) { + if samples.is_empty() { + return; + } + + let use_color = std::io::stderr().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + eprintln!("\n=== Description profanity samples ==="); + if use_color { + eprintln!( + "Detected text is shown with a bold white-on-red background." + ); + } else { + eprintln!("Detected text is enclosed in ⟦double brackets⟧."); + } + + for (index, sample) in samples.iter().enumerate() { + eprintln!("\n--- Sample {} of {} ---", index + 1, samples.len()); + eprintln!("Project: {}", sample.name); + eprintln!("ID: {}", sample.id); + eprintln!("URL: https://modrinth.com/project/{}", sample.id); + eprintln!( + "\n{}", + profanity::highlight_text(&sample.description, use_color) + ); + } + eprintln!("\n=== End description profanity samples ===\n"); +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!( + "{}", + serde_json::to_string_pretty(value).wrap_err("serializing output")? + ); + Ok(()) +} + +fn api_url(path: &[&str]) -> Result { + let mut url = Url::parse(API_BASE_URL).wrap_err("parsing API base URL")?; + url.path_segments_mut() + .map_err(|_| eyre!("API base URL cannot be a base URL"))? + .extend(path); + Ok(url) +} + +async fn fetch( + client: &reqwest::Client, + url: Url, + token: Option<&str>, +) -> Result { + let mut request = client.get(url.clone()); + if let Some(token) = token { + request = request.bearer_auth(token); + } + + let response = request + .send() + .await + .wrap_err_with(|| format!("requesting `{url}`"))? + .error_for_status() + .wrap_err_with(|| format!("requesting `{url}`"))?; + + response + .json() + .await + .wrap_err_with(|| format!("deserializing response from `{url}`")) +} + +struct DatasetSeed { + description_profanity_sample_limit: usize, +} + +impl<'de> DeserializeSeed<'de> for DatasetSeed { + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(DatasetVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) + } +} + +struct DatasetVisitor { + description_profanity_sample_limit: usize, +} + +impl<'de> Visitor<'de> for DatasetVisitor { + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut summary = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if summary.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + summary = Some(map.next_value_seed(ProjectsSeed { + description_profanity_sample_limit: + self.description_profanity_sample_limit, + })?); + } else { + map.next_value::()?; + } + } + + summary.ok_or_else(|| serde::de::Error::missing_field("projects")) + } +} + +struct ProjectsSeed { + description_profanity_sample_limit: usize, +} + +impl<'de> DeserializeSeed<'de> for ProjectsSeed { + type Value = BatchSummary; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProjectsVisitor { + description_profanity_sample_limit: self + .description_profanity_sample_limit, + }) + } +} + +struct ProjectsVisitor { + description_profanity_sample_limit: usize, +} + +impl<'de> Visitor<'de> for ProjectsVisitor { + type Value = BatchSummary; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut summary = + BatchSummary::new(self.description_profanity_sample_limit); + while let Some(project) = sequence.next_element::()? { + summary.add_project(&project); + } + Ok(summary) + } +} diff --git a/apps/labrinth/src/bin/check_project/profanity.rs b/apps/labrinth/src/bin/check_project/profanity.rs new file mode 100644 index 0000000000..58a6af8de8 --- /dev/null +++ b/apps/labrinth/src/bin/check_project/profanity.rs @@ -0,0 +1,446 @@ +use std::fmt; +use std::fs::File; +use std::io::{BufReader, BufWriter, IsTerminal}; +use std::path::Path; + +use eyre::{Result, WrapErr, eyre}; +use labrinth::models::projects::Project; +use rustrict::{Censor, Type}; +use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; + +const REPORT_FORMAT_VERSION: u32 = 1; +const CENSORED: char = '\0'; +const ANSI_START: &str = "\x1b[1;97;41m"; +const ANSI_END: &str = "\x1b[0m"; + +#[derive(Serialize, Deserialize)] +struct ProfanityReport { + format_version: u32, + matcher: String, + projects_scanned: usize, + projects_with_profanity: usize, + fields_with_profanity: usize, + fields_triggering_nags: usize, + projects: Vec, +} + +impl ProfanityReport { + fn new() -> Self { + Self { + format_version: REPORT_FORMAT_VERSION, + matcher: "rustrict: profane and moderate-or-higher".to_owned(), + projects_scanned: 0, + projects_with_profanity: 0, + fields_with_profanity: 0, + fields_triggering_nags: 0, + projects: Vec::new(), + } + } + + fn add_project(&mut self, project: &Project) { + self.projects_scanned += 1; + let mut fields = Vec::new(); + push_field( + &mut fields, + ProfanityFieldKind::ProjectName, + None, + &project.name, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectSummary, + None, + &project.summary, + ); + push_field( + &mut fields, + ProfanityFieldKind::ProjectDescription, + None, + &project.description, + ); + for (gallery_index, gallery_item) in project.gallery.iter().enumerate() + { + if let Some(name) = &gallery_item.name { + push_field( + &mut fields, + ProfanityFieldKind::GalleryName, + Some(gallery_index), + name, + ); + } + if let Some(description) = &gallery_item.description { + push_field( + &mut fields, + ProfanityFieldKind::GalleryDescription, + Some(gallery_index), + description, + ); + } + } + + if fields.is_empty() { + return; + } + + self.projects_with_profanity += 1; + self.fields_with_profanity += fields.len(); + self.fields_triggering_nags += + fields.iter().filter(|field| field.triggers_nag).count(); + self.projects.push(ProjectProfanityReport { + id: project.id.to_string(), + name: project.name.clone(), + fields, + }); + } +} + +#[derive(Serialize, Deserialize)] +struct ProjectProfanityReport { + id: String, + name: String, + fields: Vec, +} + +#[derive(Serialize, Deserialize)] +struct ProfanityFieldReport { + kind: ProfanityFieldKind, + #[serde(skip_serializing_if = "Option::is_none")] + gallery_index: Option, + text: String, + matches: Vec, + triggers_nag: bool, +} + +#[derive(Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum ProfanityFieldKind { + ProjectName, + ProjectSummary, + ProjectDescription, + GalleryName, + GalleryDescription, +} + +impl ProfanityFieldKind { + fn label(self, gallery_index: Option) -> String { + match (self, gallery_index) { + (Self::ProjectName, _) => "Project name".to_owned(), + (Self::ProjectSummary, _) => "Project summary".to_owned(), + (Self::ProjectDescription, _) => "Project description".to_owned(), + (Self::GalleryName, Some(index)) => { + format!("Gallery image {} name", index + 1) + } + (Self::GalleryDescription, Some(index)) => { + format!("Gallery image {} description", index + 1) + } + (Self::GalleryName, None) => "Gallery image name".to_owned(), + (Self::GalleryDescription, None) => { + "Gallery image description".to_owned() + } + } + } +} + +#[derive(Serialize, Deserialize)] +struct ProfanityMatch { + start_char: usize, + end_char: usize, +} + +fn push_field( + fields: &mut Vec, + kind: ProfanityFieldKind, + gallery_index: Option, + text: &str, +) { + let matches = profanity_matches(text); + if matches.is_empty() { + return; + } + + let triggers_nag = match kind { + ProfanityFieldKind::ProjectDescription => matches.len() > 2, + _ => true, + }; + fields.push(ProfanityFieldReport { + kind, + gallery_index, + text: text.to_owned(), + matches, + triggers_nag, + }); +} + +pub(super) fn write_report( + dataset_path: &Path, + report_path: &Path, +) -> Result<()> { + let report = scan_dataset(dataset_path)?; + let output = File::create(report_path) + .wrap_err_with(|| format!("creating `{}`", report_path.display()))?; + serde_json::to_writer_pretty(BufWriter::new(output), &report) + .wrap_err_with(|| format!("writing `{}`", report_path.display()))?; + + eprintln!( + "wrote {} projects with profanity across {} fields to `{}`", + report.projects_with_profanity, + report.fields_with_profanity, + report_path.display(), + ); + Ok(()) +} + +pub(super) fn render_report( + report_path: &Path, + project_limit: usize, + force_brackets: bool, +) -> Result<()> { + let input = File::open(report_path) + .wrap_err_with(|| format!("opening `{}`", report_path.display()))?; + let report: ProfanityReport = + serde_json::from_reader(BufReader::new(input)) + .wrap_err_with(|| format!("reading `{}`", report_path.display()))?; + if report.format_version != REPORT_FORMAT_VERSION { + return Err(eyre!( + "unsupported profanity report format version `{}`", + report.format_version + )); + } + + let use_color = !force_brackets + && std::io::stdout().is_terminal() + && std::env::var_os("NO_COLOR").is_none(); + println!("=== Project profanity report ==="); + println!("Matcher: {}", report.matcher); + println!("Projects scanned: {}", report.projects_scanned); + println!( + "Projects with profanity: {}", + report.projects_with_profanity + ); + println!("Fields with profanity: {}", report.fields_with_profanity); + println!( + "Fields triggering nags: {}", + report.fields_triggering_nags + ); + if use_color { + println!("Detected text uses a bold white-on-red background."); + } else { + println!("Detected text is enclosed in ⟦double brackets⟧."); + } + + let projects = if project_limit == 0 { + report.projects.as_slice() + } else { + &report.projects[..project_limit.min(report.projects.len())] + }; + for (project_index, project) in projects.iter().enumerate() { + println!( + "\n--- Project {} of {} ---", + project_index + 1, + projects.len() + ); + println!("Project: {}", sanitize_text(&project.name)); + println!("ID: {}", project.id); + println!("URL: https://modrinth.com/project/{}", project.id); + + for field in &project.fields { + println!( + "\n[{}; {} match{}; nag: {}]", + field.kind.label(field.gallery_index), + field.matches.len(), + if field.matches.len() == 1 { "" } else { "es" }, + if field.triggers_nag { "yes" } else { "no" }, + ); + println!( + "{}", + highlight_matches(&field.text, &field.matches, use_color) + ); + } + } + Ok(()) +} + +pub(super) fn highlight_text(text: &str, use_color: bool) -> String { + let matches = profanity_matches(text); + highlight_matches(text, &matches, use_color) +} + +fn scan_dataset(path: &Path) -> Result { + let input = File::open(path) + .wrap_err_with(|| format!("opening `{}`", path.display()))?; + let mut deserializer = + serde_json::Deserializer::from_reader(BufReader::new(input)); + let report = ProfanityDatasetSeed + .deserialize(&mut deserializer) + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + deserializer + .end() + .wrap_err_with(|| format!("reading `{}`", path.display()))?; + Ok(report) +} + +fn profanity_matches(text: &str) -> Vec { + let threshold = Type::PROFANE & Type::MODERATE_OR_HIGHER; + let mut censor = Censor::from_str(text); + censor + .with_ignore_self_censoring(true) + .with_censor_threshold(threshold) + .with_censor_first_character_threshold(threshold) + .with_censor_replacement(CENSORED); + let censored = censor.censor().chars().collect::>(); + + let mut matches = Vec::new(); + let mut start = None; + let mut character_count = 0; + for (index, _) in text.chars().enumerate() { + character_count = index + 1; + let is_censored = censored.get(index) == Some(&CENSORED); + match (start, is_censored) { + (None, true) => start = Some(index), + (Some(start_char), false) => { + matches.push(ProfanityMatch { + start_char, + end_char: index, + }); + start = None; + } + _ => {} + } + } + if let Some(start_char) = start { + matches.push(ProfanityMatch { + start_char, + end_char: character_count, + }); + } + matches +} + +fn highlight_matches( + text: &str, + matches: &[ProfanityMatch], + use_color: bool, +) -> String { + let mut output = String::with_capacity(text.len()); + let mut matches = matches.iter().peekable(); + let mut highlighting = false; + for (index, character) in text.chars().enumerate() { + while matches + .peek() + .is_some_and(|matched| index >= matched.end_char) + { + matches.next(); + } + let is_censored = matches.peek().is_some_and(|matched| { + index >= matched.start_char && index < matched.end_char + }); + if is_censored != highlighting { + output.push_str(if is_censored { + if use_color { ANSI_START } else { "⟦" } + } else if use_color { + ANSI_END + } else { + "⟧" + }); + highlighting = is_censored; + } + push_safe_character(&mut output, character); + } + if highlighting { + output.push_str(if use_color { ANSI_END } else { "⟧" }); + } + output +} + +fn sanitize_text(text: &str) -> String { + let mut output = String::with_capacity(text.len()); + for character in text.chars() { + push_safe_character(&mut output, character); + } + output +} + +fn push_safe_character(output: &mut String, character: char) { + match character { + '\n' | '\t' => output.push(character), + '\r' => {} + character if character.is_control() => output.push('�'), + character => output.push(character), + } +} + +struct ProfanityDatasetSeed; + +impl<'de> DeserializeSeed<'de> for ProfanityDatasetSeed { + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_map(ProfanityDatasetVisitor) + } +} + +struct ProfanityDatasetVisitor; + +impl<'de> Visitor<'de> for ProfanityDatasetVisitor { + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a moderation project dataset") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut report = None; + while let Some(key) = map.next_key::()? { + if key == "projects" { + if report.is_some() { + return Err(serde::de::Error::duplicate_field("projects")); + } + report = Some(map.next_value_seed(ProfanityProjectsSeed)?); + } else { + map.next_value::()?; + } + } + report.ok_or_else(|| serde::de::Error::missing_field("projects")) + } +} + +struct ProfanityProjectsSeed; + +impl<'de> DeserializeSeed<'de> for ProfanityProjectsSeed { + type Value = ProfanityReport; + + fn deserialize(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(ProfanityProjectsVisitor) + } +} + +struct ProfanityProjectsVisitor; + +impl<'de> Visitor<'de> for ProfanityProjectsVisitor { + type Value = ProfanityReport; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an array of projects") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut report = ProfanityReport::new(); + while let Some(project) = sequence.next_element::()? { + report.add_project(&project); + } + Ok(report) + } +} diff --git a/apps/labrinth/src/database/models/project_item.rs b/apps/labrinth/src/database/models/project_item.rs index f4d0d3118a..7d6b46afa8 100644 --- a/apps/labrinth/src/database/models/project_item.rs +++ b/apps/labrinth/src/database/models/project_item.rs @@ -576,10 +576,42 @@ impl DBProject { where E: crate::database::Acquire<'a, Database = sqlx::Postgres>, { - let val = redis.get_cached_keys_with_slug( + Self::get_many_inner(project_strings, exec, redis, true).await + } + + pub async fn get_many_uncached< + 'a, + E, + T: Display + Hash + Eq + PartialEq + Clone + Debug, + >( + project_strings: &[T], + exec: E, + redis: &RedisPool, + ) -> Result, ApiError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + Self::get_many_inner(project_strings, exec, redis, false).await + } + + async fn get_many_inner< + 'a, + E, + T: Display + Hash + Eq + PartialEq + Clone + Debug, + >( + project_strings: &[T], + exec: E, + redis: &RedisPool, + use_cache: bool, + ) -> Result, ApiError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + let val = redis.get_keys_with_slug_cache( PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, false, + use_cache, project_strings, |ids| async move { let mut exec = exec diff --git a/apps/labrinth/src/database/models/version_item.rs b/apps/labrinth/src/database/models/version_item.rs index aff56773e5..a54a933ad5 100644 --- a/apps/labrinth/src/database/models/version_item.rs +++ b/apps/labrinth/src/database/models/version_item.rs @@ -595,8 +595,32 @@ impl DBVersion { where E: crate::database::Acquire<'a, Database = sqlx::Postgres>, { - let mut val = redis.get_cached_keys( + Self::get_many_inner(version_ids, exec, redis, true).await + } + + pub async fn get_many_uncached<'a, E>( + version_ids: &[DBVersionId], + exec: E, + redis: &RedisPool, + ) -> Result, DatabaseError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + Self::get_many_inner(version_ids, exec, redis, false).await + } + + async fn get_many_inner<'a, E>( + version_ids: &[DBVersionId], + exec: E, + redis: &RedisPool, + use_cache: bool, + ) -> Result, DatabaseError> + where + E: crate::database::Acquire<'a, Database = sqlx::Postgres>, + { + let mut val = redis.get_keys_with_cache( VERSIONS_NAMESPACE, + use_cache, &version_ids.iter().map(|x| x.0).collect::>(), |version_ids| async move { let mut exec = exec.acquire().await?; diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index b456b77a67..68e0d8ef4c 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -94,6 +94,8 @@ async fn app() -> std::io::Result<()> { .install_default() .unwrap(); + dioxus_devtools::connect_subsecond(); + if args.run_background_task.is_none() { info!("Starting labrinth on {}", &ENV.BIND_ADDR); diff --git a/apps/labrinth/src/routes/v2/projects.rs b/apps/labrinth/src/routes/v2/projects.rs index 679f740ea0..23691141bd 100644 --- a/apps/labrinth/src/routes/v2/projects.rs +++ b/apps/labrinth/src/routes/v2/projects.rs @@ -691,8 +691,7 @@ pub async fn project_edit( true, ) .await - .or_else(v2_reroute::flatten_404_error) - .wrap_api_err("flattening v2 not-found response")?; + .or_else(v2_reroute::flatten_404_error)?; // If client and server side were set, we will call // the version setting route for each version to set the side types for each of them. diff --git a/apps/labrinth/src/routes/v3/disclosures.rs b/apps/labrinth/src/routes/v3/disclosures.rs index b81fefdd32..9923e7b36e 100644 --- a/apps/labrinth/src/routes/v3/disclosures.rs +++ b/apps/labrinth/src/routes/v3/disclosures.rs @@ -14,6 +14,7 @@ use crate::models::disclosures::{ ProjectDisclosureType, }; use crate::models::pats::Scopes; +use crate::models::projects::ProjectStatus; use crate::models::teams::ProjectPermissions; use crate::queue::session::AuthQueue; use crate::routes::ApiError; @@ -279,6 +280,16 @@ pub async fn modify_project_disclosures( .wrap_internal_err("failed to remove project disclosure")?; } + if project.inner.status == ProjectStatus::Processing { + super::projects::validate::ensure_project_is_valid_for_review( + project.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await diff --git a/apps/labrinth/src/routes/v3/mod.rs b/apps/labrinth/src/routes/v3/mod.rs index 46f6dba5aa..c8e95928c8 100644 --- a/apps/labrinth/src/routes/v3/mod.rs +++ b/apps/labrinth/src/routes/v3/mod.rs @@ -96,6 +96,7 @@ pub fn config(cfg: &mut web::ServiceConfig) { payouts::platform_revenue, projects::project_get, projects::project_get_check, + projects::validate::validate, projects::project_delete, projects::project_edit, projects::project_icon_edit, diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects/mod.rs similarity index 93% rename from apps/labrinth/src/routes/v3/projects.rs rename to apps/labrinth/src/routes/v3/projects/mod.rs index a919536bee..5acae44de2 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects/mod.rs @@ -6,7 +6,9 @@ use std::collections::HashMap; use crate::auth::checks::{filter_visible_versions, is_visible_project}; use crate::auth::{filter_visible_projects, get_user_from_headers}; use crate::database::models::notification_item::NotificationBuilder; -use crate::database::models::project_item::{DBGalleryItem, DBModCategory}; +use crate::database::models::project_item::{ + DBGalleryItem, DBModCategory, ProjectQueryResult, +}; use crate::database::models::thread_item::ThreadMessageBuilder; use crate::database::models::{ DBModerationLock, DBProjectId, DBTeamMember, ids as db_ids, image_item, @@ -27,7 +29,7 @@ use crate::models::projects::{ }; use crate::models::teams::{DEFAULT_ROLE, ProjectPermissions}; use crate::models::threads::MessageBody; -use crate::models::users::DELETED_USER; +use crate::models::users::{DELETED_USER, User}; use crate::models::{self, exp}; use crate::queue::session::AuthQueue; use crate::routes::ApiError; @@ -50,6 +52,8 @@ use serde::{Deserialize, Serialize}; use validator::Validate; use xredis::RedisPool; +pub mod validate; + pub fn config(cfg: &mut actix_web::web::ServiceConfig) { cfg.service(project_search) .service(project_search_post) @@ -74,7 +78,8 @@ pub fn project_config(cfg: &mut actix_web::web::ServiceConfig) { .service(super::teams::team_members_get_project) .service(super::versions::version_list) .service(super::versions::version_project_get) - .service(dependency_list); + .service(dependency_list) + .service(validate::validate); } pub async fn clear_project_cache_and_queue_search( @@ -457,10 +462,12 @@ pub async fn project_edit_internal( .wrap_auth_err("authenticating API request")? .1; - new_project - .validate() - .map_err(|err| eyre::eyre!(err)) - .wrap_request_err("validating request")?; + new_project.validate().map_err(|err| { + let message = + crate::util::validate::validation_errors_to_string(err, None) + .replacen("Field ", "field ", 1); + ApiError::Request(eyre!(message)) + })?; let Some(mut project_item) = db_models::DBProject::get(&info.into_inner().0, &**pool, &redis) @@ -491,6 +498,29 @@ pub async fn project_edit_internal( ))); }; + let submit_for_review = new_project.status + == Some(ProjectStatus::Processing) + && !user.role.is_mod(); + let leave_review = matches!( + new_project.status, + Some(ProjectStatus::Draft | ProjectStatus::Rejected) + ); + let validate_for_review = submit_for_review + || (project_item.inner.status == ProjectStatus::Processing + && !leave_review); + if submit_for_review { + if !perms.contains(ProjectPermissions::EDIT_DETAILS) { + return Err(ApiError::Auth(eyre!( + "you do not have permission to submit this project for review" + ))); + } + if project_item.inner.status.is_approved() { + return Err(ApiError::Auth(eyre!( + "you do not have permission to submit this project for review" + ))); + } + } + let mut transaction = pool .begin() .await @@ -538,7 +568,9 @@ pub async fn project_edit_internal( .wrap_internal_err("querying database for `project_edit_internal`")?; } - if let Some(status) = &new_project.status { + if let Some(status) = &new_project.status + && !submit_for_review + { if !perms.contains(ProjectPermissions::EDIT_DETAILS) { return Err(ApiError::Auth(eyre::eyre!( "You do not have the permissions to edit the status of this project!", @@ -611,12 +643,6 @@ pub async fn project_edit_internal( } if status == &ProjectStatus::Processing { - if project_item.versions.is_empty() { - return Err(ApiError::Request(eyre::eyre!(String::from( - "Project submitted for review with no initial versions", - )))); - } - sqlx::query!( " UPDATE mods @@ -701,7 +727,7 @@ pub async fn project_edit_internal( .ok(); } - if team_member.is_none_or(|x| !x.accepted) { + if team_member.as_ref().is_none_or(|x| !x.accepted) { let notified_members = sqlx::query!( " SELECT tm.user_id id @@ -1339,6 +1365,28 @@ pub async fn project_edit_internal( .await .wrap_api_err("deleting unused images")?; + if validate_for_review { + let reloaded_project = validate::ensure_project_is_valid_for_review( + id, + &pool, + &mut transaction, + &redis, + ) + .await?; + + if submit_for_review { + submit_project_for_review( + &reloaded_project, + &user, + team_member.as_ref().is_none_or(|member| !member.accepted), + sync_archival_disclosure, + &mut transaction, + &redis, + ) + .await?; + } + } + transaction .commit() .await @@ -1385,6 +1433,127 @@ pub async fn project_edit_internal( Ok(HttpResponse::NoContent().body("")) } +async fn submit_project_for_review( + project: &ProjectQueryResult, + user: &User, + notify_team_members: bool, + sync_archival_disclosure: bool, + transaction: &mut PgTransaction<'_>, + redis: &RedisPool, +) -> Result<(), ApiError> { + let archival_disclosure = + db_models::DBProjectDisclosure::get_many_for_project( + project.inner.id, + false, + &mut *transaction, + ) + .await + .wrap_internal_err("failed to fetch project disclosures")? + .into_iter() + .find(|disclosure| { + matches!(disclosure.disclosure, ProjectDisclosure::Archived { .. }) + }); + + sqlx::query!( + " + UPDATE mods + SET moderation_message = NULL, moderation_message_body = NULL, queued = NOW() + WHERE (id = $1) + ", + project.inner.id as db_ids::DBProjectId, + ) + .execute(&mut *transaction) + .await + .wrap_internal_err("querying database for `project_edit_internal`")?; + + if notify_team_members { + let notified_members = sqlx::query!( + " + SELECT tm.user_id id + FROM team_members tm + WHERE tm.team_id = $1 AND tm.accepted + ", + project.inner.team_id as db_ids::DBTeamId + ) + .fetch(&mut *transaction) + .map_ok(|row| db_models::DBUserId(row.id)) + .try_collect::>() + .await + .wrap_internal_err("fetching notified members from database")?; + + NotificationBuilder { + body: NotificationBody::StatusChange { + project_id: project.inner.id.into(), + old_status: project.inner.status, + new_status: ProjectStatus::Processing, + }, + } + .insert_many(notified_members.clone(), &mut *transaction, redis) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + + NotificationBuilder { + body: NotificationBody::ProjectStatusNeutral { + project_id: project.inner.id.into(), + old_status: project.inner.status, + new_status: ProjectStatus::Processing, + }, + } + .insert_many(notified_members, &mut *transaction, redis) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + } + + ThreadMessageBuilder { + author_id: Some(user.id.into()), + body: MessageBody::StatusChange { + new_status: ProjectStatus::Processing, + old_status: project.inner.status, + }, + thread_id: project.thread_id, + hide_identity: false, + } + .insert(&mut *transaction) + .await + .wrap_internal_err( + "inserting database records for `project_edit_internal`", + )?; + + sqlx::query!( + " + UPDATE mods + SET status = $1 + WHERE (id = $2) + ", + ProjectStatus::Processing.as_str(), + project.inner.id as db_ids::DBProjectId, + ) + .execute(&mut *transaction) + .await + .wrap_internal_err("querying database for `project_edit_internal`")?; + + if sync_archival_disclosure + && archival_disclosure + .is_some_and(|disclosure| disclosure.lock_status.allows_removal()) + { + db_models::DBProjectDisclosure::remove( + project.inner.id, + ProjectDisclosureType::Archived, + user.id.into(), + false, + &mut *transaction, + ) + .await + .wrap_internal_err("failed to remove archival disclosure")?; + } + + Ok(()) +} + pub async fn edit_project_categories( categories: &Vec, perms: &ProjectPermissions, @@ -2536,8 +2705,8 @@ pub async fn add_gallery_item_internal( } let gallery_item = vec![db_models::project_item::DBGalleryItem { - image_url: upload_result.url, - raw_image_url: upload_result.raw_url, + image_url: upload_result.url.clone(), + raw_image_url: upload_result.raw_url.clone(), featured: item.featured, name: item.name, description: item.description, @@ -2552,6 +2721,31 @@ pub async fn add_gallery_item_internal( .await .wrap_internal_err("inserting galleries into database")?; + let validation_error = match project_item.inner.status { + ProjectStatus::Processing => { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await + .err() + } + _ => None, + }; + if let Some(error) = validation_error { + delete_old_images( + Some(upload_result.url), + Some(upload_result.raw_url), + FileHostPublicity::Public, + &**file_host, + ) + .await + .wrap_api_err("deleting rejected gallery image upload")?; + return Err(error); + } + transaction .commit() .await @@ -2790,6 +2984,16 @@ pub async fn edit_gallery_item_internal( )?; } + if project_item.inner.status == ProjectStatus::Processing { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await @@ -2923,15 +3127,6 @@ pub async fn delete_gallery_item_internal( } } - delete_old_images( - Some(item.image_url), - Some(item.raw_image_url), - FileHostPublicity::Public, - &**file_host, - ) - .await - .wrap_api_err("deleting old images")?; - let mut transaction = pool .begin() .await @@ -2950,11 +3145,30 @@ pub async fn delete_gallery_item_internal( "querying database for `delete_gallery_item_internal`", )?; + if project_item.inner.status == ProjectStatus::Processing { + validate::ensure_project_is_valid_for_review( + project_item.inner.id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await .wrap_internal_err("committing database transaction")?; + delete_old_images( + Some(item.image_url), + Some(item.raw_image_url), + FileHostPublicity::Public, + &**file_host, + ) + .await + .wrap_api_err("deleting old images")?; + clear_project_cache_and_queue_search( &redis, &search_state, diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs new file mode 100644 index 0000000000..877242e7b2 --- /dev/null +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -0,0 +1,168 @@ +use actix_web::{HttpRequest, get, web}; +use eyre::eyre; +use serde::Serialize; +use xredis::RedisPool; + +use crate::auth::get_user_from_headers; +use crate::database::models::DBProjectId; +use crate::database::models::project_item::ProjectQueryResult; +use crate::database::{ + PgPool, PgTransaction, ReadOnlyPgPool, models as db_models, +}; +use crate::models::ids::ProjectId; +use crate::models::pats::Scopes; +use crate::models::projects::{Project, Version}; +use crate::models::teams::ProjectPermissions; +use crate::queue::session::AuthQueue; +use crate::routes::ApiError; +use crate::util::error::Context as _; +use crate::validate::project::{ + ProjectNag, has_required_nags_with_context, + validate_with_context as validate_project, +}; + +#[derive(Serialize, utoipa::ToSchema)] +pub struct ProjectValidationResponse { + pub nags: Vec, +} + +pub(crate) async fn ensure_project_is_valid_for_review( + project_id: DBProjectId, + pool: &PgPool, + transaction: &mut PgTransaction<'_>, + redis: &RedisPool, +) -> Result { + let mut projects = db_models::DBProject::get_many_uncached( + &[ProjectId::from(project_id)], + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project for review validation")?; + let reloaded_project = + projects.pop().wrap_not_found_err("resource not found")?; + let versions = db_models::DBVersion::get_many_uncached( + &reloaded_project.versions, + &mut *transaction, + redis, + ) + .await + .wrap_internal_err("reloading project versions for review validation")? + .into_iter() + .map(Version::from) + .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + reloaded_project.inner.id, + false, + &mut *transaction, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); + let project = Project::from(reloaded_project.clone()); + + if has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) { + return Err(ApiError::Request(eyre!( + "project must have no required validation nags before or while under review" + ))); + } + + Ok(reloaded_project) +} + +/// Validate that a project is ready to be submitted for review. +#[utoipa::path( + context_path = "/project", + tag = "projects", + responses((status = OK, body = ProjectValidationResponse)) +)] +#[get("/{id}/validate")] +pub async fn validate( + req: HttpRequest, + info: web::Path<(String,)>, + pool: web::Data, + ro_pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let user = get_user_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Scopes::PROJECT_READ, + ) + .await + .wrap_auth_err("authenticating API request")? + .1; + + let project = + db_models::DBProject::get(&info.into_inner().0, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project from database")? + .wrap_not_found_err("resource not found")?; + + let (team_member, organization_team_member) = + db_models::DBTeamMember::get_for_project_permissions( + &project.inner, + user.id.into(), + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project permissions")?; + + if ProjectPermissions::get_permissions_by_role( + &user.role, + &team_member, + &organization_team_member, + ) + .is_none() + { + return Err(ApiError::Auth(eyre!( + "you do not have permission to validate this project" + ))); + } + + let versions = + db_models::DBVersion::get_many(&project.versions, &***ro_pool, &redis) + .await + .wrap_internal_err("fetching project versions from database")? + .into_iter() + .map(Version::from) + .collect::>(); + let available_categories = + db_models::categories::Category::list(&**pool, &redis) + .await + .wrap_internal_err("fetching project categories")?; + let disclosures = db_models::DBProjectDisclosure::get_many_for_project( + project.inner.id, + false, + &***ro_pool, + ) + .await + .wrap_internal_err("fetching project disclosures")? + .into_iter() + .map(|disclosure| disclosure.disclosure) + .collect::>(); + let project = Project::from(project); + + Ok(web::Json(ProjectValidationResponse { + nags: validate_project( + &project, + &versions, + &available_categories, + &disclosures, + ), + })) +} diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 99eb84f19b..9a969bc531 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -20,7 +20,7 @@ use crate::models::ids::VersionId; use crate::models::images::ImageContext; use crate::models::pats::Scopes; use crate::models::projects::{ - Dependency, FileType, VersionStatus, VersionType, + Dependency, FileType, ProjectStatus, VersionStatus, VersionType, }; use crate::models::projects::{Loader, skip_nulls}; use crate::models::teams::ProjectPermissions; @@ -47,7 +47,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) { } // Given a project ID/slug and a version slug -/// Get a project version. +/// Get a project version. #[utoipa::path( context_path = "/project", tag = "versions", @@ -969,7 +969,7 @@ pub struct VersionListFilters { pub include_changelog: bool, } -/// List project versions. +/// List project versions. #[utoipa::path( context_path = "/project", tag = "versions", @@ -1243,6 +1243,15 @@ pub async fn version_delete( .wrap_request_err_with(|| { "the specified version does not exist!".to_string() })?; + let project = database::models::DBProject::get_id( + version.inner.project_id, + &**pool, + &redis, + ) + .await + .wrap_internal_err("fetching project from database")? + .wrap_not_found_err("resource not found")?; + let validate_for_review = project.inner.status == ProjectStatus::Processing; if !user.role.is_admin() { let team_member = @@ -1327,6 +1336,16 @@ pub async fn version_delete( "executing `tech_review_sync::sync_project_tech_review_state`", )?; + if validate_for_review { + super::projects::validate::ensure_project_is_valid_for_review( + version.inner.project_id, + &pool, + &mut transaction, + &redis, + ) + .await?; + } + transaction .commit() .await diff --git a/apps/labrinth/src/test/api_v2/request_data.rs b/apps/labrinth/src/test/api_v2/request_data.rs index e2e7156c41..4d98e9c572 100644 --- a/apps/labrinth/src/test/api_v2/request_data.rs +++ b/apps/labrinth/src/test/api_v2/request_data.rs @@ -91,7 +91,7 @@ pub fn get_public_project_creation_data_json( "slug": slug, "project_type": version_jar.as_ref().map_or("mod".to_string(), |f| f.project_type()), "description": "A dummy project for testing with.", - "body": "This project is approved, and versions are listed.", + "body": "This project is approved and its versions are listed. It contains enough descriptive information to satisfy project review validation requirements and explain what the project provides to its users.", "client_side": "required", "server_side": "optional", "initial_versions": initial_versions, diff --git a/apps/labrinth/src/test/api_v3/request_data.rs b/apps/labrinth/src/test/api_v3/request_data.rs index a3bc0eb4f7..6865f50add 100644 --- a/apps/labrinth/src/test/api_v3/request_data.rs +++ b/apps/labrinth/src/test/api_v3/request_data.rs @@ -100,7 +100,7 @@ pub fn get_public_project_creation_data_json( "name": format!("Test Project {slug}"), "slug": slug, "summary": "A dummy project for testing with.", - "description": "This project is approved, and versions are listed.", + "description": "This project is approved and its versions are listed. It contains enough descriptive information to satisfy project review validation requirements and explain what the project provides to its users.", "initial_versions": initial_versions, "is_draft": is_draft, "categories": [], diff --git a/apps/labrinth/src/test/dummy_data.rs b/apps/labrinth/src/test/dummy_data.rs index 73000cfa77..4f26c5b9aa 100644 --- a/apps/labrinth/src/test/dummy_data.rs +++ b/apps/labrinth/src/test/dummy_data.rs @@ -363,7 +363,7 @@ pub async fn add_project_beta(api: &ApiV3) -> (Project, Version) { let modify_json = serde_json::from_value(json!([ { "op": "add", "path": "/summary", "value": "A dummy project for testing with." }, - { "op": "add", "path": "/description", "value": "This project is not-yet-approved, and versions are draft." }, + { "op": "add", "path": "/description", "value": "This project is not yet approved, but it contains enough descriptive information to satisfy project review validation requirements while remaining available for testing private project behavior." }, { "op": "add", "path": "/initial_versions/0/status", "value": "unlisted" }, { "op": "add", "path": "/status", "value": "private" }, { "op": "add", "path": "/requested_status", "value": "private" }, diff --git a/apps/labrinth/src/test/search.rs b/apps/labrinth/src/test/search.rs index 9159a5a14b..0f9d0a3026 100644 --- a/apps/labrinth/src/test/search.rs +++ b/apps/labrinth/src/test/search.rs @@ -63,6 +63,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "server_only" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -140,6 +141,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server" }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -155,6 +157,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server_prefers_both" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -172,6 +175,7 @@ pub async fn setup_search_projects( { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] }, { "op": "add", "path": "/initial_versions/0/environment", "value": "client_or_server_prefers_both" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/link_urls", "value": { "source": "https://github.com/modrinth/labrinth" } }, { "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] }, ])) diff --git a/apps/labrinth/src/validate/mod.rs b/apps/labrinth/src/validate/mod.rs index 3a7aca84e2..fed2a8d077 100644 --- a/apps/labrinth/src/validate/mod.rs +++ b/apps/labrinth/src/validate/mod.rs @@ -34,6 +34,7 @@ mod liteloader; mod modpack; mod neoforge; pub mod plugin; +pub mod project; mod quilt; mod resourcepack; mod rift; diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs new file mode 100644 index 0000000000..d38e8774d9 --- /dev/null +++ b/apps/labrinth/src/validate/project/description.rs @@ -0,0 +1,171 @@ +use serde_json::json; + +use super::text::{ + ProfanityKind, contains_description_spam, description_ends_with_header, + extract_description_blocks, extract_description_text, + find_banned_description_link, has_adjacent_same_level_headers, + has_image_without_alt_text, has_sufficient_english_blocks, + js_string_length, long_header_count, non_standard_text_ratio, + normalize_project_field_text, profanity_matches, project_requires_english, +}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +use crate::models::projects::Project; + +const MIN_DESCRIPTION_CHARS: usize = 125; +const MAX_PROFANITY_COUNT: usize = 2; +const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let description = project.description.as_str(); + let normalized_description = normalize_project_field_text(description); + let text = extract_description_text(description); + let has_spam = has_description_spam(description); + let normalized_text = extract_description_text(&normalized_description); + let blocks = extract_description_blocks(description); + let profanity = profanity_matches(description); + + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .filter(|matched| matched.kind == ProfanityKind::Profanity) + .nth(MAX_PROFANITY_COUNT) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if non_standard_text_ratio(description) + >= NON_STANDARD_TEXT_FAILURE_THRESHOLD + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonStandardText, + ProjectNagSeverity::Required, + )); + } + if !has_spam && is_non_english_text(project, &text, &blocks) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionNonEnglish, + ProjectNagSeverity::Required, + )); + } + if normalized_description.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::AddDescription, + ProjectNagSeverity::Required, + )); + } else { + let description_length = js_string_length(&normalized_text); + if description_length < MIN_DESCRIPTION_CHARS { + nags.push( + ProjectNag::new( + ProjectNagKind::DescriptionTooShort, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "length": description_length, + "min_chars": MIN_DESCRIPTION_CHARS, + })), + ); + } + } + if has_spam { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionSpam, + ProjectNagSeverity::Required, + )); + } + if let Some(url) = find_banned_description_link(description) { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectDescriptionBannedLink, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "full_url": url })), + ); + } + let long_headers = long_header_count(description); + if long_headers > 0 { + nags.push( + ProjectNag::new( + ProjectNagKind::LongHeaders, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "count": long_headers })), + ); + } + if description_ends_with_header(description) { + nags.push(ProjectNag::new( + ProjectNagKind::DescriptionEndsWithHeader, + ProjectNagSeverity::Required, + )); + } + if has_adjacent_same_level_headers(description) { + nags.push(ProjectNag::new( + ProjectNagKind::AdjacentHeaders, + ProjectNagSeverity::Required, + )); + } + if has_image_without_alt_text(description) { + nags.push(ProjectNag::new( + ProjectNagKind::MissingAltText, + ProjectNagSeverity::Warning, + )); + } + + nags +} + +pub(super) fn is_non_english(project: &Project) -> bool { + let text = extract_description_text(&project.description); + let blocks = extract_description_blocks(&project.description); + is_non_english_text(project, &text, &blocks) +} + +fn is_non_english_text( + project: &Project, + text: &str, + blocks: &[String], +) -> bool { + project_requires_english(project) + && js_string_length(text) >= MIN_DESCRIPTION_CHARS + && !has_sufficient_english_blocks(blocks) +} + +fn has_description_spam(markdown: &str) -> bool { + contains_description_spam(markdown) +} + +#[cfg(test)] +mod tests { + use super::has_description_spam; + + #[test] + fn description_spam_ignores_formatting() { + assert!(!has_description_spam("======== ━━━━━━━━ !!!!!!!!")); + assert!(has_description_spam(&"a".repeat(64))); + } + + #[test] + fn description_spam_allows_isolated_repetition() { + assert!(!has_description_spam( + "aaaaaaaa This project provides configurable tools for exploring worlds, organizing resources, managing settings, sharing creations, and helping players understand every available feature through clear documentation." + )); + } +} diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs new file mode 100644 index 0000000000..3355c72099 --- /dev/null +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -0,0 +1,57 @@ +use super::text::has_paired_html_formatting; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::disclosures::ProjectDisclosure; +use crate::models::v2::projects::LegacyProject; + +pub(super) fn validate( + project: &crate::models::projects::Project, + disclosures: Option<&[ProjectDisclosure]>, +) -> Vec { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + let mut nags = vec![ + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ]; + + if disclosures.is_some_and(|disclosures| { + disclosures.iter().any(disclosure_has_paired_html) + }) { + nags.push(ProjectNag::new( + ProjectNagKind::DisclosuresSpecialFormatting, + ProjectNagSeverity::Required, + )); + } + + nags +} + +fn disclosure_has_paired_html(disclosure: &ProjectDisclosure) -> bool { + match disclosure { + ProjectDisclosure::AiContent { note, .. } + | ProjectDisclosure::Advertisements { note } + | ProjectDisclosure::EpilepsyTriggers { note } + | ProjectDisclosure::SystemInteractions { note, .. } + | ProjectDisclosure::Archived { note } => { + note.as_deref().is_some_and(has_paired_html_formatting) + } + ProjectDisclosure::Telemetry { data_collected, .. } => data_collected + .iter() + .any(|text| has_paired_html_formatting(text)), + ProjectDisclosure::DerivativeWork { sources } => { + sources.iter().any(|source| { + has_paired_html_formatting(&source.label) + || source + .note + .as_deref() + .is_some_and(has_paired_html_formatting) + }) + } + ProjectDisclosure::PaidFeatures { features } => { + features.iter().any(|text| has_paired_html_formatting(text)) + } + } +} diff --git a/apps/labrinth/src/validate/project/gallery.rs b/apps/labrinth/src/validate/project/gallery.rs new file mode 100644 index 0000000000..c40e82c04f --- /dev/null +++ b/apps/labrinth/src/validate/project/gallery.rs @@ -0,0 +1,119 @@ +use serde_json::json; + +use super::text::{ProfanityKind, has_non_standard_text, profanity_matches}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT: usize = 0; + +fn validate_text( + text: Option<&str>, + gallery_index: usize, + field: &'static str, + max_profanity_count: usize, +) -> Vec { + let text = text.unwrap_or_default(); + let mut nags = Vec::new(); + let profanity = profanity_matches(text); + + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + "value": matched.raw_text, + })), + ); + } + if let Some(matched) = profanity + .iter() + .filter(|matched| matched.kind == ProfanityKind::Profanity) + .nth(max_profanity_count) + { + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + "value": matched.raw_text, + })), + ); + } + if has_non_standard_text(text) { + nags.push( + ProjectNag::new( + ProjectNagKind::GalleryTextNonStandard, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "gallery_index": gallery_index, + "field": field, + })), + ); + } + + nags +} + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + let mut nags = Vec::new(); + let gallery_is_empty = project.gallery.is_empty(); + let is_shader = project.project_types.iter().any(|ty| ty == "shader"); + let is_resource_pack = + project.project_types.iter().any(|ty| ty == "resourcepack"); + let has_gallery_exemption = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "audio" || category == "locale"); + + if is_shader && project.gallery.len() < 3 { + nags.push( + ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "project_type": "shader" })), + ); + } else if is_resource_pack && gallery_is_empty && !has_gallery_exemption { + nags.push( + ProjectNag::new( + ProjectNagKind::UploadGalleryImage, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "project_type": "resourcepack" })), + ); + } + + let is_minecraft_server = project.components.minecraft_server.is_some(); + if !is_minecraft_server && !project.gallery.iter().any(|item| item.featured) + { + nags.push(ProjectNag::new( + ProjectNagKind::FeatureGalleryImage, + ProjectNagSeverity::Suggestion, + )); + } + + for (index, item) in project.gallery.iter().enumerate() { + nags.extend(validate_text(item.name.as_deref(), index, "name", 0)); + nags.extend(validate_text( + item.description.as_deref(), + index, + "description", + MAX_GALLERY_DESCRIPTION_PROFANITY_COUNT, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/icon.rs b/apps/labrinth/src/validate/project/icon.rs new file mode 100644 index 0000000000..566015b8fd --- /dev/null +++ b/apps/labrinth/src/validate/project/icon.rs @@ -0,0 +1,14 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + if project.icon_url.as_deref().is_none_or(str::is_empty) { + vec![ProjectNag::new( + ProjectNagKind::AddIcon, + ProjectNagSeverity::Suggestion, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/license.rs b/apps/labrinth/src/validate/project/license.rs new file mode 100644 index 0000000000..d84d154573 --- /dev/null +++ b/apps/labrinth/src/validate/project/license.rs @@ -0,0 +1,120 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::v2::projects::LegacyProject; + +const INAPPROPRIATE_LICENSE_DOMAINS: &[&str] = &[ + "youtube.com", + "youtu.be", + "modrinth.com", + "curseforge.com", + "twitter.com", + "x.com", + "discord.gg", + "discord.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "reddit.com", + "twitch.tv", + "patreon.com", + "ko-fi.com", + "paypal.com", + "buymeacoffee.com", + "google.com", + "example.com", + "t.me", +]; + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) +} + +enum InvalidLicenseUrl { + InappropriateDomain(String), + Malformed, +} + +fn get_invalid_license_url(url: &str) -> Option { + if url.is_empty() { + return None; + } + + let Ok(url) = url::Url::parse(url) else { + return Some(InvalidLicenseUrl::Malformed); + }; + let Some(hostname) = url.host_str() else { + return Some(InvalidLicenseUrl::Malformed); + }; + let hostname = hostname.to_ascii_lowercase(); + let hostname = hostname.trim_end_matches('.'); + + INAPPROPRIATE_LICENSE_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + .then(|| InvalidLicenseUrl::InappropriateDomain(hostname.to_string())) +} + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + let license = &project.license; + let has_unknown_license = matches!( + license.id.as_str(), + "LicenseRef-Unknown" | "NOASSERTION" | "LicenseRef-NOASSERTION" + ); + + if has_unknown_license && !is_minecraft_server { + nags.push( + ProjectNag::new( + ProjectNagKind::SelectLicense, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } + + let has_license_url = + license.url.as_deref().is_some_and(|url| !url.is_empty()); + let missing_custom_license_details = license.id == "LicenseRef-" + || (license.id.starts_with("LicenseRef-") + && !has_license_url + && license.id != "LicenseRef-Unknown" + && license.id != "LicenseRef-All-Rights-Reserved"); + if missing_custom_license_details && !is_minecraft_server { + nags.push(ProjectNag::new( + ProjectNagKind::AddCustomLicenseDetails, + ProjectNagSeverity::Required, + )); + } + + if let Some(invalid_url) = + license.url.as_deref().and_then(get_invalid_license_url) + { + let details = match invalid_url { + InvalidLicenseUrl::InappropriateDomain(domain) => { + serde_json::json!({ + "domain": domain, + "reason": "inappropriate_domain", + }) + } + InvalidLicenseUrl::Malformed => serde_json::json!({ + "reason": "malformed", + }), + }; + nags.push( + ProjectNag::new( + ProjectNagKind::InvalidLicenseUrl, + ProjectNagSeverity::Required, + ) + .with_details(details), + ); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/links.rs b/apps/labrinth/src/validate/project/links.rs new file mode 100644 index 0000000000..16c2819e57 --- /dev/null +++ b/apps/labrinth/src/validate/project/links.rs @@ -0,0 +1,295 @@ +use std::net::IpAddr; + +use url::Url; + +use super::{ProjectNagKind, ProjectNagSeverity}; +use crate::models::{projects::Version, v2::projects::LegacyProject}; + +const SOURCE_DOMAINS: &[&str] = &[ + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "git.sr.ht", + "tangled.org", + "git.gay", +]; +const ISSUES_DOMAINS: &[&str] = &[ + "github.com", + "gitlab.com", + "bitbucket.org", + "codeberg.org", + "docs.google.com", + "tangled.org", + "git.gay", +]; +const DISCORD_DOMAINS: &[&str] = &["discord.gg", "discord.com", "dsc.gg"]; +const URL_SHORTENERS: &[&str] = + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; +const BLOCKED_EXTERNAL_LINK_DOMAINS: &[&str] = &[ + "twitter.com", + "x.com", + "instagram.com", + "facebook.com", + "tiktok.com", + "telegram.org", + "t.me", + "bilibili.com", + "bsky.app", + "twitch.tv", + "youtube.com", + "youtu.be", + "reddit.com", + "redd.it", + "modrinth.com", + "minecraft.net", + "curseforge.com", + "planetminecraft.com", + "9minecraft.net", + "mcmod.cn", + "creativemode.net", + "orcaclient.com", + "autoforged.cn", +]; +const SOURCE_REQUIRING_LICENSES: &[&str] = &[ + "GPL-2.0", + "GPL-2.0+", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-3.0", + "GPL-3.0+", + "GPL-3.0-only", + "GPL-3.0-or-later", + "LGPL-2.1", + "LGPL-2.1+", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-3.0", + "LGPL-3.0+", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "AGPL-3.0", + "AGPL-3.0+", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "MPL-2.0", +]; + +pub(super) fn validate( + project: &crate::models::projects::Project, + versions: &[Version], +) -> Vec { + let mut nags = Vec::new(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + + if !is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinks, + ProjectNagSeverity::Suggestion, + )); + } + + if is_minecraft_server && project.link_urls.is_empty() { + nags.push(super::ProjectNag::new( + ProjectNagKind::AddLinksServer, + ProjectNagSeverity::Suggestion, + )); + } + + let link_count = project.link_urls.len(); + let unique_link_count = project + .link_urls + .values() + .map(|link| link.url.as_str()) + .collect::>() + .len(); + if unique_link_count != link_count { + nags.push(super::ProjectNag::new( + ProjectNagKind::IdenticalLinks, + ProjectNagSeverity::Required, + )); + } + + let uncommon_link_fields = [ + ("source", SOURCE_DOMAINS), + ("issues", ISSUES_DOMAINS), + ("discord", DISCORD_DOMAINS), + ] + .into_iter() + .filter_map(|(key, domains)| { + named_link_is_uncommon(project, key, domains).then_some(key) + }) + .collect::>(); + if !uncommon_link_fields.is_empty() { + nags.push( + super::ProjectNag::new( + ProjectNagKind::VerifyExternalLinks, + ProjectNagSeverity::Warning, + ) + .with_details( + serde_json::json!({ "fields": uncommon_link_fields }), + ), + ); + } + + let misused_discord_link_fields = + ["source", "issues", "wiki", "site", "store"] + .into_iter() + .filter(|key| named_link(project, key).is_some_and(is_discord_link)) + .collect::>(); + if !misused_discord_link_fields.is_empty() { + nags.push( + super::ProjectNag::new( + ProjectNagKind::MisusedDiscordLink, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "fields": misused_discord_link_fields, + })), + ); + } + + if let Some(url) = find_blocked_external_link(project) { + nags.push( + super::ProjectNag::new( + ProjectNagKind::BannedLinkUsage, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "url": url })), + ); + } + + let is_source_project = project + .project_types + .iter() + .any(|project_type| project_type == "mod" || project_type == "plugin"); + let is_datapack = project + .project_types + .iter() + .any(|project_type| project_type == "datapack"); + let has_source_link = + named_link(project, "source").is_some_and(|url| !url.is_empty()); + let every_version_has_additional_files = + versions.iter().all(|version| version.files.len() >= 2); + + if !is_datapack + && is_source_project + && SOURCE_REQUIRING_LICENSES.contains(&project.license.id.as_str()) + && !has_source_link + && !every_version_has_additional_files + { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + super::ProjectNag::new( + ProjectNagKind::GplLicenseSourceRequired, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } + + nags +} + +fn named_link<'a>( + project: &'a crate::models::projects::Project, + key: &str, +) -> Option<&'a str> { + project.link_urls.get(key).map(|link| link.url.as_str()) +} + +fn named_link_is_uncommon( + project: &crate::models::projects::Project, + key: &str, + domains: &[&str], +) -> bool { + named_link(project, key) + .is_some_and(|url| !is_link_from_domains(url, domains)) +} + +fn is_discord_link(url: &str) -> bool { + is_link_from_domains(url, DISCORD_DOMAINS) +} + +fn find_blocked_external_link( + project: &crate::models::projects::Project, +) -> Option<&str> { + const LEGACY_LINK_KEYS: &[&str] = &["source", "issues", "wiki", "discord"]; + + for key in LEGACY_LINK_KEYS { + if let Some(url) = + named_link(project, key).filter(|url| is_blocked_external_link(url)) + { + return Some(url); + } + } + if let Some(url) = project + .license + .url + .as_deref() + .filter(|url| is_blocked_external_link(url)) + { + return Some(url); + } + + let mut remaining_links = project + .link_urls + .iter() + .filter(|(key, _)| !LEGACY_LINK_KEYS.contains(&key.as_str())) + .collect::>(); + remaining_links.sort_unstable_by_key(|(key, _)| key.as_str()); + remaining_links + .into_iter() + .map(|(_, link)| link.url.as_str()) + .find(|url| is_blocked_external_link(url)) +} + +fn is_blocked_external_link(url: &str) -> bool { + let Some(hostname) = get_link_hostname(url) else { + return false; + }; + + is_ip_address(&hostname) + || URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) + || BLOCKED_EXTERNAL_LINK_DOMAINS + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) +} + +fn is_link_from_domains(url: &str, domains: &[&str]) -> bool { + let Some(hostname) = get_link_hostname(url) else { + return false; + }; + + domains + .iter() + .any(|domain| hostname_matches_domain(&hostname, domain)) +} + +fn get_link_hostname(url: &str) -> Option { + let parsed = Url::parse(url).ok()?; + let hostname = parsed + .host_str()? + .trim_end_matches('.') + .to_ascii_lowercase(); + + (!hostname.is_empty()).then_some(hostname) +} + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname == domain + || hostname + .strip_suffix(domain) + .is_some_and(|prefix| prefix.ends_with('.')) +} + +fn is_ip_address(hostname: &str) -> bool { + hostname + .trim_start_matches('[') + .trim_end_matches(']') + .parse::() + .is_ok() +} diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs new file mode 100644 index 0000000000..308870b7a7 --- /dev/null +++ b/apps/labrinth/src/validate/project/mod.rs @@ -0,0 +1,235 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::database::models::categories::Category; +use crate::models::disclosures::ProjectDisclosure; +use crate::models::projects::{Project, Version}; + +mod description; +mod disclosures; +mod gallery; +mod icon; +mod license; +mod links; +mod moderation; +mod name; +mod permissions; +mod server_settings; +mod summary; +mod tags; +mod text; +mod versions; + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + strum::EnumIter, + utoipa::ToSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum ProjectNagKind { + // Project name + ProjectNameSlur, + ProjectNameProfanity, + ProjectNameNonStandardText, + ProjectNameVersion, + MinecraftTitleClause, + + // Project summary + ProjectSummarySlur, + ProjectSummaryProfanity, + ProjectSummaryNonStandardText, + ProjectSummaryNonEnglish, + ProjectSummaryMatchesTitle, + SummaryTooShort, + ProjectSummarySpam, + SummarySpecialFormatting, + ProjectSummaryLinks, + + // Project icon + AddIcon, + + // Gallery + GalleryTextSlur, + GalleryTextProfanity, + GalleryTextNonStandard, + UploadGalleryImage, + FeatureGalleryImage, + + // Project description + ProjectDescriptionSlur, + ProjectDescriptionProfanity, + ProjectDescriptionNonStandardText, + ProjectDescriptionNonEnglish, + AddDescription, + DescriptionTooShort, + ProjectDescriptionSpam, + ProjectDescriptionBannedLink, + LongHeaders, + DescriptionEndsWithHeader, + AdjacentHeaders, + MissingAltText, + + // License + SelectLicense, + AddCustomLicenseDetails, + InvalidLicenseUrl, + + // External links + AddLinks, + AddLinksServer, + IdenticalLinks, + VerifyExternalLinks, + MisusedDiscordLink, + BannedLinkUsage, + GplLicenseSourceRequired, + + // Permissions + ReviewPermissions, + + // Server settings + SelectCountry, + AllLanguages, + AddJavaAddress, + SelectCompatibility, + TooManyLanguages, + SelectLanguage, + + // Tags + SelectTags, + TooManyTags, + TooManyTagsServer, + MultipleResolutionTags, + AllTagsSelected, + + // Versions + UploadVersion, + SelectEnvironment, + + // Disclosures + CheckDisclosures, + DisclosuresSpecialFormatting, + + // Moderation + ModeratorFeedback, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "lowercase")] +pub enum ProjectNagSeverity { + Required, + Warning, + Suggestion, +} + +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema, +)] +pub struct ProjectNag { + pub kind: ProjectNagKind, + pub severity: ProjectNagSeverity, + #[schema(value_type = serde_json::Value)] + pub details: Value, +} + +impl ProjectNag { + pub(super) fn new( + kind: ProjectNagKind, + severity: ProjectNagSeverity, + ) -> Self { + Self { + kind, + severity, + details: json!({}), + } + } + + pub(super) fn with_details(mut self, details: Value) -> Self { + debug_assert!(details.is_object()); + self.details = details; + self + } +} + +pub fn validate(project: &Project, versions: &[Version]) -> Vec { + validate_inner(project, versions, None, None) +} + +pub fn is_project_summary_non_english(project: &Project) -> bool { + summary::is_non_english(project) +} + +pub fn is_project_description_non_english(project: &Project) -> bool { + description::is_non_english(project) +} + +pub fn validate_with_context( + project: &Project, + versions: &[Version], + available_categories: &[Category], + disclosures: &[ProjectDisclosure], +) -> Vec { + validate_inner( + project, + versions, + Some(available_categories), + Some(disclosures), + ) +} + +fn validate_inner( + project: &Project, + versions: &[Version], + available_categories: Option<&[Category]>, + disclosures: Option<&[ProjectDisclosure]>, +) -> Vec { + let mut nags = [ + name::validate, + summary::validate, + icon::validate, + gallery::validate, + description::validate, + license::validate, + ] + .into_iter() + .flat_map(|validate| validate(project)) + .collect::>(); + + nags.extend(links::validate(project, versions)); + nags.extend(permissions::validate(versions)); + nags.extend(versions::validate(project, versions)); + nags.extend( + [server_settings::validate, moderation::validate] + .into_iter() + .flat_map(|validate| validate(project)), + ); + nags.extend(tags::validate(project, available_categories)); + nags.extend(disclosures::validate(project, disclosures)); + nags +} + +pub fn has_required_nags(project: &Project, versions: &[Version]) -> bool { + validate(project, versions) + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) +} + +pub fn has_required_nags_with_context( + project: &Project, + versions: &[Version], + available_categories: &[Category], + disclosures: &[ProjectDisclosure], +) -> bool { + validate_with_context(project, versions, available_categories, disclosures) + .iter() + .any(|nag| nag.severity == ProjectNagSeverity::Required) +} diff --git a/apps/labrinth/src/validate/project/moderation.rs b/apps/labrinth/src/validate/project/moderation.rs new file mode 100644 index 0000000000..8f4774a61a --- /dev/null +++ b/apps/labrinth/src/validate/project/moderation.rs @@ -0,0 +1,19 @@ +use crate::models::projects::ProjectStatus; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &crate::models::projects::Project, +) -> Vec { + if matches!( + project.status, + ProjectStatus::Rejected | ProjectStatus::Withheld + ) { + vec![ProjectNag::new( + ProjectNagKind::ModeratorFeedback, + ProjectNagSeverity::Warning, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/name.rs b/apps/labrinth/src/validate/project/name.rs new file mode 100644 index 0000000000..dae386ecc3 --- /dev/null +++ b/apps/labrinth/src/validate/project/name.rs @@ -0,0 +1,71 @@ +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::json; + +use super::text::{ + ProfanityKind, has_non_standard_text, normalize_project_field_text, + profanity_matches, +}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::projects::Project; + +static VERSION_NUMBER: LazyLock = + LazyLock::new(|| Regex::new(r"\d+(?:\.\d+)+").unwrap()); + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let name = &project.name; + let profanity = profanity_matches(name); + + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectNameSlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Profanity) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectNameProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if has_non_standard_text(name) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameNonStandardText, + ProjectNagSeverity::Required, + )); + } + + let normalized = normalize_project_field_text(name).to_lowercase(); + let is_port_or_fork = + normalized.contains("port") || normalized.contains("fork"); + if VERSION_NUMBER.is_match(&normalized) && !is_port_or_fork { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectNameVersion, + ProjectNagSeverity::Required, + )); + } + if normalized.contains("minecraft") + && normalized.split_whitespace().count() <= 3 + { + nags.push(ProjectNag::new( + ProjectNagKind::MinecraftTitleClause, + ProjectNagSeverity::Required, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/permissions.rs b/apps/labrinth/src/validate/project/permissions.rs new file mode 100644 index 0000000000..8bef6a0836 --- /dev/null +++ b/apps/labrinth/src/validate/project/permissions.rs @@ -0,0 +1,16 @@ +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; +use crate::models::projects::Version; + +pub(super) fn validate(versions: &[Version]) -> Vec { + if versions + .iter() + .any(|version| !version.files_missing_attribution.is_empty()) + { + vec![ProjectNag::new( + ProjectNagKind::ReviewPermissions, + ProjectNagSeverity::Required, + )] + } else { + Vec::new() + } +} diff --git a/apps/labrinth/src/validate/project/server_settings.rs b/apps/labrinth/src/validate/project/server_settings.rs new file mode 100644 index 0000000000..cd9a0841ed --- /dev/null +++ b/apps/labrinth/src/validate/project/server_settings.rs @@ -0,0 +1,76 @@ +use crate::models::{exp::minecraft::ServerContentQuery, projects::Project}; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const MAX_LANGUAGE_COUNT: usize = 10; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let server = project.components.minecraft_server.as_ref(); + + if server.is_some_and(|server| server.region.is_none()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCountry, + ProjectNagSeverity::Required, + )); + } + + if server.is_some() + && project + .components + .minecraft_java_server + .as_ref() + .is_none_or(|server| server.address.is_empty()) + { + nags.push(ProjectNag::new( + ProjectNagKind::AddJavaAddress, + ProjectNagSeverity::Required, + )); + } + + if project + .components + .minecraft_java_server + .as_ref() + .is_some_and(|server| { + matches!( + &server.content, + ServerContentQuery::Vanilla { + recommended_game_version, + .. + } if recommended_game_version + .as_deref() + .is_none_or(str::is_empty) + ) + }) + { + nags.push(ProjectNag::new( + ProjectNagKind::SelectCompatibility, + ProjectNagSeverity::Required, + )); + } + + if let Some(language_count) = server + .map(|server| server.languages.len()) + .filter(|language_count| *language_count > MAX_LANGUAGE_COUNT) + { + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyLanguages, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "language_count": language_count, + })), + ); + } + + if server.is_some_and(|server| server.languages.is_empty()) { + nags.push(ProjectNag::new( + ProjectNagKind::SelectLanguage, + ProjectNagSeverity::Suggestion, + )); + } + + nags +} diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs new file mode 100644 index 0000000000..518bed6dfc --- /dev/null +++ b/apps/labrinth/src/validate/project/summary.rs @@ -0,0 +1,180 @@ +use serde_json::json; + +use super::text::{ + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_likely_english_summary, js_string_length, + normalize_project_field_text, profanity_matches, project_requires_english, +}; +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +use crate::models::projects::Project; + +const MIN_SUMMARY_CHARS: usize = 25; +const MAX_SUMMARY_NAME_SIMILARITY: f64 = 0.8; + +pub(super) fn validate(project: &Project) -> Vec { + let mut nags = Vec::new(); + let summary = project.summary.as_str(); + let normalized_summary = normalize_project_field_text(summary); + let summary_link = find_link_or_ip(summary); + let contains_link = summary_link.is_some(); + let has_spam = contains_spam(&normalized_summary); + let profanity = profanity_matches(summary); + + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Slur) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummarySlur, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if let Some(matched) = profanity + .iter() + .find(|matched| matched.kind == ProfanityKind::Profanity) + { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummaryProfanity, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": matched.raw_text })), + ); + } + if has_non_standard_text(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonStandardText, + ProjectNagSeverity::Required, + )); + } + if requires_language_nag( + project, + &normalized_summary, + contains_link, + has_spam, + ) { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryNonEnglish, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() + && !contains_link + && !project.name.is_empty() + && summary_name_similarity(summary, &project.name) + >= MAX_SUMMARY_NAME_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummaryMatchesTitle, + ProjectNagSeverity::Required, + )); + } + let summary_length = js_string_length(&normalized_summary); + if !summary.is_empty() + && !contains_link + && summary_length < MIN_SUMMARY_CHARS + { + nags.push( + ProjectNag::new( + ProjectNagKind::SummaryTooShort, + ProjectNagSeverity::Required, + ) + .with_details(json!({ + "length": summary_length, + "min_chars": MIN_SUMMARY_CHARS, + })), + ); + } + if has_spam { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectSummarySpam, + ProjectNagSeverity::Required, + )); + } + if !summary.is_empty() && has_summary_formatting(summary) { + nags.push(ProjectNag::new( + ProjectNagKind::SummarySpecialFormatting, + ProjectNagSeverity::Required, + )); + } + if let Some(value) = summary_link { + nags.push( + ProjectNag::new( + ProjectNagKind::ProjectSummaryLinks, + ProjectNagSeverity::Required, + ) + .with_details(json!({ "value": value })), + ); + } + + nags +} + +pub(super) fn is_non_english(project: &Project) -> bool { + let normalized_summary = normalize_project_field_text(&project.summary); + is_non_english_text(project, &normalized_summary) +} + +fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { + project_requires_english(project) + && js_string_length(normalized_summary) >= MIN_SUMMARY_CHARS + && !is_likely_english_summary(normalized_summary) +} + +fn requires_language_nag( + project: &Project, + normalized_summary: &str, + contains_link: bool, + has_spam: bool, +) -> bool { + is_non_english_text(project, normalized_summary) + && !contains_link + && !has_spam +} + +fn summary_name_similarity(summary: &str, name: &str) -> f64 { + let summary = normalized_for_similarity(summary); + let name = normalized_for_similarity(name); + let longest_length = summary.len().max(name.len()); + if longest_length == 0 { + return 0.0; + } + + 1.0 - levenshtein_distance(&summary, &name) as f64 / longest_length as f64 +} + +fn normalized_for_similarity(text: &str) -> Vec { + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn levenshtein_distance(left: &[char], right: &[char]) -> usize { + if left.len() > right.len() { + return levenshtein_distance(right, left); + } + + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] +} diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs new file mode 100644 index 0000000000..53a8f40746 --- /dev/null +++ b/apps/labrinth/src/validate/project/tags.rs @@ -0,0 +1,102 @@ +use crate::{ + database::models::categories::Category, + models::{projects::Project, v2::projects::LegacyProject}, +}; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +const MAX_TAG_COUNT: usize = 8; +const MAX_TAG_COUNT_SERVER: usize = 18; +const RESOLUTION_TAGS: [&str; 8] = + ["8x-", "16x", "32x", "48x", "64x", "128x", "256x", "512x+"]; + +pub(super) fn validate( + project: &Project, + available_categories: Option<&[Category]>, +) -> Vec { + let mut nags = Vec::new(); + let tag_count = + project.categories.len() + project.additional_categories.len(); + let is_minecraft_server = project.components.minecraft_server.is_some(); + let (project_type, actual_project_type) = + LegacyProject::get_project_type(&project.project_types); + + if !project.versions.is_empty() && project.categories.is_empty() { + nags.push(ProjectNag::new( + ProjectNagKind::SelectTags, + ProjectNagSeverity::Suggestion, + )); + } + + if !is_minecraft_server && tag_count > MAX_TAG_COUNT { + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyTags, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "tag_count": tag_count, + "max_tag_count": MAX_TAG_COUNT, + })), + ); + } + + if is_minecraft_server && tag_count > MAX_TAG_COUNT_SERVER { + nags.push( + ProjectNag::new( + ProjectNagKind::TooManyTagsServer, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "tag_count": tag_count, + "max_tag_count": MAX_TAG_COUNT_SERVER, + })), + ); + } + + let mut resolution_tags = project + .categories + .iter() + .chain(&project.additional_categories) + .filter(|tag| RESOLUTION_TAGS.contains(&tag.as_str())) + .map(String::as_str) + .collect::>(); + resolution_tags.sort_by_key(|tag| { + RESOLUTION_TAGS + .iter() + .position(|resolution| resolution == tag) + .unwrap_or(RESOLUTION_TAGS.len()) + }); + if project_type == "resourcepack" && resolution_tags.len() > 1 { + nags.push( + ProjectNag::new( + ProjectNagKind::MultipleResolutionTags, + ProjectNagSeverity::Warning, + ) + .with_details(serde_json::json!({ + "count": resolution_tags.len(), + "tags": resolution_tags.join("|"), + })), + ); + } + + if let Some(available_categories) = available_categories { + let total_available_tags = available_categories + .iter() + .filter(|category| category.project_type == actual_project_type) + .count(); + if tag_count == total_available_tags && project_type != "project" { + nags.push( + ProjectNag::new( + ProjectNagKind::AllTagsSelected, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "total_available_tags": total_available_tags, + })), + ); + } + } + + nags +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs new file mode 100644 index 0000000000..0b04a3654f --- /dev/null +++ b/apps/labrinth/src/validate/project/text.rs @@ -0,0 +1,1204 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::LazyLock; + +use linkify::{LinkFinder, LinkKind}; +use regex::Regex; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; +use url::Url; +use whatlang::{Detector, Lang}; + +use crate::models::exp::minecraft::Language; +use crate::models::projects::Project; + +static WORD: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); +static NON_LATIN_LETTER: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); +static SPAM_TOKEN: LazyLock = LazyLock::new(|| { + Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) + .unwrap() +}); +static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder +}); +static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); +static MARKDOWN_LINK: LazyLock = + LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); +static HTML_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)|]*>").unwrap()); +static HTML_ENTITY: LazyLock = LazyLock::new(|| { + Regex::new(r"&(?:#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]+);").unwrap() +}); +static HTML_OPEN_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)<([a-z][\w:-]*)\b[^>]*>").unwrap()); +static HTML_CLOSE_TAG: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)").unwrap()); +static CODE_BLOCK: LazyLock = + LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); +static DESCRIPTION_BLOCK_BREAK: LazyLock = + LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); +static INLINE_CODE: LazyLock = + LazyLock::new(|| Regex::new(r"`([^`]*)`").unwrap()); +static MARKDOWN_IMAGE: LazyLock = + LazyLock::new(|| Regex::new(r"!\[([^\]]*)\]\([^)]+\)").unwrap()); +static HTML_IMAGE: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)]*>").unwrap()); +static ALT_ATTRIBUTE: LazyLock = LazyLock::new(|| { + Regex::new(r#"(?is)\balt\s*=\s*(?:"([^"]*)"|'([^']*)')"#).unwrap() +}); +static DESCRIPTION_LINK_FINDER: LazyLock = LazyLock::new(|| { + let mut finder = LinkFinder::new(); + finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); + finder +}); +static HEADER: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^#{1,3}[\t ]+(.+?)\s*#*\s*$").unwrap()); +static HEADER_LINE: LazyLock = + LazyLock::new(|| Regex::new(r"^([#]{1,6})[\t ]+.+?\s*#*\s*$").unwrap()); +static SETEXT_HEADER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?m)^([^\r\n]+)\r?\n[\t ]*(?:=+|-+)[\t ]*$").unwrap() +}); +static HTML_HEADER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?is)]*>(.*?)").unwrap() +}); +static ADJACENT_HTML_HEADERS: LazyLock = + LazyLock::new(|| Regex::new(r"(?is)\s* = LazyLock::new(|| { + Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() +}); + +const URL_SHORTENERS: &[&str] = + &["bit.ly", "adf.ly", "tinyurl.com", "short.io", "is.gd"]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ProfanityKind { + Profanity, + Slur, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ProfanityMatch { + pub(super) kind: ProfanityKind, + pub(super) raw_text: String, +} + +const SLUR_TERMS: &[&str] = &[ + "beaner", + "cameljockey", + "chankoro", + "chink", + "chingchong", + "coon", + "cottonpic", + "cottonpik", + "darkie", + "downie", + "dyke", + "fag", + "gook", + "jap", + "jigabo", + "junglebunny", + "kike", + "koon", + "niqa", + "nigga", + "niqqa", + "niggu", + "niqqu", + "niggr", + "nigger", + "niglet", + "nignog", + "paki", + "raghead", + "retard", + "trannie", + "tranny", + "wetback", +]; + +const PROFANITY_TERMS: &[&str] = &[ + "asshole", + "bastard", + "bitch", + "bullshit", + "cum", + "cunt", + "douchebag", + "fck", + "fuck", + "incest", + "motherfucker", + "pussy", + "shit", + "slut", + "twat", + "whore", +]; + +pub(super) fn normalize_project_field_text(text: &str) -> String { + text.trim().nfc().collect() +} + +pub(super) fn js_string_length(text: &str) -> usize { + text.encode_utf16().count() +} + +pub(super) fn profanity_matches(text: &str) -> Vec { + let (prepared, raw_ranges) = prepare_profanity_text(text); + let mut spans = term_spans(&prepared, PROFANITY_TERMS, false); + spans.extend(term_spans(&prepared, SLUR_TERMS, true)); + spans.sort_unstable_by(|left, right| { + left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1)) + }); + + let mut matches = Vec::new(); + let mut previous_end = 0; + for (start, end) in spans { + if start < previous_end { + continue; + } + push_profanity_match(text, &raw_ranges, start, end, &mut matches); + previous_end = end; + } + matches +} + +fn term_spans( + text: &str, + terms: &[&str], + match_repeated_letters: bool, +) -> Vec<(usize, usize)> { + let characters = text.chars().collect::>(); + let mut spans = Vec::new(); + + for start in 0..characters.len() { + if start > 0 && is_profanity_word_character(characters[start - 1]) { + continue; + } + + let end = terms + .iter() + .flat_map(|term| { + [ + if match_repeated_letters { + match_repeated_term(&characters, start, term) + } else { + match_exact_term(&characters, start, term) + }, + match_separated_term(&characters, start, term), + ] + }) + .flatten() + .max(); + if let Some(end) = end { + spans.push((start, end)); + } + } + + spans +} + +fn match_exact_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let mut input_index = start; + for expected in term.chars() { + if characters.get(input_index) != Some(&expected) { + return None; + } + input_index += 1; + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + +fn match_repeated_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let term = term.as_bytes(); + let mut input_index = start; + let mut term_index = 0; + + while term_index < term.len() { + let expected = term[term_index] as char; + let mut required = 1; + while term.get(term_index + required) == Some(&term[term_index]) { + required += 1; + } + + let mut matched = 0; + while characters.get(input_index) == Some(&expected) { + matched += 1; + input_index += 1; + } + if matched < required { + return None; + } + term_index += required; + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + +fn match_separated_term( + characters: &[char], + start: usize, + term: &str, +) -> Option { + let mut input_index = start; + let mut term = term.bytes().peekable(); + + while let Some(expected) = term.next() { + if characters.get(input_index) != Some(&(expected as char)) { + return None; + } + input_index += 1; + + if term.peek().is_some() { + let separator_start = input_index; + while characters.get(input_index).is_some_and(|character| { + !is_profanity_word_character(*character) + }) { + input_index += 1; + } + if input_index == separator_start { + return None; + } + } + } + + is_whole_word_end(characters, input_index).then_some(input_index) +} + +fn is_whole_word_end(characters: &[char], end: usize) -> bool { + characters + .get(end) + .is_none_or(|character| !is_profanity_word_character(*character)) +} + +fn is_profanity_word_character(character: char) -> bool { + character.is_alphanumeric() + || character == '_' + || is_in_ranges( + character as u32, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ) +} + +fn prepare_profanity_text(text: &str) -> (String, Vec<(usize, usize)>) { + let mut prepared = String::new(); + let mut raw_ranges = Vec::new(); + + for (start, grapheme) in text.grapheme_indices(true) { + let end = start + grapheme.len(); + for character in grapheme.nfkc().flat_map(char::to_lowercase) { + if is_invisible_separator(character) { + continue; + } + + prepared.push(normalize_obfuscated_character(character)); + raw_ranges.push((start, end)); + } + } + + (prepared, raw_ranges) +} + +fn push_profanity_match( + text: &str, + raw_ranges: &[(usize, usize)], + start: usize, + end: usize, + matches: &mut Vec, +) { + let Some(raw_text) = raw_text_for_span(text, raw_ranges, (start, end)) + else { + return; + }; + if normalize_project_field_text(raw_text).to_lowercase() == "кооп" { + return; + } + matches.push(ProfanityMatch { + kind: if is_slur(raw_text) { + ProfanityKind::Slur + } else { + ProfanityKind::Profanity + }, + raw_text: raw_text.to_owned(), + }); +} + +fn raw_text_for_span<'a>( + text: &'a str, + raw_ranges: &[(usize, usize)], + (start, end): (usize, usize), +) -> Option<&'a str> { + let &(raw_start, _) = raw_ranges.get(start)?; + let &(_, raw_end) = raw_ranges.get(end.checked_sub(1)?)?; + text.get(raw_start..raw_end) +} + +fn is_slur(raw_text: &str) -> bool { + let normalized = normalized_profanity_term(raw_text); + SLUR_TERMS + .iter() + .any(|term| collapse_duplicate_letters(term) == normalized) +} + +fn normalized_profanity_term(text: &str) -> String { + let (prepared, _) = prepare_profanity_text(text); + collapse_duplicate_letters( + &prepared + .chars() + .filter(char::is_ascii_alphabetic) + .collect::(), + ) +} + +fn collapse_duplicate_letters(text: &str) -> String { + let mut collapsed = String::with_capacity(text.len()); + let mut previous = None; + for character in text.chars() { + if previous != Some(character) { + collapsed.push(character); + previous = Some(character); + } + } + collapsed +} + +fn normalize_obfuscated_character(character: char) -> char { + match character { + '@' | '4' => 'a', + '8' => 'b', + '(' | '[' | '{' | 'с' | 'ϲ' => 'c', + '3' | 'е' | 'ε' => 'e', + '6' | '9' => 'g', + '!' | '1' | '/' | '|' | 'і' | 'ι' | 'ı' => 'i', + 'ј' => 'j', + 'к' | 'κ' => 'k', + 'м' | 'μ' => 'm', + 'п' => 'n', + '0' | 'о' | 'ο' => 'o', + 'р' | 'ρ' => 'p', + '$' | '5' | 'ѕ' => 's', + '+' | '7' | 'т' | 'τ' => 't', + 'υ' | 'ս' => 'u', + 'х' | 'χ' => 'x', + 'у' | 'γ' => 'y', + '2' => 'z', + _ => character, + } +} + +fn is_invisible_separator(character: char) -> bool { + let code = character as u32; + is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x034f, 0x034f), + (0x061c, 0x061c), + (0x115f, 0x1160), + (0x17b4, 0x17b5), + (0x180b, 0x180f), + (0x200b, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0x3164, 0x3164), + (0xfe00, 0xfe0f), + (0xfeff, 0xfeff), + (0xffa0, 0xffa0), + (0xe0100, 0xe01ef), + ], + ) +} + +pub(super) fn has_non_standard_text(text: &str) -> bool { + non_standard_character_count(text) > 0 +} + +pub(super) fn non_standard_text_ratio(text: &str) -> f64 { + let character_count = text.chars().count(); + if character_count == 0 { + return 0.0; + } + + non_standard_character_count(text) as f64 / character_count as f64 +} + +fn non_standard_character_count(text: &str) -> usize { + let mut count = 0; + let mut combining_marks = 0; + let mut has_base = false; + + for character in text.chars() { + let code = character as u32; + let is_combining = is_in_ranges( + code, + &[ + (0x0300, 0x036f), + (0x1ab0, 0x1aff), + (0x1dc0, 0x1dff), + (0x20d0, 0x20ff), + (0xfe20, 0xfe2f), + ], + ); + if is_combining { + combining_marks += 1; + if !has_base || combining_marks > 2 { + count += 1; + } + continue; + } + + combining_marks = 0; + has_base = !character.is_whitespace(); + let allowed_control = matches!(character, '\n' | '\r' | '\t'); + let control = character.is_control() && !allowed_control; + let private_use = is_in_ranges( + code, + &[(0xe000, 0xf8ff), (0xf0000, 0xffffd), (0x100000, 0x10fffd)], + ); + let invisible = is_in_ranges( + code, + &[ + (0x00ad, 0x00ad), + (0x061c, 0x061c), + (0x200e, 0x200f), + (0x202a, 0x202e), + (0x2060, 0x206f), + (0xfeff, 0xfeff), + ], + ); + let fancy = is_in_ranges( + code, + &[ + (0x02b0, 0x02ff), + (0x1d400, 0x1d7ff), + (0x2460, 0x24ff), + (0x2070, 0x209f), + (0x2100, 0x214f), + (0xfb00, 0xfb06), + (0xff10, 0xff19), + (0xff21, 0xff3a), + (0xff41, 0xff5a), + (0x1f100, 0x1f1ad), + ], + ) && !matches!( + code, + 0x02d6 + | 0x02d7 + | 0x02d8 + | 0x02d9 + | 0x02da + | 0x02db + | 0x02dc + | 0x02dd + | 0x207a + | 0x207b + | 0x208a + | 0x208b + | 0x2120 + | 0x2122 + | 0x2139 + ); + + if control || private_use || invisible || fancy { + count += 1; + } + } + + count +} + +fn is_in_ranges(code: u32, ranges: &[(u32, u32)]) -> bool { + ranges + .iter() + .any(|(start, end)| code >= *start && code <= *end) +} + +pub(super) fn contains_spam(text: &str) -> bool { + let normalized = + text.nfc().flat_map(char::to_lowercase).collect::(); + let mut previous = None; + let mut repeated = 0; + for character in normalized.chars() { + if !character.is_whitespace() && previous == Some(character) { + repeated += 1; + } else { + repeated = usize::from(!character.is_whitespace()); + previous = Some(character); + } + if repeated >= 8 { + return true; + } + } + + let words = WORD + .find_iter(&normalized) + .map(|word| word.as_str()) + .collect::>(); + let mut repeated_words = 1; + for pair in words.windows(2) { + repeated_words = if pair[0] == pair[1] { + repeated_words + 1 + } else { + 1 + }; + if repeated_words >= 4 { + return true; + } + } + + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for phrases in words.windows(phrase_words * 3) { + if phrases[..phrase_words] + == phrases[phrase_words..phrase_words * 2] + && phrases[..phrase_words] + == phrases[phrase_words * 2..phrase_words * 3] + { + return true; + } + } + } + + false +} + +pub(super) fn contains_description_spam(markdown: &str) -> bool { + const MIN_CHARACTER_RUN: usize = 16; + const MIN_CHARACTER_EXCESS: usize = 20; + const MIN_CHARACTER_EXCESS_PERCENT: usize = 10; + const EXTREME_CHARACTER_RUN: usize = 64; + const MIN_REPEATED_WORDS: usize = 6; + const MIN_REPEATED_PHRASE_WORDS: usize = 12; + const MIN_DUPLICATE_BLOCK_PERCENT: usize = 20; + + let blocks = extract_description_spam_blocks(markdown); + let mut block_counts = BTreeMap::, usize>::new(); + let mut visible_characters = 0; + let mut repeated_character_excess = 0; + let mut longest_character_run = 0; + let mut total_words = 0; + + for block in blocks { + let words = spam_words(&block); + if words.is_empty() { + continue; + } + total_words += words.len(); + + update_character_repetition( + &words, + &mut visible_characters, + &mut repeated_character_excess, + &mut longest_character_run, + ); + + if has_repeated_words(&words, MIN_REPEATED_WORDS) + || has_repeated_phrase(&words, MIN_REPEATED_PHRASE_WORDS) + || has_repeated_ngram_density(&words) + { + return true; + } + + if is_duplicate_spam_block_candidate(&block) { + *block_counts.entry(words).or_default() += 1; + } + } + + let duplicate_block_words = block_counts + .iter() + .filter(|(words, count)| { + **count >= 3 && words.len() * **count >= MIN_REPEATED_PHRASE_WORDS + }) + .map(|(words, count)| words.len() * (count - 1)) + .sum::(); + if duplicate_block_words > 0 + && duplicate_block_words * 100 + >= total_words * MIN_DUPLICATE_BLOCK_PERCENT + { + return true; + } + + longest_character_run >= EXTREME_CHARACTER_RUN + || (longest_character_run >= MIN_CHARACTER_RUN + && repeated_character_excess >= MIN_CHARACTER_EXCESS + && repeated_character_excess * 100 + >= visible_characters * MIN_CHARACTER_EXCESS_PERCENT) +} + +fn extract_description_spam_blocks(markdown: &str) -> Vec { + let without_code = CODE_BLOCK.replace_all(markdown, "\n\n"); + let with_inline_code = INLINE_CODE.replace_all(&without_code, "$1"); + let without_images = MARKDOWN_IMAGE.replace_all(&with_inline_code, " "); + let with_link_labels = MARKDOWN_LINK.replace_all(&without_images, "$1"); + let without_html_images = HTML_IMAGE.replace_all(&with_link_labels, " "); + let without_html = HTML_TAG.replace_all(&without_html_images, " "); + let without_entities = HTML_ENTITY.replace_all(&without_html, " "); + let without_links = text_without_explicit_links(&without_entities); + + let mut blocks = Vec::new(); + let mut paragraph = String::new(); + for line in without_links.lines() { + let line = line.trim(); + if line.is_empty() { + push_spam_block(&mut blocks, &mut paragraph); + continue; + } + + if is_standalone_spam_block(line) { + push_spam_block(&mut blocks, &mut paragraph); + blocks.push(line.to_owned()); + } else { + if !paragraph.is_empty() { + paragraph.push(' '); + } + paragraph.push_str(line); + } + } + push_spam_block(&mut blocks, &mut paragraph); + + blocks +} + +fn text_without_explicit_links(text: &str) -> String { + let mut without_links = String::with_capacity(text.len()); + let mut previous_end = 0; + + for link in DESCRIPTION_LINK_FINDER.links(text).filter(|link| { + let link = link.as_str(); + link.contains("://") || link.starts_with("www.") + }) { + without_links.push_str(&text[previous_end..link.start()]); + without_links.push(' '); + previous_end = link.end(); + } + without_links.push_str(&text[previous_end..]); + without_links +} + +fn push_spam_block(blocks: &mut Vec, paragraph: &mut String) { + if !paragraph.is_empty() { + blocks.push(std::mem::take(paragraph)); + } +} + +fn is_standalone_spam_block(line: &str) -> bool { + let line = line.trim_start_matches('>'); + let trimmed = line.trim_start(); + let is_heading = trimmed.starts_with('#'); + let is_list_item = ["- ", "* ", "+ ", "•"] + .iter() + .any(|prefix| trimmed.starts_with(prefix)) + || trimmed.split_once(". ").is_some_and(|(prefix, _)| { + prefix.chars().all(|char| char.is_ascii_digit()) + }); + let is_table_row = trimmed.contains('|'); + + is_heading || is_list_item || is_table_row +} + +fn is_duplicate_spam_block_candidate(block: &str) -> bool { + let trimmed = block.trim_start_matches('>').trim_start(); + !trimmed.starts_with('#') && !trimmed.contains('|') +} + +fn spam_words(text: &str) -> Vec { + SPAM_TOKEN + .find_iter(text) + .map(|word| word.as_str().nfc().flat_map(char::to_lowercase).collect()) + .collect() +} + +fn update_character_repetition( + words: &[String], + visible_characters: &mut usize, + repeated_character_excess: &mut usize, + longest_character_run: &mut usize, +) { + for word in words { + *visible_characters += word.chars().count(); + let mut previous = None; + let mut run: usize = 0; + for character in word.chars() { + if previous == Some(character) { + run += 1; + } else { + *repeated_character_excess += run.saturating_sub(3); + *longest_character_run = (*longest_character_run).max(run); + previous = Some(character); + run = 1; + } + } + *repeated_character_excess += run.saturating_sub(3); + *longest_character_run = (*longest_character_run).max(run); + } +} + +fn has_repeated_words(words: &[String], minimum_repetitions: usize) -> bool { + let mut repetitions = 1; + for pair in words.windows(2) { + repetitions = if pair[0] == pair[1] { + repetitions + 1 + } else { + 1 + }; + if repetitions >= minimum_repetitions { + return true; + } + } + + false +} + +fn has_repeated_phrase( + words: &[String], + minimum_repeated_words: usize, +) -> bool { + let max_phrase_words = 8.min(words.len() / 3); + for phrase_words in (2..=max_phrase_words).rev() { + for start in 0..=words.len() - phrase_words * 3 { + let phrase = &words[start..start + phrase_words]; + let mut repetitions = 1; + while start + phrase_words * (repetitions + 1) <= words.len() + && phrase + == &words[start + phrase_words * repetitions + ..start + phrase_words * (repetitions + 1)] + { + repetitions += 1; + } + if repetitions >= 3 + && phrase_words * repetitions >= minimum_repeated_words + { + return true; + } + } + } + + false +} + +fn has_repeated_ngram_density(words: &[String]) -> bool { + const NGRAM_WORDS: usize = 4; + const MIN_WORDS: usize = 30; + const MIN_DUPLICATE_NGRAM_PERCENT: usize = 50; + const MIN_COVERED_WORD_PERCENT: usize = 70; + + if words.len() < MIN_WORDS { + return false; + } + + let mut ngram_counts = BTreeMap::<&[String], usize>::new(); + for ngram in words.windows(NGRAM_WORDS) { + *ngram_counts.entry(ngram).or_default() += 1; + } + + let total_ngrams = words.len() - NGRAM_WORDS + 1; + let duplicate_ngrams = ngram_counts + .values() + .map(|count| count.saturating_sub(1)) + .sum::(); + if duplicate_ngrams * 100 < total_ngrams * MIN_DUPLICATE_NGRAM_PERCENT { + return false; + } + + let mut covered_words = vec![false; words.len()]; + for (start, ngram) in words.windows(NGRAM_WORDS).enumerate() { + if ngram_counts.get(ngram).is_some_and(|count| *count > 1) { + covered_words[start..start + NGRAM_WORDS].fill(true); + } + } + covered_words.iter().filter(|covered| **covered).count() * 100 + >= words.len() * MIN_COVERED_WORD_PERCENT +} + +pub(super) fn find_link_or_ip(text: &str) -> Option { + SUMMARY_LINK_FINDER.links(text).find_map(|link| { + let raw = link.as_str(); + if raw.contains("://") { + return Some(raw.to_owned()); + } + + Url::parse(&format!("https://{raw}")) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + hostname.parse::().is_ok() + || psl::domain(hostname.as_bytes()).is_some_and( + |domain| domain.suffix().typ().is_some(), + ) + }) + }) + .map(|_| raw.to_owned()) + }) +} + +pub(super) fn has_summary_formatting(summary: &str) -> bool { + has_paired_html_formatting(summary) + || MARKDOWN_LINK.is_match(summary) + || summary.lines().any(|line| { + let line = line.trim_start(); + line.starts_with('#') + || line.starts_with('>') + || line.starts_with("- ") + || line.starts_with("* ") + || line.starts_with("+ ") + || line.starts_with("```") + || line.contains("**") + || line.contains("__") + || line.contains("~~") + || INLINE_CODE.is_match(line) + }) +} + +pub(super) fn has_paired_html_formatting(text: &str) -> bool { + let without_code = CODE_BLOCK.replace_all(text, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + HTML_OPEN_TAG.captures_iter(&without_code).any(|opening| { + let Some(tag) = opening.get(1) else { + return false; + }; + let Some(opening_match) = opening.get(0) else { + return false; + }; + HTML_CLOSE_TAG + .captures_iter(&without_code[opening_match.end()..]) + .any(|closing| { + closing.get(1).is_some_and(|closing_tag| { + closing_tag.as_str().eq_ignore_ascii_case(tag.as_str()) + }) + }) + }) +} + +pub(super) fn extract_description_text(markdown: &str) -> String { + let without_code = CODE_BLOCK.replace_all(markdown, " "); + let without_code = INLINE_CODE.replace_all(&without_code, " "); + let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); + let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); + let with_html_image_alt = HTML_IMAGE.replace_all( + &without_links, + |captures: ®ex::Captures<'_>| { + ALT_ATTRIBUTE + .captures(&captures[0]) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) + }, + ); + let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); + without_html + .lines() + .map(|line| line.trim_start_matches(['>', '#'])) + .collect::>() + .join("\n") + .replace(['*', '_', '~', '`', '>', '-', '|'], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +pub(super) fn extract_description_blocks(markdown: &str) -> Vec { + let without_code = CODE_BLOCK.replace_all(markdown, ""); + DESCRIPTION_BLOCK_BREAK + .split(&without_code) + .map(extract_description_text) + .filter(|block| !block.is_empty()) + .collect() +} + +pub(super) fn long_header_count(markdown: &str) -> usize { + let markdown_headers = HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let setext_headers = SETEXT_HEADER + .captures_iter(markdown) + .filter(|captures| !captures[1].trim_start().starts_with('#')) + .filter(|captures| header_is_long(&captures[1])) + .count(); + let html_headers = HTML_HEADER + .captures_iter(markdown) + .filter(|captures| header_is_long(&captures[1])) + .count(); + + markdown_headers + setext_headers + html_headers +} + +fn header_is_long(header: &str) -> bool { + let with_image_alt = MARKDOWN_IMAGE.replace_all(header, "$1"); + let with_link_text = MARKDOWN_LINK.replace_all(&with_image_alt, "$1"); + let without_html = HTML_TAG.replace_all(&with_link_text, " "); + let rendered = without_html + .replace(['*', '_', '~', '`'], "") + .split_whitespace() + .collect::>() + .join(" "); + + rendered.graphemes(true).count() > 80 +} + +pub(super) fn description_ends_with_header(markdown: &str) -> bool { + let trimmed = markdown.trim_end(); + if trimmed.is_empty() { + return false; + } + + let lines = trimmed.lines().collect::>(); + let last_line = lines.last().map_or("", |line| line.trim()); + if HEADER_LINE.is_match(last_line) { + return true; + } + if lines.len() >= 2 + && is_setext_underline(last_line) + && !lines[lines.len() - 2].trim().is_empty() + { + return true; + } + + TRAILING_HTML_HEADER.is_match(trimmed) +} + +pub(super) fn has_adjacent_same_level_headers(markdown: &str) -> bool { + let lines = markdown.lines().collect::>(); + let mut previous_header = None; + let mut index = 0; + while index < lines.len() { + let line = lines[index].trim(); + if line.is_empty() { + index += 1; + continue; + } + + let mut header_level = HEADER_LINE + .captures(line) + .and_then(|captures| captures.get(1)) + .map(|hashes| hashes.as_str().len()); + if header_level.is_none() + && lines + .get(index + 1) + .is_some_and(|underline| is_setext_underline(underline.trim())) + { + header_level = + Some(if lines[index + 1].trim_start().starts_with('=') { + 1 + } else { + 2 + }); + index += 1; + } + + if let Some(level) = header_level { + if level <= 3 && previous_header == Some(level) { + return true; + } + previous_header = Some(level); + } else { + previous_header = None; + } + index += 1; + } + + ADJACENT_HTML_HEADERS + .captures_iter(markdown) + .any(|captures| { + captures.get(1).map(|level| level.as_str()) + == captures.get(2).map(|level| level.as_str()) + }) +} + +fn is_setext_underline(line: &str) -> bool { + let mut characters = line.chars(); + let Some(marker @ ('=' | '-')) = characters.next() else { + return false; + }; + characters.all(|character| character == marker) +} + +pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { + let without_code = CODE_BLOCK.replace_all(markdown, ""); + let without_code = INLINE_CODE.replace_all(&without_code, ""); + MARKDOWN_IMAGE + .captures_iter(&without_code) + .any(|captures| captures[1].trim().is_empty()) + || HTML_IMAGE.find_iter(&without_code).any(|image| { + ALT_ATTRIBUTE + .captures(image.as_str()) + .and_then(|captures| { + captures.get(1).or_else(|| captures.get(2)) + }) + .is_none_or(|alt| alt.as_str().trim().is_empty()) + }) +} + +pub(super) fn find_banned_description_link(markdown: &str) -> Option { + DESCRIPTION_LINK_FINDER.links(markdown).find_map(|link| { + let raw = link.as_str(); + let normalized = if raw.contains("://") { + raw.to_owned() + } else { + format!("http://{raw}") + }; + Url::parse(&normalized) + .ok() + .filter(|url| { + url.host_str().is_some_and(|hostname| { + URL_SHORTENERS + .iter() + .any(|domain| hostname_matches_domain(hostname, domain)) + }) + }) + .map(|_| normalized) + }) +} + +fn hostname_matches_domain(hostname: &str, domain: &str) -> bool { + hostname.eq_ignore_ascii_case(domain) + || hostname + .to_ascii_lowercase() + .ends_with(&format!(".{domain}")) +} + +pub(super) fn project_requires_english(project: &Project) -> bool { + let has_locale_tag = project + .categories + .iter() + .chain(&project.additional_categories) + .any(|category| category == "locale"); + let is_english_server = project + .components + .minecraft_server + .as_ref() + .is_some_and(|server| server.languages.contains(&Language::En)); + + (project.components.minecraft_java_server.is_none() && !has_locale_tag) + || is_english_server +} + +pub(super) fn is_likely_english_summary(text: &str) -> bool { + let detection_text = normalize_language_text(text); + if has_dominant_non_latin_script(&detection_text) { + return false; + } + + if !has_enough_language_content(&detection_text) { + return true; + } + + LANGUAGE_DETECTOR + .detect(&detection_text) + .is_none_or(|info| info.lang() == Lang::Eng || !info.is_reliable()) +} + +pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { + let mut english_chunks = 0; + let mut non_english_chunks = 0; + + for block in blocks { + let detection_text = normalize_language_text(block); + if has_dominant_non_latin_script(&detection_text) { + non_english_chunks += 1; + + let latin_text = NON_LATIN_LETTER.replace_all(&detection_text, " "); + if has_enough_language_content(&latin_text) + && LANGUAGE_DETECTOR + .detect(&latin_text) + .is_some_and(|info| info.lang() == Lang::Eng) + { + english_chunks += 1; + } + continue; + } + + for chunk in language_chunks(&detection_text) { + let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { + continue; + }; + + if info.lang() == Lang::Eng { + english_chunks += 1; + } else if info.is_reliable() { + non_english_chunks += 1; + } + } + } + + let classified_chunks = english_chunks + non_english_chunks; + classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 +} + +fn normalize_language_text(text: &str) -> String { + text.nfkc().collect() +} + +fn has_dominant_non_latin_script(text: &str) -> bool { + const MIN_NON_LATIN_LETTERS: usize = 5; + + let non_latin_letters = NON_LATIN_LETTER.find_iter(text).count(); + let alphabetic_letters = text + .chars() + .filter(|character| character.is_alphabetic()) + .count(); + non_latin_letters >= MIN_NON_LATIN_LETTERS + && non_latin_letters * 2 >= alphabetic_letters +} + +fn language_chunks(block: &str) -> Vec { + const CHUNK_WORDS: usize = 24; + const CHUNK_STRIDE_WORDS: usize = 12; + + let words = WORD + .find_iter(block) + .map(|word| word.as_str()) + .collect::>(); + if words.len() < 8 { + return Vec::new(); + } + if words.len() <= CHUNK_WORDS { + let chunk = words.join(" "); + return has_enough_language_content(&chunk) + .then_some(chunk) + .into_iter() + .collect(); + } + + let mut starts = BTreeSet::new(); + let mut start = 0; + while start + 8 <= words.len() { + starts.insert(start); + start += CHUNK_STRIDE_WORDS; + } + starts.insert(words.len() - CHUNK_WORDS); + + starts + .into_iter() + .map(|start| { + words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") + }) + .filter(|chunk| has_enough_language_content(chunk)) + .collect() +} + +fn has_enough_language_content(text: &str) -> bool { + WORD.find_iter(text).count() >= 8 + && text.trim().graphemes(true).count() >= 35 +} diff --git a/apps/labrinth/src/validate/project/versions.rs b/apps/labrinth/src/validate/project/versions.rs new file mode 100644 index 0000000000..a9feb68642 --- /dev/null +++ b/apps/labrinth/src/validate/project/versions.rs @@ -0,0 +1,45 @@ +use crate::models::projects::{Project, Version}; + +use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; + +pub(super) fn validate( + project: &Project, + _versions: &[Version], +) -> Vec { + let mut nags = Vec::new(); + + if project.versions.is_empty() + && project.components.minecraft_server.is_none() + { + nags.push(ProjectNag::new( + ProjectNagKind::UploadVersion, + ProjectNagSeverity::Required, + )); + } + + let requires_environment = project + .project_types + .iter() + .any(|project_type| matches!(project_type.as_str(), "mod" | "modpack")); + let has_valid_environment = + project + .fields + .get("environment") + .is_some_and(|environments| { + !environments.is_empty() + && environments.iter().all(|environment| { + environment.as_str().is_some_and(|environment| { + !environment.trim().is_empty() + && environment != "unknown" + }) + }) + }); + if requires_environment && !has_valid_environment { + nags.push(ProjectNag::new( + ProjectNagKind::SelectEnvironment, + ProjectNagSeverity::Required, + )); + } + + nags +} diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 239ce885f0..02f3551fa1 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -6,7 +6,9 @@ use common::dummy_data::DUMMY_CATEGORIES; use crate::common::api_common::models::CommonProject; use crate::common::api_common::request_data::ProjectCreationRequestData; -use crate::common::api_common::{ApiProject, ApiTeams, ApiVersion}; +use crate::common::api_common::{ + Api, ApiProject, ApiTeams, ApiVersion, AppendsOptionalPat, +}; use crate::common::dummy_data::{ DummyImage, DummyOrganizationZeta, DummyProjectAlpha, DummyProjectBeta, TestFile, @@ -23,6 +25,7 @@ use labrinth::database::models::project_item::{ PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE, ProjectQueryResult, }; use labrinth::models::ids::ProjectId; +use labrinth::models::projects::ProjectStatus; use labrinth::models::teams::ProjectPermissions; use labrinth::util::actix::{MultipartSegment, MultipartSegmentData}; use serde_json::json; @@ -354,6 +357,23 @@ pub async fn test_patch_project() { assert_status!(&resp, StatusCode::BAD_REQUEST); } + let resp = api + .edit_project( + alpha_project_slug, + json!({ + "summary": "", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&resp, StatusCode::BAD_REQUEST); + let body: serde_json::Value = test::read_body_json(resp).await; + assert_eq!( + body["description"], + "field summary failed validation with error: length" + ); + assert!(body.get("details").is_none()); + // Failure because these are illegal requested statuses for a normal user. for req in ["unknown", "processing", "withheld", "scheduled"] { let resp = api @@ -501,6 +521,366 @@ pub async fn test_patch_project() { .await; } +#[actix_rt::test] +async fn test_submit_invalid_project_for_review() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "draft" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_before.status, ProjectStatus::Draft); + let original_description = project_before.description; + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": "processing", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_after.status, ProjectStatus::Draft); + assert_eq!(project_after.description, original_description); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_edit_invalid_project_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_before.status, ProjectStatus::Processing); + + let response = api + .edit_project( + project_slug, + json!({ "description": "" }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project_after.status, ProjectStatus::Processing); + assert_eq!(project_after.description, project_before.description); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_leaving_review_skips_validation() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let original_project = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + + for (status, expected_status) in [ + ("draft", ProjectStatus::Draft), + ("rejected", ProjectStatus::Rejected), + ] { + let response = api + .edit_project( + project_slug, + json!({ + "description": original_project.description.clone(), + "status": "processing", + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": status, + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project = api + .get_project_deserialized(project_slug, USER_USER_PAT) + .await; + assert_eq!(project.status, expected_status); + assert!(project.description.is_empty()); + } + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_add_invalid_gallery_item_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + false, + None, + Some("%EF%BC%A1".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_edit_invalid_gallery_item_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + true, + None, + Some("valid-gallery-description".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + let gallery_url = + project_before["gallery"][0]["url"].as_str().unwrap(); + + let response = api + .edit_gallery_item( + project_slug, + gallery_url, + vec![("name".to_string(), "Ａ".to_string())] + .into_iter() + .collect(), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_delete_gallery_item_from_invalid_project_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .add_gallery_item( + project_slug, + DummyImage::SmallIcon.get_icon_data(), + true, + None, + Some("valid-gallery-description".to_string()), + None, + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "description": "", + "status": "draft", + }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let project_before = + api.get_project(project_slug, USER_USER_PAT).await; + let project_before: serde_json::Value = + test::read_body_json(project_before).await; + let gallery_url = + project_before["gallery"][0]["url"].as_str().unwrap(); + + let response = api + .remove_gallery_item(project_slug, gallery_url, USER_USER_PAT) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let project_after = + api.get_project(project_slug, USER_USER_PAT).await; + let project_after: serde_json::Value = + test::read_body_json(project_after).await; + assert_eq!(project_after["gallery"], project_before["gallery"]); + }, + ) + .await; +} + +#[actix_rt::test] +async fn test_edit_invalid_disclosures_in_review_rolls_back() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let disclosures_uri = + format!("/v3/project/{project_slug}/disclosures"); + + let response = api + .edit_project( + project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let request = test::TestRequest::get() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let disclosures_before: serde_json::Value = + test::read_body_json(response).await; + + let request = test::TestRequest::patch() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .set_json(json!({ + "set": [{ + "type": "advertisements", + "note": "sponsored", + }], + "remove": [], + })) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let request = test::TestRequest::get() + .uri(&disclosures_uri) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let disclosures_after: serde_json::Value = + test::read_body_json(response).await; + assert_eq!(disclosures_after, disclosures_before); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v3() { // Hits V3-specific patchable fields diff --git a/apps/labrinth/tests/scopes.rs b/apps/labrinth/tests/scopes.rs index 97264946a2..aa609285f5 100644 --- a/apps/labrinth/tests/scopes.rs +++ b/apps/labrinth/tests/scopes.rs @@ -547,7 +547,7 @@ pub async fn project_write_scopes() { &[beta_project_id.as_str()], json!( { - "description": "test_project_version_write_scopes Description" + "description": "This description verifies project write scopes while containing enough information to satisfy project review validation requirements for a project that is currently being processed." }), pat.as_deref(), ) diff --git a/apps/labrinth/tests/v2/project.rs b/apps/labrinth/tests/v2/project.rs index 36bb380323..91ce1f789d 100644 --- a/apps/labrinth/tests/v2/project.rs +++ b/apps/labrinth/tests/v2/project.rs @@ -405,6 +405,45 @@ async fn permissions_upload_version() { .await; } +#[actix_rt::test] +async fn invalid_review_submission_returns_validation_error() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + + let response = api + .edit_project( + project_slug, + json!({ "status": "draft" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = api + .edit_project( + project_slug, + json!({ + "body": "", + "status": "processing", + }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let error: serde_json::Value = test::read_body_json(response).await; + assert_eq!( + error["description"], + "project must have no required validation nags before or while under review" + ); + }, + ) + .await; +} + #[actix_rt::test] pub async fn test_patch_v2() { // Hits V3-specific patchable fields diff --git a/apps/labrinth/tests/v2/search.rs b/apps/labrinth/tests/v2/search.rs index 510a21fde0..f16b0ee20a 100644 --- a/apps/labrinth/tests/v2/search.rs +++ b/apps/labrinth/tests/v2/search.rs @@ -64,6 +64,7 @@ async fn search_projects() { { "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -141,6 +142,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -157,6 +159,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, ])) .unwrap(); project_creation_futures.push(create_async_future( @@ -176,6 +179,7 @@ async fn search_projects() { { "op": "add", "path": "/client_side", "value": "optional" }, { "op": "add", "path": "/server_side", "value": "required" }, { "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" }, + { "op": "add", "path": "/source_url", "value": "https://github.com/modrinth/labrinth" }, { "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] }, { "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] }, ])) diff --git a/apps/labrinth/tests/version.rs b/apps/labrinth/tests/version.rs index 0fd00c9475..19ab8fab79 100644 --- a/apps/labrinth/tests/version.rs +++ b/apps/labrinth/tests/version.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use crate::common::api_common::ApiVersion; +use crate::common::api_common::{ApiProject, ApiVersion}; use crate::common::database::*; use crate::common::dummy_data::{ DummyProjectAlpha, DummyProjectBeta, TestFile, @@ -91,6 +91,35 @@ async fn test_get_version() { .await; } +#[actix_rt::test] +async fn deleting_version_in_review_that_fails_validation_rolls_back() { + with_test_environment( + None, + |test_env: common::environment::TestEnvironment| async move { + let api = &test_env.api; + let project = &test_env.dummy.project_alpha; + + let response = api + .edit_project( + &project.project_slug, + json!({ "status": "processing" }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let response = + api.remove_version(&project.version_id, USER_USER_PAT).await; + assert_status!(&response, StatusCode::BAD_REQUEST); + + let response = + api.get_version(&project.version_id, USER_USER_PAT).await; + assert_status!(&response, StatusCode::OK); + }, + ) + .await; +} + #[actix_rt::test] async fn rejects_duplicate_filename_when_uploading_file_to_version() { with_test_environment( diff --git a/packages/xredis/src/lib.rs b/packages/xredis/src/lib.rs index 19d4afce1e..f1d08ba5ab 100644 --- a/packages/xredis/src/lib.rs +++ b/packages/xredis/src/lib.rs @@ -113,6 +113,39 @@ impl RedisPool { .wrap_err("registering Redis blocking pool metrics") } + pub async fn get_keys_with_cache( + &self, + namespace: &str, + use_cache: bool, + keys: &[K], + closure: F, + ) -> Result> + where + F: FnOnce(Vec) -> Fut, + Fut: Future, E>>, + E: std::error::Error + Send + Sync + 'static, + T: Serialize + DeserializeOwned, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize + + Debug, + { + if use_cache { + self.get_cached_keys(namespace, keys, closure).await + } else { + Ok(closure(keys.to_vec()) + .await + .wrap_err("fetching uncached values")? + .into_iter() + .map(|(_, value)| value) + .collect()) + } + } + pub async fn get_cached_keys( &self, namespace: &str, @@ -163,6 +196,49 @@ impl RedisPool { .await } + pub async fn get_keys_with_slug_cache( + &self, + namespace: &str, + slug_namespace: &str, + case_sensitive: bool, + use_cache: bool, + keys: &[I], + closure: F, + ) -> Result> + where + F: FnOnce(Vec) -> Fut, + Fut: Future, T)>, E>>, + E: std::error::Error + Send + Sync + 'static, + T: Serialize + DeserializeOwned, + I: Display + Hash + Eq + PartialEq + Clone + Debug, + K: Display + + Hash + + Eq + + PartialEq + + Clone + + DeserializeOwned + + Serialize, + S: Display + Clone + DeserializeOwned + Serialize + Debug, + { + if use_cache { + self.get_cached_keys_with_slug( + namespace, + slug_namespace, + case_sensitive, + keys, + closure, + ) + .await + } else { + Ok(closure(keys.to_vec()) + .await + .wrap_err("fetching uncached values by slug")? + .into_iter() + .map(|(_, (_, value))| value) + .collect()) + } + } + pub async fn get_cached_keys_with_slug( &self, namespace: &str,