diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b6ed5d76..502e8fd71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -360,6 +360,12 @@ jobs: - name: Generated npm provenance workflow run: scripts/ci/with-hermetic-workflow-cli.sh cargo nextest run --locked -p lpm-workflows --test publish -E 'test(publish_npm_generated_provenance_attaches_sigstore_bundle_to_registry_put)' --no-fail-fast --status-level slow --final-status-level fail + - name: LPM registry publish limits and preflight workflows + run: scripts/ci/with-hermetic-workflow-cli.sh cargo nextest run --locked -p lpm-workflows --test publish --test release -E 'test(/lpm_dev/)' --no-fail-fast --status-level slow --final-status-level fail + + - name: Whoami workflow + run: scripts/ci/with-hermetic-workflow-cli.sh cargo nextest run --locked -p lpm-workflows --test whoami --no-fail-fast --status-level slow --final-status-level fail + - name: Native Swift publication workflow run: | swift --version diff --git a/crates/lpm-cli/src/commands/download.rs b/crates/lpm-cli/src/commands/download.rs index aeab22b0a..e618f68e4 100644 --- a/crates/lpm-cli/src/commands/download.rs +++ b/crates/lpm-cli/src/commands/download.rs @@ -128,7 +128,7 @@ pub async fn run( } }; let firewall_json = firewall_verdict.settle(json_output).await?; - let size = downloaded.compressed_size as usize; + let size = downloaded.compressed_size; let integrity_verified = if let Some(sri) = integrity_str.as_ref() { if !json_output { @@ -202,7 +202,7 @@ pub async fn run( crate::install_ui::terminal_line!( " {} {}", install_ui::dim(&format!("{:<16}", "size:")), - format_bytes(size) + lpm_common::format_bytes(size) ) ); eprintln!(); @@ -263,13 +263,3 @@ fn short_integrity(integrity: &str) -> String { .collect(); format!("{prefix}…{suffix}") } - -fn format_bytes(bytes: usize) -> String { - if bytes >= 1_048_576 { - format!("{:.1} MB", bytes as f64 / 1_048_576.0) - } else if bytes >= 1024 { - format!("{:.1} KB", bytes as f64 / 1024.0) - } else { - format!("{bytes} B") - } -} diff --git a/crates/lpm-cli/src/commands/graph.rs b/crates/lpm-cli/src/commands/graph.rs index b39c0ed7b..303b3713e 100644 --- a/crates/lpm-cli/src/commands/graph.rs +++ b/crates/lpm-cli/src/commands/graph.rs @@ -250,7 +250,7 @@ pub async fn run( install_ui::done_line(crate::install_ui::terminal_line!( "Generated {} ({})", out_path.display().to_string(), - format_byte_size(html_size), + lpm_common::format_bytes(html_size), )); // Open in browser unless suppressed (headless / CI). @@ -531,21 +531,6 @@ fn prune_unreachable(graph: &mut DepGraph) { graph_render::recompute_stats(graph); } -/// Format a byte count as a short human-readable string. Uses 1024-based -/// units to match `du -h` / file managers; switches unit at the natural -/// boundary so a 900-byte HTML never reads as "0 KB". -fn format_byte_size(bytes: u64) -> String { - const KIB: u64 = 1024; - const MIB: u64 = 1024 * 1024; - if bytes >= MIB { - format!("{:.1} MB", bytes as f64 / MIB as f64) - } else if bytes >= KIB { - format!("{:.1} KB", bytes as f64 / KIB as f64) - } else { - format!("{bytes} B") - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/lpm-cli/src/commands/publish/orchestrator.rs b/crates/lpm-cli/src/commands/publish/orchestrator.rs index 20e1894a8..88dba46cd 100644 --- a/crates/lpm-cli/src/commands/publish/orchestrator.rs +++ b/crates/lpm-cli/src/commands/publish/orchestrator.rs @@ -10,6 +10,7 @@ use super::output::{ use super::prepare::read_publish_manifest; use super::prepare::{ PublishSource, prepare_publish_project_from_manifest, read_publish_manifest_from_source, + validate_lpm_registry_body_size, validate_lpm_registry_tarball_size, validate_publish_tarball_size, }; use super::provenance::{ @@ -24,14 +25,14 @@ use super::types::{ LpmPublicationStatus, NpmTargetArtifact, NpmTargetArtifactInput, PublicationWaitResult, PublishProject, PublishQualityGateInput, PublishResult, PublishTarget, ResolvedProvenance, }; -use super::upload_lpm::publish_to_lpm; +use super::upload_lpm::{LpmPublishBody, OtpFailure, publish_to_lpm, resolve_lpm_otp}; use super::version_data::{build_publish_version_data, integrity_to_sha512_hex}; use super::wait::{DEFAULT_PUBLICATION_WAIT_TIMEOUT, wait_for_lpm_publication_with_oidc}; use crate::commands::skills::author; use crate::commands::{npm_auth, publish_common, publish_npm}; use crate::{auth, install_ui, oidc, provenance, sigstore}; use lpm_common::{LpmError, PackageName}; -use lpm_registry::RegistryClient; +use lpm_registry::{PublishPreflightVerdict, RegistryClient}; use sha2::{Digest as _, Sha256}; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -315,11 +316,31 @@ impl PublishIntent { } else { client }; - let preflight = client - .publish_preflight(&resolved.name, &self.package_version) + let verdict = client + .publish_preflight(&resolved.name, &self.package_version, None) .await?; drop(light_permit); - if !preflight.package_exists { + if let PublishPreflightVerdict::Refused { + status, + code, + error, + .. + } = &verdict + && lpm_dev_has_the_version( + *status, + code.as_deref(), + error, + &self.package_version, + ) + { + return Ok(PublishTargetPreflight { + target_display: resolved.target.display_name().to_string(), + version_exists: true, + }); + } + // Any other refusal is the error the request's failure is, + // as `--json` reports it. + if !verdict.into_result()?.package_exists { false } else { let _metadata_permit = acquire_preflight_permits(permits, permit_limit).await?; @@ -1160,6 +1181,15 @@ async fn prepare_publish_manifest_with_workspace_lock_held( &tarball_data, !allow_secrets, )?; + if targets_lpm { + check_lpm_target_tarball( + &target_names, + &name, + &tarball_data, + &tarball_hashes, + &rewritten_tarballs, + )?; + } let version_data = build_publish_version_data( &pkg_json, &name, @@ -1229,6 +1259,28 @@ async fn prepare_publish_manifest_with_workspace_lock_held( } else { None }; + if targets_lpm { + check_lpm_target_body( + LpmTargetSource { + target_names: &target_names, + name: &name, + version: &version, + version_data: &version_data, + tarball_data: &tarball_data, + tarball_hashes: &tarball_hashes, + rewritten_tarballs: &rewritten_tarballs, + }, + &LpmPublishParts { + readme: readme.as_deref(), + tarball_files: &tarball_files, + quality_result: quality_result.as_ref(), + lpm_config: lpm_config.as_ref(), + detected_ecosystem: &detected_ecosystem, + swift_manifest: swift_manifest.as_ref(), + }, + matches!(provenance_request, ProvenanceRequest::Generate), + )?; + } let local_skills_digest = has_skills.then(|| author::compute_validated_digest(&validated_skills)); @@ -1399,9 +1451,12 @@ async fn execute_prepared_inner( } else { client }; - let lpm_whoami_cache = release_clients - .filter(|_| !using_registry_oidc) - .map(|clients| &clients.lpm_whoami); + let local_whoami = tokio::sync::OnceCell::new(); + let lpm_whoami_cache = Some( + release_clients + .filter(|_| !using_registry_oidc) + .map_or(&local_whoami, |clients| &clients.lpm_whoami), + ); // Skills staleness check is an LPM network read, so skip it in --check. // @@ -1447,19 +1502,34 @@ async fn execute_prepared_inner( return Ok(PublishExecutionReport::successful(json_output)); } + let lpm_source = LpmTargetSource { + target_names: &target_names, + name: &name, + version: &version, + version_data: &version_data, + tarball_data: &tarball_data, + tarball_hashes: &tarball_hashes, + rewritten_tarballs: &rewritten_tarballs, + }; if dry_run { if targets_lpm { - let resolved_lpm_name = target_names.get("lpm").ok_or_else(|| { - LpmError::Registry("no resolved LPM.dev package name for publish preflight".into()) - })?; - client - .publish_preflight(resolved_lpm_name, &version) + let request = lpm_target_request(lpm_source)?; + let refusal = match client + .publish_preflight(request.name, &version, Some(request.tarball.len() as u64)) .await - .map_err(|error| { - LpmError::Registry(format!( - "LPM.dev publish preflight failed for {resolved_lpm_name}@{version}: {error}" - )) - })?; + { + Ok(PublishPreflightVerdict::Accepted(_)) => None, + Ok(PublishPreflightVerdict::Refused { error, .. }) => { + Some(lpm_common::sanitize_terminal_inline(&error).into_owned()) + } + Err(error) => Some(error.to_string()), + }; + if let Some(refusal) = refusal { + return Err(LpmError::Registry(format!( + "LPM.dev publish preflight failed for {}@{version}: {refusal}", + request.name + ))); + } } if !npm_version_preflighted && targets.iter().any(is_npm_compatible_target) { let preflight_client = release_clients @@ -1574,6 +1644,27 @@ async fn execute_prepared_inner( } } + // lpm.dev's second factor is settled before any target publishes too, + // so one `--json` can't ask for publishes none. A version lpm.dev + // already has needs none, and an account that can't be read yet is the + // LPM target's to read again. + let mut lpm_dev_has_the_version = None; + let otp = if targets_lpm && targets.len() > 1 { + lpm_dev_has_the_version = + refuse_before_any_publish_what_lpm_dev_refuses(client, lpm_source, &version).await?; + if lpm_dev_has_the_version.is_some() { + otp + } else { + match resolve_lpm_otp(client, otp.as_deref(), json_output, lpm_whoami_cache).await { + Ok(settled) => settled, + Err(OtpFailure::AccountUnread(_)) => otp, + Err(OtpFailure::Final(error)) => return Err(error), + } + } + } else { + otp + }; + let provenance_context = match provenance_request { ProvenanceRequest::File(_) => None, request => materialize_provenance_request(request).await?, @@ -1596,26 +1687,19 @@ async fn execute_prepared_inner( PublishTarget::Lpm => { // Wrap the entire LPM publish path so any error becomes a PublishResult let lpm_result: Result = async { - let lpm_name = target_names.get("lpm").map_or(name.as_str(), |s| s.as_str()); - - let lpm_tarball = target_tarball( - lpm_name, - &name, - &tarball_data, - &tarball_hashes, - &rewritten_tarballs, - )?; - - // Recompute dist hashes from the final rewritten tarball so metadata - // matches the actual uploaded artifact (not the pre-rewrite original). - let mut lpm_version_data = version_data.clone(); - lpm_version_data["name"] = serde_json::json!(lpm_name); - lpm_version_data["_id"] = - serde_json::json!(format!("{lpm_name}@{version}")); - lpm_version_data["dist"] = serde_json::json!({ - "shasum": lpm_tarball.hashes.shasum, - "integrity": lpm_tarball.hashes.integrity, - }); + if let Some(refusal) = &lpm_dev_has_the_version { + return Err(LpmError::Registry(refusal.clone())); + } + let mut request = lpm_target_request(LpmTargetSource { + target_names: &target_names, + name: &name, + version: &version, + version_data: &version_data, + tarball_data: &tarball_data, + tarball_hashes: &tarball_hashes, + rewritten_tarballs: &rewritten_tarballs, + })?; + let lpm_name = request.name; // Generate per-target provenance from the final rewritten tarball if let Some(ref provenance) = provenance_context { @@ -1625,7 +1709,7 @@ async fn execute_prepared_inner( .into(), )); }; - let sha512_hex = integrity_to_sha512_hex(&lpm_tarball.hashes.integrity); + let sha512_hex = integrity_to_sha512_hex(&request.tarball.hashes.integrity); let slsa = provenance::build_slsa_statement( &context.ci, lpm_name, @@ -1646,11 +1730,11 @@ async fn execute_prepared_inner( install_ui::done("Sigstore provenance generated and recorded in Rekor"); } let bundle_json = serde_json::to_value(&bundle).unwrap_or_default(); - lpm_version_data["_provenance"] = bundle_json.clone(); - lpm_version_data["_npmProvenanceAttestations"] = bundle_json; + request.version_data["_provenance"] = bundle_json.clone(); + request.version_data["_npmProvenanceAttestations"] = bundle_json; } - let lpm_tarball_data = lpm_tarball.read_data()?; + let lpm_tarball_data = request.tarball.read_data()?; let upload_spinner = if json_output { None @@ -1659,19 +1743,17 @@ async fn execute_prepared_inner( }; let response = publish_to_lpm( client, - lpm_name, - &version, - &readme, + request.body(&LpmPublishParts { + readme: readme.as_deref(), + tarball_files: &tarball_files, + quality_result: quality_result.as_ref(), + lpm_config: lpm_config.as_ref(), + detected_ecosystem: &detected_ecosystem, + swift_manifest: swift_manifest.as_ref(), + }), &lpm_tarball_data, - &tarball_files, - lpm_tarball.package_json_size, - &lpm_version_data, - &quality_result, - &lpm_config, otp.as_deref(), json_output, - &detected_ecosystem, - &swift_manifest, lpm_whoami_cache, ) .await?; @@ -2239,6 +2321,13 @@ struct TargetTarball<'a> { } impl TargetTarball<'_> { + fn len(&self) -> usize { + match self.data { + TargetTarballData::Base(data) => data.len(), + TargetTarballData::Rewritten(tarball) => tarball.len(), + } + } + fn read_data(&self) -> Result>, LpmError> { match self.data { TargetTarballData::Base(data) => Ok(std::sync::Arc::clone(data)), @@ -2247,6 +2336,211 @@ impl TargetTarball<'_> { } } +/// lpm.dev's name for the package: its target's, or package.json's. +fn lpm_target_name<'a>(target_names: &'a HashMap, name: &'a str) -> &'a str { + target_names.get("lpm").map_or(name, String::as_str) +} + +/// The version metadata lpm.dev receives, named as its target and carrying +/// the hashes of the tarball it's sent. +fn lpm_target_version_data( + version_data: &serde_json::Value, + lpm_name: &str, + version: &str, + hashes: &publish_common::TarballHashes, +) -> serde_json::Value { + let mut lpm_version_data = version_data.clone(); + lpm_version_data["name"] = serde_json::json!(lpm_name); + lpm_version_data["_id"] = serde_json::json!(format!("{lpm_name}@{version}")); + lpm_version_data["dist"] = serde_json::json!({ + "shasum": hashes.shasum, + "integrity": hashes.integrity, + }); + lpm_version_data +} + +/// Refuses, before any target publishes, a tarball lpm.dev would refuse for +/// its size: the one it's sent, renamed for it where its name differs from +/// package.json's. +fn check_lpm_target_tarball( + target_names: &HashMap, + name: &str, + tarball_data: &std::sync::Arc>, + tarball_hashes: &std::sync::Arc, + rewritten_tarballs: &HashMap, +) -> Result<(), LpmError> { + let lpm_tarball = target_tarball( + lpm_target_name(target_names, name), + name, + tarball_data, + tarball_hashes, + rewritten_tarballs, + )?; + validate_lpm_registry_tarball_size(lpm_tarball.len()) +} + +/// What lpm.dev's request is made from: the package as prepared, and the +/// tarballs prepared for each target. +#[derive(Clone, Copy)] +struct LpmTargetSource<'a> { + target_names: &'a HashMap, + name: &'a str, + version: &'a str, + version_data: &'a serde_json::Value, + tarball_data: &'a std::sync::Arc>, + tarball_hashes: &'a std::sync::Arc, + rewritten_tarballs: &'a HashMap, +} + +/// What lpm.dev's request carries beside its tarball and version metadata. +struct LpmPublishParts<'a> { + readme: Option<&'a str>, + tarball_files: &'a [publish_common::TarballFile], + quality_result: Option<&'a crate::quality::QualityResult>, + lpm_config: Option<&'a serde_json::Value>, + detected_ecosystem: &'a str, + swift_manifest: Option<&'a serde_json::Value>, +} + +/// lpm.dev's request: its name for the package, the tarball it's sent, and +/// version metadata named as its target and carrying that tarball's hashes. +/// The size check and the upload both build it here, so the request checked +/// is the request sent. +struct LpmTargetRequest<'a> { + name: &'a str, + version: &'a str, + tarball: TargetTarball<'a>, + version_data: serde_json::Value, +} + +fn lpm_target_request(source: LpmTargetSource<'_>) -> Result, LpmError> { + let name = lpm_target_name(source.target_names, source.name); + let tarball = target_tarball( + name, + source.name, + source.tarball_data, + source.tarball_hashes, + source.rewritten_tarballs, + )?; + let version_data = + lpm_target_version_data(source.version_data, name, source.version, tarball.hashes); + Ok(LpmTargetRequest { + name, + version: source.version, + tarball, + version_data, + }) +} + +impl LpmTargetRequest<'_> { + fn body<'b>(&'b self, parts: &LpmPublishParts<'b>) -> LpmPublishBody<'b> { + LpmPublishBody { + name: self.name, + version: self.version, + readme: parts.readme, + version_data: &self.version_data, + tarball_files: parts.tarball_files, + package_json_size_override: self.tarball.package_json_size, + quality_result: parts.quality_result, + lpm_config: parts.lpm_config, + detected_ecosystem: parts.detected_ecosystem, + swift_manifest: parts.swift_manifest, + } + } + + /// The bytes `publish_to_lpm` sends, but for a provenance bundle. + fn len(&self, parts: &LpmPublishParts<'_>) -> Result { + self.body(parts).len_with(self.tarball.len()) + } +} + +/// The preflight's code for a version lpm.dev already has. +const VERSION_EXISTS_CODE: &str = "VERSION_EXISTS"; + +/// Whether an lpm.dev preflight refusal says lpm.dev already has `version`: +/// by its code, or, from a Registry that sends none, by the message it +/// gives that refusal. +fn lpm_dev_has_the_version(status: u16, code: Option<&str>, error: &str, version: &str) -> bool { + match code { + Some(code) => code == VERSION_EXISTS_CODE, + None => { + status == 403 + && error + .strip_prefix("Version ") + .and_then(|rest| rest.strip_prefix(version)) + .is_some_and(|rest| rest.starts_with(" already exists.")) + } + } +} + +/// Whether an lpm.dev preflight refusal is one no retry of the same publish +/// gets past: the account's storage or package limit, its billing, a second +/// factor it requires, the tarball's size, or the token's permission. A +/// version lpm.dev already has, as after a partial publish, isn't one, nor a +/// request the Registry can't read, a rate limit or a server error, which +/// the upload judges. +fn refusal_stops_every_target(status: u16, code: Option<&str>, error: &str, version: &str) -> bool { + matches!(status, 402 | 403 | 413) && !lpm_dev_has_the_version(status, code, error, version) +} + +/// Stops a publish to more than one target before any publishes when +/// lpm.dev refuses it for good, so the other targets don't publish a version +/// lpm.dev won't take. Returns lpm.dev's refusal of a version it already +/// has, as after a partial publish, which the other targets still publish. +/// Any other answer leaves each target to its own upload, as does a +/// Registry that can't be asked. +async fn refuse_before_any_publish_what_lpm_dev_refuses( + client: &RegistryClient, + source: LpmTargetSource<'_>, + version: &str, +) -> Result, LpmError> { + let request = lpm_target_request(source)?; + let verdict = client + .publish_preflight(request.name, version, Some(request.tarball.len() as u64)) + .await; + let Ok(PublishPreflightVerdict::Refused { + status, + code, + error, + .. + }) = verdict + else { + return Ok(None); + }; + let error = lpm_common::sanitize_terminal_inline(&error).into_owned(); + if lpm_dev_has_the_version(status, code.as_deref(), &error, version) { + return Ok(Some(error)); + } + if refusal_stops_every_target(status, code.as_deref(), &error, version) { + return Err(LpmError::Registry(format!( + "LPM.dev refuses {}@{version}, so no target was published: {error}", + request.name, + ))); + } + Ok(None) +} + +/// Room a `--provenance` publish keeps for its Sigstore bundle, which goes +/// into lpm.dev's version metadata twice once the upload signs it: twice +/// the 512 KiB the Registry keeps of one, while a bundle is about 10 KB. +const PROVENANCE_BODY_RESERVE_BYTES: u64 = 1024 * 1024; + +/// Refuses, before any target publishes, a publish request lpm.dev would +/// refuse for its size, keeping room for a provenance bundle when one is +/// signed. +fn check_lpm_target_body( + source: LpmTargetSource<'_>, + parts: &LpmPublishParts<'_>, + provenance: bool, +) -> Result<(), LpmError> { + let reserve = if provenance { + PROVENANCE_BODY_RESERVE_BYTES + } else { + 0 + }; + validate_lpm_registry_body_size(lpm_target_request(source)?.len(parts)?, reserve) +} + fn target_tarball<'a>( target_name: &str, package_json_name: &str, @@ -2368,6 +2662,213 @@ fn resolve_npm_compatible_publish_endpoint( })) } +#[cfg(test)] +mod lpm_target_limit_tests { + use super::{ + LpmPublishParts, LpmTargetSource, PROVENANCE_BODY_RESERVE_BYTES, check_lpm_target_body, + check_lpm_target_tarball, lpm_dev_has_the_version, lpm_target_request, publish_common, + refusal_stops_every_target, + }; + use std::collections::HashMap; + use std::sync::Arc; + + const MIB: usize = 1024 * 1024; + + #[test] + fn reads_a_version_lpm_dev_has_from_its_code_or_a_codeless_registry_s_message() { + // What a Registry without the code answers. + let deployed = "Version 1.0.0 already exists. You cannot overwrite versions."; + assert!(lpm_dev_has_the_version( + 403, + Some("VERSION_EXISTS"), + "", + "1.0.0" + )); + assert!(lpm_dev_has_the_version(403, None, deployed, "1.0.0")); + assert!(!lpm_dev_has_the_version(403, None, deployed, "1.0")); + assert!(!lpm_dev_has_the_version(403, None, deployed, "1.0.0-beta")); + assert!(!lpm_dev_has_the_version(402, None, deployed, "1.0.0")); + assert!(!lpm_dev_has_the_version( + 403, + Some("STORAGE_LIMIT_EXCEEDED"), + deployed, + "1.0.0" + )); + assert!(!lpm_dev_has_the_version( + 403, + None, + "Insufficient permissions", + "1.0.0" + )); + + assert!(!refusal_stops_every_target(403, None, deployed, "1.0.0")); + assert!(!refusal_stops_every_target( + 403, + Some("VERSION_EXISTS"), + "", + "1.0.0" + )); + assert!(refusal_stops_every_target( + 403, + None, + "Insufficient permissions", + "1.0.0" + )); + assert!(refusal_stops_every_target( + 402, + Some("ORG_BILLING_LAPSED"), + "", + "1.0.0" + )); + assert!(refusal_stops_every_target(413, None, "", "1.0.0")); + for status in [400, 404, 408, 429, 500, 503] { + assert!( + !refusal_stops_every_target(status, None, "", "1.0.0"), + "{status}" + ); + } + } + + fn hashes() -> Arc { + Arc::new(publish_common::TarballHashes { + shasum: "0".repeat(40), + integrity: "sha512-AAAA".into(), + }) + } + + fn renamed_to(lpm_name: &str) -> HashMap { + HashMap::from([("lpm".to_owned(), lpm_name.to_owned())]) + } + + #[test] + fn measures_the_tarball_lpm_dev_is_sent() { + let limit = publish_common::MAX_LPM_REGISTRY_TARBALL_BYTES as usize; + let past = Arc::new(vec![0_u8; limit + 1]); + let within = Arc::new(vec![0_u8; limit]); + let no_copies = HashMap::new(); + + let error = check_lpm_target_tarball(&HashMap::new(), "pkg", &past, &hashes(), &no_copies) + .unwrap_err(); + assert!( + error + .to_string() + .contains("tarball too large for the LPM registry"), + "{error}" + ); + check_lpm_target_tarball(&HashMap::new(), "pkg", &within, &hashes(), &no_copies).unwrap(); + + // Renamed for lpm.dev, it's sent the copy, whatever the packed one's size. + let copies = HashMap::from([( + "@lpm.dev/acme.pkg".to_owned(), + publish_common::RewrittenTarball::spool_base(&[0_u8; 16], hashes()).unwrap(), + )]); + check_lpm_target_tarball( + &renamed_to("@lpm.dev/acme.pkg"), + "pkg", + &past, + &hashes(), + &copies, + ) + .unwrap(); + } + + fn source<'a>( + tarball_data: &'a Arc>, + tarball_hashes: &'a Arc, + version_data: &'a serde_json::Value, + no_copies: &'a HashMap, + target_names: &'a HashMap, + ) -> LpmTargetSource<'a> { + LpmTargetSource { + target_names, + name: "pkg", + version: "1.0.0", + version_data, + tarball_data, + tarball_hashes, + rewritten_tarballs: no_copies, + } + } + + fn parts(tarball_files: &[publish_common::TarballFile]) -> LpmPublishParts<'_> { + LpmPublishParts { + readme: None, + tarball_files, + quality_result: None, + lpm_config: None, + detected_ecosystem: "js", + swift_manifest: None, + } + } + + #[test] + fn refuses_a_request_its_file_list_takes_past_the_body_limit_though_its_tarball_is_within_its_own() + { + // 133.33 MiB of base64 for a tarball at its 100 MiB limit. + let tarball = Arc::new(vec![0_u8; 100 * MIB]); + let version_data = serde_json::json!({ "name": "pkg", "version": "1.0.0" }); + let few: Vec<_> = (0..10) + .map(|index| publish_common::TarballFile { + path: format!("package/{index}.js"), + size: 1, + }) + .collect(); + let many: Vec<_> = (0..100_000) + .map(|index| publish_common::TarballFile { + path: format!("package/{}/{index}.js", "d".repeat(180)), + size: 1, + }) + .collect(); + let (hashes, no_copies, names) = (hashes(), HashMap::new(), HashMap::new()); + let source = source(&tarball, &hashes, &version_data, &no_copies, &names); + + check_lpm_target_body(source, &parts(&few), false).unwrap(); + let error = check_lpm_target_body(source, &parts(&many), false).unwrap_err(); + + assert!( + error + .to_string() + .contains("publish request too large for the LPM registry"), + "{error}" + ); + } + + #[test] + fn keeps_room_for_a_provenance_bundle_when_one_is_signed() { + let version_data = serde_json::json!({ "name": "pkg", "version": "1.0.0" }); + let (hashes, no_copies, names) = (hashes(), HashMap::new(), HashMap::new()); + let files = []; + let length = |size: usize| { + let tarball = Arc::new(vec![0_u8; size]); + lpm_target_request(source(&tarball, &hashes, &version_data, &no_copies, &names)) + .unwrap() + .len(&parts(&files)) + .unwrap() + }; + // Whole groups of 3 bytes, which base64 writes as 4, so the request + // grows by exactly 4 bytes a group near the size it's sized for. + let room = publish_common::MAX_LPM_REGISTRY_BODY_BYTES - PROVENANCE_BODY_RESERVE_BYTES; + let near = (room / 4 * 3) as usize; + let groups = (room as i64 - length(near) as i64).div_euclid(4); + let fits = (near as i64 + 3 * groups) as usize; + assert!(length(fits) <= room && room - length(fits) < 4); + let check = |size: usize, provenance: bool| { + let tarball = Arc::new(vec![0_u8; size]); + let source = source(&tarball, &hashes, &version_data, &no_copies, &names); + check_lpm_target_body(source, &parts(&files), provenance) + }; + + check(fits, true).unwrap(); + check(fits + 3, false).unwrap(); + let error = check(fits + 3, true).unwrap_err().to_string(); + assert!( + error.contains("publish request too large for the LPM registry") + && error.contains("less 1 MiB kept for its provenance bundle"), + "{error}" + ); + } +} + #[cfg(test)] mod npm_compatible_publish_endpoint_tests { use super::{ diff --git a/crates/lpm-cli/src/commands/publish/prepare.rs b/crates/lpm-cli/src/commands/publish/prepare.rs index b928ffa8f..7a2b963be 100644 --- a/crates/lpm-cli/src/commands/publish/prepare.rs +++ b/crates/lpm-cli/src/commands/publish/prepare.rs @@ -677,10 +677,51 @@ fn publish_project_changed_error(package_json_path: &Path) -> LpmError { } pub(crate) fn validate_publish_tarball_size(tarball_size: usize) -> Result<(), LpmError> { - if tarball_size > MAX_PUBLISH_TARBALL_BYTES { + tarball_within(tarball_size, MAX_PUBLISH_TARBALL_BYTES as u64, "") +} + +/// Refuses a tarball the LPM registry would refuse for its size. +pub(crate) fn validate_lpm_registry_tarball_size(tarball_size: usize) -> Result<(), LpmError> { + tarball_within( + tarball_size, + publish_common::MAX_LPM_REGISTRY_TARBALL_BYTES, + " for the LPM registry", + ) +} + +/// Refuses a publish request of `body_size` bytes that lpm.dev would refuse +/// for its size, keeping `reserve` bytes of the limit for what the upload +/// adds after the check, as a provenance bundle. +pub(crate) fn validate_lpm_registry_body_size( + body_size: u64, + reserve: u64, +) -> Result<(), LpmError> { + let max = publish_common::MAX_LPM_REGISTRY_BODY_BYTES; + if body_size.saturating_add(reserve) > max { + let kept = if reserve > 0 { + format!( + ", less {} kept for its provenance bundle", + lpm_common::format_bytes(reserve) + ) + } else { + String::new() + }; return Err(LpmError::Registry(format!( - "tarball too large: {} (max 500 MiB)", - lpm_common::format_bytes(tarball_size as u64) + "publish request too large for the LPM registry: {} (max {}{kept}), the tarball in base64 with the package's metadata; publish fewer or smaller files", + lpm_common::format_bytes_past(body_size), + lpm_common::format_bytes(max) + ))); + } + Ok(()) +} + +fn tarball_within(tarball_size: usize, max: u64, target: &str) -> Result<(), LpmError> { + let size = tarball_size as u64; + if size > max { + return Err(LpmError::Registry(format!( + "tarball too large{target}: {} (max {})", + lpm_common::format_bytes_past(size), + lpm_common::format_bytes(max) ))); } Ok(()) @@ -1099,12 +1140,9 @@ fn dump_swift_manifest_from_publish_artifact_with_command( .map_err(swift_inspection_error)?; let status = status.map_err(swift_inspection_error)?; if !status.success() { - let diagnostic = String::from_utf8_lossy(&stderr) - .chars() - .take(4096) - .collect::(); - let diagnostic = lpm_common::sanitize_terminal_inline(&diagnostic); - return Err(swift_inspection_error(format!("{status}: {diagnostic}"))); + return Err(swift_inspection_error( + crate::swift_manifest::swift_failure_reason(&status, &stderr), + )); } serde_json::from_slice::(&stdout).map_err(swift_inspection_error) } @@ -1965,6 +2003,33 @@ mod tests { assert!(manifest.is_err()); } + #[cfg(unix)] + #[test] + fn a_failed_swift_manifest_inspection_says_how_swift_ended() { + let fixture = swift_fixture_tarball(); + let directory = tempfile::tempdir().unwrap(); + let executable = directory.path().join("swift-fails"); + write_executable( + &executable, + "#!/bin/sh\necho 'error: invalid manifest' >&2\nexit 3\n", + ); + + let error = dump_swift_manifest_from_publish_artifact_with_command( + &fixture, + executable.as_os_str(), + std::time::Duration::from_secs(10), + 128, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("exit status: 3: error: invalid manifest"), + "{error}" + ); + } + #[cfg(unix)] #[test] fn swift_manifest_inspection_stops_a_stalled_process_at_the_timeout() { diff --git a/crates/lpm-cli/src/commands/publish/tests.rs b/crates/lpm-cli/src/commands/publish/tests.rs index 4b746f13c..784dcd19f 100644 --- a/crates/lpm-cli/src/commands/publish/tests.rs +++ b/crates/lpm-cli/src/commands/publish/tests.rs @@ -772,7 +772,7 @@ fn publish_result_display() { fn dry_run_files_value_uses_slim_value_roles() { assert_eq!( console::strip_ansi_codes(&format_dry_run_files_value(3, 2048)).into_owned(), - "3 files (2.0 KB)" + "3 files (2 KiB)" ); } @@ -819,15 +819,47 @@ fn publish_tarball_size_accepts_the_500_mib_boundary() { } #[test] -fn publish_tarball_size_rejects_one_byte_over_with_mib_wording() { +fn publish_tarball_size_states_one_byte_over_as_past_the_limit() { let error = validate_publish_tarball_size(MAX_PUBLISH_TARBALL_BYTES + 1).unwrap_err(); assert!( - error.to_string().contains("(max 500 MiB)"), + error + .to_string() + .contains("tarball too large: 500.01 MiB (max 500 MiB)"), "unexpected tarball limit error: {error}" ); } +#[test] +fn publish_tarball_size_for_the_lpm_registry_is_its_100_mib_limit() { + let limit = crate::commands::publish_common::MAX_LPM_REGISTRY_TARBALL_BYTES as usize; + assert!(super::prepare::validate_lpm_registry_tarball_size(limit).is_ok()); + + let error = super::prepare::validate_lpm_registry_tarball_size(limit + 1).unwrap_err(); + + assert!( + error + .to_string() + .contains("tarball too large for the LPM registry: 100.01 MiB (max 100 MiB)"), + "unexpected tarball limit error: {error}" + ); +} + +#[test] +fn publish_request_size_for_the_lpm_registry_is_its_150_mib_limit() { + let limit = crate::commands::publish_common::MAX_LPM_REGISTRY_BODY_BYTES; + assert!(super::prepare::validate_lpm_registry_body_size(limit, 0).is_ok()); + + let error = super::prepare::validate_lpm_registry_body_size(limit + 1, 0).unwrap_err(); + + assert!( + error + .to_string() + .contains("publish request too large for the LPM registry: 150.01 MiB (max 150 MiB)"), + "unexpected request limit error: {error}" + ); +} + #[test] fn lpm_package_url_preserves_the_canonical_dotted_name() { assert_eq!( diff --git a/crates/lpm-cli/src/commands/publish/upload_lpm.rs b/crates/lpm-cli/src/commands/publish/upload_lpm.rs index 45af0789d..58639f1bb 100644 --- a/crates/lpm-cli/src/commands/publish/upload_lpm.rs +++ b/crates/lpm-cli/src/commands/publish/upload_lpm.rs @@ -6,23 +6,103 @@ use lpm_registry::RegistryClient; use serde::ser::{SerializeMap as _, SerializeSeq as _}; use std::io::Write as _; +/// Why lpm.dev's second factor couldn't be settled. +pub(super) enum OtpFailure { + /// The account couldn't be read, as while the Registry is unavailable, + /// which a later try may get past. + AccountUnread(LpmError), + /// A malformed `--otp`, a token lpm.dev refuses, or a code `--json` + /// can't ask for, which no retry gets past. + Final(LpmError), +} + +impl From for LpmError { + fn from(failure: OtpFailure) -> Self { + match failure { + OtpFailure::AccountUnread(error) | OtpFailure::Final(error) => error, + } + } +} + +/// Whether whoami's failure is lpm.dev refusing the token, which a retry +/// with it won't get past. +fn refuses_the_token(error: &LpmError) -> bool { + matches!( + error, + LpmError::AuthRequired + | LpmError::SessionExpired + | LpmError::EnvTokenRejected + | LpmError::Forbidden(_) + | LpmError::UnsupportedAuthSource { .. } + ) +} + +/// The second factor an lpm.dev publish sends: `provided_otp`, checked, or, +/// for an account with 2FA on, one asked for, which `--json` can't do. Reads +/// the account through whoami, which also checks the token. +pub(super) async fn resolve_lpm_otp( + client: &RegistryClient, + provided_otp: Option<&str>, + json_output: bool, + whoami_cache: Option<&tokio::sync::OnceCell>, +) -> Result, OtpFailure> { + if provided_otp.is_some_and(|code| !is_valid_otp(code)) { + return Err(OtpFailure::Final(LpmError::Registry( + "--otp must be exactly six ASCII digits".into(), + ))); + } + let whoami = match whoami_cache { + Some(cache) => cache + .get_or_try_init(|| client.whoami()) + .await + .map(|account| account.mfa_enabled), + None => client.whoami().await.map(|account| account.mfa_enabled), + }; + let mfa_enabled = match whoami { + Ok(mfa_enabled) => mfa_enabled, + Err(error) => { + let refused = refuses_the_token(&error); + let error = LpmError::Registry(format!("authentication failed: {error}")); + return Err(if refused { + OtpFailure::Final(error) + } else { + OtpFailure::AccountUnread(error) + }); + } + }; + + if let Some(code) = provided_otp { + return Ok(Some(code.to_owned())); + } + if mfa_enabled != Some(true) { + return Ok(None); + } + if json_output { + return Err(OtpFailure::Final(LpmError::Registry( + "2FA required in JSON mode — pass --otp or use a CI token".into(), + ))); + } + let code: String = cliclack::input("Enter 2FA code") + .validate(|input: &String| { + if is_valid_otp(input) { + Ok(()) + } else { + Err("Must be a 6-digit code") + } + }) + .interact() + .map_err(|e| OtpFailure::Final(LpmError::Registry(e.to_string())))?; + Ok(Some(code)) +} + /// Publish to the LPM registry (existing behavior). #[allow(clippy::too_many_arguments)] pub(super) async fn publish_to_lpm( client: &RegistryClient, - name: &str, - version: &str, - readme: &Option, + body: LpmPublishBody<'_>, tarball_data: &std::sync::Arc>, - tarball_files: &[TarballFile], - package_json_size_override: Option, - version_data: &serde_json::Value, - quality_result: &Option, - lpm_config: &Option, provided_otp: Option<&str>, json_output: bool, - detected_ecosystem: &str, - swift_manifest: &Option, whoami_cache: Option<&tokio::sync::OnceCell>, ) -> Result { // Credentials must not travel over plain HTTP for LPM publish. @@ -42,78 +122,11 @@ pub(super) async fn publish_to_lpm( )); } - // Verify token has publish scope - let mfa_enabled = if let Some(cache) = whoami_cache { - cache - .get_or_try_init(|| client.whoami()) - .await - .map_err(|e| LpmError::Registry(format!("authentication failed: {e}")))? - .mfa_enabled - } else { - client - .whoami() - .await - .map_err(|e| LpmError::Registry(format!("authentication failed: {e}")))? - .mfa_enabled - }; - - // 2FA check — prompt before uploading - let otp_code: Option = if let Some(code) = provided_otp { - if !is_valid_otp(code) { - return Err(LpmError::Registry( - "--otp must be exactly six ASCII digits".into(), - )); - } - Some(code.to_owned()) - } else if mfa_enabled == Some(true) { - if json_output { - return Err(LpmError::Registry( - "2FA required in JSON mode — pass --otp or use a CI token".into(), - )); - } - let code: String = cliclack::input("Enter 2FA code") - .validate(|input: &String| { - if is_valid_otp(input) { - Ok(()) - } else { - Err("Must be a 6-digit code") - } - }) - .interact() - .map_err(|e| LpmError::Registry(e.to_string()))?; - Some(code) - } else { - None - }; + let otp_code = resolve_lpm_otp(client, provided_otp, json_output, whoami_cache).await?; - let tarball_key = format!( - "{}-{}.tgz", - name.replace('/', "-").replace('@', ""), - version - ); - let swift_metadata = swift_manifest.as_ref().map(extract_swift_metadata); - let version_object = version_data.as_object().ok_or_else(|| { - LpmError::Registry("prepared publish version metadata must be a JSON object".into()) - })?; - let pack_metadata = LpmPackMetadata::new(tarball_files, package_json_size_override); - let lpm_version = LpmVersionPayload { - base: version_object, - quality_result: quality_result.as_ref(), - pack_metadata: &pack_metadata, - lpm_config: lpm_config.as_ref(), - detected_ecosystem, - swift_metadata: swift_metadata.as_ref(), - }; - let prefix = build_lpm_publish_payload_prefix(LpmPublishPayload { - name, - version, - readme: readme.as_deref(), - detected_ecosystem, - lpm_version: &lpm_version, - tarball_key: &tarball_key, - })?; - let payload = publish_common::prepare_base64_json_publish_body(prefix, tarball_data)?; - let published_integrity = version_data + let payload = publish_common::prepare_base64_json_publish_body(body.prefix()?, tarball_data)?; + let published_integrity = body + .version_data .get("dist") .and_then(|dist| dist.get("integrity")) .and_then(serde_json::Value::as_str) @@ -121,11 +134,11 @@ pub(super) async fn publish_to_lpm( LpmError::Registry("prepared publish metadata is missing dist.integrity".into()) })?; - let encoded_name = urlencoding::encode(name); + let encoded_name = urlencoding::encode(body.name); client .publish_package( &encoded_name, - version, + body.version, published_integrity, payload.replayable(), otp_code.as_deref(), @@ -134,6 +147,59 @@ pub(super) async fn publish_to_lpm( .await } +/// What an lpm.dev publish request carries, but its tarball. +pub(super) struct LpmPublishBody<'a> { + pub(super) name: &'a str, + pub(super) version: &'a str, + pub(super) readme: Option<&'a str>, + pub(super) version_data: &'a serde_json::Value, + pub(super) tarball_files: &'a [TarballFile], + pub(super) package_json_size_override: Option, + pub(super) quality_result: Option<&'a quality::QualityResult>, + pub(super) lpm_config: Option<&'a serde_json::Value>, + pub(super) detected_ecosystem: &'a str, + pub(super) swift_manifest: Option<&'a serde_json::Value>, +} + +impl LpmPublishBody<'_> { + /// The request's JSON up to the tarball's base64. + fn prefix(&self) -> Result, LpmError> { + let tarball_key = format!( + "{}-{}.tgz", + self.name.replace('/', "-").replace('@', ""), + self.version + ); + let swift_metadata = self.swift_manifest.map(extract_swift_metadata); + let version_object = self.version_data.as_object().ok_or_else(|| { + LpmError::Registry("prepared publish version metadata must be a JSON object".into()) + })?; + let pack_metadata = + LpmPackMetadata::new(self.tarball_files, self.package_json_size_override); + let lpm_version = LpmVersionPayload { + base: version_object, + quality_result: self.quality_result, + pack_metadata: &pack_metadata, + lpm_config: self.lpm_config, + detected_ecosystem: self.detected_ecosystem, + swift_metadata: swift_metadata.as_ref(), + }; + build_lpm_publish_payload_prefix(LpmPublishPayload { + name: self.name, + version: self.version, + readme: self.readme, + detected_ecosystem: self.detected_ecosystem, + lpm_version: &lpm_version, + tarball_key: &tarball_key, + }) + } + + /// The length of the request that carries a tarball of `tarball_len` + /// bytes, as `publish_to_lpm` sends it. + pub(super) fn len_with(&self, tarball_len: usize) -> Result { + publish_common::base64_json_publish_body_len(self.prefix()?.len(), tarball_len) + } +} + struct LpmPackMetadata<'a> { files: &'a [TarballFile], package_json_size_override: Option, diff --git a/crates/lpm-cli/src/commands/publish_common.rs b/crates/lpm-cli/src/commands/publish_common.rs index 46b9f6af8..971b4b1ab 100644 --- a/crates/lpm-cli/src/commands/publish_common.rs +++ b/crates/lpm-cli/src/commands/publish_common.rs @@ -386,6 +386,15 @@ pub(crate) const MAX_UNCOMPRESSED_TARBALL_BYTES: u64 = 500 * 1024 * 1024; pub(crate) const MAX_COMPRESSED_TARBALL_BYTES: u64 = 500 * 1024 * 1024; +/// The LPM registry's limit for a compressed tarball, below the local one, +/// checked before any target publishes so a larger one fails up front. +pub(crate) const MAX_LPM_REGISTRY_TARBALL_BYTES: u64 = 100 * 1024 * 1024; + +/// The LPM registry's limit for a publish request's body, which carries the +/// tarball in base64 with the package's metadata, checked before any target +/// publishes too. +pub(crate) const MAX_LPM_REGISTRY_BODY_BYTES: u64 = 150 * 1024 * 1024; + pub(crate) const MAX_PUBLISH_ARCHIVE_ENTRIES: usize = 100_000; pub(crate) const MAX_PUBLISH_ARCHIVE_PATH_DEPTH: usize = 256; @@ -3169,6 +3178,24 @@ pub(crate) fn prepare_json_publish_body( Ok(PreparedPublishBody { body }) } +/// The length of the body `prepare_base64_json_publish_body` makes from a +/// prefix of `prefix_len` bytes and a tarball of `tarball_len`, without +/// reading the tarball. +pub(crate) fn base64_json_publish_body_len( + prefix_len: usize, + tarball_len: usize, +) -> Result { + let mut suffix = Vec::with_capacity(64); + write_json_publish_payload_suffix(&mut suffix, tarball_len, None)?; + tarball_len + .div_ceil(3) + .checked_mul(4) + .and_then(|encoded| encoded.checked_add(prefix_len)) + .and_then(|length| length.checked_add(suffix.len())) + .and_then(|length| u64::try_from(length).ok()) + .ok_or_else(|| LpmError::Registry("publish request body length overflow".into())) +} + pub(crate) fn prepare_base64_json_publish_body( prefix: Vec, tarball_data: &std::sync::Arc>, @@ -3409,6 +3436,23 @@ fn javascript_string_length(value: &str) -> usize { #[cfg(test)] mod tests { + #[test] + fn base64_json_publish_body_len_is_the_prepared_body_s() { + for tarball_len in [0, 1, 2, 3, 4, 1000] { + let prefix = b"{\"_attachments\":{\"a.tgz\":{\"data\":\"".to_vec(); + let prefix_len = prefix.len(); + let tarball = std::sync::Arc::new(vec![7_u8; tarball_len]); + + let prepared = prepare_base64_json_publish_body(prefix, &tarball).unwrap(); + + assert_eq!( + base64_json_publish_body_len(prefix_len, tarball_len).unwrap(), + prepared.len(), + "{tarball_len}" + ); + } + } + use super::*; fn build_test_npm_payload( diff --git a/crates/lpm-cli/src/commands/remote_cache.rs b/crates/lpm-cli/src/commands/remote_cache.rs index c97f0a687..62ea1b055 100644 --- a/crates/lpm-cli/src/commands/remote_cache.rs +++ b/crates/lpm-cli/src/commands/remote_cache.rs @@ -365,6 +365,14 @@ fn local_task_cache_size(path: &Path) -> Result { } pub fn cache_status_json(status: &CacheStatus) -> serde_json::Value { + // As the human line shows them: usage on its side of the limit. + let (usage, limit) = match (status.remote.usage_bytes, status.remote.limit_bytes) { + (Some(usage), Some(limit)) => { + let (usage, limit) = lpm_common::format_quota(usage, limit); + (Some(usage), Some(limit)) + } + (usage, limit) => (usage.map(lpm_common::format_usage), limit.map(format_bytes)), + }; serde_json::json!({ "success": true, "local": { @@ -378,9 +386,9 @@ pub fn cache_status_json(status: &CacheStatus) -> serde_json::Value { "team": status.remote.team, "status": status.remote.status, "usage_bytes": status.remote.usage_bytes, - "usage": status.remote.usage_bytes.map(format_bytes), + "usage": usage, "limit_bytes": status.remote.limit_bytes, - "limit": status.remote.limit_bytes.map(format_bytes), + "limit": limit, "error": status.remote.error, }, }) @@ -441,8 +449,7 @@ pub fn print_cache_status_human(status: &CacheStatus) { ); } if let (Some(usage), Some(limit)) = (status.remote.usage_bytes, status.remote.limit_bytes) { - let usage_display = format_bytes(usage); - let limit_display = format_bytes(limit); + let (usage_display, limit_display) = lpm_common::format_quota(usage, limit); let usage_bar = install_ui::usage_bar(usage, limit, 10); let usage_line = if usage_bar.is_empty() { crate::install_ui::terminal_line!( @@ -461,6 +468,16 @@ pub fn print_cache_status_human(status: &CacheStatus) { ) }; println!("{usage_line}"); + } else if let (Some(usage), None) = (status.remote.usage_bytes, status.remote.limit_bytes) { + // A plan whose cache is unlimited reports no limit. + println!( + "{}", + crate::install_ui::terminal_line!( + " {} {} / Unlimited", + cache_status_label("usage"), + lpm_common::format_usage(usage) + ) + ); } } println!(); diff --git a/crates/lpm-cli/src/commands/run/parallel.rs b/crates/lpm-cli/src/commands/run/parallel.rs index 5899de03d..241b48498 100644 --- a/crates/lpm-cli/src/commands/run/parallel.rs +++ b/crates/lpm-cli/src/commands/run/parallel.rs @@ -36,7 +36,7 @@ pub(super) fn truncate_output(output: String) -> String { let truncated = &output[..limit]; let end = truncated.rfind('\n').unwrap_or(limit); format!( - "{}...\n\n[output truncated at {}MB]", + "{}...\n\n[output truncated at {} MiB]", &output[..end], MAX_CAPTURED_OUTPUT / (1024 * 1024), ) diff --git a/crates/lpm-cli/src/commands/run/tests.rs b/crates/lpm-cli/src/commands/run/tests.rs index 413885138..eeaa6938f 100644 --- a/crates/lpm-cli/src/commands/run/tests.rs +++ b/crates/lpm-cli/src/commands/run/tests.rs @@ -155,7 +155,7 @@ fn truncate_output_large_truncated() { let large = "x".repeat(11 * 1024 * 1024); let result = truncate_output(large); assert!(result.len() <= MAX_CAPTURED_OUTPUT + 100); // +100 for the message - assert!(result.contains("[output truncated at 10MB]")); + assert!(result.contains("[output truncated at 10 MiB]")); } #[test] @@ -167,7 +167,7 @@ fn truncate_output_cuts_at_newline() { s.push_str(&line); } let result = truncate_output(s); - assert!(result.contains("[output truncated at 10MB]")); + assert!(result.contains("[output truncated at 10 MiB]")); // The truncated content (before "...") should end at a newline boundary let before_ellipsis = result.split("...\n").next().unwrap(); assert!(before_ellipsis.ends_with('\n') || before_ellipsis.ends_with('a')); diff --git a/crates/lpm-cli/src/commands/self_update.rs b/crates/lpm-cli/src/commands/self_update.rs index 0b6444a17..3c2cfe3c9 100644 --- a/crates/lpm-cli/src/commands/self_update.rs +++ b/crates/lpm-cli/src/commands/self_update.rs @@ -2992,7 +2992,7 @@ async fn verify_and_stage_for_standalone( install_ui::done_untrusted(&format!( "Verified Sigstore attestation for {} ({}, integratedTime {})", binary_name, - format_bytes(staged.byte_len), + lpm_common::format_bytes(staged.byte_len as u64), audit.integrated_time )); } @@ -3299,16 +3299,6 @@ fn humanize_seconds(secs: u64) -> String { } } -fn format_bytes(bytes: usize) -> String { - if bytes >= 1_048_576 { - format!("{:.1} MB", bytes as f64 / 1_048_576.0) - } else if bytes >= 1024 { - format!("{:.1} KB", bytes as f64 / 1024.0) - } else { - format!("{bytes} B") - } -} - #[cfg(test)] mod tests { use super::*; @@ -5482,15 +5472,6 @@ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb lpm-linux-x64 ); } - #[test] - fn format_bytes_units() { - assert_eq!(format_bytes(500), "500 B"); - assert_eq!(format_bytes(1024), "1.0 KB"); - assert_eq!(format_bytes(1536), "1.5 KB"); - assert_eq!(format_bytes(1_048_576), "1.0 MB"); - assert_eq!(format_bytes(2_621_440), "2.5 MB"); - } - /// Rate-limit lands on `SelfUpdateRateLimited`, not `Forbidden`. /// `Forbidden` is the user-permission category and reads to users /// as "you're banned" — wrong category for a quota-reset wait. diff --git a/crates/lpm-cli/src/commands/skills/author.rs b/crates/lpm-cli/src/commands/skills/author.rs index 4bd185836..d34d50999 100644 --- a/crates/lpm-cli/src/commands/skills/author.rs +++ b/crates/lpm-cli/src/commands/skills/author.rs @@ -2,7 +2,7 @@ use std::collections::BTreeSet; use std::io::Read as _; use std::path::Path; -use lpm_common::LpmError; +use lpm_common::{LpmError, format_bytes}; use lpm_security::skill_security::{self, SkillSecurityIssue}; use sha2::{Digest, Sha256}; @@ -154,8 +154,9 @@ fn validate_open_directory(skills_dir: &cap_std::fs::Dir) -> Result MAX_TOTAL_SIZE { report.errors.push(format!( - "total package skills size {} bytes exceeds 100KB limit", - report.total_size_bytes + "total package skills size {} bytes exceeds {} limit", + report.total_size_bytes, + format_bytes(MAX_TOTAL_SIZE) )); } @@ -201,9 +202,10 @@ fn validate_opened_file( let security_count = report.security_issues.len(); if size > MAX_SKILL_SIZE { - report - .errors - .push(format!("{display}: exceeds 15KB limit ({size} bytes)")); + report.errors.push(format!( + "{display}: exceeds {} limit ({size} bytes)", + format_bytes(MAX_SKILL_SIZE) + )); return Ok(()); } @@ -217,7 +219,8 @@ fn validate_opened_file( .saturating_sub(size) .saturating_add(bytes.len() as u64); report.errors.push(format!( - "{display}: exceeds 15KB limit ({} bytes)", + "{display}: exceeds {} limit ({} bytes)", + format_bytes(MAX_SKILL_SIZE), bytes.len() )); return Ok(()); @@ -597,7 +600,7 @@ mod tests { report .errors .iter() - .any(|error| error.contains("exceeds 100KB limit")) + .any(|error| error.contains("exceeds 100 KiB limit")) ); assert!(retained_bytes <= MAX_TOTAL_SIZE, "{retained_bytes}"); } diff --git a/crates/lpm-cli/src/commands/tool_execution.rs b/crates/lpm-cli/src/commands/tool_execution.rs index 1b5698f06..ff1d88263 100644 --- a/crates/lpm-cli/src/commands/tool_execution.rs +++ b/crates/lpm-cli/src/commands/tool_execution.rs @@ -8,7 +8,7 @@ use std::process::{Command, Stdio}; /// Maximum size for captured workspace stdout/stderr before truncation. /// Mirrors the `MAX_CAPTURED_OUTPUT` constant in `commands::run` so chatty /// failing members don't unbound the JSON envelope. -const MAX_CAPTURED_OUTPUT: usize = 10 * 1024 * 1024; // 10 MB +const MAX_CAPTURED_OUTPUT: usize = 10 * 1024 * 1024; // 10 MiB /// Truncate captured output if it exceeds `MAX_CAPTURED_OUTPUT`, cutting at /// the last newline boundary to avoid splitting a line. @@ -20,7 +20,7 @@ fn truncate_output(text: &str) -> String { } let end = text[..end].rfind('\n').unwrap_or(end); format!( - "{}...\n\n[output truncated at {}MB]", + "{}...\n\n[output truncated at {} MiB]", &text[..end], MAX_CAPTURED_OUTPUT / (1024 * 1024), ) @@ -244,7 +244,7 @@ pub(super) fn selected_schedule_state( } /// Emit the workspace JSON envelope. Stdout/stderr surface ONLY for failed -/// members and are truncated at the 10MB ceiling. +/// members and are truncated at the 10 MiB ceiling. pub(super) fn emit_envelope( results: &[MemberResult], total: usize, @@ -343,7 +343,7 @@ mod tests { let huge = "x".repeat(MAX_CAPTURED_OUTPUT + 2_000); let result = truncate_output(&huge); assert!( - result.ends_with("[output truncated at 10MB]"), + result.ends_with("[output truncated at 10 MiB]"), "expected truncation marker, got tail: {:?}", &result[result.len().saturating_sub(60)..] ); diff --git a/crates/lpm-cli/src/commands/tunnel.rs b/crates/lpm-cli/src/commands/tunnel.rs index 58a7389b7..c60001707 100644 --- a/crates/lpm-cli/src/commands/tunnel.rs +++ b/crates/lpm-cli/src/commands/tunnel.rs @@ -4421,7 +4421,7 @@ mod tests { assert_eq!( tunnel_limit_summary(Some(&limits)).as_deref(), - Some("1 tunnel · 4000/min · 10.0 MB body · 0 custom domains · 600/min/IP") + Some("1 tunnel · 4000/min · 10 MiB body · 0 custom domains · 600/min/IP") ); } @@ -4438,7 +4438,7 @@ mod tests { assert_eq!( tunnel_limit_summary(Some(&limits)).as_deref(), - Some("3 tunnels · 20000/min · 100.0 MB body · 3 custom domains · 600/min/IP") + Some("3 tunnels · 20000/min · 100 MiB body · 3 custom domains · 600/min/IP") ); } diff --git a/crates/lpm-cli/src/commands/whoami.rs b/crates/lpm-cli/src/commands/whoami.rs index d9664e06e..a35956c5e 100644 --- a/crates/lpm-cli/src/commands/whoami.rs +++ b/crates/lpm-cli/src/commands/whoami.rs @@ -41,12 +41,15 @@ pub async fn run(client: &RegistryClient, json_output: bool) -> Result<(), LpmEr "has_pool_access": user.has_pool_access, "usage": user.usage.as_ref().map(|u| serde_json::json!({ "storage_bytes": u.storage_bytes, + "counted_storage_bytes": u.counted_storage_bytes, "private_packages": u.private_packages, })), "limits": user.limits.as_ref().map(|l| serde_json::json!({ "storage_bytes": l.storage_bytes, "private_packages": l.private_packages, })), + "storage_overage": user.storage_overage, + "storage_excess_billed": user.storage_excess_billed, "orgs": user.organizations.iter().map(|o| serde_json::json!({ "slug": o.slug, "name": o.name, @@ -100,27 +103,82 @@ pub async fn run(client: &RegistryClient, json_output: bool) -> Result<(), LpmEr if let Some(usage) = &user.usage { whoami_ui::blank_line(); whoami_ui::section("usage"); - let storage_mb = usage.storage_bytes as f64 / (1024.0 * 1024.0); + let storage = lpm_common::format_bytes(usage.storage_bytes); + // What publishing does past the limit, as the Registry's publish + // gate decides it. A Registry that doesn't say still refuses the + // free plan's storage past its limit, but without saying how much + // of it counts: packages whose sales exempt them don't, so only + // that the refusal comes at the limit is claimed. Nothing is claimed + // of other plans. + let free_plan_unstated = user.storage_overage.is_none() + && user + .plan_tier + .as_deref() + .is_some_and(|tier| tier.eq_ignore_ascii_case("free")); + let storage_overage = user.storage_overage.as_deref(); + let on_pro = user + .plan_tier + .as_deref() + .is_some_and(|tier| tier.eq_ignore_ascii_case("pro")); + // The storage limit's figure where storage is at or past it, so any + // publish goes past it, and whether it's past. + let mut full_storage = None; + let mut over_storage = false; if let Some(limits) = &user.limits { // Storage if let Some(limit_bytes) = limits.storage_bytes { - let limit_mb = limit_bytes as f64 / (1024.0 * 1024.0); - let storage_msg = with_usage_bar( - format!("{:.2}MB / {:.0}MB", storage_mb, limit_mb), - usage.storage_bytes, - limit_bytes, - ); - if usage.storage_bytes > limit_bytes { - whoami_ui::warn_line(crate::install_ui::terminal_line!( - "Storage: {} (OVER LIMIT)", - storage_msg - )); - } else { + // The free plan's limit leaves out the storage of packages + // whose sales exempt them, as publishing does; a Registry + // that doesn't say how much counts is shown all of it. + let counted = usage + .counted_storage_bytes + .unwrap_or(usage.storage_bytes) + .min(usage.storage_bytes); + let (used, limit) = lpm_common::format_quota(counted, limit_bytes); + let storage_msg = with_usage_bar(format!("{used} / {limit}"), counted, limit_bytes); + if counted >= limit_bytes { + full_storage = Some(limit.clone()); + } + if counted <= limit_bytes { whoami_ui::detail("storage", storage_msg); + } else { + over_storage = true; + match storage_overage { + Some("billed") => { + whoami_ui::detail("storage", storage_msg); + } + Some("paused" | "blocked") => { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "Storage: {} (OVER LIMIT)", + storage_msg + )); + } + _ => { + whoami_ui::detail("storage", storage_msg); + whoami_ui::detail("limit", format!("past the {limit} limit")); + } + } + } + // At the limit too, so the publish that starts the overage + // isn't its first notice. + if counted >= limit_bytes && storage_overage == Some("billed") { + whoami_ui::detail( + "overage", + format!("storage past {limit} is billed per GiB a month, rounded up"), + ); + } + if counted < usage.storage_bytes { + whoami_ui::detail( + "exempt", + format!( + "{} more, the tarballs of packages whose sales exempt them", + lpm_common::format_bytes(usage.storage_bytes - counted) + ), + ); } } else { - whoami_ui::detail("storage", format!("{:.2}MB", storage_mb)); + whoami_ui::detail("storage", storage); } // Package count @@ -149,21 +207,71 @@ pub async fn run(client: &RegistryClient, json_output: bool) -> Result<(), LpmEr whoami_ui::detail("private pkgs", format!("{}", usage.private_packages)); } - // Over-limit warning - let over_storage = limits - .storage_bytes - .is_some_and(|l| usage.storage_bytes > l); - let over_packages = limits + // The Registry refuses a new private package once the scope + // holds its limit, and a publish once storage is at its own. + let package_limit = limits .private_packages - .is_some_and(|l| l > 0 && l != u32::MAX && usage.private_packages > l); - - if over_storage || over_packages { + .filter(|&limit| limit > 0 && limit != u32::MAX); + let packages_full = package_limit.is_some_and(|limit| usage.private_packages >= limit); + let over_packages = package_limit.is_some_and(|limit| usage.private_packages > limit); + // A Pro plan without a subscription, as staff grant one, has + // nothing to bill storage on, and no subscription to take out. + let unbilled_pro = on_pro && storage_overage == Some("blocked"); + let storage_refused = + full_storage.is_some() && storage_overage == Some("blocked") && !unbilled_pro; + + match (&full_storage, storage_overage) { + (Some(limit), Some("paused")) => { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "Publishing past {} is refused until billing is active again.", + limit.as_str() + )); + match (over_storage, user.storage_excess_billed) { + (true, Some(true)) => whoami_ui::warn( + "Storage already past it is still billed while the payment is retried.", + ), + (true, Some(false)) => { + whoami_ui::warn("Storage already past it isn't billed meanwhile.") + } + _ => {} + } + whoami_ui::warn("Manage billing: https://lpm.dev/dashboard/settings/billing"); + } + (Some(limit), Some("blocked")) if unbilled_pro => { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "Storage past {} isn't billed on this plan, so publishing to your personal scope past it is refused.", + limit.as_str() + )); + } + _ => {} + } + if (over_storage && storage_refused) || over_packages { whoami_ui::warn("Your account is over its plan limits."); - whoami_ui::warn("Write access (publishing, inviting members) is restricted."); + } + if let (Some(limit), true) = (&full_storage, storage_refused) { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "Publishing to your personal scope is refused while its storage is at or past {}.", + limit.as_str() + )); + } + if let (Some(limit), true) = (package_limit, packages_full) { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "New private packages in your personal scope are refused once it holds {}.", + limit + )); + } + let storage_may_be_refused = free_plan_unstated && full_storage.is_some(); + if let (Some(limit), true) = (&full_storage, storage_may_be_refused) { + whoami_ui::warn_line(crate::install_ui::terminal_line!( + "Publishing to your personal scope is refused while its storage is at or past {}, apart from packages whose sales exempt them.", + limit.as_str() + )); + } + if storage_refused || storage_may_be_refused || packages_full { whoami_ui::warn("Upgrade your plan: https://lpm.dev/dashboard/settings/billing"); } } else { - whoami_ui::detail("storage", format!("{:.2}MB", storage_mb)); + whoami_ui::detail("storage", storage); whoami_ui::detail("private pkgs", format!("{}", usage.private_packages)); } } @@ -255,11 +363,18 @@ fn login_command_for_registry(registry_url: &str) -> String { } } +/// The plan's name as lpm.dev shows it: the free plan is Hobby. fn format_plan_tier(tier: &str) -> String { - let mut chars = tier.chars(); - match chars.next() { - Some(first) => first.to_uppercase().chain(chars).collect(), - None => String::new(), + match tier.to_ascii_lowercase().as_str() { + "free" => "Hobby".to_string(), + "org" => "Organization".to_string(), + _ => { + let mut chars = tier.chars(); + match chars.next() { + Some(first) => first.to_uppercase().chain(chars).collect(), + None => String::new(), + } + } } } @@ -295,6 +410,15 @@ mod tests { use super::*; use std::ffi::OsString; + #[test] + fn names_each_plan_as_lpm_dev_does() { + assert_eq!(format_plan_tier("free"), "Hobby"); + assert_eq!(format_plan_tier("FREE"), "Hobby"); + assert_eq!(format_plan_tier("pro"), "Pro"); + assert_eq!(format_plan_tier("org"), "Organization"); + assert_eq!(format_plan_tier(""), ""); + } + fn scoped_home_dir() -> (tempfile::TempDir, crate::test_env::ScopedEnv) { let dir = tempfile::tempdir().unwrap(); let env = crate::test_env::ScopedEnv::update([ diff --git a/crates/lpm-cli/src/swift_manifest.rs b/crates/lpm-cli/src/swift_manifest.rs index b1cc350f9..af5098dbe 100644 --- a/crates/lpm-cli/src/swift_manifest.rs +++ b/crates/lpm-cli/src/swift_manifest.rs @@ -46,7 +46,14 @@ pub fn find_package_swift(dir: &Path) -> Option { /// Get non-test target names from the current SPM package. /// Runs `swift package dump-package` and parses the JSON output. pub fn get_spm_targets(project_dir: &Path) -> Result, LpmError> { - let mut command = swift_command(); + spm_targets_with_program(project_dir, std::ffi::OsStr::new("swift")) +} + +fn spm_targets_with_program( + project_dir: &Path, + program: &std::ffi::OsStr, +) -> Result, LpmError> { + let mut command = swift_command_for(program); command .args(["package", "dump-package"]) .current_dir(project_dir); @@ -54,10 +61,9 @@ pub fn get_spm_targets(project_dir: &Path) -> Result, LpmError> { let output = run_bounded_swift_output(command, "swift package dump-package")?; if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); return Err(LpmError::Registry(format!( "swift package dump-package failed: {}", - stderr.trim() + swift_failure_reason(&output.status, &output.stderr) ))); } @@ -294,8 +300,32 @@ pub(crate) fn sanitize_swift_environment(command: &mut std::process::Command) { } } +/// The longest stretch of a failed Swift command's diagnostics a message shows. +const SWIFT_DIAGNOSTIC_CHARS: usize = 4096; + +/// Why a Swift command failed: how it ended (an exit status, or the signal +/// that killed it), then its diagnostics made safe for one terminal line, or +/// that it printed none. +pub(crate) fn swift_failure_reason(status: &std::process::ExitStatus, stderr: &[u8]) -> String { + let diagnostic = String::from_utf8_lossy(stderr) + .trim() + .chars() + .take(SWIFT_DIAGNOSTIC_CHARS) + .collect::(); + let diagnostic = lpm_common::sanitize_terminal_inline(&diagnostic); + if diagnostic.trim().is_empty() { + format!("{status}, without output") + } else { + format!("{status}: {}", diagnostic.trim()) + } +} + pub(crate) fn swift_command() -> std::process::Command { - let mut command = std::process::Command::new("swift"); + swift_command_for(std::ffi::OsStr::new("swift")) +} + +fn swift_command_for(program: &std::ffi::OsStr) -> std::process::Command { + let mut command = std::process::Command::new(program); sanitize_swift_environment(&mut command); command } @@ -1754,6 +1784,58 @@ pub fn remove_wrapper_dependencies( mod tests { use super::*; + #[cfg(unix)] + #[test] + fn a_swift_failure_names_how_the_process_ended_even_without_output() { + use std::os::unix::process::ExitStatusExt; + + let killed = swift_failure_reason(&std::process::ExitStatus::from_raw(9), b""); + assert!(killed.starts_with("signal: 9"), "{killed}"); + assert!(killed.ends_with("without output"), "{killed}"); + + let failed = swift_failure_reason( + &std::process::ExitStatus::from_raw(1 << 8), + b" error: no such module 'Missing'\n\x1b[31mred\x1b[0m\n", + ); + assert!( + failed.starts_with("exit status: 1: error: no such module 'Missing'"), + "{failed}" + ); + assert!(!failed.contains('\x1b'), "{failed}"); + + let long = swift_failure_reason( + &std::process::ExitStatus::from_raw(1 << 8), + "x".repeat(10_000).as_bytes(), + ); + assert!(long.chars().count() < 4_200, "{}", long.len()); + } + + #[cfg(unix)] + #[test] + fn a_failed_dump_package_says_how_swift_ended() { + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().unwrap(); + let program = directory.path().join("swift-fails"); + std::fs::write( + &program, + "#!/bin/sh\necho 'error: invalid manifest' >&2\nexit 3\n", + ) + .unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let error = spm_targets_with_program(directory.path(), program.as_os_str()) + .unwrap_err() + .to_string(); + + assert!( + error.contains( + "swift package dump-package failed: exit status: 3: error: invalid manifest" + ), + "{error}" + ); + } + #[test] fn swift_environment_scrub_covers_the_shared_baseline_and_its_own_credentials() { for key in [ diff --git a/crates/lpm-common/src/lib.rs b/crates/lpm-common/src/lib.rs index 66d75191d..673f4fe9d 100644 --- a/crates/lpm-common/src/lib.rs +++ b/crates/lpm-common/src/lib.rs @@ -427,23 +427,207 @@ fn apfs_held_back(size: u64) -> u64 { .min(33 * 1024 * 1024) } -/// Format bytes into a human-readable string (e.g., "1.2 KB", "3.4 MB"). +/// Format bytes into a human-readable string in binary units, as they're +/// counted (e.g., "1.2 KiB", "3.4 MiB", "5 GiB"), as the LPM.dev Registry +/// labels them. Switches unit at each 1024, so a 900-byte file never reads +/// as "0.9 KiB", and a value whose tenth would read 1024.0 takes the next +/// unit. A whole number only for an exact size, so 5 GiB and a few bytes +/// reads "5.0 GiB". A tenth halfway between two rounds up, as the +/// Registry's does. pub fn format_bytes(bytes: u64) -> String { + format_scaled(bytes, 1, UNITS.len() - 1, Rounding::Nearest, false) +} + +/// Formats a size past a limit it's shown against: to two decimals, rounded +/// up, so a size a byte past its limit never reads as within it ("500.01 +/// MiB" against 500 MiB), as the LPM.dev Registry states such sizes. +pub fn format_bytes_past(bytes: u64) -> String { + format_scaled(bytes, 2, UNITS.len() - 1, Rounding::Up, false) +} + +/// Formats a usage and the limit it's shown against as the LPM.dev +/// Registry's billing page does: to two decimals, in GiB at most with +/// thousands grouped, as limits and prices are stated per GiB. The usage +/// rounds toward the side of the limit it's on, so usage a few bytes past +/// its limit reads as past it, as overage bills it ("5.01 GiB / 5 GiB"), +/// and usage a few bytes short reads as short of it. +pub fn format_quota(used: u64, limit: u64) -> (String, String) { + let rounding = match used.cmp(&limit) { + std::cmp::Ordering::Greater => Rounding::Up, + std::cmp::Ordering::Less => Rounding::Down, + std::cmp::Ordering::Equal => Rounding::Nearest, + }; + ( + format_scaled(used, 2, GIB_UNIT, rounding, true), + format_scaled(limit, 2, GIB_UNIT, Rounding::Nearest, true), + ) +} + +/// Formats a usage with no limit to show it against, as `format_quota` +/// shows usage: to two decimals, in GiB at most with thousands grouped. +pub fn format_usage(used: u64) -> String { + format_scaled(used, 2, GIB_UNIT, Rounding::Nearest, true) +} + +const UNITS: [&str; 6] = ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]; +const GIB_UNIT: usize = 3; + +#[derive(Clone, Copy)] +enum Rounding { + Nearest, + Up, + Down, +} + +/// `bytes` in its largest unit up to `max_unit` whose shown value is at +/// least 1, to `decimals` places, rounded as asked in exact integer +/// arithmetic; a whole number for an exact size. +fn format_scaled( + bytes: u64, + decimals: u32, + max_unit: usize, + rounding: Rounding, + grouped: bool, +) -> String { if bytes < 1024 { - format!("{bytes} B") - } else if bytes < 1024 * 1024 { - format!("{:.1} KB", bytes as f64 / 1024.0) - } else if bytes < 1024 * 1024 * 1024 { - format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) - } else { - format!("{:.1} GB", bytes as f64 / (1024.0 * 1024.0 * 1024.0)) + let shown = if grouped { + group_thousands(u128::from(bytes)) + } else { + bytes.to_string() + }; + return format!("{shown} B"); + } + let places = 10_u128.pow(decimals); + let shown = |scale: u128| { + let numerator = u128::from(bytes) * places; + match rounding { + Rounding::Nearest => (numerator + scale / 2) / scale, + Rounding::Up => numerator.div_ceil(scale), + Rounding::Down => numerator / scale, + } + }; + let mut unit = 1; + let mut scale = 1024_u128; + while unit < max_unit && shown(scale) >= 1024 * places { + unit += 1; + scale *= 1024; + } + let whole = |value: u128| { + if grouped { + group_thousands(value) + } else { + value.to_string() + } + }; + if u128::from(bytes) % scale == 0 { + return format!("{} {}", whole(u128::from(bytes) / scale), UNITS[unit]); + } + let value = shown(scale); + format!( + "{}.{:0width$} {}", + whole(value / places), + value % places, + UNITS[unit], + width = decimals as usize + ) +} + +fn group_thousands(value: u128) -> String { + let digits = value.to_string(); + let mut grouped = String::with_capacity(digits.len() + digits.len() / 3); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + grouped.push(','); + } + grouped.push(digit); } + grouped } #[cfg(test)] mod tests { use super::*; + #[test] + fn format_bytes_labels_binary_units() { + assert_eq!(format_bytes(500), "500 B"); + assert_eq!(format_bytes(1024), "1 KiB"); + assert_eq!(format_bytes(1536), "1.5 KiB"); + assert_eq!(format_bytes(1_048_576), "1 MiB"); + assert_eq!(format_bytes(2_621_440), "2.5 MiB"); + assert_eq!(format_bytes(5 * 1024 * 1024 * 1024), "5 GiB"); + assert_eq!(format_bytes(5 * 1024 * 1024 * 1024 + 100), "5.0 GiB"); + assert_eq!(format_bytes(1025 * 1024_u64.pow(3)), "1.0 TiB"); + assert_eq!(format_bytes(2 * 1024_u64.pow(4)), "2 TiB"); + assert_eq!(format_bytes(3 * 1024_u64.pow(5)), "3 PiB"); + assert_eq!(format_bytes(u64::MAX), "16384.0 PiB"); + // Whose tenth would read 1024.0 takes the next unit. + assert_eq!(format_bytes(1_048_525), "1.0 MiB"); + assert_eq!(format_bytes(1_048_524), "1023.9 KiB"); + } + + #[test] + fn format_bytes_rounds_a_halfway_tenth_up_as_the_registry_does() { + assert_eq!(format_bytes(1280), "1.3 KiB"); + assert_eq!(format_bytes(1229), "1.2 KiB"); + assert_eq!( + format_bytes(5 * 1024_u64.pow(3) + 1024_u64.pow(3) / 4), + "5.3 GiB" + ); + } + + #[test] + fn format_quota_reads_usage_on_its_side_of_the_limit() { + const GIB: u64 = 1024 * 1024 * 1024; + let quota = |used, limit| { + let (used, limit) = format_quota(used, limit); + format!("{used} / {limit}") + }; + assert_eq!(quota(5 * GIB + 1, 5 * GIB), "5.01 GiB / 5 GiB"); + assert_eq!( + quota(5 * GIB + 51 * 1024 * 1024, 5 * GIB), + "5.05 GiB / 5 GiB" + ); + assert_eq!(quota(5 * GIB - 1, 5 * GIB), "4.99 GiB / 5 GiB"); + assert_eq!(quota(5 * GIB, 5 * GIB), "5 GiB / 5 GiB"); + assert_eq!( + quota(512 * 1024 * 1024 + 104_858, 5 * GIB), + "512.10 MiB / 5 GiB" + ); + assert_eq!(quota(0, 5 * GIB), "0 B / 5 GiB"); + } + + #[test] + fn format_usage_reads_usage_as_quotas_do_without_a_limit() { + let gib = 1024_u64.pow(3); + assert_eq!(format_usage(1253 * gib + gib / 2), "1,253.50 GiB"); + assert_eq!(format_usage(1023), "1,023 B"); + assert_eq!(format_usage(2048), "2 KiB"); + assert_eq!(format_usage(2049), "2.00 KiB"); + // To nearest: 1.0059 KiB reads 1.01, not 1.00. + assert_eq!(format_usage(1030), "1.01 KiB"); + } + + #[test] + fn format_quota_states_sizes_in_gib_at_most_with_thousands_grouped() { + const GIB: u64 = 1024 * 1024 * 1024; + assert_eq!( + format_quota(1253 * GIB, 1250 * GIB), + ("1,253 GiB".into(), "1,250 GiB".into()) + ); + assert_eq!( + format_quota(1253 * GIB + GIB / 2, 1250 * GIB).0, + "1,253.50 GiB" + ); + assert_eq!(format_quota(u64::MAX, u64::MAX).0, "17,179,869,184.00 GiB"); + assert_eq!(format_bytes_past(500 * 1024 * 1024 + 1), "500.01 MiB"); + assert_eq!(format_bytes_past(500 * 1024 * 1024), "500 MiB"); + assert_eq!(format_bytes_past(5 * GIB + 1), "5.01 GiB"); + // As the Registry groups bytes too ("1,000 B"). + assert_eq!(format_quota(1000, 2048), ("1,000 B".into(), "2 KiB".into())); + assert_eq!(format_bytes(1000), "1000 B"); + } + /// `df`'s figure for the filesystem holding `path`: its size and the /// space free to this user, in bytes. #[cfg(unix)] diff --git a/crates/lpm-inspect/src/db.rs b/crates/lpm-inspect/src/db.rs index fa9f4ecdb..78d473985 100644 --- a/crates/lpm-inspect/src/db.rs +++ b/crates/lpm-inspect/src/db.rs @@ -4547,7 +4547,7 @@ mod tests { assert!( error .to_string() - .contains("needs about 3.0 MB free on its disk"), + .contains("needs about 3 MiB free on its disk"), "{error}" ); } diff --git a/crates/lpm-registry/src/client/api.rs b/crates/lpm-registry/src/client/api.rs index 7d9deb697..d4bddd89d 100644 --- a/crates/lpm-registry/src/client/api.rs +++ b/crates/lpm-registry/src/client/api.rs @@ -1,3 +1,4 @@ +use super::transport::{Answered, FailedResponse}; use super::*; impl RegistryClient { @@ -240,22 +241,70 @@ impl RegistryClient { .await } - /// Validate publish authorization and version availability without - /// reserving a version or creating Registry state. + /// Asks the publish preflight whether the Registry would take + /// `name@version`: the token's publish authorization and the version's + /// availability, and given the `size` of the tarball it would be sent, + /// the size cap and the account's storage. It creates no Registry state. + /// A refusal comes back with its status and code; a timeout, a rate + /// limit, a server error or a network failure is retried first. pub async fn publish_preflight( &self, name: &str, version: &str, - ) -> Result { - let url = format!( + size: Option, + ) -> Result { + let mut url = format!( "{}/api/registry/-/package/publish-preflight?name={}&version={}", self.base_url, urlencoding::encode(name), urlencoding::encode(version) ); - let response: PublishPreflightResponse = self - .execute_with_recovery(AuthPosture::AuthRequired, || self.get_json(&url)) + if let Some(size) = size { + url.push_str(&format!("&size={size}")); + } + let answer = self + .execute_with_recovery(AuthPosture::AuthRequired, || async { + match self + .send_with_retry_keeping_status(self.build_get(&url).await?) + .await? + { + Answered::Failed(FailedResponse { status: 401, .. }) => { + Err(LpmError::AuthRequired) + } + answer => Ok(answer), + } + }) .await?; + let response = match answer { + Answered::Success(response) => response, + Answered::Failed(FailedResponse { + status, + retry_after_secs, + body, + }) => { + let json = serde_json::from_str::(&body).ok(); + let text = |field: &str| { + json.as_ref() + .and_then(|json| json.get(field)) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }; + let error = text("error").unwrap_or_else(|| { + let status = reqwest::StatusCode::from_u16(status) + .map_or_else(|_| status.to_string(), |status| status.to_string()); + format!("HTTP {status}") + }); + return Ok(PublishPreflightVerdict::Refused { + status, + code: text("code"), + error, + body, + retry_after_secs, + }); + } + }; + let response: PublishPreflightResponse = + parse_capped_api_json(response, "publish preflight response").await?; if !response.success { return Err(LpmError::Registry(format!( "publish preflight was denied for {name}@{version}" @@ -267,7 +316,7 @@ impl RegistryClient { response.name, response.version ))); } - Ok(response) + Ok(PublishPreflightVerdict::Accepted(response)) } /// Read the current publication lifecycle state for an uploaded LPM version. @@ -727,3 +776,31 @@ fn validate_search_registry_base(base_url: &str) -> Result<(), LpmError> { } Ok(()) } + +impl PublishPreflightVerdict { + /// The accepted answer, or the refusal as the error any other request's + /// failure is (see [`super::transport::failed_answer_error`]), with the + /// Registry's message where its body gives one, and the body otherwise. + pub fn into_result(self) -> Result { + match self { + Self::Accepted(response) => Ok(response), + Self::Refused { + status, + body, + retry_after_secs, + .. + } => { + let message = serde_json::from_str::(&body) + .ok() + .and_then(|json| json.get("error")?.as_str().map(str::to_owned)) + .unwrap_or_else(|| body.clone()); + Err(super::transport::failed_answer_error( + status, + &body, + message, + retry_after_secs, + )) + } + } + } +} diff --git a/crates/lpm-registry/src/client/metadata.rs b/crates/lpm-registry/src/client/metadata.rs index 340267816..f77d51283 100644 --- a/crates/lpm-registry/src/client/metadata.rs +++ b/crates/lpm-registry/src/client/metadata.rs @@ -805,8 +805,15 @@ impl RegistryClient { .map_err(|error| LpmError::Network(format!("failed to build request: {error}")))?; let provider = super::http::PolicyMetadataClientProvider(self.http.as_ref()); let response = self - .send_request_with_retry_and_client_provider(request, None, None, &provider) - .await?; + .send_request_with_retry_and_client_provider( + request, + None, + None, + &provider, + super::transport::FailedAnswer::Error, + ) + .await + .and_then(super::transport::Answered::into_response)?; crate::timing::record_metadata_http_version(response.version()); Ok(response) } diff --git a/crates/lpm-registry/src/client/tests/api.rs b/crates/lpm-registry/src/client/tests/api.rs index cd5de3feb..ffcc04243 100644 --- a/crates/lpm-registry/src/client/tests/api.rs +++ b/crates/lpm-registry/src/client/tests/api.rs @@ -101,14 +101,293 @@ async fn publish_preflight_sends_resolved_identity_and_bearer_without_mutation() let client = RegistryClient::new() .with_base_url(server.uri()) .with_token("publish-token"); - let response = client - .publish_preflight("@lpm.dev/owner.resolved", "1.2.3") + let verdict = client + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) .await .unwrap(); + let PublishPreflightVerdict::Accepted(response) = verdict else { + panic!("expected an accepted preflight, got {verdict:?}"); + }; assert!(response.success); assert!(response.package_exists); } +#[tokio::test] +async fn publish_preflight_sends_the_size_it_is_given() { + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .and(query_param("size", "1048576")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "success": true, + "name": "@lpm.dev/owner.resolved", + "version": "1.2.3", + "packageExists": false + }))) + .expect(1) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", Some(1024 * 1024)) + .await + .unwrap(); + + assert!( + matches!(verdict, PublishPreflightVerdict::Accepted(_)), + "{verdict:?}" + ); +} + +#[tokio::test] +async fn publish_preflight_keeps_a_refusal_s_status_code_and_message() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + for (status, body, code, error) in [ + ( + 402, + serde_json::json!({ "error": "Billing is past due.", "code": "ORG_BILLING_LAPSED" }), + Some("ORG_BILLING_LAPSED"), + "Billing is past due.", + ), + ( + 403, + serde_json::json!({ "error": "Token does not have publish permissions" }), + None, + "Token does not have publish permissions", + ), + ] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(status).set_body_json(body)) + .expect(1) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", Some(1)) + .await + .unwrap(); + + let PublishPreflightVerdict::Refused { + status: refused, + code: refused_code, + error: message, + .. + } = verdict + else { + panic!("expected a refusal, got {verdict:?}"); + }; + assert_eq!( + (refused, refused_code.as_deref(), message.as_str()), + (status, code, error) + ); + } +} + +#[tokio::test] +async fn publish_preflight_names_the_status_of_an_answer_that_isnt_json() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(413).set_body_string("too large")) + .expect(1) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", Some(1)) + .await + .unwrap(); + + let PublishPreflightVerdict::Refused { + status, + code, + error, + body, + .. + } = verdict + else { + panic!("expected a refusal, got {verdict:?}"); + }; + assert_eq!( + (status, code, error.as_str(), body.as_str()), + ( + 413, + None, + "HTTP 413 Payload Too Large", + "too large" + ) + ); +} + +#[tokio::test] +async fn a_publish_preflight_refusal_is_the_error_any_request_s_failure_is() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + for (status, body) in [ + (403, "Insufficient permissions"), + (404, "no such package"), + (429, "slow down"), + (409, "ordered-alpha-error"), + ] { + let server = MockServer::start().await; + if status == 429 { + // Retried at once three times; the last answer's wait is the + // one reported. + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "0")) + .up_to_n_times(3) + .mount(&server) + .await; + } + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with( + ResponseTemplate::new(status) + .insert_header("retry-after", "7") + .set_body_string(body), + ) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) + .await + .unwrap(); + + let error = verdict.into_result().unwrap_err(); + match status { + 403 => assert!( + matches!(&error, LpmError::Forbidden(message) if message == body), + "{error:?}" + ), + 404 => assert!( + matches!(&error, LpmError::NotFound(message) if message == body), + "{error:?}" + ), + 429 => assert!( + matches!( + error, + LpmError::RateLimited { + retry_after_secs: 7 + } + ), + "{error:?}" + ), + _ => assert!( + matches!(&error, LpmError::Http { status: 409, message } if message == body), + "{error:?}" + ), + } + } +} + +#[tokio::test] +async fn publish_preflight_redacts_a_refusal_s_reflection_of_its_bearer() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let token = "lpm-preflight-reflection-secret"; + for (status, body) in [ + ( + 403, + serde_json::json!({ "error": format!("bearer {token} may not publish") }).to_string(), + ), + (409, format!("submitted bearer was {token}")), + // The last of the retried answers. + (429, format!("slow down, {token}")), + ] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with( + ResponseTemplate::new(status) + .insert_header("retry-after", "0") + .set_body_string(body), + ) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token(token) + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) + .await + .unwrap(); + + let PublishPreflightVerdict::Refused { error, body, .. } = &verdict else { + panic!("expected a refusal, got {verdict:?}"); + }; + for text in [error, body] { + assert!(!text.contains(token), "{status}: bearer leaked: {text}"); + } + assert!(body.contains(""), "{status}: {body}"); + let error = format!("{:?}", verdict.into_result().unwrap_err()); + assert!(!error.contains(token), "{status}: bearer leaked: {error}"); + } +} + +#[tokio::test] +async fn publish_preflight_retries_a_server_error_before_reading_the_answer() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + for status in [408, 500, 502, 503, 504] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(status)) + .up_to_n_times(1) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({ + "error": "Version 1.2.3 already exists. You cannot overwrite versions.", + "code": "VERSION_EXISTS", + }))) + .expect(1) + .mount(&server) + .await; + + let verdict = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) + .await + .unwrap(); + + assert!( + matches!( + &verdict, + PublishPreflightVerdict::Refused { status: 403, code: Some(code), .. } + if code == "VERSION_EXISTS" + ), + "{status}: {verdict:?}" + ); + } +} + #[tokio::test] async fn publication_status_sends_package_identity_and_publish_bearer() { use wiremock::matchers::{header, method, path, query_param}; @@ -190,7 +469,7 @@ async fn publish_preflight_rejects_an_unsuccessful_success_envelope() { let error = RegistryClient::new() .with_base_url(server.uri()) .with_token("publish-token") - .publish_preflight("@lpm.dev/owner.resolved", "1.2.3") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) .await .unwrap_err() .to_string(); @@ -203,27 +482,36 @@ async fn publish_preflight_rejects_a_mismatched_response_identity() { use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path("/api/registry/-/package/publish-preflight")) - .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ - "success": true, - "name": "@lpm.dev/owner.other", - "version": "9.9.9", - "packageExists": false - }))) - .mount(&server) - .await; - - let error = RegistryClient::new() - .with_base_url(server.uri()) - .with_token("publish-token") - .publish_preflight("@lpm.dev/owner.resolved", "1.2.3") - .await - .unwrap_err() - .to_string(); - - assert!(error.contains("mismatched package identity"), "{error}"); + // Another package, or another version of this one. + for (name, version) in [ + ("@lpm.dev/owner.other", "1.2.3"), + ("@lpm.dev/owner.resolved", "9.9.9"), + ] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "success": true, + "name": name, + "version": version, + "packageExists": false + }))) + .mount(&server) + .await; + + let error = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) + .await + .unwrap_err() + .to_string(); + + assert!( + error.contains("mismatched package identity"), + "{name}@{version}: {error}" + ); + } } #[tokio::test] @@ -245,7 +533,7 @@ async fn publish_preflight_rejects_a_response_without_package_existence() { let error = RegistryClient::new() .with_base_url(server.uri()) .with_token("publish-token") - .publish_preflight("@lpm.dev/owner.resolved", "1.2.3") + .publish_preflight("@lpm.dev/owner.resolved", "1.2.3", None) .await .expect_err("a missing packageExists field must fail closed") .to_string(); @@ -842,6 +1130,46 @@ async fn publish_package_replays_the_exact_body_after_gateway_failure() { } } +#[tokio::test] +async fn publish_package_redacts_a_registry_reflection_of_its_one_time_code() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let code = "731905"; + for status in [403, 404, 422] { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .and(path("/api/registry/@lpm.dev/test.reflection")) + .respond_with( + ResponseTemplate::new(status).set_body_string(format!("code {code} was refused")), + ) + .expect(1) + .mount(&server) + .await; + let payload = replayable_json(&serde_json::json!({ + "name": "@lpm.dev/test.reflection" + })); + + let error = RegistryClient::new() + .with_base_url(server.uri()) + .with_token("publish-token") + .publish_package( + "@lpm.dev/test.reflection", + "1.0.0", + "sha512-test", + &payload, + Some(code), + 0, + ) + .await + .expect_err("registry rejection must remain an error") + .to_string(); + + assert!(!error.contains(code), "{status}: code leaked: {error}"); + assert!(error.contains(""), "{status}: {error}"); + } +} + #[tokio::test] async fn publish_package_redacts_a_registry_reflection_of_its_bearer() { use wiremock::matchers::{method, path}; diff --git a/crates/lpm-registry/src/client/transport.rs b/crates/lpm-registry/src/client/transport.rs index db3bb1e5a..6f0376b2c 100644 --- a/crates/lpm-registry/src/client/transport.rs +++ b/crates/lpm-registry/src/client/transport.rs @@ -1,7 +1,7 @@ use super::*; mod error_redaction; -use error_redaction::read_request_error_text; +use error_redaction::{read_error_text_redacting, read_request_error_text}; pub(super) enum PublishSafeResponse { Response(reqwest::Response), @@ -473,6 +473,7 @@ impl RegistryClient { let req = request_factory()? .build() .map_err(|e| LpmError::Network(format!("failed to build request: {e}")))?; + let headers = req.headers().clone(); match lpm_http::send_with_replayable_redirects(self.http.as_ref(), req, Some(payload)) .await @@ -486,7 +487,7 @@ impl RegistryClient { // Non-retryable client errors — fail immediately 401 => return Err(LpmError::AuthRequired), 403 => { - let body = read_publish_error_text(response, bearer).await; + let body = read_publish_error_text(response, &headers, bearer).await; let duplicate_message = format!( "Version {published_version} already exists. You cannot overwrite versions." ); @@ -507,14 +508,15 @@ impl RegistryClient { return Err(forbidden_error_from_body(body)); } 404 => { - let body = read_publish_error_text(response, bearer).await; + let body = read_publish_error_text(response, &headers, bearer).await; return Err(LpmError::NotFound(body)); } // S4: 500 — do NOT retry. Server may have stored the version. // Check if the version now exists on the registry. 500 => { - let body_text = read_publish_error_text(response, bearer).await; + let body_text = + read_publish_error_text(response, &headers, bearer).await; tracing::warn!("publish got HTTP 500 — checking if version was stored"); match self @@ -561,7 +563,7 @@ impl RegistryClient { // Retryable gateway errors only (NOT 500) 502..=504 => { - let body = read_publish_error_text(response, bearer).await; + let body = read_publish_error_text(response, &headers, bearer).await; last_error = Some(LpmError::Http { status, message: body, @@ -594,7 +596,7 @@ impl RegistryClient { // Other errors — fail immediately _ => { - let body = read_publish_error_text(response, bearer).await; + let body = read_publish_error_text(response, &headers, bearer).await; return Err(LpmError::Http { status, message: body, @@ -727,6 +729,31 @@ impl RegistryClient { client_override, auth, self.http.as_ref(), + FailedAnswer::Error, + ) + .await + .and_then(Answered::into_response) + } + + /// Sends a request as [`send_with_retry`](Self::send_with_retry) does, + /// retrying a timeout, a rate limit, a server error or a network + /// failure, but hands back any other answer, and the last one retried, + /// with its status, and its body with the request's credentials + /// redacted. + pub(super) async fn send_with_retry_keeping_status( + &self, + request_builder: reqwest::RequestBuilder, + ) -> Result { + self.validate_base_url()?; + let request = request_builder + .build() + .map_err(|e| LpmError::Network(format!("failed to build request: {e}")))?; + self.send_request_with_retry_and_client_provider( + request, + None, + None, + self.http.as_ref(), + FailedAnswer::Response, ) .await } @@ -743,7 +770,8 @@ impl RegistryClient { client_override: Option, auth: Option<&crate::npmrc::RegistryAuth>, provider: &P, - ) -> Result + failed: FailedAnswer, + ) -> Result where P: lpm_http::ReplayableHttpClientProvider + Sync, { @@ -800,27 +828,34 @@ impl RegistryClient { Ok(response) => { let status = response.status().as_u16(); tracing::event!(name: "headers_observed", target: "lpm_install_timeline", tracing::Level::TRACE, attempt = attempt as u64, status = u64::from(status)); + let retried = matches!(status, 408 | 429 | 500 | 502 | 503 | 504) + && attempt < MAX_RETRIES; + if failed == FailedAnswer::Response + && !matches!(status, 200..=299 | 304) + && !retried + { + return Ok(Answered::Failed(FailedResponse { + status, + retry_after_secs: parse_retry_after(&response), + body: read_request_error_text(response, &request).await, + })); + } match status { - 200..=299 | 304 => return Ok(response), + 200..=299 | 304 => return Ok(Answered::Success(response)), // Non-retryable errors — fail immediately 401 => return Err(LpmError::AuthRequired), - 403 => { + 403 | 404 => { let body = read_request_error_text(response, &request).await; - return Err(forbidden_error_from_body(body)); - } - 404 => { - let body = read_request_error_text(response, &request).await; - return Err(LpmError::NotFound(body)); + return Err(failed_answer_error(status, &body, body.clone(), 0)); } // Retryable: rate limit 429 => { let retry_after = parse_retry_after(&response); - last_error = Some(LpmError::RateLimited { - retry_after_secs: retry_after, - }); + last_error = + Some(failed_answer_error(status, "", String::new(), retry_after)); if attempt < MAX_RETRIES { // Honor the test-only backoff override here // too — see the sibling site in publish-path. @@ -841,10 +876,7 @@ impl RegistryClient { ) .await .unwrap_or_default(); - last_error = Some(LpmError::Http { - status, - message: body, - }); + last_error = Some(failed_answer_error(status, &body, body.clone(), 0)); if attempt < MAX_RETRIES { let delay = backoff_delay(attempt); tracing::event!(name: "backoff_start", target: "lpm_install_timeline", tracing::Level::TRACE, attempt = attempt as u64); @@ -857,10 +889,7 @@ impl RegistryClient { // Other errors — fail immediately _ => { let body = read_request_error_text(response, &request).await; - return Err(LpmError::Http { - status, - message: body, - }); + return Err(failed_answer_error(status, &body, body.clone(), 0)); } } } @@ -900,14 +929,88 @@ impl RegistryClient { } } -async fn read_publish_error_text(response: reqwest::Response, bearer: Option<&str>) -> String { - let body = read_capped_error_text(response).await; +async fn read_publish_error_text( + response: reqwest::Response, + headers: &reqwest::header::HeaderMap, + bearer: Option<&str>, +) -> String { + let body = read_error_text_redacting(response, headers).await; match bearer { Some(secret) => lpm_common::redact_exact_secret(&body, secret).into_owned(), None => body, } } +/// The error an answer that isn't a success, nor a 401, is: a 403 as +/// forbidden, or as the error an LPM Worker's `body` names, a 404 as not +/// found, a 429 as rate limited for `retry_after_secs`, any other as +/// `Http`, each with `message`. +pub(super) fn failed_answer_error( + status: u16, + body: &str, + message: String, + retry_after_secs: u64, +) -> LpmError { + match status { + 403 => { + super::body::parse_lpm_worker_error_body(body).unwrap_or(LpmError::Forbidden(message)) + } + 404 => LpmError::NotFound(message), + 429 => LpmError::RateLimited { retry_after_secs }, + _ => LpmError::Http { status, message }, + } +} + +/// What the retry loop hands back for an answer that isn't a success. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum FailedAnswer { + /// An error: a 401 as `AuthRequired`, any other as + /// [`failed_answer_error`] makes it. + Error, + /// [`Answered::Failed`]: one that isn't retried, and the last one that + /// was. + Response, +} + +/// The retry loop's answer. +#[derive(Debug)] +pub(super) enum Answered { + Success(reqwest::Response), + /// An answer that isn't a success, which only [`FailedAnswer::Response`] + /// hands back. + Failed(FailedResponse), +} + +impl Answered { + /// The success, or the error a failed answer is. + pub(super) fn into_response(self) -> Result { + match self { + Self::Success(response) => Ok(response), + Self::Failed(FailedResponse { status: 401, .. }) => Err(LpmError::AuthRequired), + Self::Failed(FailedResponse { + status, + retry_after_secs, + body, + }) => Err(failed_answer_error( + status, + &body, + body.clone(), + retry_after_secs, + )), + } + } +} + +/// An answer that isn't a success, read: its status, the wait its +/// `Retry-After` asks, and its body, capped, with the request's credentials +/// redacted, as a server can echo them. +#[derive(Debug)] +pub(super) struct FailedResponse { + pub(super) status: u16, + pub(super) retry_after_secs: u64, + pub(super) body: String, +} + /// Parse `Retry-After` header from a 429 response. /// Returns seconds to wait. Falls back to 1 second if header is missing/unparseable. pub(super) fn parse_retry_after(response: &reqwest::Response) -> u64 { diff --git a/crates/lpm-registry/src/client/transport/error_redaction.rs b/crates/lpm-registry/src/client/transport/error_redaction.rs index c257be126..165ef446f 100644 --- a/crates/lpm-registry/src/client/transport/error_redaction.rs +++ b/crates/lpm-registry/src/client/transport/error_redaction.rs @@ -4,9 +4,17 @@ use reqwest::header::{AUTHORIZATION, HeaderMap, PROXY_AUTHORIZATION}; pub(super) async fn read_request_error_text( response: reqwest::Response, request: &reqwest::Request, +) -> String { + read_error_text_redacting(response, request.headers()).await +} + +/// The answer's body, capped, with the credentials `headers` sent redacted. +pub(super) async fn read_error_text_redacting( + response: reqwest::Response, + headers: &HeaderMap, ) -> String { let mut text = super::read_capped_error_text(response).await; - redact_credentials(&mut text, request.headers()); + redact_credentials(&mut text, headers); text } diff --git a/crates/lpm-registry/src/types.rs b/crates/lpm-registry/src/types.rs index d7fa5ca45..1ab999441 100644 --- a/crates/lpm-registry/src/types.rs +++ b/crates/lpm-registry/src/types.rs @@ -993,6 +993,18 @@ pub struct WhoamiResponse { #[serde(default)] pub limits: Option, + + /// What publishing does with storage past the plan's limit: `billed`, + /// `paused` while billing isn't active, or `blocked`. A Registry that + /// predates it sends none. + #[serde(default, rename = "storage_overage")] + pub storage_overage: Option, + + /// Whether storage already past the plan's limit is billed meanwhile, + /// as it is while a subscription is past due, and not while it's paused + /// or ended. A Registry that predates it sends none. + #[serde(default, rename = "storage_excess_billed")] + pub storage_excess_billed: Option, } /// GET /api/registry/-/package/publish-preflight @@ -1005,6 +1017,24 @@ pub struct PublishPreflightResponse { pub package_exists: bool, } +/// What the publish preflight says of a publish of a given size. +#[derive(Debug, Clone)] +pub enum PublishPreflightVerdict { + /// The Registry would take the publish. + Accepted(PublishPreflightResponse), + /// The Registry refuses it: its HTTP status, its `code` where it gives + /// one, its message, or `HTTP ` where its body has none, the + /// body as sent, which a request's failure reports, and a 429's + /// `Retry-After` in seconds. + Refused { + status: u16, + code: Option, + error: String, + body: String, + retry_after_secs: u64, + }, +} + /// GET /api/registry/-/package/publication-status #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PublicationStatusResponse { @@ -1022,6 +1052,12 @@ pub struct WhoamiUsage { #[serde(default, rename = "storage_bytes")] pub storage_bytes: u64, + /// The storage the plan's limit counts: on the free plan, all but that + /// of packages whose sales exempt them. A Registry that predates it + /// sends none. + #[serde(default, rename = "counted_storage_bytes")] + pub counted_storage_bytes: Option, + #[serde(default, rename = "private_packages")] pub private_packages: u32, } diff --git a/crates/lpm-runtime/src/download.rs b/crates/lpm-runtime/src/download.rs index d8f5c5939..4747f8513 100644 --- a/crates/lpm-runtime/src/download.rs +++ b/crates/lpm-runtime/src/download.rs @@ -367,12 +367,7 @@ async fn read_download_body( } fn checked_download_size(size: u64) -> Result { - let size = usize::try_from(size).map_err(|_| { - LpmError::Script(format!( - "download size {size} bytes exceeds maximum allowed size of {MAX_DOWNLOAD_SIZE} bytes ({}MB)", - MAX_DOWNLOAD_SIZE / (1024 * 1024) - )) - })?; + let size = usize::try_from(size).map_err(|_| LpmError::Script(download_too_large(size)))?; validate_download_size(size)?; Ok(size) } @@ -762,14 +757,18 @@ const MAX_DOWNLOAD_SIZE: usize = 200 * 1024 * 1024; /// after buffering bytes (defence against lying headers / chunked encoding). fn validate_download_size(size: usize) -> Result<(), LpmError> { if size > MAX_DOWNLOAD_SIZE { - return Err(LpmError::Network(format!( - "download size {size} bytes exceeds maximum allowed size of {MAX_DOWNLOAD_SIZE} bytes ({}MB)", - MAX_DOWNLOAD_SIZE / (1024 * 1024) - ))); + return Err(LpmError::Network(download_too_large(size))); } Ok(()) } +fn download_too_large(size: impl std::fmt::Display) -> String { + format!( + "download size {size} bytes exceeds maximum allowed size of {MAX_DOWNLOAD_SIZE} bytes ({})", + lpm_common::format_bytes(MAX_DOWNLOAD_SIZE as u64) + ) +} + /// Compare an expected hex-encoded SHA-256 hash against the SHA-256 of raw bytes. /// /// Returns `Ok(())` on match, `Err` on mismatch. @@ -1543,7 +1542,12 @@ mod tests { let result = validate_download_size(MAX_DOWNLOAD_SIZE + 1); assert!(result.is_err(), "oversized download must be rejected"); let err_msg = result.unwrap_err().to_string(); - assert!(err_msg.contains("exceeds maximum"), "got: {err_msg}"); + assert!( + err_msg.contains( + "download size 209715201 bytes exceeds maximum allowed size of 209715200 bytes (200 MiB)" + ), + "got: {err_msg}" + ); } // --- Directory and file permissions --- diff --git a/crates/lpm-tunnel/src/webhook.rs b/crates/lpm-tunnel/src/webhook.rs index 18265e605..71ce4fe1c 100644 --- a/crates/lpm-tunnel/src/webhook.rs +++ b/crates/lpm-tunnel/src/webhook.rs @@ -210,21 +210,10 @@ fn extract_json_type(body: &[u8]) -> Option { /// Generic summary: `METHOD /path (size)`. fn format_generic(webhook: &CapturedWebhook) -> String { - let size = format_size(webhook.body_size()); + let size = lpm_common::format_bytes(webhook.body_size() as u64); format!("{} {} ({size})", webhook.method, webhook.path) } -/// Format byte size into a human-readable string. -fn format_size(bytes: usize) -> String { - if bytes < 1024 { - format!("{bytes}B") - } else if bytes < 1024 * 1024 { - format!("{:.1}KB", bytes as f64 / 1024.0) - } else { - format!("{:.1}MB", bytes as f64 / (1024.0 * 1024.0)) - } -} - /// Serde helper for base64-encoding `Vec` fields. mod base64_bytes { use base64::Engine; @@ -414,7 +403,7 @@ mod tests { w.path = "/api/hook".to_string(); w.request_body = vec![0u8; 2500]; }); - assert_eq!(summarize_webhook(&wh), "POST /api/hook (2.4KB)"); + assert_eq!(summarize_webhook(&wh), "POST /api/hook (2.4 KiB)"); } // -- CapturedWebhook methods -- @@ -523,24 +512,4 @@ mod tests { assert_eq!(WebhookProvider::GitHub.to_string(), "GitHub"); assert_eq!(WebhookProvider::SendGrid.to_string(), "SendGrid"); } - - // -- format_size -- - - #[test] - fn format_size_bytes() { - assert_eq!(format_size(0), "0B"); - assert_eq!(format_size(512), "512B"); - assert_eq!(format_size(1023), "1023B"); - } - - #[test] - fn format_size_kilobytes() { - assert_eq!(format_size(1024), "1.0KB"); - assert_eq!(format_size(2560), "2.5KB"); - } - - #[test] - fn format_size_megabytes() { - assert_eq!(format_size(1024 * 1024), "1.0MB"); - } } diff --git a/crates/lpm-vault/src/keychain.rs b/crates/lpm-vault/src/keychain.rs index 744763411..8c3c7a26d 100644 --- a/crates/lpm-vault/src/keychain.rs +++ b/crates/lpm-vault/src/keychain.rs @@ -1145,11 +1145,20 @@ pub(crate) fn mutate_vault_env( } fn warn_if_large(size: usize) { - if size > MAX_VAULT_SIZE_WARNING { - tracing::warn!("vault data is {size} bytes (approaching ~100KB Keychain limit)"); + if let Some(warning) = large_env_warning(size) { + tracing::warn!("{warning}"); } } +fn large_env_warning(size: usize) -> Option { + (size > MAX_VAULT_SIZE_WARNING).then(|| { + format!( + "env data is {} (approaching the Keychain's ~100 KiB limit)", + lpm_common::format_bytes(size as u64) + ) + }) +} + fn write_all_environments_with_storage_after_recovery( storage: &mut impl ProjectStorage, vault_id: &str, @@ -1813,6 +1822,15 @@ fn project_discovery_shard(probe: (usize, usize), attempt: usize) -> usize { mod tests { use super::*; + #[test] + fn large_env_data_is_warned_of_in_binary_units() { + assert_eq!( + large_env_warning(95 * 1024).as_deref(), + Some("env data is 95 KiB (approaching the Keychain's ~100 KiB limit)") + ); + assert_eq!(large_env_warning(MAX_VAULT_SIZE_WARNING), None); + } + #[derive(serde::Deserialize)] #[serde(rename_all = "camelCase")] struct ProjectIndexFixture { diff --git a/tests/workflows/tests/cache.rs b/tests/workflows/tests/cache.rs index 06816a402..b0ebac01e 100644 --- a/tests/workflows/tests/cache.rs +++ b/tests/workflows/tests/cache.rs @@ -84,7 +84,7 @@ async fn cache_status_json_reports_local_usage_and_remote_status() { .and(path("/v8/artifacts/status")) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "status": "enabled", - "usageBytes": 1024, + "usageBytes": 2049, "limitBytes": 2048 }))) .mount(&server) @@ -126,8 +126,72 @@ async fn cache_status_json_reports_local_usage_and_remote_status() { assert!(envelope["local"]["bytes"].as_u64().unwrap_or(0) >= 11); assert_eq!(envelope["remote"]["enabled"], serde_json::json!(true)); assert_eq!(envelope["remote"]["status"], serde_json::json!("enabled")); - assert_eq!(envelope["remote"]["usage_bytes"], serde_json::json!(1024)); + assert_eq!(envelope["remote"]["usage_bytes"], serde_json::json!(2049)); assert_eq!(envelope["remote"]["limit_bytes"], serde_json::json!(2048)); + // A byte past the limit reads as past it, as the human line shows it. + assert_eq!(envelope["remote"]["usage"], serde_json::json!("2.01 KiB")); + assert_eq!(envelope["remote"]["limit"], serde_json::json!("2 KiB")); +} + +#[tokio::test] +async fn cache_status_reads_an_unlimited_cache_s_usage_in_gib_against_no_limit() { + let server = MockServer::start().await; + let usage: u64 = 1253 * 1024 * 1024 * 1024 + 512 * 1024 * 1024; + Mock::given(method("GET")) + .and(path("/v8/artifacts/status")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "status": "enabled", + "usageBytes": usage, + "limitBytes": null + }))) + .mount(&server) + .await; + let project = TempProject::empty(r#"{"name":"cache-status","version":"1.0.0"}"#); + project.write_file( + "lpm.json", + &format!( + r#"{{ + "remoteCache": {{ + "enabled": true, + "url": "{}/v8" + }} + }}"#, + server.uri(), + ), + ); + let run = |json: bool| { + let mut command = lpm(&project); + command + .env("LPM_REMOTE_CACHE_TOKEN", "remote-token") + .env("LPM_REMOTE_CACHE_SIGNATURE_KEY", "signing-key"); + if json { + command.arg("--json"); + } + let output = command + .args(["cache", "status"]) + .output() + .expect("failed to run lpm cache status"); + assert!( + output.status.success(), + "cache status failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8_lossy(&output.stdout).into_owned() + }; + + let envelope: serde_json::Value = serde_json::from_str(&run(true)).unwrap(); + let human = run(false); + + assert_eq!( + envelope["remote"]["usage"], + serde_json::json!("1,253.50 GiB") + ); + assert_eq!(envelope["remote"]["limit"], serde_json::Value::Null); + assert!( + human.contains("1,253.50 GiB / Unlimited"), + "an unlimited cache should read as its dashboard does, got:\n{human}" + ); } #[tokio::test] @@ -577,7 +641,7 @@ async fn cache_status_human_renders_tables_and_slim_completion() { "cache status must render local and remote sections, got:\n{stdout}" ); assert!( - stdout.contains("1.0 KB / 2.0 KB █████░░░░░"), + stdout.contains("1 KiB / 2 KiB █████░░░░░"), "cache status must render remote usage with a quota bar, got:\n{stdout}" ); diff --git a/tests/workflows/tests/download.rs b/tests/workflows/tests/download.rs index a4cc82018..9d0485a94 100644 --- a/tests/workflows/tests/download.rs +++ b/tests/workflows/tests/download.rs @@ -527,6 +527,15 @@ async fn download_human_output_uses_slim_progress_and_completion() { stderr.contains("output:") && stderr.contains("files extracted:"), "download must report extraction details on stderr, got:\n{stderr}" ); + let size_row = format!( + "{:<16} {}", + "size:", + lpm_common::format_bytes(tarball.len() as u64) + ); + assert!( + stderr.contains(&size_row), + "download must state the tarball's size in binary units, {size_row:?}, got:\n{stderr}" + ); assert!( stderr.contains("✓ Done · tarball extracted in "), "download must report slim timed completion, got:\n{stderr}" diff --git a/tests/workflows/tests/graph.rs b/tests/workflows/tests/graph.rs index 5dc50e319..8ec0da286 100644 --- a/tests/workflows/tests/graph.rs +++ b/tests/workflows/tests/graph.rs @@ -349,16 +349,18 @@ fn graph_html_writes_human_readable_size() { String::from_utf8_lossy(&output.stderr) ); - // The size suffix must read as KB or MB — never "0 KB" via integer - // division. The fixture renders well over 1 KB of HTML, so KB is the + // The size suffix must read as KiB or MiB — never "0 KiB" via integer + // division. The fixture renders well over 1 KiB of HTML, so KiB is the // expected unit. assert!( - combined.contains(" KB)") || combined.contains(" MB)"), + combined.contains(" KiB)") || combined.contains(" MiB)"), "size in success message should use a human-readable unit: {combined}" ); + let written = project.read_file(".lpm/graph.html").len() as u64; + let stated = format!("({})", lpm_common::format_bytes(written)); assert!( - !combined.contains("(0 KB)"), - "size must never collapse to 0 KB via integer truncation: {combined}" + combined.contains(&stated), + "the success message must state the written file's size, {stated}: {combined}" ); } diff --git a/tests/workflows/tests/publish.rs b/tests/workflows/tests/publish.rs index 835fe7175..81cc16e36 100644 --- a/tests/workflows/tests/publish.rs +++ b/tests/workflows/tests/publish.rs @@ -250,6 +250,644 @@ fn publish_dry_run_validates_package() { ); } +#[test] +fn publish_refuses_a_tarball_past_lpm_dev_s_limit_before_any_target_publishes() { + let project = TempProject::empty( + r#"{ + "name": "@lpm.dev/testuser.too-big", + "version": "1.0.0", + "main": "index.js", + "license": "MIT" + }"#, + ); + project.write_file("index.js", "module.exports = {}"); + // Incompressible, so the gzipped tarball stays past lpm.dev's 100 MiB. + std::fs::write( + project.path().join("blob.bin"), + incompressible(101 * 1024 * 1024), + ) + .unwrap(); + + let output = lpm(&project) + .args(["publish", "--dry-run", "--yes", "--json"]) + .output() + .expect("failed to run lpm publish --dry-run"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + let envelope: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|error| panic!("publish --json must print JSON: {error}\n{stdout}")); + assert!( + envelope["error"] + .as_str() + .is_some_and(|error| error.contains("tarball too large for the LPM registry")), + "{stdout}" + ); +} + +/// `bytes` that gzip can't shrink. +fn incompressible(bytes: usize) -> Vec { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + (0..bytes.div_ceil(8)) + .flat_map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state.to_le_bytes() + }) + .take(bytes) + .collect() +} + +#[test] +fn publish_refuses_a_request_past_lpm_dev_s_limit_whose_tarball_is_within_its_own() { + // The tarball stays under 100 MiB. Its base64, package.json's long field + // and a README whose control characters JSON writes six bytes each, sent + // twice, take the request past 150 MiB. + let filler = "x".repeat(15 * 1024 * 1024); + let project = TempProject::empty(&format!( + r#"{{ + "name": "@lpm.dev/testuser.too-big-request", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "filler": "{filler}" + }}"# + )); + project.write_file("index.js", "module.exports = {}"); + project.write_file("README.md", &"\u{1}".repeat(1024 * 1024)); + std::fs::write( + project.path().join("blob.bin"), + incompressible(96 * 1024 * 1024), + ) + .unwrap(); + + let output = lpm(&project) + .args(["publish", "--dry-run", "--yes", "--json"]) + .output() + .expect("failed to run lpm publish --dry-run"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + let envelope: serde_json::Value = serde_json::from_str(&stdout) + .unwrap_or_else(|error| panic!("publish --json must print JSON: {error}\n{stdout}")); + assert!( + envelope["error"] + .as_str() + .is_some_and(|error| error.contains("publish request too large for the LPM registry")), + "{stdout}" + ); +} + +/// A project publishing to lpm.dev and to an npm-compatible registry at +/// `registry`. +fn lpm_and_npm_project(name: &str, registry: &str) -> TempProject { + let project = TempProject::empty(&format!( + r#"{{ + "name": "@lpm.dev/testuser.{name}", + "version": "1.0.0", + "main": "index.js", + "license": "MIT" + }}"# + )); + project.write_file("index.js", "module.exports = {}"); + project.write_file( + "lpm.json", + &format!( + r#"{{"publish":{{"npm":{{"name":"{name}","registry":"{registry}","access":"public"}}}}}}"# + ), + ); + project +} + +fn publish_to_lpm_and_npm(project: &TempProject, registry: &str) -> std::process::Output { + let login = lpm(project) + .args([ + "--json", + "login", + "--login-registry", + registry, + "--token", + CUSTOM_REGISTRY_TOKEN, + ]) + .output() + .expect("store the npm registry's token"); + assert!(login.status.success(), "{login:?}"); + lpm_with_registry(project, registry) + .env("LPM_RETRY_BACKOFF_MS_OVERRIDE", "0") + .args([ + "--json", + "publish", + "--yes", + "--token", + "test-token-123", + "--lpm", + "--npm", + ]) + .output() + .expect("run publish") +} + +async fn mount_preflight(mock: &MockRegistry, status: u16, body: serde_json::Value) { + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(status).set_body_json(body)) + .mount(mock.server()) + .await; +} + +async fn preflight_sizes(mock: &MockRegistry) -> Vec { + mock.server() + .received_requests() + .await + .unwrap() + .iter() + .filter(|request| request.url.path() == "/api/registry/-/package/publish-preflight") + .filter_map(|request| { + request + .url + .query_pairs() + .find(|(key, _)| key == "size") + .map(|(_, value)| value.parse().unwrap()) + }) + .collect() +} + +#[tokio::test] +async fn publish_publishes_no_target_when_lpm_dev_refuses_it_for_good() { + for (status, code, error) in [ + ( + 403, + Some("STORAGE_LIMIT_EXCEEDED"), + "Storage limit exceeded (100 MiB). Upgrade to Pro for more storage.", + ), + ( + 402, + Some("STORAGE_OVERAGE_PAUSED"), + "Storage past what your plan includes is refused until billing is active again.", + ), + ( + 402, + Some("ORG_BILLING_LAPSED"), + "Organization billing is past due. Package publishing is blocked.", + ), + ( + 403, + Some("PACKAGE_LIMIT_EXCEEDED"), + "Private package limit exceeded (2). Upgrade to Pro for unlimited packages.", + ), + ( + 413, + Some("TARBALL_TOO_LARGE"), + "Tarball too large (101 MiB). Maximum single package size is 100 MiB.", + ), + (403, None, "Token does not have publish permissions"), + ] { + let mock = MockRegistry::start().await; + let mut body = serde_json::json!({ "error": error }); + if let Some(code) = code { + body["code"] = serde_json::json!(code); + } + mount_preflight(&mock, status, body).await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({}))) + .expect(0) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("refused", &mock.url()); + + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{code:?}: {stdout}"); + assert!( + stdout.contains( + "LPM.dev refuses @lpm.dev/testuser.refused@1.0.0, so no target was published" + ) && stdout.contains(error), + "{code:?}: {stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(preflight_sizes(&mock).await.len(), 1, "{code:?}"); + } +} + +#[tokio::test] +async fn publish_asks_for_lpm_dev_s_second_factor_before_any_target_publishes() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/whoami")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "username": "test@example.com", + "profile_username": "testuser", + "mfa_enabled": true, + }))) + .mount(mock.server()) + .await; + mount_preflight( + &mock, + 200, + serde_json::json!({ + "success": true, + "name": "@lpm.dev/testuser.guarded", + "version": "1.0.0", + "packageExists": false, + }), + ) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({}))) + .expect(0) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("guarded", &mock.url()); + + // `--json` can't ask for the code lpm.dev needs, so no target publishes. + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + // The publish stops as a whole, before any target's result. + let envelope = parse_json_output(&output.stdout); + assert!( + envelope["error"] + .as_str() + .is_some_and(|error| error.contains("2FA required in JSON mode")) + && envelope.get("results").is_none(), + "{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +/// The uploads a publish made: to lpm.dev's publish route, and to the npm +/// registry's. +async fn uploads(mock: &MockRegistry) -> (usize, usize) { + let requests = mock.server().received_requests().await.unwrap(); + let puts = requests + .iter() + .filter(|request| request.method.as_str() == "PUT"); + let lpm = puts + .clone() + .filter(|request| request.url.path().starts_with("/api/registry/")) + .count(); + (lpm, puts.count() - lpm) +} + +async fn mount_accepted_preflight(mock: &MockRegistry, name: &str) { + mount_preflight( + mock, + 200, + serde_json::json!({ + "success": true, + "name": format!("@lpm.dev/testuser.{name}"), + "version": "1.0.0", + "packageExists": false, + }), + ) + .await; +} + +#[tokio::test] +async fn publish_publishes_the_other_targets_when_lpm_dev_s_account_can_t_be_read() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/whoami")) + .respond_with(ResponseTemplate::new(503).set_body_json(serde_json::json!({ + "error": "Service unavailable", + }))) + .mount(mock.server()) + .await; + mount_accepted_preflight(&mock, "unread").await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({}))) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("unread", &mock.url()); + + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + let envelope = parse_json_output(&output.stdout); + let results = envelope["results"].as_array(); + assert!( + results.is_some_and(|results| results.len() == 2), + "each target should report its own result:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(uploads(&mock).await, (0, 1), "{stdout}"); +} + +#[tokio::test] +async fn publish_publishes_no_target_when_lpm_dev_refuses_the_token() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/whoami")) + .respond_with(ResponseTemplate::new(401).set_body_json(serde_json::json!({ + "error": "Unauthorized", + }))) + .mount(mock.server()) + .await; + mount_accepted_preflight(&mock, "unauthorized").await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({}))) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("unauthorized", &mock.url()); + + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + let envelope = parse_json_output(&output.stdout); + assert!( + envelope["error"] + .as_str() + .is_some_and(|error| error.contains("authentication failed")) + && envelope.get("results").is_none(), + "{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(uploads(&mock).await, (0, 0), "{stdout}"); +} + +#[tokio::test] +async fn publish_asks_no_second_factor_for_a_version_lpm_dev_already_has() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/whoami")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "username": "test@example.com", + "profile_username": "testuser", + "mfa_enabled": true, + }))) + .mount(mock.server()) + .await; + mount_preflight( + &mock, + 403, + serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + "code": "VERSION_EXISTS", + }), + ) + .await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(201).set_body_json(serde_json::json!({}))) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("partial-mfa", &mock.url()); + + // As after a partial publish: lpm.dev has the version, npm doesn't. + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!stdout.contains("2FA required"), "{stdout}"); + let envelope = parse_json_output(&output.stdout); + let lpm_error = envelope["results"] + .as_array() + .and_then(|results| { + results + .iter() + .find(|result| result["registry"] == serde_json::json!("lpm")) + }) + .and_then(|result| result["error"].as_str()); + assert!( + lpm_error.is_some_and(|error| error.contains("already exists")), + "{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(uploads(&mock).await, (0, 1), "{stdout}"); +} + +#[tokio::test] +async fn publish_leaves_each_target_to_its_upload_when_lpm_dev_s_answer_isnt_final() { + // A version lpm.dev already has, as after a partial publish, also from + // a Registry that answers it without a code, and a server error that + // outlasts the retries. + for (status, body) in [ + ( + 403, + serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + "code": "VERSION_EXISTS", + }), + ), + ( + 403, + serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + }), + ), + (503, serde_json::json!({ "error": "Service unavailable" })), + // A request it can't read, an unknown package, a rate limit, and an + // answer that can't be read, which leaves the Registry unasked. + (400, serde_json::json!({ "error": "Invalid version" })), + (404, serde_json::json!({ "error": "Not found" })), + (429, serde_json::json!({ "error": "Too many requests" })), + (200, serde_json::json!("not a preflight answer")), + ] { + let mock = MockRegistry::start().await; + mock.with_whoami("testuser", "test@example.com").await; + mount_preflight(&mock, status, body).await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(409).set_body_json(serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + "code": "VERSION_EXISTS", + }))) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("partial", &mock.url()); + + let output = publish_to_lpm_and_npm(&project, &mock.url()); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("so no target was published"), + "{status}: {stdout}" + ); + let requests = mock.server().received_requests().await.unwrap(); + assert!( + requests + .iter() + .any(|request| request.method.as_str() == "PUT"), + "{status}: a target should have uploaded, got:\n{stdout}\n{}", + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[tokio::test] +async fn publish_asks_lpm_dev_about_the_tarball_it_is_sent() { + let refusing = MockRegistry::start().await; + mount_preflight( + &refusing, + 403, + serde_json::json!({ + "error": "Storage limit exceeded (100 MiB). Upgrade to Pro for more storage.", + "code": "STORAGE_LIMIT_EXCEEDED", + }), + ) + .await; + let project = lpm_and_npm_project("sized", &refusing.url()); + project.write_file("data.bin", &"lpm ".repeat(40_000)); + + // A publish to both targets, and a dry run, ask with a size. + let multi = publish_to_lpm_and_npm(&project, &refusing.url()); + assert!(!multi.status.success()); + let dry_run = lpm_with_registry(&project, &refusing.url()) + .args(["--json", "publish", "--dry-run", "--yes", "--lpm"]) + .output() + .expect("run publish dry-run"); + assert!(!dry_run.status.success()); + let sizes = preflight_sizes(&refusing).await; + + // The upload of the same project sends a tarball of that size. + let accepting = MockRegistry::start().await; + accepting.with_publish_endpoint().await; + accepting.with_whoami("testuser", "test@example.com").await; + let published = lpm_with_registry(&project, &accepting.url()) + .args(["publish", "--yes", "--token", "test-token-123", "--lpm"]) + .output() + .expect("run publish"); + assert!( + published.status.success(), + "{}", + String::from_utf8_lossy(&published.stderr) + ); + let requests = accepting.server().received_requests().await.unwrap(); + let upload = requests + .iter() + .find(|request| request.method.as_str() == "PUT") + .expect("publish must upload"); + let payload: serde_json::Value = serde_json::from_slice(&upload.body).unwrap(); + let attachment = payload["_attachments"] + .as_object() + .and_then(|attachments| attachments.values().next()) + .expect("the upload carries its tarball"); + let uploaded = BASE64 + .decode(attachment["data"].as_str().unwrap()) + .unwrap() + .len() as u64; + + assert!(uploaded > 0); + assert_eq!(sizes, vec![uploaded, uploaded]); +} + +#[tokio::test] +async fn publish_dry_run_retries_lpm_dev_s_preflight_and_names_its_status() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(503).set_body_string("unavailable")) + .expect(4) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("unavailable", &mock.url()); + + let output = lpm_with_registry(&project, &mock.url()) + .env("LPM_RETRY_BACKOFF_MS_OVERRIDE", "0") + .args(["--json", "publish", "--dry-run", "--yes", "--lpm"]) + .output() + .expect("run publish dry-run"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + assert!( + stdout.contains("LPM.dev publish preflight failed for @lpm.dev/testuser.unavailable@1.0.0") + && stdout.contains("HTTP 503") + && !stdout.contains(""), + "{stdout}" + ); + mock.server().verify().await; +} + +#[tokio::test] +async fn publish_dry_run_fails_when_lpm_dev_s_preflight_answer_cant_be_read() { + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(200).set_body_string("not a preflight answer")) + .mount(mock.server()) + .await; + let project = lpm_and_npm_project("unreadable", &mock.url()); + + let output = lpm_with_registry(&project, &mock.url()) + .args(["--json", "publish", "--dry-run", "--yes", "--lpm"]) + .output() + .expect("run publish dry-run"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + assert!( + stdout.contains("LPM.dev publish preflight failed for @lpm.dev/testuser.unreadable@1.0.0"), + "{stdout}" + ); +} + +#[tokio::test] +async fn publish_dry_run_reports_what_lpm_dev_refuses() { + let mock = MockRegistry::start().await; + mount_preflight( + &mock, + 402, + serde_json::json!({ + "error": "Storage past what your plan includes is refused until billing is active again.", + "code": "STORAGE_OVERAGE_PAUSED", + }), + ) + .await; + let project = lpm_and_npm_project("paused", &mock.url()); + + let output = lpm_with_registry(&project, &mock.url()) + .args(["--json", "publish", "--dry-run", "--yes", "--lpm"]) + .output() + .expect("run publish dry-run"); + + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "{stdout}"); + assert!( + stdout.contains("LPM.dev publish preflight failed for @lpm.dev/testuser.paused@1.0.0") + && stdout.contains("refused until billing is active again"), + "{stdout}" + ); +} + +#[tokio::test] +async fn publish_redacts_lpm_dev_s_reflection_of_its_bearer_from_a_refusal() { + let token = "test-token-123"; + let mock = MockRegistry::start().await; + mount_preflight( + &mock, + 403, + serde_json::json!({ "error": format!("Bearer {token} may not publish") }), + ) + .await; + let project = lpm_and_npm_project("reflected", &mock.url()); + + for json in [true, false] { + let mut command = lpm_with_registry(&project, &mock.url()); + if json { + command.arg("--json"); + } + let output = command + .env("LPM_RETRY_BACKOFF_MS_OVERRIDE", "0") + .args(["publish", "--yes", "--token", token, "--lpm", "--npm"]) + .output() + .expect("run publish"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "{stdout}"); + assert!( + !stdout.contains(token) && !stderr.contains(token), + "bearer leaked (json: {json})\nstdout: {stdout}\nstderr: {stderr}" + ); + assert!( + format!("{stdout}{stderr}").contains("Bearer may not publish"), + "json: {json}\nstdout: {stdout}\nstderr: {stderr}" + ); + } +} + #[tokio::test] async fn publish_dry_run_with_authored_skills_leaves_project_files_unchanged() { let mock = MockRegistry::start().await; diff --git a/tests/workflows/tests/release.rs b/tests/workflows/tests/release.rs index 4418d8fcb..80b063744 100644 --- a/tests/workflows/tests/release.rs +++ b/tests/workflows/tests/release.rs @@ -1762,6 +1762,204 @@ async fn release_publish_checks_the_requested_version_when_the_lpm_package_exist server.verify().await; } +#[tokio::test] +async fn release_publish_skips_a_version_lpm_dev_says_it_already_has() { + let project = TempProject::empty( + r#"{"name":"root","private":true,"version":"0.0.0","workspaces":["packages/*"]}"#, + ); + let name = "@lpm.dev/acme.published"; + project.write_file( + "packages/published/package.json", + &format!(r#"{{"name":"{name}","version":"1.0.0"}}"#), + ); + // As the Registry answers a version it has: a refusal with its code, + // or, from one that sends no code, its message alone. + for answer in [ + serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + "code": "VERSION_EXISTS", + }), + serde_json::json!({ + "error": "Version 1.0.0 already exists. You cannot overwrite versions.", + }), + ] { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(403).set_body_json(&answer)) + .expect(1) + .mount(&server) + .await; + + let output = lpm_with_registry(&project, &server.uri()) + .args([ + "release", + "publish", + "--all", + "--lpm", + "--dry-run", + "--json", + ]) + .output() + .expect("run release publish against a version lpm.dev has"); + + assert!( + output.status.success(), + "release publish failed for {answer}\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + let result = &parse_json_output(&output.stdout)["results"][0]; + assert_eq!(result["status"], "skipped", "{answer}: {result}"); + assert_eq!(result["reason"], "already_published", "{answer}: {result}"); + server.verify().await; + } +} + +#[tokio::test] +async fn release_publish_reports_what_lpm_dev_refuses_in_its_preflight() { + let project = TempProject::empty( + r#"{"name":"root","private":true,"version":"0.0.0","workspaces":["packages/*"]}"#, + ); + project.write_file( + "packages/lapsed/package.json", + r#"{"name":"@lpm.dev/acme.lapsed","version":"1.0.0"}"#, + ); + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(ResponseTemplate::new(402).set_body_json(serde_json::json!({ + "error": "Organization billing is past due. Package publishing is blocked.", + "code": "ORG_BILLING_LAPSED", + }))) + .mount(&server) + .await; + + let output = lpm_with_registry(&project, &server.uri()) + .args([ + "release", + "publish", + "--all", + "--lpm", + "--dry-run", + "--json", + ]) + .output() + .expect("run release publish against a refused member"); + + assert!(!output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + // The refusal is the error any failed request is, with the Registry's + // message rather than its body. + let envelope = parse_json_output(&output.stdout); + assert_eq!(envelope["error_code"], "http", "{stdout}"); + assert!( + stdout.contains("Organization billing is past due.") && !stdout.contains("{\\\"error"), + "{stdout}" + ); +} + +#[tokio::test] +async fn release_publish_redacts_lpm_dev_s_reflection_of_its_bearer_from_a_refusal() { + let token = "test-token-123"; + for (status, json_body) in [(402, true), (403, true), (403, false), (500, false)] { + let project = TempProject::empty( + r#"{"name":"root","private":true,"version":"0.0.0","workspaces":["packages/*"]}"#, + ); + project.write_file( + "packages/reflected/package.json", + r#"{"name":"@lpm.dev/acme.reflected","version":"1.0.0"}"#, + ); + let server = MockServer::start().await; + let reflected = format!("Authorization: Bearer {token} may not publish"); + let response = if json_body { + ResponseTemplate::new(status).set_body_json(serde_json::json!({ "error": reflected })) + } else { + ResponseTemplate::new(status).set_body_string(reflected) + }; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with(response) + .mount(&server) + .await; + + for json in [true, false] { + let mut command = lpm_with_registry(&project, &server.uri()); + command + .env("LPM_RETRY_BACKOFF_MS_OVERRIDE", "0") + .args(["release", "publish", "--all", "--lpm", "--dry-run"]) + .args(["--token", token]); + if json { + command.arg("--json"); + } + let output = command.output().expect("run release publish"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "{status}: {stdout}"); + assert!( + !stdout.contains(token) && !stderr.contains(token), + "bearer leaked ({status}, JSON body: {json_body}, --json: {json})\nstdout: {stdout}\nstderr: {stderr}" + ); + if json_body { + assert!( + format!("{stdout}{stderr}").contains("Bearer may not publish"), + "{status}, --json: {json}\nstdout: {stdout}\nstderr: {stderr}" + ); + } + } + } +} + +#[tokio::test] +async fn release_publish_reports_a_preflight_refusal_as_the_error_its_status_is() { + for (status, error_code) in [ + (401, "auth_required"), + (403, "forbidden"), + (404, "not_found"), + (429, "rate_limited"), + ] { + let project = TempProject::empty( + r#"{"name":"root","private":true,"version":"0.0.0","workspaces":["packages/*"]}"#, + ); + project.write_file( + "packages/refused/package.json", + r#"{"name":"@lpm.dev/acme.refused","version":"1.0.0"}"#, + ); + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/package/publish-preflight")) + .respond_with( + ResponseTemplate::new(status) + .insert_header("retry-after", "0") + .set_body_json(serde_json::json!({ "error": "Refused by the registry" })), + ) + .mount(&server) + .await; + + let output = lpm_with_registry(&project, &server.uri()) + .args([ + "release", + "publish", + "--all", + "--lpm", + "--dry-run", + "--json", + ]) + .output() + .expect("run release publish against a refused member"); + + assert!(!output.status.success(), "{status}"); + let envelope = parse_json_output(&output.stdout); + assert_eq!( + envelope["error_code"], + error_code, + "{status}: {}", + String::from_utf8_lossy(&output.stdout) + ); + } +} + #[tokio::test] async fn release_publish_revalidates_fresh_metadata_before_deciding_version_absence() { let project = TempProject::empty( diff --git a/tests/workflows/tests/skills.rs b/tests/workflows/tests/skills.rs index 6894b0203..5913006f4 100644 --- a/tests/workflows/tests/skills.rs +++ b/tests/workflows/tests/skills.rs @@ -2887,7 +2887,7 @@ fn skills_validate_rejects_skill_exceeding_size_limit() { String::from_utf8_lossy(&output.stderr), ); assert!( - combined.contains("15KB") || combined.contains("exceed") || combined.contains("limit"), + combined.contains("exceeds 15 KiB limit ("), "validate output must surface the per-skill size-limit warning, got:\n{combined}", ); let stderr = String::from_utf8_lossy(&output.stderr); @@ -2940,7 +2940,7 @@ fn skills_validate_json_reports_success_false_and_exits_non_zero_on_error() { .as_str() .expect("each error entry must be a string"); assert!( - error_text.contains("15KB") || error_text.contains("limit"), + error_text.contains("exceeds 15 KiB limit ("), "error text must name the violated limit, got {error_text:?}", ); } diff --git a/tests/workflows/tests/snapshots/auth_lifecycle__whoami_json_envelope_recovers_from_refresh_only.snap b/tests/workflows/tests/snapshots/auth_lifecycle__whoami_json_envelope_recovers_from_refresh_only.snap index 06abd321a..adf5033b0 100644 --- a/tests/workflows/tests/snapshots/auth_lifecycle__whoami_json_envelope_recovers_from_refresh_only.snap +++ b/tests/workflows/tests/snapshots/auth_lifecycle__whoami_json_envelope_recovers_from_refresh_only.snap @@ -1,6 +1,5 @@ --- source: tests/workflows/tests/auth_lifecycle.rs -assertion_line: 278 expression: json --- { @@ -23,8 +22,11 @@ expression: json ], "storage_backend": "encrypted_file_fallback", "storage_degraded": true, + "storage_excess_billed": null, + "storage_overage": null, "success": true, "usage": { + "counted_storage_bytes": null, "private_packages": 3, "storage_bytes": 52428800 }, diff --git a/tests/workflows/tests/whoami.rs b/tests/workflows/tests/whoami.rs index fbe9903fc..317f45081 100644 --- a/tests/workflows/tests/whoami.rs +++ b/tests/workflows/tests/whoami.rs @@ -74,11 +74,11 @@ async fn whoami_human_output_uses_slim_account_summary() { "mfa_enabled": false, "has_pool_access": true, "usage": { - "storage_bytes": 1024 * 1024 * 50, + "storage_bytes": 1024 * 1024 * 512 + 1024 * 100, "private_packages": 3 }, "limits": { - "storageBytes": 1024 * 1024 * 500, + "storageBytes": 1024_u64 * 1024 * 1024 * 5, "privatePackages": 100 }, "organizations": [ @@ -149,7 +149,8 @@ async fn whoami_human_output_uses_slim_account_summary() { "authenticated whoami should warn when auth storage uses the encrypted file fallback, got:\n{stderr}" ); assert!( - stderr.contains("usage") && stderr.contains("storage 50.00MB / 500MB █░░░░░░░░░"), + stderr.contains("usage") + && stderr.contains("storage 512.09 MiB / 5 GiB ██░░░░░░░░"), "authenticated whoami should show storage usage with a quota bar, got:\n{stderr}" ); assert!( @@ -204,6 +205,434 @@ async fn whoami_human_output_uses_slim_account_summary() { ); } +/// Human `lpm whoami` stderr for an account the registry describes as +/// `account`. +async fn whoami_stderr(account: serde_json::Value) -> String { + let project = TempProject::empty(r#"{"name":"whoami","version":"1.0.0"}"#); + let mock = MockRegistry::start().await; + Mock::given(method("GET")) + .and(path("/api/registry/-/whoami")) + .and(header("authorization", "Bearer access-primary")) + .respond_with(ResponseTemplate::new(200).set_body_json(account)) + .mount(mock.server()) + .await; + seed_sessions( + project.home(), + &[SessionSeed { + registry_url: &mock.url(), + access_token: Some("access-primary"), + refresh_token: None, + session_access_expires_at: None, + }], + ); + + let output = lpm_with_registry(&project, &mock.url()) + .arg("whoami") + .output() + .expect("failed to run lpm whoami"); + assert!( + output.status.success(), + "whoami must exit 0, stderr:\n{}", + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +const GIB: u64 = 1024 * 1024 * 1024; + +#[tokio::test] +async fn whoami_tells_a_pro_account_past_its_storage_that_it_is_billed_not_restricted() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB + 51 * 1024 * 1024, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "billed", + })) + .await; + + assert!( + stderr.contains("storage 5.05 GiB / 5 GiB") + && stderr.contains( + "overage storage past 5 GiB is billed per GiB a month, rounded up" + ), + "a Pro account past its storage should read as past it and billed, got:\n{stderr}" + ); + assert!( + !stderr.contains("OVER LIMIT") + && !stderr.contains("restricted") + && !stderr.contains("Upgrade"), + "a Pro account's billed storage must not read as restricted, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_a_pro_account_at_its_storage_that_the_next_publish_is_billed() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "billed", + })) + .await; + + assert!( + stderr.contains("storage 5 GiB / 5 GiB") + && stderr.contains( + "overage storage past 5 GiB is billed per GiB a month, rounded up" + ), + "a Pro account at its storage should be told what past it costs, got:\n{stderr}" + ); + let below = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB - 1, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "billed", + })) + .await; + assert!(!below.contains("overage "), "{below}"); +} + +#[tokio::test] +async fn whoami_says_whether_storage_already_past_the_limit_is_billed_while_billing_isnt_active() { + for (billed, said) in [ + ( + serde_json::json!(true), + Some("Storage already past it is still billed while the payment is retried."), + ), + ( + serde_json::json!(false), + Some("Storage already past it isn't billed meanwhile."), + ), + (serde_json::Value::Null, None), + ] { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB + 1, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "paused", + "storage_excess_billed": billed, + })) + .await; + + match said { + Some(said) => assert!(stderr.contains(said), "{billed}: {stderr}"), + None => assert!(!stderr.contains("Storage already past it"), "{stderr}"), + } + } + + // At the limit, nothing is past it yet, but the next publish is refused. + let at = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "paused", + "storage_excess_billed": true, + })) + .await; + assert!( + at.contains("Publishing past 5 GiB is refused until billing is active again.") + && !at.contains("Storage already past it"), + "{at}" + ); +} + +#[tokio::test] +async fn whoami_tells_a_pro_account_without_a_subscription_at_its_storage_that_past_it_is_refused() +{ + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains( + "Storage past 5 GiB isn't billed on this plan, so publishing to your personal scope past it is refused." + ) && !stderr.contains("OVER LIMIT"), + "{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_a_free_account_past_its_storage_that_publishing_is_restricted() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { "storage_bytes": 100 * 1024 * 1024 + 1, "private_packages": 1 }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("Storage: 100.01 MiB / 100 MiB") + && stderr.contains("(OVER LIMIT)") + && stderr.contains( + "Publishing to your personal scope is refused while its storage is at or past 100 MiB." + ), + "a free account past its storage should read as over its limit and refused, got:\n{stderr}" + ); + assert!( + !stderr.contains("New private packages"), + "an account within its private packages is told nothing of them, got:\n{stderr}" + ); + assert!( + !stderr.contains("inviting members"), + "a personal account invites no members, got:\n{stderr}" + ); + assert!( + stderr.contains("Hobby") && !stderr.contains("Free") && !stderr.contains("exempt"), + "the free plan reads as Hobby, and storage nothing exempts shows no exempt line, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_reads_a_free_account_s_storage_as_its_limit_counts_it() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { + "storage_bytes": 150 * 1024 * 1024, + "counted_storage_bytes": 50 * 1024 * 1024, + "private_packages": 1, + }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("storage 50 MiB / 100 MiB") + && stderr.contains( + "exempt 100 MiB more, the tarballs of packages whose sales exempt them" + ), + "storage the limit doesn't count should read apart from it, got:\n{stderr}" + ); + assert!( + !stderr.contains("OVER LIMIT") && !stderr.contains("refused"), + "exempt storage takes no account past its limit, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_an_account_past_its_private_packages_that_only_new_ones_are_refused() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { "storage_bytes": 1024, "private_packages": 3 }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("Private Packages: 3 / 2") + && stderr.contains( + "New private packages in your personal scope are refused once it holds 2." + ), + "an account past its private packages should read as refusing new ones, got:\n{stderr}" + ); + assert!( + !stderr.contains("Publishing to your personal scope is refused"), + "storage within its limit refuses nothing, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_an_account_whose_billing_isnt_active_that_publishing_past_its_storage_waits_for_it() + { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB + 1, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "paused", + })) + .await; + + assert!( + stderr.contains("Storage: 5.01 GiB / 5 GiB") + && stderr.contains("Publishing past 5 GiB is refused until billing is active again.") + && stderr.contains("Manage billing"), + "an account whose billing isn't active should read as refused until it is, got:\n{stderr}" + ); + assert!( + !stderr.contains("payment") && !stderr.contains("overage is paused"), + "a subscription that ended in its grace owes no payment, got:\n{stderr}" + ); + assert!( + !stderr.contains("is billed per GiB") && !stderr.contains("Upgrade your plan"), + "publishing past storage that waits for billing is neither billed nor an upgrade's to fix, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_claims_nothing_of_storage_past_its_limit_when_the_registry_doesnt_say() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB + 1, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + })) + .await; + + assert!( + stderr.contains("storage 5.01 GiB / 5 GiB") + && stderr.contains("limit past the 5 GiB limit"), + "storage past its limit should read as past it, got:\n{stderr}" + ); + assert!( + !stderr.contains("billed") + && !stderr.contains("overage") + && !stderr.contains("included") + && !stderr.contains("OVER LIMIT") + && !stderr.contains("restricted"), + "without the Registry's word, whoami should claim neither billing nor a refusal, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_a_free_account_past_its_storage_only_where_publishing_stops_when_the_registry_doesnt_say_what_counts() + { + // Packages whose sales exempt them don't count against the limit, and + // this Registry doesn't say how much of the storage is theirs. + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { "storage_bytes": 100 * 1024 * 1024 + 1, "private_packages": 1 }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + })) + .await; + + assert!( + stderr.contains("storage 100.01 MiB / 100 MiB") + && stderr.contains("limit past the 100 MiB limit") + && stderr.contains( + "Publishing to your personal scope is refused while its storage is at or past 100 MiB, apart from packages whose sales exempt them." + ), + "the refusal should be told with what it leaves out, got:\n{stderr}" + ); + assert!( + !stderr.contains("OVER LIMIT") + && !stderr.contains("over its plan limits") + && !stderr.contains("refused while its storage is at or past 100 MiB.\n"), + "without the counted storage, the account isn't said to be refused, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_a_pro_account_without_a_subscription_that_its_storage_past_the_limit_isnt_billed() + { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 5 * GIB + 1, "private_packages": 3 }, + "limits": { "storageBytes": 5 * GIB, "privatePackages": 100 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("Storage: 5.01 GiB / 5 GiB") + && stderr.contains("(OVER LIMIT)") + && stderr.contains( + "Storage past 5 GiB isn't billed on this plan, so publishing to your personal scope past it is refused." + ), + "a Pro account without a subscription should read as refused past its storage, got:\n{stderr}" + ); + assert!( + !stderr.contains("Upgrade your plan") && !stderr.contains("Subscribe"), + "an account on Pro without a subscription can neither upgrade nor take one out, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_tells_an_account_at_its_private_packages_that_new_ones_are_refused() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { "storage_bytes": 1024, "private_packages": 2 }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("private pkgs 2 / 2") + && stderr.contains( + "New private packages in your personal scope are refused once it holds 2." + ) + && stderr.contains("Upgrade your plan"), + "an account at its private packages should read as refusing new ones, got:\n{stderr}" + ); + assert!( + !stderr.contains("OVER LIMIT") && !stderr.contains("over its plan limits"), + "an account at its limit isn't over it, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_reads_storage_at_its_limit_as_within_it() { + let stderr = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "free", + "usage": { "storage_bytes": 100 * 1024 * 1024, "private_packages": 1 }, + "limits": { "storageBytes": 100 * 1024 * 1024, "privatePackages": 2 }, + "storage_overage": "blocked", + })) + .await; + + assert!( + stderr.contains("storage 100 MiB / 100 MiB"), + "storage at its limit should read as within it, got:\n{stderr}" + ); + assert!( + !stderr.contains("OVER LIMIT") + && !stderr.contains("restricted") + && !stderr.contains("over its plan limits"), + "storage at its limit isn't past it, got:\n{stderr}" + ); + // But any publish would take it past. + assert!( + stderr.contains( + "Publishing to your personal scope is refused while its storage is at or past 100 MiB." + ), + "storage at its limit refuses the next publish, got:\n{stderr}" + ); +} + +#[tokio::test] +async fn whoami_shows_storage_alone_without_a_storage_limit() { + let with_other_limits = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 3 * GIB + GIB / 2, "private_packages": 3 }, + "limits": { "privatePackages": 100 }, + })) + .await; + let without_limits = whoami_stderr(serde_json::json!({ + "username": "testuser", + "plan_tier": "pro", + "usage": { "storage_bytes": 3 * GIB + GIB / 2, "private_packages": 3 }, + })) + .await; + + for stderr in [with_other_limits, without_limits] { + assert!( + stderr.contains("storage 3.5 GiB") && !stderr.contains("3.5 GiB /"), + "storage without a limit should read as its size alone, got:\n{stderr}" + ); + } +} + #[tokio::test] async fn whoami_json_reports_file_backed_auth_storage_backend() { let project = TempProject::empty(r#"{"name":"whoami","version":"1.0.0"}"#); @@ -214,7 +643,14 @@ async fn whoami_json_reports_file_backed_auth_storage_backend() { .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "username": "test@example.com", "profile_username": "testuser", - "email": "test@example.com" + "email": "test@example.com", + "storage_overage": "paused", + "storage_excess_billed": false, + "usage": { + "storage_bytes": 150 * 1024 * 1024, + "counted_storage_bytes": 50 * 1024 * 1024, + "private_packages": 1, + }, }))) .mount(mock.server()) .await; @@ -247,6 +683,12 @@ async fn whoami_json_reports_file_backed_auth_storage_backend() { serde_json::json!("encrypted_file_fallback") ); assert_eq!(json["storage_degraded"], serde_json::json!(true)); + assert_eq!(json["storage_overage"], serde_json::json!("paused")); + assert_eq!(json["storage_excess_billed"], serde_json::json!(false)); + assert_eq!( + json["usage"]["counted_storage_bytes"], + serde_json::json!(50 * 1024 * 1024) + ); assert_eq!( json["registries"][0]["storage_backend"], serde_json::json!("encrypted_file_fallback")