From dd58419bdc4e0708ddb0b826fcae2bada3474bb6 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 22 Jul 2026 17:56:46 -0700 Subject: [PATCH 1/2] feat: re-export reqwest and add opt-in reqwest-gzip feature Re-export reqwest as oneio::reqwest (under the http feature) so downstream crates can name HTTP types exposed in oneio's public API (StatusCode, header, blocking::Response) without declaring their own reqwest dependency and risking version skew. Add opt-in reqwest-gzip feature that enables reqwest's gzip content-encoding: requests advertise Accept-Encoding: gzip and gzipped responses are transparently decoded. Off by default, so dependency trees are unchanged unless explicitly enabled. Distinct from the gz family, which is URL-suffix-based file decompression. Motivation: bgpkit-commons#33 (conditional RPKI loading) currently needs a direct reqwest dependency solely to name these types and enable gzip. --- .github/workflows/rust.yml | 4 + CHANGELOG.md | 4 + Cargo.toml | 5 + README.md | 4 +- specs/03-reqwest-reexport-http-gzip/README.md | 129 +++++++++++++++ specs/README.md | 1 + src/lib.rs | 43 +++++ tests/http_advanced_tests.rs | 147 ++++++++++++++++++ 8 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 specs/03-reqwest-reexport-http-gzip/README.md create mode 100644 tests/http_advanced_tests.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5cecb56..21a4754 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -114,6 +114,7 @@ jobs: cargo build --no-default-features --features http,rustls cargo build --no-default-features --features ftp,http,rustls cargo build --no-default-features --features s3,http,rustls + cargo build --no-default-features --features reqwest-gzip,rustls - name: Build CLI feature run: cargo build --no-default-features --features cli,http,gz,bz,rustls @@ -137,6 +138,9 @@ jobs: - name: Test with no features run: cargo test --no-default-features + - name: Test reqwest-gzip feature + run: cargo test --no-default-features --features reqwest-gzip,rustls --test http_advanced_tests + - name: Test documentation run: cargo test --doc --all-features diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c34de..cd45373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable changes to this project will be documented in this file. ### Bug fixes - Preserve leading-slash S3 object keys when using path-style endpoints such as Cloudflare R2. +### Added +- Re-exported `reqwest` as `oneio::reqwest` (under the `http` feature) so downstream crates can name HTTP types (`StatusCode`, `header`, `blocking::Response`) without declaring their own reqwest dependency and risking version skew. Note: this makes reqwest part of oneio's public API contract; a reqwest major-version bump is a breaking oneio change. +- New opt-in `reqwest-gzip` feature: advertises `Accept-Encoding: gzip` and transparently decodes `Content-Encoding: gzip` responses (e.g. ~97 MB to ~4.6 MB for `rpki.cloudflare.com/rpki.json`). Distinct from the `gz` family, which is URL-suffix-based file decompression. Off by default; no dependency-tree change unless enabled. + ## v0.23.0 -- 2026-05-12 ### Added diff --git a/Cargo.toml b/Cargo.toml index d315fdc..ac733dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,11 @@ https = ["http", "rustls"] # https needs http ftp = ["https", "suppaftp"] # ftp needs https s3 = ["rusty-s3", "http", "quick-xml", "percent-encoding", "dep:sha2", "dep:hmac", "dep:hex"] +# HTTP content-encoding (opt-in, additive passthrough to reqwest) +# Advertises `Accept-Encoding: gzip` and transparently decodes gzipped responses. +# Distinct from the `gz` family below, which is URL-suffix-based file decompression. +reqwest-gzip = ["http", "reqwest/gzip"] + gz = ["gz-zlib-rs"] # internal feature to enable gzip support any_gz = [] diff --git a/README.md b/README.md index 59875e0..c58d789 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ oneio = { version = "0.23", features = ["async"] } - `https` - HTTP/HTTPS with rustls TLS backend (equivalent to `http` + `rustls`) - `ftp` - FTP support (requires `http` + TLS backend) - `s3` - S3-compatible storage +- `reqwest-gzip` - Opt-in HTTP gzip content-encoding: advertises `Accept-Encoding: gzip` and transparently decodes gzipped responses (distinct from `gz`, which is URL-suffix-based file decompression) **TLS Backends** (for HTTPS - mutually exclusive): - `rustls` - Pure Rust TLS (use with `http`). Uses both system certificates and bundled Mozilla certificates for maximum compatibility. @@ -160,7 +161,8 @@ The `OneIo` client allows you to configure headers, TLS certificates, timeouts, ```rust use oneio::OneIo; -use reqwest::header::{HeaderName, HeaderValue}; +// reqwest is re-exported for naming HTTP types without a direct reqwest dependency +use oneio::reqwest::header::{HeaderName, HeaderValue}; // Build a reusable client with custom headers and certificates let oneio = OneIo::builder() diff --git a/specs/03-reqwest-reexport-http-gzip/README.md b/specs/03-reqwest-reexport-http-gzip/README.md new file mode 100644 index 0000000..15f0c4f --- /dev/null +++ b/specs/03-reqwest-reexport-http-gzip/README.md @@ -0,0 +1,129 @@ +# Spec: reqwest Re-export and Opt-in HTTP gzip + +**Status**: In Progress +**Author**: Mingwei Zhang +**Created**: 2026-07-22 +**Target Branch**: `dev/reqwest-reexport-http-gzip` + +## 1. Overview + +Re-export `reqwest` so downstream crates can name the HTTP types exposed through +oneio's public API, and add an opt-in `reqwest-gzip` feature that enables +reqwest's transparent gzip content-encoding support. + +**Non-goals:** +- Changing any default behavior — the gzip feature is off unless explicitly enabled +- Brotli/deflate content-encoding — can follow the same pattern later if requested +- Replacing oneio's suffix-based file decompression (`gz` family) — orthogonal concern +- Wrapping or abstracting reqwest's API — the goal is to expose it, not hide it + +**Success criteria:** +- [ ] `oneio::reqwest` resolves when the `http` feature is enabled (e.g. + `oneio::reqwest::StatusCode::NOT_MODIFIED` compiles downstream) +- [ ] With `reqwest-gzip`, requests advertise `Accept-Encoding: gzip` and + `Content-Encoding: gzip` responses are transparently decoded +- [ ] Without `reqwest-gzip`, dependency tree is unchanged (no gzip decoder crates) +- [ ] Conditional-GET workflow (send `If-None-Match`, read `304` status and + `ETag`/`Last-Modified` response headers) is possible using only oneio's API +- [ ] All quality gates pass: fmt, build, test, clippy (`-D warnings`) + +## 2. Current State + +`OneIo::get_http_reader_raw()` (since 0.21) returns `reqwest::blocking::Response` +in its public signature, and `OneIo::builder()` / `OneIo::http_client()` expose +reqwest types as well. However, reqwest is an internal dependency: downstream +crates cannot `use reqwest::...` unless they declare reqwest themselves, which +risks version skew (two reqwest major versions in the tree; the `Response` they +receive is not the `Response` they imported). Working around it via type +inference (`.status().as_u16() == 304`) is fragile and implicit. + +Separately, oneio's reqwest build enables `blocking`, `http2`, `charset`, +`stream` — but not `gzip`. Requests therefore do not advertise +`Accept-Encoding: gzip`, and `Content-Encoding: gzip` responses are not decoded. +oneio's own decompression is inferred from the URL suffix, so payload-gzipped +endpoints like `https://rpki.cloudflare.com/rpki.json` (~97 MB plain, ~4.6 MB +gzipped) cannot benefit. + +Concrete downstream need: bgpkit-commons#33 (conditional RPKI loading) currently +adds a direct reqwest dependency solely to name these types and enable gzip. + +## 3. Proposed Solution + +### Re-export + +```rust +#[cfg(feature = "http")] +pub use reqwest; +``` + +Downstream then uses `oneio::reqwest::{blocking::Client, StatusCode, header}` +guaranteed to be the exact reqwest oneio was built against. + +**Key decision — full re-export vs curated module:** full `pub use reqwest` +(chosen) is simpler and the type already leaks via `get_http_reader_raw`; a +curated `oneio::http` module would not actually reduce the semver exposure. + +**Semver implication (documented in the PR):** reqwest becomes part of oneio's +public API contract. A future reqwest 0.13 upgrade becomes a breaking oneio +change (same trade-off as tokio↔bytes). Acceptable: oneio is pre-1.0 and +already exposes reqwest types in signatures. + +### Feature flag + +```toml +reqwest-gzip = ["http", "reqwest/gzip"] +``` + +reqwest's `gzip` feature is purely additive: it advertises +`Accept-Encoding: gzip` and transparently decodes gzipped responses. No oneio +code changes or `#[cfg]` gating are required — the flag only forwards. + +**Key decision — naming:** `reqwest-gzip` (not `gzip`) to avoid confusion with +the existing `gz`/`gz-*` family, which is suffix-based file decompression. +Pattern extends naturally to `reqwest-brotli` / `reqwest-deflate` later. + +## 4. Implementation Plan + +1. **Cargo.toml**: add `reqwest-gzip = ["http", "reqwest/gzip"]` feature. + Acceptance: `cargo tree --no-default-features --features reqwest-gzip` shows + a gzip decoder backend; without the feature it does not. +2. **src/lib.rs**: add the gated re-export with doc comment; add `reqwest-gzip` + row to the feature table in the crate docs. + Acceptance: doc test using `oneio::reqwest::StatusCode` compiles. +3. **tests/**: gzip integration test (mock server, gzipped body with + `Content-Encoding: gzip`, assert transparent decode) gated on + `reqwest-gzip`; conditional-GET example test using the re-export. + Acceptance: tests pass under `--features reqwest-gzip` and are absent/skipped + otherwise. +4. **Docs/CI/changelog**: README feature mention, CI build/clippy legs for the + new feature, CHANGELOG entry under Unreleased. + Acceptance: CI matrix covers `reqwest-gzip`. + +## 5. Testing Strategy + +- Integration test with in-process `TcpListener` mock server (pattern already + used in `tests/basic_integration.rs`): + - serve a gzip-compressed body (compressed with flate2, already in-tree via + the `gz` features) with `Content-Encoding: gzip`; assert decoded content + matches and the request carried `Accept-Encoding: gzip` + - serve `304 Not Modified` after an initial `200` with `ETag`; assert the + client can read status and validators via `oneio::reqwest` types +- Doc test on the re-export (compile-time check downstream usage works) + +## 6. Risks + +- **Semver coupling**: reqwest version becomes public API → documented in + CHANGELOG and PR; mitigated by oneio being pre-1.0 with reqwest already in + public signatures. +- **Feature unification surprises**: a downstream crate enabling + `reqwest/gzip` itself would silently change oneio's wire behavior (gzip is + additive) — this is standard Cargo feature semantics, noted in docs. +- **MSRV**: reqwest's gzip feature pulls `async-compression`/`flate2`; flate2 + is already a oneio dependency via the `gz` features, so MSRV is unaffected. + +## 7. Decision Log + +- 2026-07-22: Full `pub use reqwest` over curated re-export module (the type + already leaks; curation adds maintenance without reducing exposure) +- 2026-07-22: Feature named `reqwest-gzip` to disambiguate from the + suffix-based `gz` family; passthrough-only, no cfg-gated code diff --git a/specs/README.md b/specs/README.md index 9cd91f5..8914f94 100644 --- a/specs/README.md +++ b/specs/README.md @@ -6,6 +6,7 @@ This directory contains design specifications for oneio features. |---|---|------| | 01 | [rusty-s3-migration](rusty-s3-migration/README.md) | Complete | | 02 | [lossy-read](02-lossy-read/README.md) | Draft | +| 03 | [reqwest-reexport-http-gzip](03-reqwest-reexport-http-gzip/README.md) | In Progress | --- diff --git a/src/lib.rs b/src/lib.rs index b435658..12a1a70 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ Enable only what you need: | `xz` | XZ compression | | `zstd` | Zstandard compression | | `http` | HTTP/HTTPS support | +| `reqwest-gzip` | Opt-in HTTP gzip content-encoding (advertises `Accept-Encoding: gzip`, transparently decodes responses) | | `ftp` | FTP support | | `s3` | S3-compatible storage | | `async` | Async I/O support | @@ -241,6 +242,48 @@ pub use builder::OneIoBuilder; pub use client::OneIo; pub use error::OneIoError; +/// Re-export of the exact `reqwest` crate oneio is built against. +/// +/// HTTP response types (e.g. [`OneIo::get_http_reader_raw`] returning +/// `reqwest::blocking::Response`) appear in oneio's public API. This re-export +/// lets downstream crates name those types (`oneio::reqwest::StatusCode`, +/// `oneio::reqwest::header`, ...) without declaring their own reqwest +/// dependency, avoiding version skew between the reqwest oneio uses and the +/// reqwest a consumer imports. +/// +/// Note: this makes reqwest part of oneio's public API contract; a reqwest +/// major-version bump is a breaking oneio change. +/// +/// # Example: conditional GET with HTTP validators +/// +/// ```rust,no_run +/// # #[cfg(feature = "http")] +/// # fn main() -> Result<(), Box> { +/// use oneio::reqwest::StatusCode; +/// use oneio::OneIo; +/// +/// let client = OneIo::builder() +/// .header_str("If-None-Match", "\"some-etag\"") +/// .build()?; +/// let response = client.get_http_reader_raw("https://example.com/data.json")?; +/// if response.status() == StatusCode::NOT_MODIFIED { +/// // cached copy is still current; skip re-processing +/// } else { +/// let etag = response +/// .headers() +/// .get("etag") +/// .and_then(|v| v.to_str().ok()) +/// .map(String::from); +/// // read `response` (implements `std::io::Read`) and store `etag` +/// } +/// # Ok(()) +/// # } +/// # #[cfg(not(feature = "http"))] +/// # fn main() {} +/// ``` +#[cfg(feature = "http")] +pub use reqwest; + #[cfg(feature = "async")] pub mod async_reader; #[cfg(feature = "rustls")] diff --git a/tests/http_advanced_tests.rs b/tests/http_advanced_tests.rs new file mode 100644 index 0000000..abc1285 --- /dev/null +++ b/tests/http_advanced_tests.rs @@ -0,0 +1,147 @@ +//! Integration tests for HTTP advanced features: +//! - `reqwest` re-export (`oneio::reqwest`) for naming HTTP types downstream +//! - `reqwest-gzip` feature: transparent gzip content-encoding +//! +//! Uses an in-process mock HTTP server; no external network access required. + +#![cfg(feature = "http")] + +use std::io::{Read, Write}; + +/// Spawn a minimal HTTP/1.1 server that answers one canned response per +/// incoming connection, in order. Returns the base URL and a handle that +/// yields the raw request texts it received. +fn mock_server(responses: Vec>) -> (String, std::thread::JoinHandle>) { + use std::net::TcpListener; + use std::time::Duration; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + + let handle = std::thread::spawn(move || { + let mut requests = Vec::new(); + for response in responses { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let bytes_read = stream.read(&mut buffer).unwrap(); + if bytes_read == 0 { + break; + } + request.extend_from_slice(&buffer[..bytes_read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + requests.push(String::from_utf8(request).unwrap()); + + stream.write_all(&response).unwrap(); + stream.flush().unwrap(); + } + requests + }); + + (format!("http://{addr}"), handle) +} + +fn http_response(status: &str, headers: &[(&str, &str)], body: &[u8]) -> Vec { + let mut response = format!("HTTP/1.1 {status}\r\n"); + for (name, value) in headers { + response.push_str(&format!("{name}: {value}\r\n")); + } + response.push_str(&format!( + "Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + )); + let mut response = response.into_bytes(); + response.extend_from_slice(body); + response +} + +#[test] +fn test_reqwest_reexport_conditional_get() { + use oneio::reqwest::StatusCode; + use oneio::OneIo; + + let body = b"fresh data"; + let (base_url, server) = mock_server(vec![ + http_response( + "200 OK", + &[ + ("ETag", "\"v1\""), + ("Last-Modified", "Wed, 01 Jan 2025 00:00:00 GMT"), + ], + body, + ), + http_response("304 Not Modified", &[], &[]), + ]); + let url = format!("{base_url}/data.txt"); + + // Initial unconditional fetch: 200 with validators exposed via re-exported types. + let client = OneIo::new().unwrap(); + let response = client.get_http_reader_raw(&url).unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let etag = response + .headers() + .get(oneio::reqwest::header::ETAG) + .and_then(|v| v.to_str().ok()) + .map(String::from); + assert_eq!(etag.as_deref(), Some("\"v1\"")); + + // Conditional fetch with the validator: 304 maps to "keep cached copy". + let client = OneIo::builder() + .header_str("If-None-Match", etag.as_deref().unwrap()) + .build() + .unwrap(); + let response = client.get_http_reader_raw(&url).unwrap(); + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 2); + assert!( + requests[1].to_lowercase().contains("if-none-match: \"v1\""), + "second request should carry If-None-Match, got: {}", + requests[1] + ); +} + +/// gzip-compressed bytes of `GZIP_TEST_TEXT` (pre-compressed so this test does +/// not require a compression feature to be enabled). +#[cfg(feature = "reqwest-gzip")] +const GZIP_TEST_TEXT: &str = "OneIO test file.\nThis is gzip-encoded content."; + +#[cfg(feature = "reqwest-gzip")] +const GZIP_TEST_BODY: [u8; 64] = [ + 31, 139, 8, 0, 0, 0, 0, 0, 2, 255, 243, 207, 75, 245, 244, 87, 40, 73, 45, 46, 81, 72, 203, + 204, 73, 213, 227, 10, 201, 200, 44, 86, 0, 162, 244, 170, 204, 2, 221, 212, 188, 228, 252, + 148, 212, 20, 133, 228, 252, 188, 146, 212, 188, 18, 61, 0, 97, 22, 228, 9, 46, 0, 0, 0, +]; + +#[cfg(feature = "reqwest-gzip")] +#[test] +fn test_reqwest_gzip_transparent_decode() { + let (base_url, server) = mock_server(vec![http_response( + "200 OK", + &[("Content-Encoding", "gzip")], + &GZIP_TEST_BODY, + )]); + // No compression suffix in the URL: decoding must come from the + // Content-Encoding header, not oneio's suffix-based decompression. + let url = format!("{base_url}/data.txt"); + + let text = oneio::read_to_string_lossy(&url).unwrap(); + assert_eq!(text.as_str(), GZIP_TEST_TEXT); + + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 1); + assert!( + requests[0].to_lowercase().contains("accept-encoding: gzip"), + "request should advertise Accept-Encoding: gzip, got: {}", + requests[0] + ); +} From 89aec0b787d535851d6c091d31a2a203908da982 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Wed, 22 Jul 2026 18:09:12 -0700 Subject: [PATCH 2/2] docs: mark spec 03 status as Complete --- specs/03-reqwest-reexport-http-gzip/README.md | 2 +- specs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/03-reqwest-reexport-http-gzip/README.md b/specs/03-reqwest-reexport-http-gzip/README.md index 15f0c4f..fa1502b 100644 --- a/specs/03-reqwest-reexport-http-gzip/README.md +++ b/specs/03-reqwest-reexport-http-gzip/README.md @@ -1,6 +1,6 @@ # Spec: reqwest Re-export and Opt-in HTTP gzip -**Status**: In Progress +**Status**: Complete **Author**: Mingwei Zhang **Created**: 2026-07-22 **Target Branch**: `dev/reqwest-reexport-http-gzip` diff --git a/specs/README.md b/specs/README.md index 8914f94..2baa177 100644 --- a/specs/README.md +++ b/specs/README.md @@ -6,7 +6,7 @@ This directory contains design specifications for oneio features. |---|---|------| | 01 | [rusty-s3-migration](rusty-s3-migration/README.md) | Complete | | 02 | [lossy-read](02-lossy-read/README.md) | Draft | -| 03 | [reqwest-reexport-http-gzip](03-reqwest-reexport-http-gzip/README.md) | In Progress | +| 03 | [reqwest-reexport-http-gzip](03-reqwest-reexport-http-gzip/README.md) | Complete | ---