diff --git a/.changeset/current-clippy-baseline.md b/.changeset/current-clippy-baseline.md new file mode 100644 index 000000000..a14e86242 --- /dev/null +++ b/.changeset/current-clippy-baseline.md @@ -0,0 +1,5 @@ +--- +"@googleworkspace/cli": patch +--- + +Keep Apps Script file selection compatible with the current Clippy checks. diff --git a/.changeset/docs-review-bundle.md b/.changeset/docs-review-bundle.md new file mode 100644 index 000000000..286e9e7f0 --- /dev/null +++ b/.changeset/docs-review-bundle.md @@ -0,0 +1,11 @@ +--- +"@googleworkspace/cli": minor +--- + +Add a standalone Python companion for visual Google Docs review bundles with +native exports, safe DOCX raster extraction, local HTML, optional PDF page +previews with explicitly unverified page coverage, revision observations, +and an offline fixture workflow. Preserve nested image occurrences and +legitimate asset reuse, and keep oversized optional comments from failing +the required bundle. Verify each export's exact canonical destination against +the real CLI receipt. diff --git a/.github/workflows/docs-review-bundle.yml b/.github/workflows/docs-review-bundle.yml new file mode 100644 index 000000000..a0f651c61 --- /dev/null +++ b/.github/workflows/docs-review-bundle.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Docs Review Bundle + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + docs-review-bundle: + name: Docs Review Bundle (Python) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@d1031067263f94b142dd6c0ce24c5eb9d02d52a0 # master + with: + toolchain: stable + - name: Cache cargo + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: docs-review-bundle-${{ matrix.os }} + - name: Build CLI for export contract regression + run: cargo build --locked + - name: Test stdlib companion with synthetic fixtures + run: python3 -B -m unittest discover -s examples/docs-review-bundle -p 'test_*.py' -v + env: + GWS_TEST_BINARY: ${{ github.workspace }}/target/debug/gws diff --git a/crates/google-workspace-cli/src/helpers/script.rs b/crates/google-workspace-cli/src/helpers/script.rs index 11bcdebec..4b31db62d 100644 --- a/crates/google-workspace-cli/src/helpers/script.rs +++ b/crates/google-workspace-cli/src/helpers/script.rs @@ -169,13 +169,7 @@ fn process_file(path: &Path) -> Result, GwsError> { filename.trim_end_matches(".js").trim_end_matches(".gs"), ), "html" => ("HTML", filename.trim_end_matches(".html")), - "json" => { - if filename == "appsscript.json" { - ("JSON", "appsscript") - } else { - return Ok(None); - } - } + "json" if filename == "appsscript.json" => ("JSON", "appsscript"), _ => return Ok(None), }; diff --git a/examples/docs-review-bundle/README.md b/examples/docs-review-bundle/README.md new file mode 100644 index 000000000..7975497f3 --- /dev/null +++ b/examples/docs-review-bundle/README.md @@ -0,0 +1,212 @@ +# Visual Docs review bundle + +This standalone companion combines native Docs JSON, Drive PDF/DOCX/Markdown +exports, DOCX raster assets, and a local HTML review index. It uses only the +Python standard library and existing `gws` commands. It does not change `gws`, +authenticate separately, or download document hyperlinks or image `contentUri`s. + +## Run + +Requires Python 3.10+ on Linux or macOS and an authenticated `gws` executable +with read access to the document through both Docs and Drive. + +From the repository root: + +```sh +python3 examples/docs-review-bundle/docs_review_bundle.py \ + --document-id DOCUMENT_ID review-bundle +``` + +`review-bundle` must be a **new relative directory inside the current working +directory**. Existing directories, absolute paths, `..`, symlink components, +control characters, and names outside the portable ASCII subset are refused. +Nested paths work when their parents already exist. The final directory is +created with mode `0700`; keep its parents under your control. + +Optional orchestration flags: + +```sh +python3 examples/docs-review-bundle/docs_review_bundle.py \ + --document-id DOCUMENT_ID --include-comments --render-pages \ + --timeout 120 --gws /trusted/path/to/gws another-review-bundle +``` + +- `--include-comments`: collect every returned Drive comments page. The entire + optional artifact is omitted and marked unavailable if retrieval is incomplete + or fails, or if the final serialized artifact exceeds 20 MiB. Size is checked + before publication, so an oversized optional result does not fail the required + bundle. Comments are a separate observation, not revision-bound or mapped to + PDF coordinates. Deleted comments are not requested. +- `--render-pages`: use a trusted `pdftoppm` on `PATH` to generate 96 DPI PNGs. + Its path is resolved before running inside the bundle, including relative + `PATH` entries. Rendering is opt-in. Missing tools, failures, timeouts, invalid + output and noncontiguous page numbers retain the PDF and report no available + raster previews. Outputs from these detected failures are discarded. After a + successful process exit and validation, previews are labeled `available` + with `coverage: "unverified"`; a missing page suffix cannot be detected. +- `--timeout`: positive finite seconds per subprocess; default 60. This is not + a total workflow deadline. +- `--gws`: trusted executable, resolved before changing subprocess working + directories. Existing `gws` authentication and Model Armor environment + settings are inherited. No credential values or raw process diagnostics + are inserted into HTML or error messages. + +The companion requests `docs documents get` with `includeTabsContent: true` +and uses `drive files export` with `--format json` and fixed relative +`--output` filenames. Every `gws` subprocess runs inside the new bundle. +Export success, MIME type, destination and byte count are checked against the +written artifact. The receipt must name the exact canonical absolute destination, +as returned by `gws`; a matching basename alone is insufficient. This requires +the existing `gws` binary-export receipt format. + +Open `index.html` locally. The index has no JavaScript or remote dependencies. +It contains a sandboxed PDF frame, optional page images, a paragraph/table +outline grouped by native tabs, DOCX figures and nearby text, native object +metadata, and escaped Markdown source. Some browsers block local PDF frames; +use the PDF artifact link in that case. Markdown is readable escaped source, +not rendered Markdown. + +## Artifacts and completion + +| File | Meaning | +| --- | --- | +| `source.json` | Original Docs JSON response, including all returned tabs | +| `revision-after.json` | Final revision observation, when available | +| `document.pdf`, `document.docx`, `document.md` | Required native Drive exports | +| `comments.json` | Optional fully retrieved comments result | +| `assets/` | Recognized DOCX PNG, JPEG, GIF and WebP media | +| `pages/` | Optional successful local page-rendering output | +| `index.html` | Local review index | +| `manifest.json` | State, versions, capabilities, limitations, mappings and hashes | + +The manifest begins as `in-progress` and becomes `complete` only after all +required exports, validation, index generation and artifact hashing succeed. +`complete` means the required bundle files were produced; it does **not** mean +an atomic snapshot, successful optional rendering, or verified export tab +coverage. Check `revisions`, `comments`, and `rendering` independently. + +Page previews are never labeled `complete`. `rendering.status: "available"` +means that local PNG files passed validation, while +`rendering.coverage: "unverified"` means the original PDF page count was not +independently checked. Even a contiguous list beginning with page 1 may omit +later pages. The HTML displays this same coverage limitation. + +Revision status is `mixed` for differing observed Docs revision IDs, `unchanged` +for equal nonempty IDs, and `unknown` if either is missing. Even `unchanged` +does not prove atomicity or that every export represents the same revision. +The manifest always records `atomic_snapshot: false`. + +Required failures return exit code 1 and leave a `failed` manifest with a safe +error code and stage. If writing the failure state also fails (for example, +a full disk), the earlier `in-progress` state can remain. Process termination +can also leave that state. No such bundle should be treated as complete. +Usage errors return 2. Optional failures and mixed/unknown revisions return 0 +when the required bundle completes. Existing output directories are never +overwritten; retries need a new name. + +SHA-256 and byte counts cover each produced artifact, including the index and +assets. The manifest does not hash itself and is not an authenticity signature. + +## Scope and safety limits + +- Native JSON traversal includes nested tabs, body paragraphs, tables and + tables of contents. The outline identifies paragraph styles but does not + recreate full layout. Headers, footers, notes, lists, equations, charts, + suggestions and other document features are not fully represented. +- Google determines the PDF/DOCX/Markdown export layout and tab coverage. + This companion cannot verify that every tab appears in those formats or + associate a PDF page/DOCX figure with an exact native tab. +- DOCX figure order follows individual image occurrences in the main document, + including drawings in tables and nested text boxes. Each occurrence uses its + nearest drawing's alt text and nearest paragraph's text; legitimate repeated + uses of an asset remain separate figures. Alt text and nearby paragraphs are + context, **not exact captions**. Native object IDs are recorded separately; + the companion never fabricates a matching native ID or source tab ID. +- Media basenames are replaced with distinct generated local filenames. + External, missing, traversing and unsupported relationships remain visibly + unavailable. Unreferenced recognized rasters are retained as artifacts. +- ZIP validation rejects traversal, absolute/Windows/control-character paths, + duplicate names (case-insensitive), symlinks and other special files, + encryption and unsupported compression. It never calls `extractall`. + Limits are 2,000 members, 20 MiB per member and 100 MiB total uncompressed. + ZIP paths are restricted to printable ASCII. Only stored/deflate compression + is accepted. +- Every XML/relationship member is parsed after rejecting DTDs, entities, + UTF-16/32 and non-UTF-8 encodings. XML trees are limited to 100 levels and + 100,000 nodes per part. The supported DOCX vocabulary is transitional OOXML + main-document drawings; unsupported content can remain unmapped. +- Each source/export/JSON artifact is capped at 20 MiB. Page output is limited + to 500 files and 100 MiB total. PDF header/EOF and raster signatures are + checked; these are **not full format validation**. Renderer exit success and + contiguous numbering do not independently prove the original PDF page count, + so preview coverage is always labeled unverified. +- Subprocess capture is file-backed, with size checks after exit. Timeouts and + post-render limits do not enforce disk or memory quotas on external tools. + `pdftoppm`, `gws`, local viewers and the operator-controlled parent directory + are trusted. This is not a sandbox against a concurrent local attacker. +- Display text is HTML/attribute escaped; URI references use generated local + allowlisted names. The index has a restrictive content-security policy and + no scripts. Source URLs and recognizable bearer/token strings are redacted + from display text, but this is not a general secret scanner or Model Armor + replacement. Existing `gws` sanitization behavior is preserved; binary + exports are not made safe by JSON sanitization. +- Raw artifacts deliberately retain original content, including any temporary + URLs or sensitive text. Treat the whole directory as sensitive. Review + external links and active content in native viewers separately; the + companion never follows them automatically. + +## Offline fixtures and visual QA + +Offline mode performs no `gws` calls. Supply a relative directory containing +`source.json`, `document.pdf`, `document.docx`, and UTF-8 `document.md`. +`revision-after.json` and `comments.json` are optional; a missing revision +observation remains unknown. Files and path components must not be symlinks. + +Generate entirely synthetic fixtures using the test utility, then create a +review bundle suitable for inspecting the HTML: + +```sh +python3 -B - <<'PY' +import sys +from pathlib import Path +sys.path.insert(0, "examples/docs-review-bundle") +from test_docs_review_bundle import fixtures +fixtures(Path("synthetic-docs-fixture")) +PY + +python3 -B examples/docs-review-bundle/docs_review_bundle.py \ + --from-fixture synthetic-docs-fixture --render-pages synthetic-docs-review +``` + +These generated fixtures intentionally contain HTML injection strings, an +untrusted synthetic URL, duplicate image basenames, missing image URIs, +nested tabs and table content. They are hand-built test data, not an actual +Google export or proof of cross-format fidelity. No real documents, +credentials or network access are needed. Omit `--render-pages` to avoid +running external software. + +## Tests + +```sh +python3 -B -m unittest discover \ + -s examples/docs-review-bundle -p 'test_*.py' -v +``` + +Tests use generated JSON/PDF/DOCX files and explicit `gws`/renderer executables +as stubs. Their subprocess environment omits real authentication settings; +no installed `gws`, real document, or network request is needed for those tests. + +To include the real CLI export-contract regression: + +```sh +cargo build --locked +GWS_TEST_BINARY="$PWD/target/debug/gws" python3 -B -m unittest discover \ + -s examples/docs-review-bundle -p 'test_*.py' -v +``` + +This additional test uses cached synthetic Discovery, a dummy token, isolated +configuration and ADC paths, and a loopback HTTP server. It downloads all three +generated exports through the real CLI and checks that the bundle completes. +It never contacts Google or uses real credentials. The dedicated Linux/macOS CI +job builds `gws` and always enables this test; local runs without +`GWS_TEST_BINARY` explicitly skip it. diff --git a/examples/docs-review-bundle/docs_review_bundle.py b/examples/docs-review-bundle/docs_review_bundle.py new file mode 100755 index 000000000..2ba9ee7c0 --- /dev/null +++ b/examples/docs-review-bundle/docs_review_bundle.py @@ -0,0 +1,783 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at https://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build a local visual review bundle using Python's standard library and gws.""" + +import argparse +import hashlib +import html +import io +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +import zipfile + + +VERSION = "1.0" +MIB = 1024 * 1024 +FILE_LIMIT = 20 * MIB +TOTAL_LIMIT = 100 * MIB +MEMBER_COUNT = 2000 +MAX_PAGES = 500 +EXPORTS = { + "document.pdf": "application/pdf", + "document.docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "document.md": "text/markdown", +} +W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +A = "{http://schemas.openxmlformats.org/drawingml/2006/main}" +WP = "{http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing}" +R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}" +REL = "{http://schemas.openxmlformats.org/package/2006/relationships}" +LIMITATIONS = [ + "Exports are sequential, not an atomic snapshot; unchanged revisions are observations only.", + "Native outline includes body paragraphs, tables and nested tabs, not full layout or styling.", + "Drive export tab coverage is not verified; pages and DOCX figures have no reliable tab mapping.", + "Page preview coverage is unverified; available previews may omit pages.", + "DOCX relationships provide document order and nearby text, " + "not exact captions or native Docs IDs.", + "Only recognized PNG, JPEG, GIF and WebP media are previewed; " + "signatures are not full validation.", + "Headers, footers, notes, charts, vectors and unsupported drawings " + "may be absent from the outline or figures.", + "Content URIs and document hyperlinks are never fetched; URLs are redacted from display text.", + "Raw exports and JSON are sensitive, unsanitized source artifacts " + "and may contain temporary URLs.", + "The companion is not a document sanitizer or a sandbox for PDF/image viewers or pdftoppm.", +] + + +class BundleError(Exception): + """A fixed, non-sensitive failure code safe for manifests and terminals.""" + + +def relative_parts(value): + """Conservative portable path subset; do not normalize away traversal.""" + parts = value.split("/") + if not parts or any( + part in ("", ".", "..") or not re.fullmatch(r"[A-Za-z0-9_. -]+", part) + for part in parts + ): + raise BundleError("unsafe-relative-path") + return parts + + +def directory(value, *, create=False): + """Walk existing parents with no-follow descriptors (Linux/macOS).""" + parts = relative_parts(value) + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + fd = os.open(".", flags) + try: + for index, part in enumerate(parts): + if create and index == len(parts) - 1: + os.mkdir(part, mode=0o700, dir_fd=fd) + next_fd = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = next_fd + except OSError: + raise BundleError("directory-exists-or-unsafe") from None + finally: + os.close(fd) + return Path.cwd().joinpath(*parts) + + +def read_bytes(path, limit=FILE_LIMIT): + """Bound reads and reject special files, including symlinks.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > limit: + raise BundleError("file-type-or-size-limit") + data = stream.read(limit + 1) + except OSError: + raise BundleError("missing-or-unsafe-file") from None + if len(data) > limit: + raise BundleError("file-size-limit") + return data + + +def write_bytes(path, data): + # Files are new, inside the newly created private bundle directory. + with path.open("xb") as stream: + stream.write(data) + + +def json_bytes(value): + return (json.dumps(value, ensure_ascii=True, indent=2) + "\n").encode("utf-8") + + +def parse_json(data): + try: + value = json.loads(data) + except (ValueError, UnicodeError): + raise BundleError("invalid-json") from None + if not isinstance(value, dict) or "error" in value: + raise BundleError("invalid-json-response") + return value + + +def publish_manifest(bundle, manifest): + temporary = bundle / ".manifest.tmp" + write_bytes(temporary, json_bytes(manifest)) + os.replace(temporary, bundle / "manifest.json") + + +def local_uri(value): + """HTML references are generated file names, never document-supplied URIs.""" + if not re.fullmatch(r"[A-Za-z0-9_-]+(?:[./][A-Za-z0-9_-]+)*", value): + raise BundleError("unsafe-local-uri") + return value + + +def display(value): + text = str(value or "") + text = re.sub(r"(?i)\b(?:https?|ftp|file|data|javascript):[^\s<>\"']+", "[URL omitted]", text) + text = re.sub(r"(?i)\bBearer\s+\S+", "[credential omitted]", text) + text = re.sub( + r"(?i)\b(?:access_token|authorization|token)\s*[:=]\s*[^\s<>\"']+", + "[credential omitted]", + text, + ) + text = "".join(char for char in text if char in "\n\t" or ord(char) >= 32) + return html.escape(text, quote=True) + + +def safe_xml(data): + # Reject UTF-16/32 (including declaration smuggling via NUL bytes), DTDs and + # entities before giving XML to the stdlib parser. DOCX normally uses UTF-8. + if b"\x00" in data or re.search(br"]*\bencoding\s*=\s*['\"]([^'\"]+)", text, re.I) + if declaration and declaration[1].lower() not in ("utf-8", "utf8", "us-ascii"): + raise BundleError("unsupported-xml-encoding") + root = ET.fromstring(text) + except (ET.ParseError, UnicodeError): + raise BundleError("invalid-xml") from None + pending = [(root, 0)] + count = 0 + while pending: + node, depth = pending.pop() + count += 1 + if depth > 100 or count > 100_000: + raise BundleError("xml-complexity-limit") + pending.extend((child, depth + 1) for child in node) + return root + + +def raster_extension(data): + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "png" + if data.startswith(b"\xff\xd8\xff"): + return "jpg" + if data.startswith((b"GIF87a", b"GIF89a")): + return "gif" + if data.startswith(b"RIFF") and data[8:12] == b"WEBP": + return "webp" + return None + + +def zip_members(source, member_limit, total_limit, member_count): + """Read a bounded archive without ever extracting its member paths.""" + try: + with zipfile.ZipFile(io.BytesIO(read_bytes(source))) as archive: + infos = archive.infolist() + if len(infos) > member_count: + raise BundleError("zip-member-count-limit") + seen = set() + declared_total = 0 + for info in infos: + name = info.orig_filename + parts = name.rstrip("/").split("/") + kind = stat.S_IFMT(info.external_attr >> 16) + if ( + name != info.filename + or name.startswith("/") + or "\\" in name + or ":" in name + or any(part in ("", ".", "..") for part in parts) + or any(ord(char) < 32 or ord(char) > 126 for char in name) + or kind not in (0, stat.S_IFREG, stat.S_IFDIR) + or info.flag_bits & 1 + or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) + or name.casefold() in seen + ): + raise BundleError("unsafe-zip-member") + seen.add(name.casefold()) + declared_total += info.file_size + if info.file_size > member_limit or declared_total > total_limit: + raise BundleError("zip-size-limit") + members = {} + actual_total = 0 + for info in infos: + if info.is_dir(): + continue + with archive.open(info) as stream: + data = stream.read(member_limit + 1) + actual_total += len(data) + if ( + len(data) > member_limit + or actual_total > total_limit + or len(data) != info.file_size + ): + raise BundleError("zip-size-limit") + members[info.filename] = data + return members + except (zipfile.BadZipFile, RuntimeError, NotImplementedError, OSError, EOFError): + raise BundleError("invalid-zip") from None + + +def docx_image_occurrences(document): + """Visit each blip once, retaining its nearest paragraph and drawing.""" + paragraph_text = {} + paragraph_index = {} + drawing_alt = {} + occurrences = [] + pending = [(document, None, None)] + while pending: + node, paragraph, drawing = pending.pop() + if node.tag == W + "p": + paragraph = node + paragraph_index[node] = len(paragraph_text) + paragraph_text[node] = [] + elif node.tag == W + "drawing": + drawing = node + elif node.tag == W + "t" and paragraph is not None: + paragraph_text[paragraph].append(node.text or "") + elif node.tag == WP + "docPr" and drawing is not None: + drawing_alt.setdefault( + drawing, " ".join(node.get(field, "") for field in ("title", "descr")).strip() + ) + elif node.tag == A + "blip" and paragraph is not None and drawing is not None: + occurrences.append((node, paragraph, drawing)) + pending.extend((child, paragraph, drawing) for child in reversed(node)) + + texts = ["".join(parts) for parts in paragraph_text.values()] + for blip, paragraph, drawing in occurrences: + index = paragraph_index[paragraph] + nearby = texts[index] or " ".join( + texts[max(0, index - 1):index] + texts[index + 1:index + 2] + ) + yield blip, drawing_alt.get(drawing, ""), nearby + + +def extract_docx( + source, output, *, + member_limit=FILE_LIMIT, total_limit=TOTAL_LIMIT, member_count=MEMBER_COUNT, +): + members = zip_members(source, member_limit, total_limit, member_count) + # Validate even unused XML before any asset writes. + trees = { + name: safe_xml(data) + for name, data in members.items() + if name.lower().endswith((".xml", ".rels")) + } + document = trees.get("word/document.xml") + if document is None or document.tag != W + "document": + raise BundleError("missing-docx-document") + relationships = {} + rels = trees.get("word/_rels/document.xml.rels") + if rels is not None: + for rel in rels.findall(REL + "Relationship"): + identity = rel.get("Id") + if not identity or identity in relationships: + raise BundleError("ambiguous-docx-relationship") + relationships[identity] = rel.attrib + output.mkdir(mode=0o700) + assets = {} + for name, data in members.items(): + extension = raster_extension(data) + if name.startswith("word/media/") and extension: + filename = f"image-{len(assets) + 1}.{extension}" + write_bytes(output / filename, data) + assets[name] = local_uri(f"{output.name}/{filename}") + figures = [] + for blip, alt, nearby in docx_image_occurrences(document): + identity = blip.get(R + "embed") or blip.get(R + "link") + relationship = relationships.get(identity, {}) + asset = None + availability = "missing-or-unsupported" + if relationship.get("TargetMode", "").lower() == "external": + availability = "external-not-fetched" + elif relationship.get("Type") == R[1:-1] + "/image": + target = relationship.get("Target", "") + # Allow only relative media targets in the document's part. + if ( + target.startswith("media/") + and not any(p in ("", ".", "..") for p in target.split("/")) + and not any(char in target for char in "\\:%?#") + ): + asset = assets.get(str(PurePosixPath("word") / target)) + if asset: + availability = "available" + figures.append({ + "order": len(figures) + 1, + "relationship_id": identity, + "asset": asset, + "alt": alt, + "nearby_text": nearby[:1000], + "availability": availability, + "mapping_confidence": "docx-relationship-only", + "native_object_id": None, + "source_tab_id": None, + }) + return {"figures": figures, "assets": list(assets.values())} + + +def native_view(source): + if not isinstance(source.get("body"), dict) and not isinstance(source.get("tabs"), list): + raise BundleError("missing-document-content") + tabs = [] + images = [] + + def add_tab(tab, depth): + if depth > 50 or len(tabs) >= 1000: + raise BundleError("native-tab-limit") + properties = tab.get("tabProperties", {}) + document = tab.get("documentTab", {}) + tab_id = properties.get("tabId") + tabs.append({ + "id": tab_id, + "title": properties.get("title", "Document"), + "depth": depth, + "blocks": document.get("body", {}).get("content", []), + }) + for collection, property_name in ( + ("inlineObjects", "inlineObjectProperties"), + ("positionedObjects", "positionedObjectProperties"), + ): + for object_id, obj in document.get(collection, {}).items(): + embedded = obj.get(property_name, {}).get("embeddedObject", {}) + images.append({ + "object_id": object_id, + "tab_id": tab_id, + "title": embedded.get("title", ""), + "description": embedded.get("description", ""), + "content_uri_available": bool( + embedded.get("imageProperties", {}).get("contentUri") + ), + "asset": None, + "mapping_confidence": "unmapped", + }) + for child in tab.get("childTabs", []): + add_tab(child, depth + 1) + + if source.get("tabs"): + for tab in source["tabs"]: + add_tab(tab, 0) + else: + add_tab({"documentTab": source}, 0) + return tabs, images + + +def outline_html(blocks, depth=0): + if depth > 50: + raise BundleError("native-outline-depth-limit") + parts = [] + for block in blocks: + if "paragraph" in block: + paragraph = block["paragraph"] + text = "".join( + element.get("textRun", {}).get("content", "") + for element in paragraph.get("elements", []) + ) + style = paragraph.get("paragraphStyle", {}).get("namedStyleType", "NORMAL_TEXT") + parts.append(f"

{display(style)} {display(text)}

") + elif "table" in block: + parts.append("") + for row in block["table"].get("tableRows", []): + parts.append("") + for cell in row.get("tableCells", []): + parts.append( + "" + ) + parts.append("") + parts.append("
" + outline_html(cell.get("content", []), depth + 1) + "
") + elif "tableOfContents" in block: + parts.append(outline_html(block["tableOfContents"].get("content", []), depth + 1)) + return "".join(parts) + + +def index_html(source, markdown, tabs, manifest, artifacts): + parts = [ + '', + '', + '", + "Docs review bundle", + "", + f"

{display(source.get('title', 'Docs review bundle'))}

", + f"

Revision observation: {display(manifest['revisions']['status'])}. " + "Sequential exports; not an atomic snapshot.

", + "

Artifacts

Raw files may contain sensitive content and temporary URLs.

", + "

Native PDF

", + '', + "

If your browser blocks the embedded viewer, open the local PDF artifact.

", + f"

Raster previews: {display(manifest['rendering']['status'])}; " + f"{display(manifest['rendering'].get('reason', ''))}

", + ] + ) + if manifest["rendering"]["pages"]: + parts.append( + f"

Page coverage: {display(manifest['rendering']['coverage'])}. " + "The PDF page count has not been verified; previews may omit pages.

" + ) + for page in manifest["rendering"]["pages"]: + parts.append(f'{display(page)}') + parts.append("

Native outline and tabs

") + for tab in tabs: + parts.append( + f"

{display(tab['title'])}

" + f"

Tab: {display(tab['id'] or 'legacy body')}; " + f"depth: {tab['depth']}

{outline_html(tab['blocks'])}" + ) + parts.append("

DOCX figures

") + for figure in manifest["figures"]: + parts.append(f"

Figure {figure['order']}

") + if figure["asset"]: + parts.append( + f'' + ) + parts.append( + f"
{display(figure['alt'])}
" + f"

Availability: {display(figure['availability'])}

" + f"

Nearby text (not an exact caption): {display(figure['nearby_text'])}

" + "

DOCX relationship only; native object and source tab mapping unknown.

" + ) + parts.append("

Native image metadata

") + for image in manifest["native_images"]: + parts.append( + f"

Object {display(image['object_id'])}, tab {display(image['tab_id'])}: " + f"{display(image['title'])} {display(image['description'])}. " + f"contentUri available: {image['content_uri_available']}; asset mapping: unmapped.

" + ) + parts.append( + "

Readable Markdown source

" + f"
{display(markdown)}
" + ) + parts.append( + f"

Comments

{display(manifest['comments']['status'])}

" + ) + parts.append("

Capabilities and limitations

") + return "".join(parts).encode("utf-8") + + +def run_process(argv, bundle, timeout): + # File-backed capture bounds memory. No shell and no untrusted diagnostics + # echoed to the terminal or copied into the manifest/display HTML. + with tempfile.TemporaryFile(dir=bundle) as output: + try: + process = subprocess.run( + argv, + cwd=bundle, + stdout=output, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + raise BundleError("timeout") from None + except OSError: + raise BundleError("process-unavailable") from None + if process.returncode: + raise BundleError("process-failed") + if output.tell() > FILE_LIMIT: + raise BundleError("process-output-limit") + output.seek(0) + return output.read(FILE_LIMIT + 1) + + +def gws_json(executable, bundle, timeout, command, params, output=None): + argv = [executable, *command, "--params", json.dumps(params), "--format", "json"] + if output: + argv.extend(["--output", local_uri(output)]) + data = run_process(argv, bundle, timeout) + return data, parse_json(data) + + +def collect_comments(executable, bundle, timeout, document_id): + comments = [] + seen = set() + token = None + total = 0 + for _ in range(100): + params = {"fileId": document_id, "fields": "nextPageToken,comments", "pageSize": 100} + if token: + params["pageToken"] = token + raw, page = gws_json(executable, bundle, timeout, ["drive", "comments", "list"], params) + total += len(raw) + if total > FILE_LIMIT or not isinstance(page.get("comments", []), list): + raise BundleError("invalid-or-oversized-comments") + comments.extend(page.get("comments", [])) + token = page.get("nextPageToken") + if not token: + return json_bytes({"comments": comments}) + if not isinstance(token, str) or token in seen: + raise BundleError("incomplete-comments") + seen.add(token) + raise BundleError("incomplete-comments") + + +def render_pages(bundle, timeout, requested): + result = {"status": "not-requested", "pages": []} + if not requested: + return result + renderer = shutil.which("pdftoppm") + if not renderer: + return {"status": "unavailable", "reason": "pdftoppm-not-found; PDF retained", "pages": []} + try: + renderer = str(Path(renderer).resolve()) + with tempfile.TemporaryDirectory(prefix=".render-", dir=bundle) as temp: + staging = Path(temp) + prefix = str(staging.relative_to(bundle) / "page") + run_process([renderer, "-png", "-r", "96", "document.pdf", prefix], bundle, timeout) + pages = {} + total = 0 + for path in staging.iterdir(): + match = re.fullmatch(r"page-([0-9]+)\.png", path.name) + if not match: + raise BundleError("invalid-render") + number = int(match[1]) + data = read_bytes(path) + total += len(data) + if number in pages or raster_extension(data) != "png" or total > TOTAL_LIMIT: + raise BundleError("invalid-render") + pages[number] = path.name + if ( + not pages + or len(pages) > MAX_PAGES + or sorted(pages) != list(range(1, len(pages) + 1)) + ): + raise BundleError("invalid-render") + staging.rename(bundle / "pages") + return { + "status": "available", + "coverage": "unverified", + "pages": [local_uri("pages/" + pages[number]) for number in sorted(pages)], + } + except (BundleError, OSError) as error: + return { + "status": "failed", "pages": [], + "reason": "timeout" if str(error) == "timeout" else "invalid-or-failed-render", + } + + +def revision_observation(before, after): + before_id = before.get("revisionId") + after_id = after.get("revisionId") + before_id = before_id if isinstance(before_id, str) and before_id else None + after_id = after_id if isinstance(after_id, str) and after_id else None + status = "unknown" + if before_id and after_id: + status = "unchanged" if before_id == after_id else "mixed" + return {"before": before_id, "after": after_id, "status": status, "atomic_snapshot": False} + + +def build_bundle(args, bundle, manifest): + artifacts = ["source.json", *EXPORTS] + manifest["stage"] = "exports" + if args.from_fixture: + fixture = directory(args.from_fixture) + for name in artifacts: + write_bytes(bundle / name, read_bytes(fixture / name)) + source = parse_json(read_bytes(bundle / "source.json")) + after_path = fixture / "revision-after.json" + after = parse_json(read_bytes(after_path)) if after_path.exists() else {} + if after_path.exists(): + write_bytes(bundle / "revision-after.json", read_bytes(after_path)) + artifacts.append("revision-after.json") + executable = None + else: + executable = shutil.which(args.gws) + if not executable: + raise BundleError("gws-not-found") + executable = str(Path(executable).resolve()) + try: + version = run_process([executable, "--version"], bundle, args.timeout).decode("utf-8") + match = re.fullmatch(r"gws ([0-9][A-Za-z0-9.+-]*)\s*", version) + manifest["versions"]["gws"] = match[1] if match else "unknown" + except BundleError: + manifest["versions"]["gws"] = "unknown" + params = {"documentId": args.document_id, "includeTabsContent": True} + raw, source = gws_json( + executable, bundle, args.timeout, ["docs", "documents", "get"], params + ) + if source.get("documentId") != args.document_id: + raise BundleError("document-id-mismatch") + write_bytes(bundle / "source.json", raw) + for name, mime in EXPORTS.items(): + _, receipt = gws_json( + executable, bundle, args.timeout, ["drive", "files", "export"], + {"fileId": args.document_id, "mimeType": mime}, output=name, + ) + data = read_bytes(bundle / name) + if ( + receipt.get("status") != "success" + or receipt.get("saved_file") != str((bundle / name).resolve()) + or str(receipt.get("mimeType", "")).split(";")[0].strip().lower() != mime + or type(receipt.get("bytes")) is not int + or receipt["bytes"] != len(data) + ): + raise BundleError("invalid-export-receipt") + raw, after = gws_json( + executable, bundle, args.timeout, ["docs", "documents", "get"], + {**params, "fields": "revisionId"}, + ) + write_bytes(bundle / "revision-after.json", raw) + artifacts.append("revision-after.json") + + manifest["revisions"] = revision_observation(source, after) + manifest["comments"] = {"status": "not-requested"} + if args.include_comments: + try: + if executable: + comments = collect_comments(executable, bundle, args.timeout, args.document_id) + else: + comments = read_bytes(fixture / "comments.json") + value = parse_json(comments) + if not isinstance(value.get("comments"), list) or value.get("nextPageToken"): + raise BundleError("incomplete-comments") + if len(comments) > FILE_LIMIT: + raise BundleError("comments-size-limit") + write_bytes(bundle / "comments.json", comments) + artifacts.append("comments.json") + manifest["comments"] = {"status": "available"} + except (BundleError, OSError) as error: + (bundle / "comments.json").unlink(missing_ok=True) + manifest["comments"] = { + "status": "unavailable", + "reason": ( + "comments-size-limit" + if isinstance(error, BundleError) and str(error) == "comments-size-limit" + else "retrieval-incomplete-or-failed" + ), + } + + manifest["stage"] = "validate-and-extract" + pdf = read_bytes(bundle / "document.pdf") + if not pdf.startswith(b"%PDF-") or b"%%EOF" not in pdf[-1024:]: + raise BundleError("invalid-or-truncated-pdf") + markdown = read_bytes(bundle / "document.md").decode("utf-8") + tabs, native_images = native_view(source) + extracted = extract_docx(bundle / "document.docx", bundle / "assets") + manifest["figures"] = extracted["figures"] + manifest["native_images"] = native_images + manifest["tabs"] = [{k: v for k, v in tab.items() if k != "blocks"} for tab in tabs] + artifacts.extend(extracted["assets"]) + manifest["stage"] = "render" + manifest["rendering"] = render_pages(bundle, args.timeout, args.render_pages) + artifacts.extend(manifest["rendering"]["pages"]) + manifest["stage"] = "index" + write_bytes(bundle / "index.html", index_html(source, markdown, tabs, manifest, artifacts)) + artifacts.append("index.html") + manifest["artifacts"] = {} + for name in artifacts: + data = read_bytes(bundle / local_uri(name)) + manifest["artifacts"][name] = { + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + manifest["status"] = "complete" + manifest["stage"] = "complete" + publish_manifest(bundle, manifest) + + +def positive_timeout(value): + number = float(value) + if not math.isfinite(number) or number <= 0: + raise argparse.ArgumentTypeError("timeout must be a positive finite number") + return number + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--document-id", help="Google Docs ID (not a URL)") + source.add_argument( + "--from-fixture", help="Relative directory containing captured export files" + ) + parser.add_argument("output_dir", help="New relative directory within CWD; parents must exist") + parser.add_argument("--include-comments", action="store_true") + parser.add_argument( + "--render-pages", action="store_true", help="Opt in to local pdftoppm rendering" + ) + parser.add_argument( + "--timeout", type=positive_timeout, default=60.0, + help="Seconds per process (default: 60)", + ) + parser.add_argument( + "--gws", default="gws", help="Trusted gws executable (default: PATH lookup)" + ) + args = parser.parse_args(argv) + if args.document_id and not re.fullmatch(r"[A-Za-z0-9_-]+", args.document_id): + parser.error("document-id must be a Google Docs ID, not a URL or path") + bundle = None + manifest = { + "schema_version": 1, "status": "in-progress", "stage": "initialize", + "mode": "offline" if args.from_fixture else "gws", + "versions": {"companion": VERSION, "python": sys.version.split()[0], "gws": None}, + "capabilities": { + "all_native_tabs": True, "native_pdf": True, "docx_raster_assets": True, + "native_markdown": True, "network_image_fetch": False, + "exact_native_asset_mapping": False, "atomic_snapshot": False, + }, + "limitations": LIMITATIONS, + } + try: + bundle = directory(args.output_dir, create=True) + publish_manifest(bundle, manifest) + build_bundle(args, bundle, manifest) + except (Exception, KeyboardInterrupt) as error: + code = str(error) if isinstance(error, BundleError) else "invalid-or-incomplete-bundle" + if bundle is not None: + manifest["status"] = "failed" + manifest["error"] = code + try: + publish_manifest(bundle, manifest) + except OSError: + # An earlier in-progress manifest remains non-complete if storage fails. + pass + print(f"Review bundle failed: {code}.", file=sys.stderr) + return 1 + print("Review bundle complete. Open index.html in the new output directory.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/docs-review-bundle/test_docs_review_bundle.py b/examples/docs-review-bundle/test_docs_review_bundle.py new file mode 100644 index 000000000..299ab0614 --- /dev/null +++ b/examples/docs-review-bundle/test_docs_review_bundle.py @@ -0,0 +1,1058 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at https://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic behavior tests: generated documents and executable process stubs only.""" + +import base64 +import contextlib +import hashlib +import http.server +from html.parser import HTMLParser +import importlib.util +import io +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest import mock +import urllib.parse +import zipfile + + +SCRIPT = Path(__file__).with_name("docs_review_bundle.py") +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8" + "/x8AAwMCAO+jRZkAAAAASUVORK5CYII=" +) +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +A = "http://schemas.openxmlformats.org/drawingml/2006/main" +WP = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" +REL = "http://schemas.openxmlformats.org/package/2006/relationships" +REMOTE = "https://untrusted.invalid/image?token=SIGNED_SECRET" + + +def paragraph(text, style="NORMAL_TEXT"): + return { + "paragraph": { + "paragraphStyle": {"namedStyleType": style}, + "elements": [ + {"textRun": {"content": text, "textStyle": {"link": {"url": REMOTE}}}} + ], + } + } + + +def native_document(): + return { + "documentId": "synthetic-doc", + "title": 'Review ', + "revisionId": "revision-one", + "tabs": [ + { + "tabProperties": {"tabId": "tab-main", "title": "Main"}, + "documentTab": { + "body": { + "content": [ + paragraph("Heading ", "HEADING_1"), + { + "table": { + "tableRows": [ + { + "tableCells": [ + {"content": [paragraph("Table cell")]} + ] + } + ] + } + }, + { + "paragraph": { + "elements": [ + { + "inlineObjectElement": { + "inlineObjectId": "native-image" + } + } + ] + } + }, + ] + }, + "inlineObjects": { + "native-image": { + "inlineObjectProperties": { + "embeddedObject": { + "title": "Native figure", + "description": "No exact DOCX mapping", + "imageProperties": {"contentUri": REMOTE}, + } + } + }, + "missing-uri": { + "inlineObjectProperties": { + "embeddedObject": { + "description": "No downloadable URI", + "imageProperties": {}, + } + } + }, + }, + }, + "childTabs": [ + { + "tabProperties": {"tabId": "tab-child", "title": "Child"}, + "documentTab": { + "body": {"content": [paragraph("Nested tab paragraph")]} + }, + } + ], + } + ], + } + + +def pdf_bytes(page_count=1): + """A complete synthetic PDF, also usable by real pdftoppm.""" + content = ( + b"BT /F1 22 Tf 48 720 Td (Synthetic Docs review) Tj ET\n" + b"BT /F1 12 Tf 48 687 Td (Local fixture - no Google document) Tj ET\n" + b"0.85 0.92 1 rg 48 435 516 210 re f\n" + b"0.08 0.25 0.5 rg 72 459 120 162 re f\n" + b"0.12 0.45 0.6 rg 216 459 120 105 re f\n" + b"0.1 0.6 0.45 rg 360 459 120 140 re f\n" + b"0 0 0 rg BT /F1 12 Tf 48 402 Td (Synthetic figure and table context) Tj ET\n" + b"0.5 G 48 270 516 90 re S 48 315 m 564 315 l S\n" + b"306 270 m 306 360 l S\n" + b"BT /F1 12 Tf 60 333 Td (Column A) Tj 258 0 Td (Column B) Tj ET\n" + b"BT /F1 12 Tf 60 288 Td (Cell one) Tj 258 0 Td (Cell two) Tj ET\n" + ) + content_id = page_count + 3 + font_id = page_count + 4 + kids = " ".join(f"{number} 0 R" for number in range(3, page_count + 3)) + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + f"<< /Type /Pages /Kids [{kids}] /Count {page_count} >>".encode(), + ] + objects.extend( + ( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + f"/Resources << /Font << /F1 {font_id} 0 R >> >> /Contents {content_id} 0 R >>" + ).encode() + for _ in range(page_count) + ) + objects.extend([ + f"<< /Length {len(content)} >>\nstream\n".encode() + content + b"endstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ]) + data = b"%PDF-1.4\n" + offsets = [0] + for index, obj in enumerate(objects, 1): + offsets.append(len(data)) + data += f"{index} 0 obj\n".encode() + obj + b"\nendobj\n" + startxref = len(data) + object_count = len(objects) + 1 + data += f"xref\n0 {object_count}\n0000000000 65535 f \n".encode() + for offset in offsets[1:]: + data += f"{offset:010d} 00000 n \n".encode() + data += ( + f"trailer\n<< /Size {object_count} /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n" + ).encode() + return data + + +def docx_members(): + xml = f""" + + + +Nearby <script>bad()</script> + + +Table image context + + + + + +""" + rels = f""" + + +""" + return { + "word/document.xml": xml.encode(), + "word/_rels/document.xml.rels": rels.encode(), + "word/styles.xml": f''.encode(), + "word/media/image1.png": PNG, + "word/media/nested/image1.png": PNG + b"different", + } + + +def nested_docx_members(*, outer_image=False): + members = docx_members() + outer_blip = '' if outer_image else "" + members["word/document.xml"] = f""" + +Outer paragraph + +Inner paragraph + + + +{outer_blip} +""".encode() + return members + + +def write_docx(path, members=None): + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in (members if members is not None else docx_members()).items(): + archive.writestr(name, data) + + +def fixtures(path): + path.mkdir() + (path / "source.json").write_text(json.dumps(native_document()), encoding="utf-8") + (path / "revision-after.json").write_text( + '{"revisionId":"revision-one"}', encoding="utf-8" + ) + (path / "document.pdf").write_bytes(pdf_bytes()) + (path / "document.md").write_text( + "# Markdown\n\n![remote](" + REMOTE + ")\n", + encoding="utf-8", + ) + write_docx(path / "document.docx") + return path + + +GWS_STUB = r''' +import json +import os +from pathlib import Path +import shutil +import sys +import time + +args = sys.argv[1:] +mode = os.environ.get("STUB_MODE", "") +fixture = Path(os.environ["STUB_FIXTURES"]) +with open(os.environ["STUB_LOG"], "a", encoding="utf-8") as log: + log.write(json.dumps({"args": args, "cwd": os.getcwd(), + "sanitize": os.environ.get("GOOGLE_WORKSPACE_CLI_SANITIZE_MODE")}) + "\n") +if args == ["--version"]: + print("gws 0.0.0-synthetic") + sys.exit(0) +params = json.loads(args[args.index("--params") + 1]) +assert args[args.index("--format") + 1] == "json" +if mode == "timeout": + time.sleep(10) +if args[:3] == ["docs", "documents", "get"]: + assert params["documentId"] == "synthetic-doc" + assert params["includeTabsContent"] is True + if params.get("fields") == "revisionId": + source = {"revisionId": "revision-two" if mode == "mixed" else "revision-one"} + if mode == "missing-revision": + source = {} + else: + source = json.loads((fixture / "source.json").read_text()) + print(json.dumps(source)) +elif args[:3] == ["drive", "files", "export"]: + assert params["fileId"] == "synthetic-doc" + expected = { + "application/pdf": "document.pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "document.docx", + "text/markdown": "document.md", + } + name = args[args.index("--output") + 1] + assert expected[params["mimeType"]] == name + assert "/" not in name and "\\" not in name + if mode == "failed-export" and name == "document.docx": + Path(name).write_bytes(b"partial") + print("Bearer PRIVATE_TOKEN " + "\x1b[31m", file=sys.stderr) + sys.exit(7) + if mode != "no-export-file": + shutil.copyfile(fixture / name, name) + print(json.dumps({ + "status": "error" if mode == "bad-export-status" else "success", + "saved_file": ( + str(Path.cwd().parent / "other" / name) if mode == "wrong-export-path" + else name if mode == "relative-export-path" + else str(Path(name).resolve()) + ), + "mimeType": params["mimeType"], + "bytes": 1 if mode == "wrong-export-size" else (fixture / name).stat().st_size, + })) +elif args[:3] == ["drive", "comments", "list"]: + assert params["fileId"] == "synthetic-doc" + assert "nextPageToken" in params["fields"] + if mode == "expanded-comments": + # ~8 MiB on the wire; >24 MiB after ensure_ascii=True serialization. + print(json.dumps( + {"comments": [{"id": "large", "content": "é" * (4 * 1024 * 1024)}]}, + ensure_ascii=False, separators=(",", ":"), + )) + sys.exit(0) + if mode == "failed-comments" and params.get("pageToken"): + sys.exit(9) + if params.get("pageToken"): + print(json.dumps({"comments": [{"id": "two", "content": "Second comment"}]})) + else: + print(json.dumps({"nextPageToken": "page-two", "comments": [{"id": "one"}]})) +else: + raise AssertionError(args) +''' + +RENDER_STUB = r''' +import base64 +import os +from pathlib import Path +import sys +import time +assert sys.argv[1:4] == ["-png", "-r", "96"] +assert sys.argv[-2] == "document.pdf" +prefix = Path(sys.argv[-1]) +png = base64.b64decode(os.environ["STUB_PNG"]) +mode = os.environ.get("RENDER_MODE", "") +Path(str(prefix) + "-1.png").write_bytes(png) +if mode == "failed": + sys.exit(2) +if mode == "timeout": + time.sleep(10) +if mode == "gap": + Path(str(prefix) + "-3.png").write_bytes(png) +''' + + +class HTMLInspection(HTMLParser): + def __init__(self, text): + super().__init__() + self.tags = [] + self.references = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + for name, value in attrs: + if name in ("src", "href", "data"): + self.references.append(value) + + +class BundleTestCase(unittest.TestCase): + def setUp(self): + # All generated files stay inside this assigned worktree and are removed. + self.temp = tempfile.TemporaryDirectory(dir=SCRIPT.parent) + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.fixture = fixtures(self.root / "fixtures") + self.bin = self.root / "bin" + self.bin.mkdir() + self.env = { + "PATH": str(self.bin), + "PYTHONDONTWRITEBYTECODE": "1", + "STUB_FIXTURES": str(self.fixture), + "STUB_LOG": str(self.root / "gws.log"), + "STUB_PNG": base64.b64encode(PNG).decode(), + "GOOGLE_WORKSPACE_CLI_CONFIG_DIR": str(self.root / "unused-config"), + "GOOGLE_WORKSPACE_CLI_SANITIZE_MODE": "block", + } + + def api(self): + self.assertTrue(SCRIPT.is_file(), "Missing executable review bundle companion") + if not hasattr(self, "_api"): + spec = importlib.util.spec_from_file_location("docs_review_bundle", SCRIPT) + self._api = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self._api) + return self._api + + def executable(self, name, source): + path = self.bin / name + path.write_text(f"#!{sys.executable}\n" + source, encoding="utf-8") + path.chmod(0o700) + return path + + def run_bundle(self, *args, live=False, **env): + self.assertTrue(SCRIPT.is_file(), "Missing executable review bundle companion") + if live: + self.executable("gws", GWS_STUB) + source = ["--document-id", "synthetic-doc"] + else: + source = ["--from-fixture", "fixtures"] + return subprocess.run( + [sys.executable, "-B", str(SCRIPT), *source, *args], + cwd=self.root, + env={**self.env, **env}, + text=True, + capture_output=True, + timeout=15, + ) + + def manifest(self, directory="review"): + return json.loads((self.root / directory / "manifest.json").read_text()) + + +class ExportReceiptTests(BundleTestCase): + def test_canonical_receipts_complete_all_exports(self): + result = self.run_bundle("review", live=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.manifest()["status"], "complete") + + def test_wrong_destination_or_relative_receipt_is_refused(self): + for mode in ["wrong-export-path", "relative-export-path"]: + with self.subTest(mode=mode): + result = self.run_bundle(mode, live=True, STUB_MODE=mode) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(self.manifest(mode)["error"], "invalid-export-receipt") + + +@unittest.skipUnless(os.environ.get("GWS_TEST_BINARY"), "Set GWS_TEST_BINARY for real CLI coverage") +class RealCliExportTests(BundleTestCase): + def test_real_cli_exports_complete_bundle_with_canonical_receipts(self): + binary = Path(os.environ["GWS_TEST_BINARY"]).resolve(strict=True) + fixture = self.fixture + requests = [] + exports = { + "application/pdf": "document.pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "document.docx", + "text/markdown": "document.md", + } + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + path = urllib.parse.unquote(parsed.path) + params = urllib.parse.parse_qs(parsed.query) + requests.append((path, params, self.headers.get("Authorization"))) + if path == "/documents/synthetic-doc": + data = (fixture / "source.json").read_bytes() + mime = "application/json" + elif path == "/files/synthetic-doc/export": + mime = params.get("mimeType", [""])[0] + if mime not in exports: + self.send_error(400) + return + data = (fixture / exports[mime]).read_bytes() + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", mime) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + config = self.root / "real-cli-config" + cache = config / "cache" + cache.mkdir(parents=True) + (self.root / ".env").write_text("") + for service, version, resource, method, id_field, path in [ + ("docs", "v1", "documents", "get", "documentId", "documents/{documentId}"), + ("drive", "v3", "files", "export", "fileId", "files/{fileId}/export"), + ]: + discovery = { + "name": service, "version": version, + "rootUrl": f"http://127.0.0.1:{server.server_port}/", + "resources": {resource: {"methods": {method: { + "httpMethod": "GET", "path": path, + "parameters": {id_field: { + "type": "string", "location": "path", "required": True, + }}, + }}}}, + } + (cache / f"{service}_{version}.json").write_text(json.dumps(discovery)) + env = { + **self.env, + "GOOGLE_WORKSPACE_CLI_CONFIG_DIR": str(config), + "GOOGLE_WORKSPACE_CLI_TOKEN": "synthetic-loopback-token", + "GOOGLE_APPLICATION_CREDENTIALS": str(self.root / "absent-adc.json"), + "GOOGLE_WORKSPACE_CLI_KEYRING_BACKEND": "file", + "GOOGLE_WORKSPACE_PROJECT_ID": "synthetic-loopback-project", + "HTTP_PROXY": "http://127.0.0.1:1", + "HTTPS_PROXY": "http://127.0.0.1:1", + "ALL_PROXY": "http://127.0.0.1:1", + "NO_PROXY": "127.0.0.1,localhost", + } + result = subprocess.run( + [sys.executable, "-B", str(SCRIPT), "--document-id", "synthetic-doc", + "--gws", str(binary), "--timeout", "10", "review"], + cwd=self.root, env=env, text=True, capture_output=True, timeout=25, + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["revisions"]["status"], "unchanged") + for name in exports.values(): + data = (self.root / "review" / name).read_bytes() + self.assertEqual(data, (fixture / name).read_bytes()) + self.assertEqual(manifest["artifacts"][name]["bytes"], len(data)) + self.assertEqual(len(requests), 5) + self.assertTrue(all(auth == "Bearer synthetic-loopback-token" + for _, _, auth in requests)) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +class ExtractionTests(BundleTestCase): + def extract(self, **limits): + return self.api().extract_docx( + self.fixture / "document.docx", self.root / "assets", **limits + ) + + def test_relationships_preserve_order_alt_text_and_table_context(self): + result = self.extract() + figures = result["figures"] + self.assertEqual([f["order"] for f in figures], [1, 2, 3]) + self.assertIn('Figure " onerror="bad()', figures[0]["alt"]) + self.assertIn("Table image context", figures[1]["nearby_text"]) + self.assertIsNone(figures[2]["asset"]) + self.assertEqual(figures[2]["availability"], "external-not-fetched") + self.assertIsNone(figures[0]["native_object_id"]) + self.assertEqual(figures[0]["mapping_confidence"], "docx-relationship-only") + + def test_duplicate_basenames_get_distinct_local_assets(self): + result = self.extract() + paths = [f["asset"] for f in result["figures"][:2]] + self.assertNotEqual(*paths) + self.assertEqual((self.root / paths[0]).read_bytes(), PNG) + self.assertEqual((self.root / paths[1]).read_bytes(), PNG + b"different") + + def test_nested_text_box_image_has_one_occurrence_with_inner_ownership(self): + write_docx(self.fixture / "document.docx", nested_docx_members()) + figures = self.extract()["figures"] + self.assertEqual(len(figures), 1) + self.assertEqual(figures[0]["order"], 1) + self.assertEqual(figures[0]["alt"], "Inner image") + self.assertEqual(figures[0]["nearby_text"], "Inner paragraph") + self.assertEqual((self.root / figures[0]["asset"]).read_bytes(), PNG) + + def test_nested_image_order_and_context_follow_nearest_owners(self): + write_docx(self.fixture / "document.docx", nested_docx_members(outer_image=True)) + figures = self.extract()["figures"] + self.assertEqual([f["relationship_id"] for f in figures], ["rId1", "rId2"]) + self.assertEqual([f["alt"] for f in figures], ["Inner image", "Outer text box"]) + self.assertEqual( + [f["nearby_text"] for f in figures], ["Inner paragraph", "Outer paragraph"] + ) + self.assertEqual([f["order"] for f in figures], [1, 2]) + + def test_repeated_asset_uses_remain_distinct_figure_occurrences(self): + members = docx_members() + members["word/document.xml"] = members["word/document.xml"].replace( + b'r:embed="rId2"', b'r:embed="rId1"' + ) + write_docx(self.fixture / "document.docx", members) + figures = self.extract()["figures"] + self.assertEqual(len(figures), 3) + self.assertEqual([f["order"] for f in figures], [1, 2, 3]) + self.assertEqual(figures[0]["asset"], figures[1]["asset"]) + self.assertEqual([f["relationship_id"] for f in figures[:2]], ["rId1", "rId1"]) + self.assertNotEqual(figures[0]["alt"], figures[1]["alt"]) + self.assertEqual(figures[1]["nearby_text"], "Table image context") + + def test_zip_traversal_absolute_windows_and_control_paths_are_rejected(self): + api = self.api() + for index, name in enumerate( + ["../escape", "/escape", "C:/escape", r"..\escape", "word/../escape", "bad\x01"] + ): + with self.subTest(name=name): + members = {**docx_members(), name: b"bad"} + write_docx(self.fixture / "document.docx", members) + with self.assertRaises(api.BundleError): + api.extract_docx( + self.fixture / "document.docx", self.root / f"assets-{index}" + ) + self.assertFalse((self.root / "escape").exists()) + + def test_zip_symlink_rejected(self): + api = self.api() + with zipfile.ZipFile(self.fixture / "document.docx", "a") as archive: + link = zipfile.ZipInfo("word/media/link.png") + link.create_system = 3 + link.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(link, "../../../escape") + with self.assertRaises(api.BundleError): + self.extract() + + def test_member_total_and_count_limits_reject_before_writing_assets(self): + api = self.api() + for limits in ( + {"member_limit": 32}, + {"total_limit": 32}, + {"member_count": 2}, + ): + with self.subTest(limits=limits), self.assertRaises(api.BundleError): + self.extract(**limits) + self.assertFalse((self.root / "assets").exists()) + + def test_duplicate_archive_member_rejected(self): + api = self.api() + with zipfile.ZipFile(self.fixture / "document.docx", "a") as archive: + archive.writestr("WORD/MEDIA/IMAGE1.PNG", PNG) + with self.assertRaises(api.BundleError): + self.extract() + + def test_doctype_entities_and_utf16_are_rejected_even_in_unused_xml(self): + api = self.api() + for data in ( + b']>&x;', + b'', + ']>&x;'.encode("utf-16"), + ): + with self.subTest(data=data[:30]): + members = {**docx_members(), "word/unused.xml": data} + write_docx(self.fixture / "document.docx", members) + with self.assertRaises(api.BundleError): + self.extract() + self.assertFalse((self.root / "assets").exists()) + + def test_xml_encoding_allowlist_handles_declaration_whitespace(self): + api = self.api() + for encoding in ("UTF-7", "UTF-16", "UTF-32", "ISO-8859-1"): + for assignment in (f' = "{encoding}"', f"= '{encoding}'", f'\t=\n"{encoding}"'): + with self.subTest(encoding=encoding, assignment=assignment): + data = f''.encode("utf-8") + with self.assertRaises(api.BundleError) as raised: + api.safe_xml(data) + self.assertEqual(str(raised.exception), "unsupported-xml-encoding") + + def test_xml_utf8_bom_is_accepted_and_utf16_utf32_are_rejected(self): + api = self.api() + data = 'café'.encode("utf-8-sig") + root = api.safe_xml(data) + self.assertEqual(root.tag, "x") + self.assertEqual(root.text, "café") + for encoding in ("utf-16", "utf-32"): + with self.subTest(encoding=encoding), self.assertRaises(api.BundleError): + api.safe_xml("".encode(encoding)) + + def test_relationship_traversal_and_svg_are_not_local_html_assets(self): + members = docx_members() + members["word/_rels/document.xml.rels"] = members[ + "word/_rels/document.xml.rels" + ].replace(b"media/image1.png", b"../outside.png") + members["word/media/nested/image1.png"] = b"" + write_docx(self.fixture / "document.docx", members) + figures = self.extract()["figures"] + self.assertIsNone(figures[0]["asset"]) + self.assertIsNone(figures[1]["asset"]) + # The safe, now unreferenced raster is still preserved; the SVG is not. + files = list((self.root / "assets").iterdir()) + self.assertEqual(len(files), 1) + self.assertEqual(files[0].read_bytes(), PNG) + + def test_malformed_xml_and_corrupt_zip_fail_closed(self): + api = self.api() + for data in (b"not a zip", None): + if data is None: + write_docx( + self.fixture / "document.docx", + {**docx_members(), "word/document.xml": b"", html) + self.assertIn("<script>", html) + self.assertNotIn("SIGNED_SECRET", html) + self.assertNotIn("untrusted.invalid", html) + self.assertTrue(any(tag == "iframe" for tag, _ in inspection.tags)) + for tag, attrs in inspection.tags: + self.assertNotEqual(tag, "script") + self.assertFalse(any(name.startswith("on") for name in attrs)) + if tag == "iframe": + self.assertIn("sandbox", attrs) + for ref in inspection.references: + self.assertNotIn(":", ref) + self.assertNotIn("..", ref) + self.assertNotIn("%", ref) + self.assertTrue((self.root / "review" / ref).is_file(), ref) + + def test_native_outline_includes_tables_child_tabs_styles_and_uncertain_images(self): + result = self.run_bundle("review") + self.assertEqual(result.returncode, 0, result.stderr) + html = (self.root / "review/index.html").read_text() + for text in ("Table cell", "Nested tab paragraph", "HEADING_1", "tab-child"): + self.assertIn(text, html) + self.assertIn("