From 9da870ebedb45e9477c19336887545c9fbfb40db Mon Sep 17 00:00:00 2001 From: ratovarius Date: Fri, 11 Sep 2026 12:04:09 -0300 Subject: [PATCH 1/5] feat(docs): add visual review bundle companion --- .changeset/docs-review-bundle.md | 7 + .github/workflows/ci.yml | 11 + examples/docs-review-bundle/README.md | 182 ++++ .../docs-review-bundle/docs_review_bundle.py | 749 ++++++++++++++++ .../test_docs_review_bundle.py | 813 ++++++++++++++++++ 5 files changed, 1762 insertions(+) create mode 100644 .changeset/docs-review-bundle.md create mode 100644 examples/docs-review-bundle/README.md create mode 100755 examples/docs-review-bundle/docs_review_bundle.py create mode 100644 examples/docs-review-bundle/test_docs_review_bundle.py diff --git a/.changeset/docs-review-bundle.md b/.changeset/docs-review-bundle.md new file mode 100644 index 000000000..bac3f4983 --- /dev/null +++ b/.changeset/docs-review-bundle.md @@ -0,0 +1,7 @@ +--- +"@googleworkspace/cli": minor +--- + +Add a standalone Python companion for visual Google Docs review bundles with +native exports, safe DOCX raster extraction, local HTML, optional PDF page +previews, revision observations, and an offline fixture workflow. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eedc9972..a7fae1a7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,17 @@ env: SCCACHE_IGNORE_SERVER_IO_ERROR: "true" jobs: + docs-review-bundle: + name: Docs Review Bundle (Python) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Test stdlib companion with synthetic fixtures + run: python3 -B -m unittest discover -s examples/docs-review-bundle -p 'test_*.py' -v + changes: name: Detect Changes runs-on: ubuntu-latest diff --git a/examples/docs-review-bundle/README.md b/examples/docs-review-bundle/README.md new file mode 100644 index 000000000..fcc7f58ca --- /dev/null +++ b/examples/docs-review-bundle/README.md @@ -0,0 +1,182 @@ +# Visual Docs review bundle + +This standalone companion combines native Docs JSON, Drive PDF/DOCX/Markdown +exports, DOCX raster assets, and a local HTML review index. It uses only the +Python standard library and existing `gws` commands. It does not change `gws`, +authenticate separately, or download document hyperlinks or image `contentUri`s. + +## Run + +Requires Python 3.10+ on Linux or macOS and an authenticated `gws` executable +with read access to the document through both Docs and Drive. + +From the repository root: + +```sh +python3 examples/docs-review-bundle/docs_review_bundle.py \ + --document-id DOCUMENT_ID review-bundle +``` + +`review-bundle` must be a **new relative directory inside the current working +directory**. Existing directories, absolute paths, `..`, symlink components, +control characters, and names outside the portable ASCII subset are refused. +Nested paths work when their parents already exist. The final directory is +created with mode `0700`; keep its parents under your control. + +Optional orchestration flags: + +```sh +python3 examples/docs-review-bundle/docs_review_bundle.py \ + --document-id DOCUMENT_ID --include-comments --render-pages \ + --timeout 120 --gws /trusted/path/to/gws another-review-bundle +``` + +- `--include-comments`: collect every returned Drive comments page. The entire + optional artifact is omitted and marked unavailable if retrieval is incomplete + or fails. Comments are a separate observation, not revision-bound or mapped to + PDF coordinates. Deleted comments are not requested. +- `--render-pages`: use a trusted `pdftoppm` on `PATH` to generate 96 DPI PNGs. + Rendering is opt-in. Missing tools, failures, timeouts, invalid output and + noncontiguous page numbers retain the PDF and report no available raster + previews. Partial page output is discarded. +- `--timeout`: positive finite seconds per subprocess; default 60. This is not + a total workflow deadline. +- `--gws`: trusted executable, resolved before changing subprocess working + directories. Existing `gws` authentication and Model Armor environment + settings are inherited. No credential values or raw process diagnostics + are inserted into HTML or error messages. + +The companion requests `docs documents get` with `includeTabsContent: true` +and uses `drive files export` with `--format json` and fixed relative +`--output` filenames. Every `gws` subprocess runs inside the new bundle. +Export success, MIME type, destination and byte count are checked against the +written artifact. This requires the existing `gws` binary-export receipt format. + +Open `index.html` locally. The index has no JavaScript or remote dependencies. +It contains a sandboxed PDF frame, optional page images, a paragraph/table +outline grouped by native tabs, DOCX figures and nearby text, native object +metadata, and escaped Markdown source. Some browsers block local PDF frames; +use the PDF artifact link in that case. Markdown is readable escaped source, +not rendered Markdown. + +## Artifacts and completion + +| File | Meaning | +| --- | --- | +| `source.json` | Original Docs JSON response, including all returned tabs | +| `revision-after.json` | Final revision observation, when available | +| `document.pdf`, `document.docx`, `document.md` | Required native Drive exports | +| `comments.json` | Optional fully retrieved comments result | +| `assets/` | Recognized DOCX PNG, JPEG, GIF and WebP media | +| `pages/` | Optional successful local page-rendering output | +| `index.html` | Local review index | +| `manifest.json` | State, versions, capabilities, limitations, mappings and hashes | + +The manifest begins as `in-progress` and becomes `complete` only after all +required exports, validation, index generation and artifact hashing succeed. +`complete` means the required bundle files were produced; it does **not** mean +an atomic snapshot, successful optional rendering, or verified export tab +coverage. Check `revisions`, `comments`, and `rendering` independently. + +Revision status is `mixed` for differing observed Docs revision IDs, `unchanged` +for equal nonempty IDs, and `unknown` if either is missing. Even `unchanged` +does not prove atomicity or that every export represents the same revision. +The manifest always records `atomic_snapshot: false`. + +Required failures return exit code 1 and leave a `failed` manifest with a safe +error code and stage. If writing the failure state also fails (for example, +a full disk), the earlier `in-progress` state can remain. Process termination +can also leave that state. No such bundle should be treated as complete. +Usage errors return 2. Optional failures and mixed/unknown revisions return 0 +when the required bundle completes. Existing output directories are never +overwritten; retries need a new name. + +SHA-256 and byte counts cover each produced artifact, including the index and +assets. The manifest does not hash itself and is not an authenticity signature. + +## Scope and safety limits + +- Native JSON traversal includes nested tabs, body paragraphs, tables and + tables of contents. The outline identifies paragraph styles but does not + recreate full layout. Headers, footers, notes, lists, equations, charts, + suggestions and other document features are not fully represented. +- Google determines the PDF/DOCX/Markdown export layout and tab coverage. + This companion cannot verify that every tab appears in those formats or + associate a PDF page/DOCX figure with an exact native tab. +- DOCX figure order comes from main-document drawing relationships, including + drawings in tables. Alt text and nearby paragraphs are context, **not exact + captions**. Native object IDs are recorded separately; the companion never + fabricates a matching native ID or a source tab ID for a DOCX image. +- Media basenames are replaced with distinct generated local filenames. + External, missing, traversing and unsupported relationships remain visibly + unavailable. Unreferenced recognized rasters are retained as artifacts. +- ZIP validation rejects traversal, absolute/Windows/control-character paths, + duplicate names (case-insensitive), symlinks and other special files, + encryption and unsupported compression. It never calls `extractall`. + Limits are 2,000 members, 20 MiB per member and 100 MiB total uncompressed. + ZIP paths are restricted to printable ASCII. Only stored/deflate compression + is accepted. +- Every XML/relationship member is parsed after rejecting DTDs, entities, + UTF-16/32 and non-UTF-8 encodings. XML trees are limited to 100 levels and + 100,000 nodes per part. The supported DOCX vocabulary is transitional OOXML + main-document drawings; unsupported content can remain unmapped. +- Each source/export/JSON artifact is capped at 20 MiB. Page output is limited + to 500 files and 100 MiB total. PDF header/EOF and raster signatures are + checked; these are **not full format validation**. Renderer exit success and + contiguous numbering do not independently prove the original PDF page count. +- Subprocess capture is file-backed, with size checks after exit. Timeouts and + post-render limits do not enforce disk or memory quotas on external tools. + `pdftoppm`, `gws`, local viewers and the operator-controlled parent directory + are trusted. This is not a sandbox against a concurrent local attacker. +- Display text is HTML/attribute escaped; URI references use generated local + allowlisted names. The index has a restrictive content-security policy and + no scripts. Source URLs and recognizable bearer/token strings are redacted + from display text, but this is not a general secret scanner or Model Armor + replacement. Existing `gws` sanitization behavior is preserved; binary + exports are not made safe by JSON sanitization. +- Raw artifacts deliberately retain original content, including any temporary + URLs or sensitive text. Treat the whole directory as sensitive. Review + external links and active content in native viewers separately; the + companion never follows them automatically. + +## Offline fixtures and visual QA + +Offline mode performs no `gws` calls. Supply a relative directory containing +`source.json`, `document.pdf`, `document.docx`, and UTF-8 `document.md`. +`revision-after.json` and `comments.json` are optional; a missing revision +observation remains unknown. Files and path components must not be symlinks. + +Generate entirely synthetic fixtures using the test utility, then create a +review bundle suitable for inspecting the HTML: + +```sh +python3 -B - <<'PY' +import sys +from pathlib import Path +sys.path.insert(0, "examples/docs-review-bundle") +from test_docs_review_bundle import fixtures +fixtures(Path("synthetic-docs-fixture")) +PY + +python3 -B examples/docs-review-bundle/docs_review_bundle.py \ + --from-fixture synthetic-docs-fixture --render-pages synthetic-docs-review +``` + +These generated fixtures intentionally contain HTML injection strings, an +untrusted synthetic URL, duplicate image basenames, missing image URIs, +nested tabs and table content. They are hand-built test data, not an actual +Google export or proof of cross-format fidelity. No real documents, +credentials or network access are needed. Omit `--render-pages` to avoid +running external software. + +## Tests + +```sh +python3 -B -m unittest discover \ + -s examples/docs-review-bundle -p 'test_*.py' -v +``` + +Tests use generated JSON/PDF/DOCX files and explicit `gws`/renderer executables +as stubs. Their subprocess environment omits real authentication settings; +no installed `gws`, real document, or network request is needed. CI runs this +same command in a dedicated Linux/macOS job. diff --git a/examples/docs-review-bundle/docs_review_bundle.py b/examples/docs-review-bundle/docs_review_bundle.py new file mode 100755 index 000000000..fd54477d4 --- /dev/null +++ b/examples/docs-review-bundle/docs_review_bundle.py @@ -0,0 +1,749 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at https://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build a local visual review bundle using Python's standard library and gws.""" + +import argparse +import hashlib +import html +import io +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +import zipfile + + +VERSION = "1.0" +MIB = 1024 * 1024 +FILE_LIMIT = 20 * MIB +TOTAL_LIMIT = 100 * MIB +MEMBER_COUNT = 2000 +MAX_PAGES = 500 +EXPORTS = { + "document.pdf": "application/pdf", + "document.docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "document.md": "text/markdown", +} +W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +A = "{http://schemas.openxmlformats.org/drawingml/2006/main}" +WP = "{http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing}" +R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}" +REL = "{http://schemas.openxmlformats.org/package/2006/relationships}" +LIMITATIONS = [ + "Exports are sequential, not an atomic snapshot; unchanged revisions are observations only.", + "Native outline includes body paragraphs, tables and nested tabs, not full layout or styling.", + "Drive export tab coverage is not verified; pages and DOCX figures have no reliable tab mapping.", + "DOCX relationships provide document order and nearby text, " + "not exact captions or native Docs IDs.", + "Only recognized PNG, JPEG, GIF and WebP media are previewed; " + "signatures are not full validation.", + "Headers, footers, notes, charts, vectors and unsupported drawings " + "may be absent from the outline or figures.", + "Content URIs and document hyperlinks are never fetched; URLs are redacted from display text.", + "Raw exports and JSON are sensitive, unsanitized source artifacts " + "and may contain temporary URLs.", + "The companion is not a document sanitizer or a sandbox for PDF/image viewers or pdftoppm.", +] + + +class BundleError(Exception): + """A fixed, non-sensitive failure code safe for manifests and terminals.""" + + +def relative_parts(value): + """Conservative portable path subset; do not normalize away traversal.""" + parts = value.split("/") + if not parts or any( + part in ("", ".", "..") or not re.fullmatch(r"[A-Za-z0-9_. -]+", part) + for part in parts + ): + raise BundleError("unsafe-relative-path") + return parts + + +def directory(value, *, create=False): + """Walk existing parents with no-follow descriptors (Linux/macOS).""" + parts = relative_parts(value) + flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + fd = os.open(".", flags) + try: + for index, part in enumerate(parts): + if create and index == len(parts) - 1: + os.mkdir(part, mode=0o700, dir_fd=fd) + next_fd = os.open(part, flags, dir_fd=fd) + os.close(fd) + fd = next_fd + except OSError: + raise BundleError("directory-exists-or-unsafe") from None + finally: + os.close(fd) + return Path.cwd().joinpath(*parts) + + +def read_bytes(path, limit=FILE_LIMIT): + """Bound reads and reject special files, including symlinks.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(fd, "rb") as stream: + info = os.fstat(stream.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_size > limit: + raise BundleError("file-type-or-size-limit") + data = stream.read(limit + 1) + except OSError: + raise BundleError("missing-or-unsafe-file") from None + if len(data) > limit: + raise BundleError("file-size-limit") + return data + + +def write_bytes(path, data): + # Files are new, inside the newly created private bundle directory. + with path.open("xb") as stream: + stream.write(data) + + +def json_bytes(value): + return (json.dumps(value, ensure_ascii=True, indent=2) + "\n").encode("utf-8") + + +def parse_json(data): + try: + value = json.loads(data) + except (ValueError, UnicodeError): + raise BundleError("invalid-json") from None + if not isinstance(value, dict) or "error" in value: + raise BundleError("invalid-json-response") + return value + + +def publish_manifest(bundle, manifest): + temporary = bundle / ".manifest.tmp" + write_bytes(temporary, json_bytes(manifest)) + os.replace(temporary, bundle / "manifest.json") + + +def local_uri(value): + """HTML references are generated file names, never document-supplied URIs.""" + if not re.fullmatch(r"[A-Za-z0-9_-]+(?:[./][A-Za-z0-9_-]+)*", value): + raise BundleError("unsafe-local-uri") + return value + + +def display(value): + text = str(value or "") + text = re.sub(r"(?i)\b(?:https?|ftp|file|data|javascript):[^\s<>\"']+", "[URL omitted]", text) + text = re.sub(r"(?i)\bBearer\s+\S+", "[credential omitted]", text) + text = re.sub( + r"(?i)\b(?:access_token|authorization|token)\s*[:=]\s*[^\s<>\"']+", + "[credential omitted]", + text, + ) + text = "".join(char for char in text if char in "\n\t" or ord(char) >= 32) + return html.escape(text, quote=True) + + +def safe_xml(data): + # Reject UTF-16/32 (including declaration smuggling via NUL bytes), DTDs and + # entities before giving XML to the stdlib parser. DOCX normally uses UTF-8. + if b"\x00" in data or re.search(br"]*encoding=['\"]([^'\"]+)", text, re.I) + if declaration and declaration[1].lower() not in ("utf-8", "utf8", "us-ascii"): + raise BundleError("unsupported-xml-encoding") + root = ET.fromstring(text) + except (ET.ParseError, UnicodeError): + raise BundleError("invalid-xml") from None + pending = [(root, 0)] + count = 0 + while pending: + node, depth = pending.pop() + count += 1 + if depth > 100 or count > 100_000: + raise BundleError("xml-complexity-limit") + pending.extend((child, depth + 1) for child in node) + return root + + +def raster_extension(data): + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "png" + if data.startswith(b"\xff\xd8\xff"): + return "jpg" + if data.startswith((b"GIF87a", b"GIF89a")): + return "gif" + if data.startswith(b"RIFF") and data[8:12] == b"WEBP": + return "webp" + return None + + +def zip_members(source, member_limit, total_limit, member_count): + """Read a bounded archive without ever extracting its member paths.""" + try: + with zipfile.ZipFile(io.BytesIO(read_bytes(source))) as archive: + infos = archive.infolist() + if len(infos) > member_count: + raise BundleError("zip-member-count-limit") + seen = set() + declared_total = 0 + for info in infos: + name = info.orig_filename + parts = name.rstrip("/").split("/") + kind = stat.S_IFMT(info.external_attr >> 16) + if ( + name != info.filename + or name.startswith("/") + or "\\" in name + or ":" in name + or any(part in ("", ".", "..") for part in parts) + or any(ord(char) < 32 or ord(char) > 126 for char in name) + or kind not in (0, stat.S_IFREG, stat.S_IFDIR) + or info.flag_bits & 1 + or info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) + or name.casefold() in seen + ): + raise BundleError("unsafe-zip-member") + seen.add(name.casefold()) + declared_total += info.file_size + if info.file_size > member_limit or declared_total > total_limit: + raise BundleError("zip-size-limit") + members = {} + actual_total = 0 + for info in infos: + if info.is_dir(): + continue + with archive.open(info) as stream: + data = stream.read(member_limit + 1) + actual_total += len(data) + if ( + len(data) > member_limit + or actual_total > total_limit + or len(data) != info.file_size + ): + raise BundleError("zip-size-limit") + members[info.filename] = data + return members + except (zipfile.BadZipFile, RuntimeError, NotImplementedError, OSError, EOFError): + raise BundleError("invalid-zip") from None + + +def extract_docx( + source, output, *, + member_limit=FILE_LIMIT, total_limit=TOTAL_LIMIT, member_count=MEMBER_COUNT, +): + members = zip_members(source, member_limit, total_limit, member_count) + # Validate even unused XML before any asset writes. + trees = { + name: safe_xml(data) + for name, data in members.items() + if name.lower().endswith((".xml", ".rels")) + } + document = trees.get("word/document.xml") + if document is None or document.tag != W + "document": + raise BundleError("missing-docx-document") + relationships = {} + rels = trees.get("word/_rels/document.xml.rels") + if rels is not None: + for rel in rels.findall(REL + "Relationship"): + identity = rel.get("Id") + if not identity or identity in relationships: + raise BundleError("ambiguous-docx-relationship") + relationships[identity] = rel.attrib + output.mkdir(mode=0o700) + assets = {} + for name, data in members.items(): + extension = raster_extension(data) + if name.startswith("word/media/") and extension: + filename = f"image-{len(assets) + 1}.{extension}" + write_bytes(output / filename, data) + assets[name] = local_uri(f"{output.name}/{filename}") + paragraphs = list(document.iter(W + "p")) + texts = [ + "".join(text.text or "" for text in paragraph.iter(W + "t")) + for paragraph in paragraphs + ] + figures = [] + for index, paragraph in enumerate(paragraphs): + nearby = texts[index] or " ".join( + texts[max(0, index - 1):index] + texts[index + 1:index + 2] + ) + for drawing in paragraph.iter(W + "drawing"): + properties = drawing.find(".//" + WP + "docPr") + alt = "" + if properties is not None: + alt = " ".join(properties.get(field, "") for field in ("title", "descr")).strip() + for blip in drawing.iter(A + "blip"): + identity = blip.get(R + "embed") or blip.get(R + "link") + relationship = relationships.get(identity, {}) + asset = None + availability = "missing-or-unsupported" + if relationship.get("TargetMode", "").lower() == "external": + availability = "external-not-fetched" + elif relationship.get("Type") == R[1:-1] + "/image": + target = relationship.get("Target", "") + # Allow only relative media targets in the document's part. + if ( + target.startswith("media/") + and not any(p in ("", ".", "..") for p in target.split("/")) + and not any(char in target for char in "\\:%?#") + ): + asset = assets.get(str(PurePosixPath("word") / target)) + if asset: + availability = "available" + figures.append({ + "order": len(figures) + 1, + "relationship_id": identity, + "asset": asset, + "alt": alt, + "nearby_text": nearby[:1000], + "availability": availability, + "mapping_confidence": "docx-relationship-only", + "native_object_id": None, + "source_tab_id": None, + }) + return {"figures": figures, "assets": list(assets.values())} + + +def native_view(source): + if not isinstance(source.get("body"), dict) and not isinstance(source.get("tabs"), list): + raise BundleError("missing-document-content") + tabs = [] + images = [] + + def add_tab(tab, depth): + if depth > 50 or len(tabs) >= 1000: + raise BundleError("native-tab-limit") + properties = tab.get("tabProperties", {}) + document = tab.get("documentTab", {}) + tab_id = properties.get("tabId") + tabs.append({ + "id": tab_id, + "title": properties.get("title", "Document"), + "depth": depth, + "blocks": document.get("body", {}).get("content", []), + }) + for collection, property_name in ( + ("inlineObjects", "inlineObjectProperties"), + ("positionedObjects", "positionedObjectProperties"), + ): + for object_id, obj in document.get(collection, {}).items(): + embedded = obj.get(property_name, {}).get("embeddedObject", {}) + images.append({ + "object_id": object_id, + "tab_id": tab_id, + "title": embedded.get("title", ""), + "description": embedded.get("description", ""), + "content_uri_available": bool( + embedded.get("imageProperties", {}).get("contentUri") + ), + "asset": None, + "mapping_confidence": "unmapped", + }) + for child in tab.get("childTabs", []): + add_tab(child, depth + 1) + + if source.get("tabs"): + for tab in source["tabs"]: + add_tab(tab, 0) + else: + add_tab({"documentTab": source}, 0) + return tabs, images + + +def outline_html(blocks, depth=0): + if depth > 50: + raise BundleError("native-outline-depth-limit") + parts = [] + for block in blocks: + if "paragraph" in block: + paragraph = block["paragraph"] + text = "".join( + element.get("textRun", {}).get("content", "") + for element in paragraph.get("elements", []) + ) + style = paragraph.get("paragraphStyle", {}).get("namedStyleType", "NORMAL_TEXT") + parts.append(f"

{display(style)} {display(text)}

") + elif "table" in block: + parts.append("") + for row in block["table"].get("tableRows", []): + parts.append("") + for cell in row.get("tableCells", []): + parts.append( + "" + ) + parts.append("") + parts.append("
" + outline_html(cell.get("content", []), depth + 1) + "
") + elif "tableOfContents" in block: + parts.append(outline_html(block["tableOfContents"].get("content", []), depth + 1)) + return "".join(parts) + + +def index_html(source, markdown, tabs, manifest, artifacts): + parts = [ + '', + '', + '", + "Docs review bundle", + "", + f"

{display(source.get('title', 'Docs review bundle'))}

", + f"

Revision observation: {display(manifest['revisions']['status'])}. " + "Sequential exports; not an atomic snapshot.

", + "

Artifacts

Raw files may contain sensitive content and temporary URLs.

", + "

Native PDF

", + '', + "

If your browser blocks the embedded viewer, open the local PDF artifact.

", + f"

Raster previews: {display(manifest['rendering']['status'])}; " + f"{display(manifest['rendering'].get('reason', ''))}

", + ] + ) + for page in manifest["rendering"]["pages"]: + parts.append(f'{display(page)}') + parts.append("

Native outline and tabs

") + for tab in tabs: + parts.append( + f"

{display(tab['title'])}

" + f"

Tab: {display(tab['id'] or 'legacy body')}; " + f"depth: {tab['depth']}

{outline_html(tab['blocks'])}" + ) + parts.append("

DOCX figures

") + for figure in manifest["figures"]: + parts.append(f"

Figure {figure['order']}

") + if figure["asset"]: + parts.append( + f'' + ) + parts.append( + f"
{display(figure['alt'])}
" + f"

Availability: {display(figure['availability'])}

" + f"

Nearby text (not an exact caption): {display(figure['nearby_text'])}

" + "

DOCX relationship only; native object and source tab mapping unknown.

" + ) + parts.append("

Native image metadata

") + for image in manifest["native_images"]: + parts.append( + f"

Object {display(image['object_id'])}, tab {display(image['tab_id'])}: " + f"{display(image['title'])} {display(image['description'])}. " + f"contentUri available: {image['content_uri_available']}; asset mapping: unmapped.

" + ) + parts.append( + "

Readable Markdown source

" + f"
{display(markdown)}
" + ) + parts.append( + f"

Comments

{display(manifest['comments']['status'])}

" + ) + parts.append("

Capabilities and limitations

") + return "".join(parts).encode("utf-8") + + +def run_process(argv, bundle, timeout): + # File-backed capture bounds memory. No shell and no untrusted diagnostics + # echoed to the terminal or copied into the manifest/display HTML. + with tempfile.TemporaryFile(dir=bundle) as output: + try: + process = subprocess.run( + argv, + cwd=bundle, + stdout=output, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + raise BundleError("timeout") from None + except OSError: + raise BundleError("process-unavailable") from None + if process.returncode: + raise BundleError("process-failed") + if output.tell() > FILE_LIMIT: + raise BundleError("process-output-limit") + output.seek(0) + return output.read(FILE_LIMIT + 1) + + +def gws_json(executable, bundle, timeout, command, params, output=None): + argv = [executable, *command, "--params", json.dumps(params), "--format", "json"] + if output: + argv.extend(["--output", local_uri(output)]) + data = run_process(argv, bundle, timeout) + return data, parse_json(data) + + +def collect_comments(executable, bundle, timeout, document_id): + comments = [] + seen = set() + token = None + total = 0 + for _ in range(100): + params = {"fileId": document_id, "fields": "nextPageToken,comments", "pageSize": 100} + if token: + params["pageToken"] = token + raw, page = gws_json(executable, bundle, timeout, ["drive", "comments", "list"], params) + total += len(raw) + if total > FILE_LIMIT or not isinstance(page.get("comments", []), list): + raise BundleError("invalid-or-oversized-comments") + comments.extend(page.get("comments", [])) + token = page.get("nextPageToken") + if not token: + return json_bytes({"comments": comments}) + if not isinstance(token, str) or token in seen: + raise BundleError("incomplete-comments") + seen.add(token) + raise BundleError("incomplete-comments") + + +def render_pages(bundle, timeout, requested): + result = {"status": "not-requested", "pages": []} + if not requested: + return result + renderer = shutil.which("pdftoppm") + if not renderer: + return {"status": "unavailable", "reason": "pdftoppm-not-found; PDF retained", "pages": []} + try: + with tempfile.TemporaryDirectory(prefix=".render-", dir=bundle) as temp: + staging = Path(temp) + prefix = str(staging.relative_to(bundle) / "page") + run_process([renderer, "-png", "-r", "96", "document.pdf", prefix], bundle, timeout) + pages = {} + total = 0 + for path in staging.iterdir(): + match = re.fullmatch(r"page-([0-9]+)\.png", path.name) + if not match: + raise BundleError("invalid-render") + number = int(match[1]) + data = read_bytes(path) + total += len(data) + if number in pages or raster_extension(data) != "png" or total > TOTAL_LIMIT: + raise BundleError("invalid-render") + pages[number] = path.name + if ( + not pages + or len(pages) > MAX_PAGES + or sorted(pages) != list(range(1, len(pages) + 1)) + ): + raise BundleError("invalid-render") + staging.rename(bundle / "pages") + return { + "status": "complete", + "pages": [local_uri("pages/" + pages[number]) for number in sorted(pages)], + } + except (BundleError, OSError) as error: + return { + "status": "failed", "pages": [], + "reason": "timeout" if str(error) == "timeout" else "invalid-or-failed-render", + } + + +def revision_observation(before, after): + before_id = before.get("revisionId") + after_id = after.get("revisionId") + before_id = before_id if isinstance(before_id, str) and before_id else None + after_id = after_id if isinstance(after_id, str) and after_id else None + status = "unknown" + if before_id and after_id: + status = "unchanged" if before_id == after_id else "mixed" + return {"before": before_id, "after": after_id, "status": status, "atomic_snapshot": False} + + +def build_bundle(args, bundle, manifest): + artifacts = ["source.json", *EXPORTS] + manifest["stage"] = "exports" + if args.from_fixture: + fixture = directory(args.from_fixture) + for name in artifacts: + write_bytes(bundle / name, read_bytes(fixture / name)) + source = parse_json(read_bytes(bundle / "source.json")) + after_path = fixture / "revision-after.json" + after = parse_json(read_bytes(after_path)) if after_path.exists() else {} + if after_path.exists(): + write_bytes(bundle / "revision-after.json", read_bytes(after_path)) + artifacts.append("revision-after.json") + executable = None + else: + executable = shutil.which(args.gws) + if not executable: + raise BundleError("gws-not-found") + executable = str(Path(executable).resolve()) + try: + version = run_process([executable, "--version"], bundle, args.timeout).decode("utf-8") + match = re.fullmatch(r"gws ([0-9][A-Za-z0-9.+-]*)\s*", version) + manifest["versions"]["gws"] = match[1] if match else "unknown" + except BundleError: + manifest["versions"]["gws"] = "unknown" + params = {"documentId": args.document_id, "includeTabsContent": True} + raw, source = gws_json( + executable, bundle, args.timeout, ["docs", "documents", "get"], params + ) + if source.get("documentId") != args.document_id: + raise BundleError("document-id-mismatch") + write_bytes(bundle / "source.json", raw) + for name, mime in EXPORTS.items(): + _, receipt = gws_json( + executable, bundle, args.timeout, ["drive", "files", "export"], + {"fileId": args.document_id, "mimeType": mime}, output=name, + ) + data = read_bytes(bundle / name) + if ( + receipt.get("status") != "success" + or receipt.get("saved_file") != name + or str(receipt.get("mimeType", "")).split(";")[0].strip().lower() != mime + or type(receipt.get("bytes")) is not int + or receipt["bytes"] != len(data) + ): + raise BundleError("invalid-export-receipt") + raw, after = gws_json( + executable, bundle, args.timeout, ["docs", "documents", "get"], + {**params, "fields": "revisionId"}, + ) + write_bytes(bundle / "revision-after.json", raw) + artifacts.append("revision-after.json") + + manifest["revisions"] = revision_observation(source, after) + manifest["comments"] = {"status": "not-requested"} + if args.include_comments: + try: + if executable: + comments = collect_comments(executable, bundle, args.timeout, args.document_id) + else: + comments = read_bytes(fixture / "comments.json") + value = parse_json(comments) + if not isinstance(value.get("comments"), list) or value.get("nextPageToken"): + raise BundleError("incomplete-comments") + write_bytes(bundle / "comments.json", comments) + artifacts.append("comments.json") + manifest["comments"] = {"status": "available"} + except (BundleError, OSError): + (bundle / "comments.json").unlink(missing_ok=True) + manifest["comments"] = { + "status": "unavailable", + "reason": "retrieval-incomplete-or-failed", + } + + manifest["stage"] = "validate-and-extract" + pdf = read_bytes(bundle / "document.pdf") + if not pdf.startswith(b"%PDF-") or b"%%EOF" not in pdf[-1024:]: + raise BundleError("invalid-or-truncated-pdf") + markdown = read_bytes(bundle / "document.md").decode("utf-8") + tabs, native_images = native_view(source) + extracted = extract_docx(bundle / "document.docx", bundle / "assets") + manifest["figures"] = extracted["figures"] + manifest["native_images"] = native_images + manifest["tabs"] = [{k: v for k, v in tab.items() if k != "blocks"} for tab in tabs] + artifacts.extend(extracted["assets"]) + manifest["stage"] = "render" + manifest["rendering"] = render_pages(bundle, args.timeout, args.render_pages) + artifacts.extend(manifest["rendering"]["pages"]) + manifest["stage"] = "index" + write_bytes(bundle / "index.html", index_html(source, markdown, tabs, manifest, artifacts)) + artifacts.append("index.html") + manifest["artifacts"] = {} + for name in artifacts: + data = read_bytes(bundle / local_uri(name)) + manifest["artifacts"][name] = { + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + manifest["status"] = "complete" + manifest["stage"] = "complete" + publish_manifest(bundle, manifest) + + +def positive_timeout(value): + number = float(value) + if not math.isfinite(number) or number <= 0: + raise argparse.ArgumentTypeError("timeout must be a positive finite number") + return number + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--document-id", help="Google Docs ID (not a URL)") + source.add_argument( + "--from-fixture", help="Relative directory containing captured export files" + ) + parser.add_argument("output_dir", help="New relative directory within CWD; parents must exist") + parser.add_argument("--include-comments", action="store_true") + parser.add_argument( + "--render-pages", action="store_true", help="Opt in to local pdftoppm rendering" + ) + parser.add_argument( + "--timeout", type=positive_timeout, default=60.0, + help="Seconds per process (default: 60)", + ) + parser.add_argument( + "--gws", default="gws", help="Trusted gws executable (default: PATH lookup)" + ) + args = parser.parse_args(argv) + if args.document_id and not re.fullmatch(r"[A-Za-z0-9_-]+", args.document_id): + parser.error("document-id must be a Google Docs ID, not a URL or path") + bundle = None + manifest = { + "schema_version": 1, "status": "in-progress", "stage": "initialize", + "mode": "offline" if args.from_fixture else "gws", + "versions": {"companion": VERSION, "python": sys.version.split()[0], "gws": None}, + "capabilities": { + "all_native_tabs": True, "native_pdf": True, "docx_raster_assets": True, + "native_markdown": True, "network_image_fetch": False, + "exact_native_asset_mapping": False, "atomic_snapshot": False, + }, + "limitations": LIMITATIONS, + } + try: + bundle = directory(args.output_dir, create=True) + publish_manifest(bundle, manifest) + build_bundle(args, bundle, manifest) + except (Exception, KeyboardInterrupt) as error: + code = str(error) if isinstance(error, BundleError) else "invalid-or-incomplete-bundle" + if bundle is not None: + manifest["status"] = "failed" + manifest["error"] = code + try: + publish_manifest(bundle, manifest) + except OSError: + # An earlier in-progress manifest remains non-complete if storage fails. + pass + print(f"Review bundle failed: {code}.", file=sys.stderr) + return 1 + print("Review bundle complete. Open index.html in the new output directory.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/docs-review-bundle/test_docs_review_bundle.py b/examples/docs-review-bundle/test_docs_review_bundle.py new file mode 100644 index 000000000..56ede7b74 --- /dev/null +++ b/examples/docs-review-bundle/test_docs_review_bundle.py @@ -0,0 +1,813 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at https://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hermetic behavior tests: generated documents and executable process stubs only.""" + +import base64 +import contextlib +import hashlib +from html.parser import HTMLParser +import importlib.util +import io +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +import zipfile + + +SCRIPT = Path(__file__).with_name("docs_review_bundle.py") +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8" + "/x8AAwMCAO+jRZkAAAAASUVORK5CYII=" +) +W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +R = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +A = "http://schemas.openxmlformats.org/drawingml/2006/main" +WP = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" +REL = "http://schemas.openxmlformats.org/package/2006/relationships" +REMOTE = "https://untrusted.invalid/image?token=SIGNED_SECRET" + + +def paragraph(text, style="NORMAL_TEXT"): + return { + "paragraph": { + "paragraphStyle": {"namedStyleType": style}, + "elements": [ + {"textRun": {"content": text, "textStyle": {"link": {"url": REMOTE}}}} + ], + } + } + + +def native_document(): + return { + "documentId": "synthetic-doc", + "title": 'Review ', + "revisionId": "revision-one", + "tabs": [ + { + "tabProperties": {"tabId": "tab-main", "title": "Main"}, + "documentTab": { + "body": { + "content": [ + paragraph("Heading ", "HEADING_1"), + { + "table": { + "tableRows": [ + { + "tableCells": [ + {"content": [paragraph("Table cell")]} + ] + } + ] + } + }, + { + "paragraph": { + "elements": [ + { + "inlineObjectElement": { + "inlineObjectId": "native-image" + } + } + ] + } + }, + ] + }, + "inlineObjects": { + "native-image": { + "inlineObjectProperties": { + "embeddedObject": { + "title": "Native figure", + "description": "No exact DOCX mapping", + "imageProperties": {"contentUri": REMOTE}, + } + } + }, + "missing-uri": { + "inlineObjectProperties": { + "embeddedObject": { + "description": "No downloadable URI", + "imageProperties": {}, + } + } + }, + }, + }, + "childTabs": [ + { + "tabProperties": {"tabId": "tab-child", "title": "Child"}, + "documentTab": { + "body": {"content": [paragraph("Nested tab paragraph")]} + }, + } + ], + } + ], + } + + +def pdf_bytes(): + """A complete synthetic one-page PDF, also usable by real pdftoppm.""" + content = ( + b"BT /F1 22 Tf 48 720 Td (Synthetic Docs review) Tj ET\n" + b"BT /F1 12 Tf 48 687 Td (Local fixture - no Google document) Tj ET\n" + b"0.85 0.92 1 rg 48 435 516 210 re f\n" + b"0.08 0.25 0.5 rg 72 459 120 162 re f\n" + b"0.12 0.45 0.6 rg 216 459 120 105 re f\n" + b"0.1 0.6 0.45 rg 360 459 120 140 re f\n" + b"0 0 0 rg BT /F1 12 Tf 48 402 Td (Synthetic figure and table context) Tj ET\n" + b"0.5 G 48 270 516 90 re S 48 315 m 564 315 l S\n" + b"306 270 m 306 360 l S\n" + b"BT /F1 12 Tf 60 333 Td (Column A) Tj 258 0 Td (Column B) Tj ET\n" + b"BT /F1 12 Tf 60 288 Td (Cell one) Tj 258 0 Td (Cell two) Tj ET\n" + ) + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + f"<< /Length {len(content)} >>\nstream\n".encode() + content + b"endstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + data = b"%PDF-1.4\n" + offsets = [0] + for index, obj in enumerate(objects, 1): + offsets.append(len(data)) + data += f"{index} 0 obj\n".encode() + obj + b"\nendobj\n" + startxref = len(data) + data += b"xref\n0 6\n0000000000 65535 f \n" + for offset in offsets[1:]: + data += f"{offset:010d} 00000 n \n".encode() + data += ( + f"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n" + ).encode() + return data + + +def docx_members(): + xml = f""" + + + +Nearby <script>bad()</script> + + +Table image context + + + + + +""" + rels = f""" + + +""" + return { + "word/document.xml": xml.encode(), + "word/_rels/document.xml.rels": rels.encode(), + "word/styles.xml": f''.encode(), + "word/media/image1.png": PNG, + "word/media/nested/image1.png": PNG + b"different", + } + + +def write_docx(path, members=None): + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in (members if members is not None else docx_members()).items(): + archive.writestr(name, data) + + +def fixtures(path): + path.mkdir() + (path / "source.json").write_text(json.dumps(native_document()), encoding="utf-8") + (path / "revision-after.json").write_text( + '{"revisionId":"revision-one"}', encoding="utf-8" + ) + (path / "document.pdf").write_bytes(pdf_bytes()) + (path / "document.md").write_text( + "# Markdown\n\n![remote](" + REMOTE + ")\n", + encoding="utf-8", + ) + write_docx(path / "document.docx") + return path + + +GWS_STUB = r''' +import json +import os +from pathlib import Path +import shutil +import sys +import time + +args = sys.argv[1:] +mode = os.environ.get("STUB_MODE", "") +fixture = Path(os.environ["STUB_FIXTURES"]) +with open(os.environ["STUB_LOG"], "a", encoding="utf-8") as log: + log.write(json.dumps({"args": args, "cwd": os.getcwd(), + "sanitize": os.environ.get("GOOGLE_WORKSPACE_CLI_SANITIZE_MODE")}) + "\n") +if args == ["--version"]: + print("gws 0.0.0-synthetic") + sys.exit(0) +params = json.loads(args[args.index("--params") + 1]) +assert args[args.index("--format") + 1] == "json" +if mode == "timeout": + time.sleep(10) +if args[:3] == ["docs", "documents", "get"]: + assert params["documentId"] == "synthetic-doc" + assert params["includeTabsContent"] is True + if params.get("fields") == "revisionId": + source = {"revisionId": "revision-two" if mode == "mixed" else "revision-one"} + if mode == "missing-revision": + source = {} + else: + source = json.loads((fixture / "source.json").read_text()) + print(json.dumps(source)) +elif args[:3] == ["drive", "files", "export"]: + assert params["fileId"] == "synthetic-doc" + expected = { + "application/pdf": "document.pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "document.docx", + "text/markdown": "document.md", + } + name = args[args.index("--output") + 1] + assert expected[params["mimeType"]] == name + assert "/" not in name and "\\" not in name + if mode == "failed-export" and name == "document.docx": + Path(name).write_bytes(b"partial") + print("Bearer PRIVATE_TOKEN " + "\x1b[31m", file=sys.stderr) + sys.exit(7) + if mode != "no-export-file": + shutil.copyfile(fixture / name, name) + print(json.dumps({ + "status": "error" if mode == "bad-export-status" else "success", + "saved_file": name, + "mimeType": params["mimeType"], + "bytes": 1 if mode == "wrong-export-size" else (fixture / name).stat().st_size, + })) +elif args[:3] == ["drive", "comments", "list"]: + assert params["fileId"] == "synthetic-doc" + assert "nextPageToken" in params["fields"] + if mode == "failed-comments" and params.get("pageToken"): + sys.exit(9) + if params.get("pageToken"): + print(json.dumps({"comments": [{"id": "two", "content": "Second comment"}]})) + else: + print(json.dumps({"nextPageToken": "page-two", "comments": [{"id": "one"}]})) +else: + raise AssertionError(args) +''' + +RENDER_STUB = r''' +import base64 +import os +from pathlib import Path +import sys +import time +assert sys.argv[1:4] == ["-png", "-r", "96"] +assert sys.argv[-2] == "document.pdf" +prefix = Path(sys.argv[-1]) +png = base64.b64decode(os.environ["STUB_PNG"]) +mode = os.environ.get("RENDER_MODE", "") +Path(str(prefix) + "-1.png").write_bytes(png) +if mode == "failed": + sys.exit(2) +if mode == "timeout": + time.sleep(10) +if mode == "gap": + Path(str(prefix) + "-3.png").write_bytes(png) +''' + + +class HTMLInspection(HTMLParser): + def __init__(self, text): + super().__init__() + self.tags = [] + self.references = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + self.tags.append((tag, dict(attrs))) + for name, value in attrs: + if name in ("src", "href", "data"): + self.references.append(value) + + +class BundleTestCase(unittest.TestCase): + def setUp(self): + # All generated files stay inside this assigned worktree and are removed. + self.temp = tempfile.TemporaryDirectory(dir=SCRIPT.parent) + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.fixture = fixtures(self.root / "fixtures") + self.bin = self.root / "bin" + self.bin.mkdir() + self.env = { + "PATH": str(self.bin), + "PYTHONDONTWRITEBYTECODE": "1", + "STUB_FIXTURES": str(self.fixture), + "STUB_LOG": str(self.root / "gws.log"), + "STUB_PNG": base64.b64encode(PNG).decode(), + "GOOGLE_WORKSPACE_CLI_CONFIG_DIR": str(self.root / "unused-config"), + "GOOGLE_WORKSPACE_CLI_SANITIZE_MODE": "block", + } + + def api(self): + self.assertTrue(SCRIPT.is_file(), "Missing executable review bundle companion") + if not hasattr(self, "_api"): + spec = importlib.util.spec_from_file_location("docs_review_bundle", SCRIPT) + self._api = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self._api) + return self._api + + def executable(self, name, source): + path = self.bin / name + path.write_text(f"#!{sys.executable}\n" + source, encoding="utf-8") + path.chmod(0o700) + return path + + def run_bundle(self, *args, live=False, **env): + self.assertTrue(SCRIPT.is_file(), "Missing executable review bundle companion") + if live: + self.executable("gws", GWS_STUB) + source = ["--document-id", "synthetic-doc"] + else: + source = ["--from-fixture", "fixtures"] + return subprocess.run( + [sys.executable, "-B", str(SCRIPT), *source, *args], + cwd=self.root, + env={**self.env, **env}, + text=True, + capture_output=True, + timeout=15, + ) + + def manifest(self, directory="review"): + return json.loads((self.root / directory / "manifest.json").read_text()) + + +class ExtractionTests(BundleTestCase): + def extract(self, **limits): + return self.api().extract_docx( + self.fixture / "document.docx", self.root / "assets", **limits + ) + + def test_relationships_preserve_order_alt_text_and_table_context(self): + result = self.extract() + figures = result["figures"] + self.assertEqual([f["order"] for f in figures], [1, 2, 3]) + self.assertIn('Figure " onerror="bad()', figures[0]["alt"]) + self.assertIn("Table image context", figures[1]["nearby_text"]) + self.assertIsNone(figures[2]["asset"]) + self.assertEqual(figures[2]["availability"], "external-not-fetched") + self.assertIsNone(figures[0]["native_object_id"]) + self.assertEqual(figures[0]["mapping_confidence"], "docx-relationship-only") + + def test_duplicate_basenames_get_distinct_local_assets(self): + result = self.extract() + paths = [f["asset"] for f in result["figures"][:2]] + self.assertNotEqual(*paths) + self.assertEqual((self.root / paths[0]).read_bytes(), PNG) + self.assertEqual((self.root / paths[1]).read_bytes(), PNG + b"different") + + def test_zip_traversal_absolute_windows_and_control_paths_are_rejected(self): + api = self.api() + for index, name in enumerate( + ["../escape", "/escape", "C:/escape", r"..\escape", "word/../escape", "bad\x01"] + ): + with self.subTest(name=name): + members = {**docx_members(), name: b"bad"} + write_docx(self.fixture / "document.docx", members) + with self.assertRaises(api.BundleError): + api.extract_docx( + self.fixture / "document.docx", self.root / f"assets-{index}" + ) + self.assertFalse((self.root / "escape").exists()) + + def test_zip_symlink_rejected(self): + api = self.api() + with zipfile.ZipFile(self.fixture / "document.docx", "a") as archive: + link = zipfile.ZipInfo("word/media/link.png") + link.create_system = 3 + link.external_attr = (stat.S_IFLNK | 0o777) << 16 + archive.writestr(link, "../../../escape") + with self.assertRaises(api.BundleError): + self.extract() + + def test_member_total_and_count_limits_reject_before_writing_assets(self): + api = self.api() + for limits in ( + {"member_limit": 32}, + {"total_limit": 32}, + {"member_count": 2}, + ): + with self.subTest(limits=limits), self.assertRaises(api.BundleError): + self.extract(**limits) + self.assertFalse((self.root / "assets").exists()) + + def test_duplicate_archive_member_rejected(self): + api = self.api() + with zipfile.ZipFile(self.fixture / "document.docx", "a") as archive: + archive.writestr("WORD/MEDIA/IMAGE1.PNG", PNG) + with self.assertRaises(api.BundleError): + self.extract() + + def test_doctype_entities_and_utf16_are_rejected_even_in_unused_xml(self): + api = self.api() + for data in ( + b']>&x;', + b'', + ']>&x;'.encode("utf-16"), + ): + with self.subTest(data=data[:30]): + members = {**docx_members(), "word/unused.xml": data} + write_docx(self.fixture / "document.docx", members) + with self.assertRaises(api.BundleError): + self.extract() + self.assertFalse((self.root / "assets").exists()) + + def test_relationship_traversal_and_svg_are_not_local_html_assets(self): + members = docx_members() + members["word/_rels/document.xml.rels"] = members[ + "word/_rels/document.xml.rels" + ].replace(b"media/image1.png", b"../outside.png") + members["word/media/nested/image1.png"] = b"" + write_docx(self.fixture / "document.docx", members) + figures = self.extract()["figures"] + self.assertIsNone(figures[0]["asset"]) + self.assertIsNone(figures[1]["asset"]) + # The safe, now unreferenced raster is still preserved; the SVG is not. + files = list((self.root / "assets").iterdir()) + self.assertEqual(len(files), 1) + self.assertEqual(files[0].read_bytes(), PNG) + + def test_malformed_xml_and_corrupt_zip_fail_closed(self): + api = self.api() + for data in (b"not a zip", None): + if data is None: + write_docx( + self.fixture / "document.docx", + {**docx_members(), "word/document.xml": b"", html) + self.assertIn("<script>", html) + self.assertNotIn("SIGNED_SECRET", html) + self.assertNotIn("untrusted.invalid", html) + self.assertTrue(any(tag == "iframe" for tag, _ in inspection.tags)) + for tag, attrs in inspection.tags: + self.assertNotEqual(tag, "script") + self.assertFalse(any(name.startswith("on") for name in attrs)) + if tag == "iframe": + self.assertIn("sandbox", attrs) + for ref in inspection.references: + self.assertNotIn(":", ref) + self.assertNotIn("..", ref) + self.assertNotIn("%", ref) + self.assertTrue((self.root / "review" / ref).is_file(), ref) + + def test_native_outline_includes_tables_child_tabs_styles_and_uncertain_images(self): + result = self.run_bundle("review") + self.assertEqual(result.returncode, 0, result.stderr) + html = (self.root / "review/index.html").read_text() + for text in ("Table cell", "Nested tab paragraph", "HEADING_1", "tab-child"): + self.assertIn(text, html) + self.assertIn(" Date: Fri, 11 Sep 2026 11:59:05 -0300 Subject: [PATCH 2/5] fix(script): keep current Clippy checks passing --- .changeset/current-clippy-baseline.md | 5 +++++ crates/google-workspace-cli/src/helpers/script.rs | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 .changeset/current-clippy-baseline.md diff --git a/.changeset/current-clippy-baseline.md b/.changeset/current-clippy-baseline.md new file mode 100644 index 000000000..a14e86242 --- /dev/null +++ b/.changeset/current-clippy-baseline.md @@ -0,0 +1,5 @@ +--- +"@googleworkspace/cli": patch +--- + +Keep Apps Script file selection compatible with the current Clippy checks. diff --git a/crates/google-workspace-cli/src/helpers/script.rs b/crates/google-workspace-cli/src/helpers/script.rs index 11bcdebec..4b31db62d 100644 --- a/crates/google-workspace-cli/src/helpers/script.rs +++ b/crates/google-workspace-cli/src/helpers/script.rs @@ -169,13 +169,7 @@ fn process_file(path: &Path) -> Result, GwsError> { filename.trim_end_matches(".js").trim_end_matches(".gs"), ), "html" => ("HTML", filename.trim_end_matches(".html")), - "json" => { - if filename == "appsscript.json" { - ("JSON", "appsscript") - } else { - return Ok(None); - } - } + "json" if filename == "appsscript.json" => ("JSON", "appsscript"), _ => return Ok(None), }; From 933abfc0a1e0888a775cb11464f46d940f5b61e8 Mon Sep 17 00:00:00 2001 From: ratovarius Date: Fri, 11 Sep 2026 12:23:54 -0300 Subject: [PATCH 3/5] fix(docs): correct bundle mapping and optional output states --- .changeset/docs-review-bundle.md | 5 +- examples/docs-review-bundle/README.md | 32 ++-- .../docs-review-bundle/docs_review_bundle.py | 128 +++++++++------ .../test_docs_review_bundle.py | 146 ++++++++++++++++-- 4 files changed, 245 insertions(+), 66 deletions(-) diff --git a/.changeset/docs-review-bundle.md b/.changeset/docs-review-bundle.md index bac3f4983..e8504e8ce 100644 --- a/.changeset/docs-review-bundle.md +++ b/.changeset/docs-review-bundle.md @@ -4,4 +4,7 @@ Add a standalone Python companion for visual Google Docs review bundles with native exports, safe DOCX raster extraction, local HTML, optional PDF page -previews, revision observations, and an offline fixture workflow. +previews with explicitly unverified page coverage, revision observations, +and an offline fixture workflow. Preserve nested image occurrences and +legitimate asset reuse, and keep oversized optional comments from failing +the required bundle. diff --git a/examples/docs-review-bundle/README.md b/examples/docs-review-bundle/README.md index fcc7f58ca..1943ac148 100644 --- a/examples/docs-review-bundle/README.md +++ b/examples/docs-review-bundle/README.md @@ -33,12 +33,17 @@ python3 examples/docs-review-bundle/docs_review_bundle.py \ - `--include-comments`: collect every returned Drive comments page. The entire optional artifact is omitted and marked unavailable if retrieval is incomplete - or fails. Comments are a separate observation, not revision-bound or mapped to + or fails, or if the final serialized artifact exceeds 20 MiB. Size is checked + before publication, so an oversized optional result does not fail the required + bundle. Comments are a separate observation, not revision-bound or mapped to PDF coordinates. Deleted comments are not requested. - `--render-pages`: use a trusted `pdftoppm` on `PATH` to generate 96 DPI PNGs. - Rendering is opt-in. Missing tools, failures, timeouts, invalid output and - noncontiguous page numbers retain the PDF and report no available raster - previews. Partial page output is discarded. + Its path is resolved before running inside the bundle, including relative + `PATH` entries. Rendering is opt-in. Missing tools, failures, timeouts, invalid + output and noncontiguous page numbers retain the PDF and report no available + raster previews. Outputs from these detected failures are discarded. After a + successful process exit and validation, previews are labeled `available` + with `coverage: "unverified"`; a missing page suffix cannot be detected. - `--timeout`: positive finite seconds per subprocess; default 60. This is not a total workflow deadline. - `--gws`: trusted executable, resolved before changing subprocess working @@ -78,6 +83,12 @@ required exports, validation, index generation and artifact hashing succeed. an atomic snapshot, successful optional rendering, or verified export tab coverage. Check `revisions`, `comments`, and `rendering` independently. +Page previews are never labeled `complete`. `rendering.status: "available"` +means that local PNG files passed validation, while +`rendering.coverage: "unverified"` means the original PDF page count was not +independently checked. Even a contiguous list beginning with page 1 may omit +later pages. The HTML displays this same coverage limitation. + Revision status is `mixed` for differing observed Docs revision IDs, `unchanged` for equal nonempty IDs, and `unknown` if either is missing. Even `unchanged` does not prove atomicity or that every export represents the same revision. @@ -103,10 +114,12 @@ assets. The manifest does not hash itself and is not an authenticity signature. - Google determines the PDF/DOCX/Markdown export layout and tab coverage. This companion cannot verify that every tab appears in those formats or associate a PDF page/DOCX figure with an exact native tab. -- DOCX figure order comes from main-document drawing relationships, including - drawings in tables. Alt text and nearby paragraphs are context, **not exact - captions**. Native object IDs are recorded separately; the companion never - fabricates a matching native ID or a source tab ID for a DOCX image. +- DOCX figure order follows individual image occurrences in the main document, + including drawings in tables and nested text boxes. Each occurrence uses its + nearest drawing's alt text and nearest paragraph's text; legitimate repeated + uses of an asset remain separate figures. Alt text and nearby paragraphs are + context, **not exact captions**. Native object IDs are recorded separately; + the companion never fabricates a matching native ID or source tab ID. - Media basenames are replaced with distinct generated local filenames. External, missing, traversing and unsupported relationships remain visibly unavailable. Unreferenced recognized rasters are retained as artifacts. @@ -123,7 +136,8 @@ assets. The manifest does not hash itself and is not an authenticity signature. - Each source/export/JSON artifact is capped at 20 MiB. Page output is limited to 500 files and 100 MiB total. PDF header/EOF and raster signatures are checked; these are **not full format validation**. Renderer exit success and - contiguous numbering do not independently prove the original PDF page count. + contiguous numbering do not independently prove the original PDF page count, + so preview coverage is always labeled unverified. - Subprocess capture is file-backed, with size checks after exit. Timeouts and post-render limits do not enforce disk or memory quotas on external tools. `pdftoppm`, `gws`, local viewers and the operator-controlled parent directory diff --git a/examples/docs-review-bundle/docs_review_bundle.py b/examples/docs-review-bundle/docs_review_bundle.py index fd54477d4..841dbe2d3 100755 --- a/examples/docs-review-bundle/docs_review_bundle.py +++ b/examples/docs-review-bundle/docs_review_bundle.py @@ -49,6 +49,7 @@ "Exports are sequential, not an atomic snapshot; unchanged revisions are observations only.", "Native outline includes body paragraphs, tables and nested tabs, not full layout or styling.", "Drive export tab coverage is not verified; pages and DOCX figures have no reliable tab mapping.", + "Page preview coverage is unverified; available previews may omit pages.", "DOCX relationships provide document order and nearby text, " "not exact captions or native Docs IDs.", "Only recognized PNG, JPEG, GIF and WebP media are previewed; " @@ -165,7 +166,7 @@ def safe_xml(data): raise BundleError("unsafe-xml") try: text = data.decode("utf-8-sig") - declaration = re.search(r"<\?xml[^>]*encoding=['\"]([^'\"]+)", text, re.I) + declaration = re.search(r"<\?xml[^>]*\bencoding\s*=\s*['\"]([^'\"]+)", text, re.I) if declaration and declaration[1].lower() not in ("utf-8", "utf8", "us-ascii"): raise BundleError("unsupported-xml-encoding") root = ET.fromstring(text) @@ -244,6 +245,40 @@ def zip_members(source, member_limit, total_limit, member_count): raise BundleError("invalid-zip") from None +def docx_image_occurrences(document): + """Visit each blip once, retaining its nearest paragraph and drawing.""" + paragraph_text = {} + paragraph_index = {} + drawing_alt = {} + occurrences = [] + pending = [(document, None, None)] + while pending: + node, paragraph, drawing = pending.pop() + if node.tag == W + "p": + paragraph = node + paragraph_index[node] = len(paragraph_text) + paragraph_text[node] = [] + elif node.tag == W + "drawing": + drawing = node + elif node.tag == W + "t" and paragraph is not None: + paragraph_text[paragraph].append(node.text or "") + elif node.tag == WP + "docPr" and drawing is not None: + drawing_alt.setdefault( + drawing, " ".join(node.get(field, "") for field in ("title", "descr")).strip() + ) + elif node.tag == A + "blip" and paragraph is not None and drawing is not None: + occurrences.append((node, paragraph, drawing)) + pending.extend((child, paragraph, drawing) for child in reversed(node)) + + texts = ["".join(parts) for parts in paragraph_text.values()] + for blip, paragraph, drawing in occurrences: + index = paragraph_index[paragraph] + nearby = texts[index] or " ".join( + texts[max(0, index - 1):index] + texts[index + 1:index + 2] + ) + yield blip, drawing_alt.get(drawing, ""), nearby + + def extract_docx( source, output, *, member_limit=FILE_LIMIT, total_limit=TOTAL_LIMIT, member_count=MEMBER_COUNT, @@ -274,50 +309,36 @@ def extract_docx( filename = f"image-{len(assets) + 1}.{extension}" write_bytes(output / filename, data) assets[name] = local_uri(f"{output.name}/{filename}") - paragraphs = list(document.iter(W + "p")) - texts = [ - "".join(text.text or "" for text in paragraph.iter(W + "t")) - for paragraph in paragraphs - ] figures = [] - for index, paragraph in enumerate(paragraphs): - nearby = texts[index] or " ".join( - texts[max(0, index - 1):index] + texts[index + 1:index + 2] - ) - for drawing in paragraph.iter(W + "drawing"): - properties = drawing.find(".//" + WP + "docPr") - alt = "" - if properties is not None: - alt = " ".join(properties.get(field, "") for field in ("title", "descr")).strip() - for blip in drawing.iter(A + "blip"): - identity = blip.get(R + "embed") or blip.get(R + "link") - relationship = relationships.get(identity, {}) - asset = None - availability = "missing-or-unsupported" - if relationship.get("TargetMode", "").lower() == "external": - availability = "external-not-fetched" - elif relationship.get("Type") == R[1:-1] + "/image": - target = relationship.get("Target", "") - # Allow only relative media targets in the document's part. - if ( - target.startswith("media/") - and not any(p in ("", ".", "..") for p in target.split("/")) - and not any(char in target for char in "\\:%?#") - ): - asset = assets.get(str(PurePosixPath("word") / target)) - if asset: - availability = "available" - figures.append({ - "order": len(figures) + 1, - "relationship_id": identity, - "asset": asset, - "alt": alt, - "nearby_text": nearby[:1000], - "availability": availability, - "mapping_confidence": "docx-relationship-only", - "native_object_id": None, - "source_tab_id": None, - }) + for blip, alt, nearby in docx_image_occurrences(document): + identity = blip.get(R + "embed") or blip.get(R + "link") + relationship = relationships.get(identity, {}) + asset = None + availability = "missing-or-unsupported" + if relationship.get("TargetMode", "").lower() == "external": + availability = "external-not-fetched" + elif relationship.get("Type") == R[1:-1] + "/image": + target = relationship.get("Target", "") + # Allow only relative media targets in the document's part. + if ( + target.startswith("media/") + and not any(p in ("", ".", "..") for p in target.split("/")) + and not any(char in target for char in "\\:%?#") + ): + asset = assets.get(str(PurePosixPath("word") / target)) + if asset: + availability = "available" + figures.append({ + "order": len(figures) + 1, + "relationship_id": identity, + "asset": asset, + "alt": alt, + "nearby_text": nearby[:1000], + "availability": availability, + "mapping_confidence": "docx-relationship-only", + "native_object_id": None, + "source_tab_id": None, + }) return {"figures": figures, "assets": list(assets.values())} @@ -428,6 +449,11 @@ def index_html(source, markdown, tabs, manifest, artifacts): f"{display(manifest['rendering'].get('reason', ''))}

", ] ) + if manifest["rendering"]["pages"]: + parts.append( + f"

Page coverage: {display(manifest['rendering']['coverage'])}. " + "The PDF page count has not been verified; previews may omit pages.

" + ) for page in manifest["rendering"]["pages"]: parts.append(f'{display(page)}') parts.append("

Native outline and tabs

") @@ -536,6 +562,7 @@ def render_pages(bundle, timeout, requested): if not renderer: return {"status": "unavailable", "reason": "pdftoppm-not-found; PDF retained", "pages": []} try: + renderer = str(Path(renderer).resolve()) with tempfile.TemporaryDirectory(prefix=".render-", dir=bundle) as temp: staging = Path(temp) prefix = str(staging.relative_to(bundle) / "page") @@ -560,7 +587,8 @@ def render_pages(bundle, timeout, requested): raise BundleError("invalid-render") staging.rename(bundle / "pages") return { - "status": "complete", + "status": "available", + "coverage": "unverified", "pages": [local_uri("pages/" + pages[number]) for number in sorted(pages)], } except (BundleError, OSError) as error: @@ -645,14 +673,20 @@ def build_bundle(args, bundle, manifest): value = parse_json(comments) if not isinstance(value.get("comments"), list) or value.get("nextPageToken"): raise BundleError("incomplete-comments") + if len(comments) > FILE_LIMIT: + raise BundleError("comments-size-limit") write_bytes(bundle / "comments.json", comments) artifacts.append("comments.json") manifest["comments"] = {"status": "available"} - except (BundleError, OSError): + except (BundleError, OSError) as error: (bundle / "comments.json").unlink(missing_ok=True) manifest["comments"] = { "status": "unavailable", - "reason": "retrieval-incomplete-or-failed", + "reason": ( + "comments-size-limit" + if isinstance(error, BundleError) and str(error) == "comments-size-limit" + else "retrieval-incomplete-or-failed" + ), } manifest["stage"] = "validate-and-extract" diff --git a/examples/docs-review-bundle/test_docs_review_bundle.py b/examples/docs-review-bundle/test_docs_review_bundle.py index 56ede7b74..84e9a6e0e 100644 --- a/examples/docs-review-bundle/test_docs_review_bundle.py +++ b/examples/docs-review-bundle/test_docs_review_bundle.py @@ -122,8 +122,8 @@ def native_document(): } -def pdf_bytes(): - """A complete synthetic one-page PDF, also usable by real pdftoppm.""" +def pdf_bytes(page_count=1): + """A complete synthetic PDF, also usable by real pdftoppm.""" content = ( b"BT /F1 22 Tf 48 720 Td (Synthetic Docs review) Tj ET\n" b"BT /F1 12 Tf 48 687 Td (Local fixture - no Google document) Tj ET\n" @@ -137,25 +137,36 @@ def pdf_bytes(): b"BT /F1 12 Tf 60 333 Td (Column A) Tj 258 0 Td (Column B) Tj ET\n" b"BT /F1 12 Tf 60 288 Td (Cell one) Tj 258 0 Td (Cell two) Tj ET\n" ) + content_id = page_count + 3 + font_id = page_count + 4 + kids = " ".join(f"{number} 0 R" for number in range(3, page_count + 3)) objects = [ b"<< /Type /Catalog /Pages 2 0 R >>", - b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " - b"/Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + f"<< /Type /Pages /Kids [{kids}] /Count {page_count} >>".encode(), + ] + objects.extend( + ( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + f"/Resources << /Font << /F1 {font_id} 0 R >> >> /Contents {content_id} 0 R >>" + ).encode() + for _ in range(page_count) + ) + objects.extend([ f"<< /Length {len(content)} >>\nstream\n".encode() + content + b"endstream", b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", - ] + ]) data = b"%PDF-1.4\n" offsets = [0] for index, obj in enumerate(objects, 1): offsets.append(len(data)) data += f"{index} 0 obj\n".encode() + obj + b"\nendobj\n" startxref = len(data) - data += b"xref\n0 6\n0000000000 65535 f \n" + object_count = len(objects) + 1 + data += f"xref\n0 {object_count}\n0000000000 65535 f \n".encode() for offset in offsets[1:]: data += f"{offset:010d} 00000 n \n".encode() data += ( - f"trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n" + f"trailer\n<< /Size {object_count} /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n" ).encode() return data @@ -190,6 +201,22 @@ def docx_members(): } +def nested_docx_members(*, outer_image=False): + members = docx_members() + outer_blip = '' if outer_image else "" + members["word/document.xml"] = f""" + +Outer paragraph + +Inner paragraph + + + +{outer_blip} +""".encode() + return members + + def write_docx(path, members=None): with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: for name, data in (members if members is not None else docx_members()).items(): @@ -267,6 +294,13 @@ def fixtures(path): elif args[:3] == ["drive", "comments", "list"]: assert params["fileId"] == "synthetic-doc" assert "nextPageToken" in params["fields"] + if mode == "expanded-comments": + # ~8 MiB on the wire; >24 MiB after ensure_ascii=True serialization. + print(json.dumps( + {"comments": [{"id": "large", "content": "é" * (4 * 1024 * 1024)}]}, + ensure_ascii=False, separators=(",", ":"), + )) + sys.exit(0) if mode == "failed-comments" and params.get("pageToken"): sys.exit(9) if params.get("pageToken"): @@ -389,6 +423,39 @@ def test_duplicate_basenames_get_distinct_local_assets(self): self.assertEqual((self.root / paths[0]).read_bytes(), PNG) self.assertEqual((self.root / paths[1]).read_bytes(), PNG + b"different") + def test_nested_text_box_image_has_one_occurrence_with_inner_ownership(self): + write_docx(self.fixture / "document.docx", nested_docx_members()) + figures = self.extract()["figures"] + self.assertEqual(len(figures), 1) + self.assertEqual(figures[0]["order"], 1) + self.assertEqual(figures[0]["alt"], "Inner image") + self.assertEqual(figures[0]["nearby_text"], "Inner paragraph") + self.assertEqual((self.root / figures[0]["asset"]).read_bytes(), PNG) + + def test_nested_image_order_and_context_follow_nearest_owners(self): + write_docx(self.fixture / "document.docx", nested_docx_members(outer_image=True)) + figures = self.extract()["figures"] + self.assertEqual([f["relationship_id"] for f in figures], ["rId1", "rId2"]) + self.assertEqual([f["alt"] for f in figures], ["Inner image", "Outer text box"]) + self.assertEqual( + [f["nearby_text"] for f in figures], ["Inner paragraph", "Outer paragraph"] + ) + self.assertEqual([f["order"] for f in figures], [1, 2]) + + def test_repeated_asset_uses_remain_distinct_figure_occurrences(self): + members = docx_members() + members["word/document.xml"] = members["word/document.xml"].replace( + b'r:embed="rId2"', b'r:embed="rId1"' + ) + write_docx(self.fixture / "document.docx", members) + figures = self.extract()["figures"] + self.assertEqual(len(figures), 3) + self.assertEqual([f["order"] for f in figures], [1, 2, 3]) + self.assertEqual(figures[0]["asset"], figures[1]["asset"]) + self.assertEqual([f["relationship_id"] for f in figures[:2]], ["rId1", "rId1"]) + self.assertNotEqual(figures[0]["alt"], figures[1]["alt"]) + self.assertEqual(figures[1]["nearby_text"], "Table image context") + def test_zip_traversal_absolute_windows_and_control_paths_are_rejected(self): api = self.api() for index, name in enumerate( @@ -445,6 +512,26 @@ def test_doctype_entities_and_utf16_are_rejected_even_in_unused_xml(self): self.extract() self.assertFalse((self.root / "assets").exists()) + def test_xml_encoding_allowlist_handles_declaration_whitespace(self): + api = self.api() + for encoding in ("UTF-7", "UTF-16", "UTF-32", "ISO-8859-1"): + for assignment in (f' = "{encoding}"', f"= '{encoding}'", f'\t=\n"{encoding}"'): + with self.subTest(encoding=encoding, assignment=assignment): + data = f''.encode("utf-8") + with self.assertRaises(api.BundleError) as raised: + api.safe_xml(data) + self.assertEqual(str(raised.exception), "unsupported-xml-encoding") + + def test_xml_utf8_bom_is_accepted_and_utf16_utf32_are_rejected(self): + api = self.api() + data = 'café'.encode("utf-8-sig") + root = api.safe_xml(data) + self.assertEqual(root.tag, "x") + self.assertEqual(root.text, "café") + for encoding in ("utf-16", "utf-32"): + with self.subTest(encoding=encoding), self.assertRaises(api.BundleError): + api.safe_xml("".encode(encoding)) + def test_relationship_traversal_and_svg_are_not_local_html_assets(self): members = docx_members() members["word/_rels/document.xml.rels"] = members[ @@ -715,10 +802,38 @@ def test_successful_renderer_publishes_local_page_previews(self): self.executable("pdftoppm", RENDER_STUB) result = self.run_bundle("--render-pages", "review") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(self.manifest()["rendering"]["status"], "complete") + self.assertEqual(self.manifest()["rendering"]["status"], "available") + self.assertEqual(self.manifest()["rendering"]["coverage"], "unverified") self.assertEqual(self.manifest()["rendering"]["pages"], ["pages/page-1.png"]) self.assertIn("pages/page-1.png", self.manifest()["artifacts"]) + def test_zero_exit_page_prefix_has_unverified_coverage_not_complete(self): + (self.fixture / "document.pdf").write_bytes(pdf_bytes(page_count=2)) + self.executable("pdftoppm", RENDER_STUB) + result = self.run_bundle("--render-pages", "review") + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["rendering"]["status"], "available") + self.assertEqual(manifest["rendering"]["coverage"], "unverified") + self.assertEqual(manifest["rendering"]["pages"], ["pages/page-1.png"]) + html = (self.root / "review/index.html").read_text() + self.assertIn("Page coverage: unverified", html) + self.assertIn("previews may omit pages", html) + self.assertTrue((self.root / "review/document.pdf").is_file()) + + def test_renderer_discovered_on_relative_path_runs_in_bundle(self): + self.executable("pdftoppm", RENDER_STUB) + for output, search_path in (("relative-path", "bin"), ("absolute-path", str(self.bin))): + with self.subTest(search_path=search_path): + result = self.run_bundle("--render-pages", output, PATH=search_path) + self.assertEqual(result.returncode, 0, result.stderr) + rendering = self.manifest(output)["rendering"] + self.assertEqual(rendering["status"], "available") + self.assertEqual(rendering["coverage"], "unverified") + self.assertEqual(rendering["pages"], ["pages/page-1.png"]) + self.assertEqual((self.root / output / "pages/page-1.png").read_bytes(), PNG) + def test_renderer_failure_timeout_and_gap_publish_no_partial_page_list(self): self.executable("pdftoppm", RENDER_STUB) for mode in ("failed", "timeout", "gap"): @@ -808,6 +923,19 @@ def test_partial_comments_are_unavailable_not_published(self): self.assertEqual(self.manifest()["comments"]["status"], "unavailable") self.assertFalse((self.root / "review/comments.json").exists()) + def test_serialized_comments_over_limit_remain_an_optional_failure(self): + result = self.run_bundle( + "--include-comments", "review", live=True, STUB_MODE="expanded-comments" + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["comments"]["status"], "unavailable") + self.assertEqual(manifest["comments"]["reason"], "comments-size-limit") + self.assertNotIn("comments.json", manifest["artifacts"]) + self.assertFalse((self.root / "review/comments.json").exists()) + self.assertTrue((self.root / "review/index.html").is_file()) + if __name__ == "__main__": unittest.main() From ac635126b5b6f961f500edce01535ecda2df73a4 Mon Sep 17 00:00:00 2001 From: ratovarius Date: Fri, 11 Sep 2026 12:39:06 -0300 Subject: [PATCH 4/5] fix(docs): accept canonical export receipts --- .changeset/docs-review-bundle.md | 3 +- .github/workflows/ci.yml | 10 ++ examples/docs-review-bundle/README.md | 22 +++- .../docs-review-bundle/docs_review_bundle.py | 2 +- .../test_docs_review_bundle.py | 119 +++++++++++++++++- 5 files changed, 150 insertions(+), 6 deletions(-) diff --git a/.changeset/docs-review-bundle.md b/.changeset/docs-review-bundle.md index e8504e8ce..286e9e7f0 100644 --- a/.changeset/docs-review-bundle.md +++ b/.changeset/docs-review-bundle.md @@ -7,4 +7,5 @@ native exports, safe DOCX raster extraction, local HTML, optional PDF page previews with explicitly unverified page coverage, revision observations, and an offline fixture workflow. Preserve nested image occurrences and legitimate asset reuse, and keep oversized optional comments from failing -the required bundle. +the required bundle. Verify each export's exact canonical destination against +the real CLI receipt. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7fae1a7b..7918eb92d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,18 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable + - name: Cache cargo + uses: Swatinem/rust-cache@ad397744b0d591a723ab90405b7247fac0e6b8db # v2 + with: + key: docs-review-bundle-${{ matrix.os }} + - name: Build CLI for export contract regression + run: cargo build --locked - name: Test stdlib companion with synthetic fixtures run: python3 -B -m unittest discover -s examples/docs-review-bundle -p 'test_*.py' -v + env: + GWS_TEST_BINARY: ${{ github.workspace }}/target/debug/gws changes: name: Detect Changes diff --git a/examples/docs-review-bundle/README.md b/examples/docs-review-bundle/README.md index 1943ac148..7975497f3 100644 --- a/examples/docs-review-bundle/README.md +++ b/examples/docs-review-bundle/README.md @@ -55,7 +55,9 @@ The companion requests `docs documents get` with `includeTabsContent: true` and uses `drive files export` with `--format json` and fixed relative `--output` filenames. Every `gws` subprocess runs inside the new bundle. Export success, MIME type, destination and byte count are checked against the -written artifact. This requires the existing `gws` binary-export receipt format. +written artifact. The receipt must name the exact canonical absolute destination, +as returned by `gws`; a matching basename alone is insufficient. This requires +the existing `gws` binary-export receipt format. Open `index.html` locally. The index has no JavaScript or remote dependencies. It contains a sandboxed PDF frame, optional page images, a paragraph/table @@ -192,5 +194,19 @@ python3 -B -m unittest discover \ Tests use generated JSON/PDF/DOCX files and explicit `gws`/renderer executables as stubs. Their subprocess environment omits real authentication settings; -no installed `gws`, real document, or network request is needed. CI runs this -same command in a dedicated Linux/macOS job. +no installed `gws`, real document, or network request is needed for those tests. + +To include the real CLI export-contract regression: + +```sh +cargo build --locked +GWS_TEST_BINARY="$PWD/target/debug/gws" python3 -B -m unittest discover \ + -s examples/docs-review-bundle -p 'test_*.py' -v +``` + +This additional test uses cached synthetic Discovery, a dummy token, isolated +configuration and ADC paths, and a loopback HTTP server. It downloads all three +generated exports through the real CLI and checks that the bundle completes. +It never contacts Google or uses real credentials. The dedicated Linux/macOS CI +job builds `gws` and always enables this test; local runs without +`GWS_TEST_BINARY` explicitly skip it. diff --git a/examples/docs-review-bundle/docs_review_bundle.py b/examples/docs-review-bundle/docs_review_bundle.py index 841dbe2d3..2ba9ee7c0 100755 --- a/examples/docs-review-bundle/docs_review_bundle.py +++ b/examples/docs-review-bundle/docs_review_bundle.py @@ -649,7 +649,7 @@ def build_bundle(args, bundle, manifest): data = read_bytes(bundle / name) if ( receipt.get("status") != "success" - or receipt.get("saved_file") != name + or receipt.get("saved_file") != str((bundle / name).resolve()) or str(receipt.get("mimeType", "")).split(";")[0].strip().lower() != mime or type(receipt.get("bytes")) is not int or receipt["bytes"] != len(data) diff --git a/examples/docs-review-bundle/test_docs_review_bundle.py b/examples/docs-review-bundle/test_docs_review_bundle.py index 84e9a6e0e..299ab0614 100644 --- a/examples/docs-review-bundle/test_docs_review_bundle.py +++ b/examples/docs-review-bundle/test_docs_review_bundle.py @@ -14,6 +14,7 @@ import base64 import contextlib import hashlib +import http.server from html.parser import HTMLParser import importlib.util import io @@ -24,8 +25,10 @@ import subprocess import sys import tempfile +import threading import unittest from unittest import mock +import urllib.parse import zipfile @@ -287,7 +290,11 @@ def fixtures(path): shutil.copyfile(fixture / name, name) print(json.dumps({ "status": "error" if mode == "bad-export-status" else "success", - "saved_file": name, + "saved_file": ( + str(Path.cwd().parent / "other" / name) if mode == "wrong-export-path" + else name if mode == "relative-export-path" + else str(Path(name).resolve()) + ), "mimeType": params["mimeType"], "bytes": 1 if mode == "wrong-export-size" else (fixture / name).stat().st_size, })) @@ -399,6 +406,116 @@ def manifest(self, directory="review"): return json.loads((self.root / directory / "manifest.json").read_text()) +class ExportReceiptTests(BundleTestCase): + def test_canonical_receipts_complete_all_exports(self): + result = self.run_bundle("review", live=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.manifest()["status"], "complete") + + def test_wrong_destination_or_relative_receipt_is_refused(self): + for mode in ["wrong-export-path", "relative-export-path"]: + with self.subTest(mode=mode): + result = self.run_bundle(mode, live=True, STUB_MODE=mode) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(self.manifest(mode)["error"], "invalid-export-receipt") + + +@unittest.skipUnless(os.environ.get("GWS_TEST_BINARY"), "Set GWS_TEST_BINARY for real CLI coverage") +class RealCliExportTests(BundleTestCase): + def test_real_cli_exports_complete_bundle_with_canonical_receipts(self): + binary = Path(os.environ["GWS_TEST_BINARY"]).resolve(strict=True) + fixture = self.fixture + requests = [] + exports = { + "application/pdf": "document.pdf", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "document.docx", + "text/markdown": "document.md", + } + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + path = urllib.parse.unquote(parsed.path) + params = urllib.parse.parse_qs(parsed.query) + requests.append((path, params, self.headers.get("Authorization"))) + if path == "/documents/synthetic-doc": + data = (fixture / "source.json").read_bytes() + mime = "application/json" + elif path == "/files/synthetic-doc/export": + mime = params.get("mimeType", [""])[0] + if mime not in exports: + self.send_error(400) + return + data = (fixture / exports[mime]).read_bytes() + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Type", mime) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + config = self.root / "real-cli-config" + cache = config / "cache" + cache.mkdir(parents=True) + (self.root / ".env").write_text("") + for service, version, resource, method, id_field, path in [ + ("docs", "v1", "documents", "get", "documentId", "documents/{documentId}"), + ("drive", "v3", "files", "export", "fileId", "files/{fileId}/export"), + ]: + discovery = { + "name": service, "version": version, + "rootUrl": f"http://127.0.0.1:{server.server_port}/", + "resources": {resource: {"methods": {method: { + "httpMethod": "GET", "path": path, + "parameters": {id_field: { + "type": "string", "location": "path", "required": True, + }}, + }}}}, + } + (cache / f"{service}_{version}.json").write_text(json.dumps(discovery)) + env = { + **self.env, + "GOOGLE_WORKSPACE_CLI_CONFIG_DIR": str(config), + "GOOGLE_WORKSPACE_CLI_TOKEN": "synthetic-loopback-token", + "GOOGLE_APPLICATION_CREDENTIALS": str(self.root / "absent-adc.json"), + "GOOGLE_WORKSPACE_CLI_KEYRING_BACKEND": "file", + "GOOGLE_WORKSPACE_PROJECT_ID": "synthetic-loopback-project", + "HTTP_PROXY": "http://127.0.0.1:1", + "HTTPS_PROXY": "http://127.0.0.1:1", + "ALL_PROXY": "http://127.0.0.1:1", + "NO_PROXY": "127.0.0.1,localhost", + } + result = subprocess.run( + [sys.executable, "-B", str(SCRIPT), "--document-id", "synthetic-doc", + "--gws", str(binary), "--timeout", "10", "review"], + cwd=self.root, env=env, text=True, capture_output=True, timeout=25, + ) + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest["status"], "complete") + self.assertEqual(manifest["revisions"]["status"], "unchanged") + for name in exports.values(): + data = (self.root / "review" / name).read_bytes() + self.assertEqual(data, (fixture / name).read_bytes()) + self.assertEqual(manifest["artifacts"][name]["bytes"], len(data)) + self.assertEqual(len(requests), 5) + self.assertTrue(all(auth == "Bearer synthetic-loopback-token" + for _, _, auth in requests)) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + class ExtractionTests(BundleTestCase): def extract(self, **limits): return self.api().extract_docx( From 52e47c93e7cd98118c29a9d2e9f6beddeef26293 Mon Sep 17 00:00:00 2001 From: ratovarius Date: Fri, 11 Sep 2026 12:51:16 -0300 Subject: [PATCH 5/5] ci(docs): isolate companion checks with verified action pins --- .github/workflows/ci.yml | 21 --------- .github/workflows/docs-review-bundle.yml | 54 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/docs-review-bundle.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7918eb92d..1eedc9972 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,27 +30,6 @@ env: SCCACHE_IGNORE_SERVER_IO_ERROR: "true" jobs: - docs-review-bundle: - name: Docs Review Bundle (Python) - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable - - name: Cache cargo - uses: Swatinem/rust-cache@ad397744b0d591a723ab90405b7247fac0e6b8db # v2 - with: - key: docs-review-bundle-${{ matrix.os }} - - name: Build CLI for export contract regression - run: cargo build --locked - - name: Test stdlib companion with synthetic fixtures - run: python3 -B -m unittest discover -s examples/docs-review-bundle -p 'test_*.py' -v - env: - GWS_TEST_BINARY: ${{ github.workspace }}/target/debug/gws - changes: name: Detect Changes runs-on: ubuntu-latest diff --git a/.github/workflows/docs-review-bundle.yml b/.github/workflows/docs-review-bundle.yml new file mode 100644 index 000000000..a0f651c61 --- /dev/null +++ b/.github/workflows/docs-review-bundle.yml @@ -0,0 +1,54 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Docs Review Bundle + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + docs-review-bundle: + name: Docs Review Bundle (Python) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@d1031067263f94b142dd6c0ce24c5eb9d02d52a0 # master + with: + toolchain: stable + - name: Cache cargo + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: docs-review-bundle-${{ matrix.os }} + - name: Build CLI for export contract regression + run: cargo build --locked + - name: Test stdlib companion with synthetic fixtures + run: python3 -B -m unittest discover -s examples/docs-review-bundle -p 'test_*.py' -v + env: + GWS_TEST_BINARY: ${{ github.workspace }}/target/debug/gws