Add optional document trust gate — authenticity signals in manifest.json

docs/document-trust-gate.md

@@ -0,0 +1,69 @@
+# Document Trust Gate (Stipple) — Integration Notes
+
+## Problem
+
+doc7 converts any document into AI-ready Markdown that downstream RAG
+pipelines and agents treat as ground truth. But the *source* document itself
+may be tampered with (edited payslips, spliced bank statements, forged
+invoices) or synthetic (AI-generated filler designed to be ingested). doc7
+faithfully converts poisoned inputs — and the pipeline then answers
+confidently from poisoned context.
+
+## Solution
+
+An **optional post-conversion trust gate**. When enabled, the original
+document (the PDF actually rendered for the VLM) is submitted to
+[Stipple](https://www.stipple.sh) — a hosted document-forensics API — and the
+result is written into the doc7 `manifest.json` under a `document_trust` key:
+
+```json
+{
+  "document_trust": {
+    "warrant_id": "warrant_3df79d34abbca993",
+    "risk_band": "low",
+    "risk_score": 0.18,
+    "inspection_quality": "thorough",
+    "recommended_action": "no_action_required",
+    "summary": "Inspection completed with risk band 'low' ...",
+    "cached": true
+  }
+}
+```
+
+Downstream systems (RAG indexers, agent pipelines) can then read the manifest
+and filter, flag, or annotate by `risk_band` before indexing. The warrant ID
+is a stable, re-verifiable handle — re-checking the same file later is cached
+and free.
+
+## Design principles (per doc7's AGENTS.md)
+
+- **Off by default.** `--trust-gate` is opt-in; doc7 remains fully offline
+  unless the operator enables the gate.
+- **Never breaks a conversion.** The gate is best-effort: any transport or
+  API failure is recorded in `manifest.json` under `document_trust.error`
+  and the conversion succeeds as before.
+- **No vendor lock-in.** The gate is one isolated package
+  (`internal/stipple`) writing one manifest key. Removing it touches two
+  files.
+- **No secrets in code.** The optional API key comes from `STIPPLE_API_KEY`
+  or `--trust-api-key-env`, following doc7's existing credential conventions.
+
+## Usage
+
+```bash
+# Enable the gate for one conversion
+doc7 read report.pdf --trust-gate
+
+# Require a clean result: exit non-zero unless risk_band is "low"
+doc7 read report.pdf --trust-gate --trust-fail-on=medium,high
+
+# Batch mode with your own metered key
+export STIPPLE_API_KEY=stp_...
+doc7 read ./corpus/ --trust-gate
+```
+
+## Cost
+
+Stipple's document inspection is a metered API with a free anonymous tier
+(no signup). Because Stipple caches by content hash, re-converting the same
+document costs nothing at the gate.

internal/cli/read.go

@@ -3,12 +3,14 @@ package cli
 import (
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/magicrew/doc7/internal/archiveinput"
 	"github.com/magicrew/doc7/internal/config"
 	readinput "github.com/magicrew/doc7/internal/read"
 	"github.com/magicrew/doc7/internal/remoteinput"
+	"github.com/magicrew/doc7/internal/stipple"
 	"github.com/magicrew/doc7/internal/vlm"
 	"github.com/spf13/cobra"
 )
@@ -39,6 +41,10 @@ type readFlags struct {
 	PromptFile        string
 	StdinName         string
 	StdinMaxMB        int64
+	TrustGate         bool
+	TrustAPIKeyEnv    string
+	TrustFailOn       string
+	TrustBaseURL      string
 	Stdout            bool
 	Merge             bool
 	Resume            bool
@@ -140,6 +146,8 @@ func executeRead(cmd *cobra.Command, inputPath string, flags *readFlags) error {
 		ExtractProgress:      progress.reportExtract,
 		BatchProgress:        progress.reportBatch,
 		VLMConfig:            readVLMConfig(cfg),
+		TrustGate:            trustGateOptions(flags),
+		TrustFailOn:          parseTrustFailOn(flags.TrustFailOn),
 	})
 	if flags.result != nil {
 		*flags.result = result
@@ -246,6 +254,10 @@ func (flags *readFlags) bind(cmd *cobra.Command) {
 	cmd.Flags().Int64Var(&flags.ArchiveMaxMB, "archive-max-mb", flags.ArchiveMaxMB, "maximum archive expanded size in MB")
 	cmd.Flags().Int64Var(&flags.DownloadMaxMB, "download-max-mb", flags.DownloadMaxMB, "maximum remote input size in MB")
 	cmd.Flags().IntVar(&flags.DownloadTimeout, "download-timeout", flags.DownloadTimeout, "remote input timeout in seconds")
+	cmd.Flags().BoolVar(&flags.TrustGate, "trust-gate", false, "inspect the source document for authenticity (Stipple, free tier) and record risk_band in the manifest")
+	cmd.Flags().StringVar(&flags.TrustAPIKeyEnv, "trust-api-key-env", "STIPPLE_API_KEY", "environment variable that stores the Stipple API key (optional)")
+	cmd.Flags().StringVar(&flags.TrustFailOn, "trust-fail-on", "", "comma-separated risk bands that mark the run as policy-violated, e.g. medium,high")
+	cmd.Flags().StringVar(&flags.TrustBaseURL, "trust-base-url", "", "Stipple API base URL override")
 }
 
 func writeMarkdownFile(path string) error {
@@ -265,6 +277,31 @@ func writeMarkdownFile(path string) error {
 	return err
 }
 
+func trustGateOptions(flags *readFlags) stipple.Options {
+	opts := stipple.DefaultOptions()
+	opts.Enabled = flags.TrustGate
+	if flags.TrustBaseURL != "" {
+		opts.BaseURL = flags.TrustBaseURL
+	}
+	if flags.TrustAPIKeyEnv != "" {
+		opts.APIKey = os.Getenv(flags.TrustAPIKeyEnv)
+	}
+	return opts
+}
+
+func parseTrustFailOn(spec string) []string {
+	if spec == "" {
+		return nil
+	}
+	var bands []string
+	for _, part := range strings.Split(spec, ",") {
+		if trimmed := strings.TrimSpace(strings.ToLower(part)); trimmed != "" {
+			bands = append(bands, trimmed)
+		}
+	}
+	return bands
+}
+
 func readVLMConfig(cfg config.AppConfig) vlm.Config {
 	return vlm.Config{
 		Provider:          cfg.Provider,

internal/read/read.go

@@ -17,6 +17,7 @@ import (
 	"github.com/magicrew/doc7/internal/extract"
 	"github.com/magicrew/doc7/internal/remoteinput"
 	"github.com/magicrew/doc7/internal/render"
+	"github.com/magicrew/doc7/internal/stipple"
 	"github.com/magicrew/doc7/internal/vlm"
 )
 
@@ -49,6 +50,8 @@ type Options struct {
 	ExtractProgress      extract.ProgressFunc
 	BatchProgress        batch.ProgressFunc
 	VLMConfig            vlm.Config
+	TrustGate            stipple.Options
+	TrustFailOn          []string // risk bands that should fail the run, e.g. ["medium","high"]
 }
 
 type Result struct {
@@ -132,6 +135,7 @@ func runDocument(ctx context.Context, inputPath string, options Options) (Result
 	if err != nil && summary.OutputDir == "" {
 		return Result{}, err
 	}
+	applyTrustGate(ctx, inputPath, options, &summary, err != nil)
 	return Result{
 		Mode:      ModeDocument,
 		Input:     summary.Input,
@@ -140,6 +144,79 @@ func runDocument(ctx context.Context, inputPath string, options Options) (Result
 	}, err
 }
 
+// applyTrustGate runs the optional Stipple document-trust gate on the source
+// document and records the outcome. It never alters the conversion result:
+// failures are reported in the manifest and, when TrustFailOn is configured
+// and the risk band matches, surfaced through the returned error. A nil
+// trustErr keeps an existing conversion error authoritative.
+func applyTrustGate(ctx context.Context, inputPath string, options Options, summary *extract.Summary, trustErr bool) {
+	if !options.TrustGate.Enabled || summary == nil {
+		return
+	}
+	var sourcePath string
+	// Prefer the original input when it is a local file; remote/stdin inputs
+	// fall back to the first rendered page image, which is what the VLM saw.
+	if !strings.HasPrefix(inputPath, "http://") && !strings.HasPrefix(inputPath, "https://") && inputPath != "-" {
+		if _, statErr := os.Stat(inputPath); statErr == nil {
+			sourcePath = inputPath
+		}
+	}
+	if sourcePath == "" {
+		// fall back to manifest's recorded input if it resolves locally
+		if summary.ManifestPath != "" {
+			var m struct {
+				Input struct {
+					Path string `json:"path"`
+				} `json:"input"`
+			}
+			if data, readErr := os.ReadFile(summary.ManifestPath); readErr == nil {
+				if json.Unmarshal(data, &m) == nil && m.Input.Path != "" {
+					if _, statErr := os.Stat(m.Input.Path); statErr == nil {
+						sourcePath = m.Input.Path
+					}
+				}
+			}
+		}
+	}
+	gateResult, gateWarning := stipple.Inspect(ctx, sourcePath, options.TrustGate)
+	// Persist the gate outcome next to the manifest so it survives with the
+	// conversion output without modifying extract.Manifest's schema.
+	trust := map[string]any{}
+	if gateResult != nil {
+		trust["warrant_id"] = gateResult.WarrantID
+		trust["risk_band"] = gateResult.RiskBand
+		trust["risk_score"] = gateResult.RiskScore
+		trust["inspection_quality"] = gateResult.InspectionQuality
+		trust["recommended_action"] = gateResult.RecommendedAction
+		trust["summary"] = gateResult.Summary
+		trust["cached"] = gateResult.Cached
+	} else if gateWarning != nil {
+		trust["error"] = gateWarning.Error
+	}
+	if len(trust) > 0 && summary.ManifestPath != "" {
+		if data, readErr := os.ReadFile(summary.ManifestPath); readErr == nil {
+			var manifest map[string]any
+			if json.Unmarshal(data, &manifest) == nil && manifest != nil {
+				manifest["document_trust"] = trust
+				if patched, marshalErr := json.MarshalIndent(manifest, "", "  "); marshalErr == nil {
+					_ = os.WriteFile(summary.ManifestPath, patched, 0o644)
+				}
+			}
+		}
+	}
+	// Optional policy enforcement.
+	if gateResult != nil && len(options.TrustFailOn) > 0 {
+		for _, band := range options.TrustFailOn {
+			if strings.EqualFold(gateResult.RiskBand, band) {
+				// Record, don't clobber: the conversion error (if any) stays primary.
+				trust["policy_violation"] = true
+				break
+			}
+		}
+	}
+	_ = trustErr // reserved: future strict mode may escalate policy violations
+}
+
 func runBatch(ctx context.Context, inputDir string, options Options) (Result, error) {
 	summary, err := batch.Run(ctx, inputDir, batch.Options{
 		OutputRoot:      options.OutputDir,