From d5ec8d0b6bb083a97887d9b4750293bad833c11c Mon Sep 17 00:00:00 2001 From: Sketchjar Date: Mon, 31 Aug 2026 12:21:28 +1000 Subject: [PATCH] Add optional document trust gate (Stipple) with manifest integration - New internal/stipple package: best-effort forensic authenticity inspection via the Stipple REST API (free anonymous tier, no signup required) - Off by default; enable with --trust-gate on doc7 read - Records risk_band, risk_score, inspection_quality, recommended_action, and warrant_id into manifest.json under document_trust - Gate failures never break a conversion (recorded as document_trust.error) - Optional policy enforcement: --trust-fail-on=medium,high flags violations - Design doc: docs/document-trust-gate.md --- docs/document-trust-gate.md | 69 +++++++++++++++++ internal/cli/read.go | 37 +++++++++ internal/read/read.go | 77 +++++++++++++++++++ internal/stipple/stipple.go | 149 ++++++++++++++++++++++++++++++++++++ 4 files changed, 332 insertions(+) create mode 100644 docs/document-trust-gate.md create mode 100644 internal/stipple/stipple.go diff --git a/docs/document-trust-gate.md b/docs/document-trust-gate.md new file mode 100644 index 0000000..da7be80 --- /dev/null +++ b/docs/document-trust-gate.md @@ -0,0 +1,69 @@ +# Document Trust Gate (Stipple) — Integration Notes + +## Problem + +doc7 converts any document into AI-ready Markdown that downstream RAG +pipelines and agents treat as ground truth. But the *source* document itself +may be tampered with (edited payslips, spliced bank statements, forged +invoices) or synthetic (AI-generated filler designed to be ingested). doc7 +faithfully converts poisoned inputs — and the pipeline then answers +confidently from poisoned context. + +## Solution + +An **optional post-conversion trust gate**. When enabled, the original +document (the PDF actually rendered for the VLM) is submitted to +[Stipple](https://www.stipple.sh) — a hosted document-forensics API — and the +result is written into the doc7 `manifest.json` under a `document_trust` key: + +```json +{ + "document_trust": { + "warrant_id": "warrant_3df79d34abbca993", + "risk_band": "low", + "risk_score": 0.18, + "inspection_quality": "thorough", + "recommended_action": "no_action_required", + "summary": "Inspection completed with risk band 'low' ...", + "cached": true + } +} +``` + +Downstream systems (RAG indexers, agent pipelines) can then read the manifest +and filter, flag, or annotate by `risk_band` before indexing. The warrant ID +is a stable, re-verifiable handle — re-checking the same file later is cached +and free. + +## Design principles (per doc7's AGENTS.md) + +- **Off by default.** `--trust-gate` is opt-in; doc7 remains fully offline + unless the operator enables the gate. +- **Never breaks a conversion.** The gate is best-effort: any transport or + API failure is recorded in `manifest.json` under `document_trust.error` + and the conversion succeeds as before. +- **No vendor lock-in.** The gate is one isolated package + (`internal/stipple`) writing one manifest key. Removing it touches two + files. +- **No secrets in code.** The optional API key comes from `STIPPLE_API_KEY` + or `--trust-api-key-env`, following doc7's existing credential conventions. + +## Usage + +```bash +# Enable the gate for one conversion +doc7 read report.pdf --trust-gate + +# Require a clean result: exit non-zero unless risk_band is "low" +doc7 read report.pdf --trust-gate --trust-fail-on=medium,high + +# Batch mode with your own metered key +export STIPPLE_API_KEY=stp_... +doc7 read ./corpus/ --trust-gate +``` + +## Cost + +Stipple's document inspection is a metered API with a free anonymous tier +(no signup). Because Stipple caches by content hash, re-converting the same +document costs nothing at the gate. diff --git a/internal/cli/read.go b/internal/cli/read.go index 9c84db0..8fa88df 100644 --- a/internal/cli/read.go +++ b/internal/cli/read.go @@ -3,12 +3,14 @@ package cli import ( "fmt" "os" + "strings" "time" "github.com/magicrew/doc7/internal/archiveinput" "github.com/magicrew/doc7/internal/config" readinput "github.com/magicrew/doc7/internal/read" "github.com/magicrew/doc7/internal/remoteinput" + "github.com/magicrew/doc7/internal/stipple" "github.com/magicrew/doc7/internal/vlm" "github.com/spf13/cobra" ) @@ -39,6 +41,10 @@ type readFlags struct { PromptFile string StdinName string StdinMaxMB int64 + TrustGate bool + TrustAPIKeyEnv string + TrustFailOn string + TrustBaseURL string Stdout bool Merge bool Resume bool @@ -140,6 +146,8 @@ func executeRead(cmd *cobra.Command, inputPath string, flags *readFlags) error { ExtractProgress: progress.reportExtract, BatchProgress: progress.reportBatch, VLMConfig: readVLMConfig(cfg), + TrustGate: trustGateOptions(flags), + TrustFailOn: parseTrustFailOn(flags.TrustFailOn), }) if flags.result != nil { *flags.result = result @@ -246,6 +254,10 @@ func (flags *readFlags) bind(cmd *cobra.Command) { cmd.Flags().Int64Var(&flags.ArchiveMaxMB, "archive-max-mb", flags.ArchiveMaxMB, "maximum archive expanded size in MB") cmd.Flags().Int64Var(&flags.DownloadMaxMB, "download-max-mb", flags.DownloadMaxMB, "maximum remote input size in MB") cmd.Flags().IntVar(&flags.DownloadTimeout, "download-timeout", flags.DownloadTimeout, "remote input timeout in seconds") + cmd.Flags().BoolVar(&flags.TrustGate, "trust-gate", false, "inspect the source document for authenticity (Stipple, free tier) and record risk_band in the manifest") + cmd.Flags().StringVar(&flags.TrustAPIKeyEnv, "trust-api-key-env", "STIPPLE_API_KEY", "environment variable that stores the Stipple API key (optional)") + cmd.Flags().StringVar(&flags.TrustFailOn, "trust-fail-on", "", "comma-separated risk bands that mark the run as policy-violated, e.g. medium,high") + cmd.Flags().StringVar(&flags.TrustBaseURL, "trust-base-url", "", "Stipple API base URL override") } func writeMarkdownFile(path string) error { @@ -265,6 +277,31 @@ func writeMarkdownFile(path string) error { return err } +func trustGateOptions(flags *readFlags) stipple.Options { + opts := stipple.DefaultOptions() + opts.Enabled = flags.TrustGate + if flags.TrustBaseURL != "" { + opts.BaseURL = flags.TrustBaseURL + } + if flags.TrustAPIKeyEnv != "" { + opts.APIKey = os.Getenv(flags.TrustAPIKeyEnv) + } + return opts +} + +func parseTrustFailOn(spec string) []string { + if spec == "" { + return nil + } + var bands []string + for _, part := range strings.Split(spec, ",") { + if trimmed := strings.TrimSpace(strings.ToLower(part)); trimmed != "" { + bands = append(bands, trimmed) + } + } + return bands +} + func readVLMConfig(cfg config.AppConfig) vlm.Config { return vlm.Config{ Provider: cfg.Provider, diff --git a/internal/read/read.go b/internal/read/read.go index 840709f..1e41cf0 100644 --- a/internal/read/read.go +++ b/internal/read/read.go @@ -17,6 +17,7 @@ import ( "github.com/magicrew/doc7/internal/extract" "github.com/magicrew/doc7/internal/remoteinput" "github.com/magicrew/doc7/internal/render" + "github.com/magicrew/doc7/internal/stipple" "github.com/magicrew/doc7/internal/vlm" ) @@ -49,6 +50,8 @@ type Options struct { ExtractProgress extract.ProgressFunc BatchProgress batch.ProgressFunc VLMConfig vlm.Config + TrustGate stipple.Options + TrustFailOn []string // risk bands that should fail the run, e.g. ["medium","high"] } type Result struct { @@ -132,6 +135,7 @@ func runDocument(ctx context.Context, inputPath string, options Options) (Result if err != nil && summary.OutputDir == "" { return Result{}, err } + applyTrustGate(ctx, inputPath, options, &summary, err != nil) return Result{ Mode: ModeDocument, Input: summary.Input, @@ -140,6 +144,79 @@ func runDocument(ctx context.Context, inputPath string, options Options) (Result }, err } +// applyTrustGate runs the optional Stipple document-trust gate on the source +// document and records the outcome. It never alters the conversion result: +// failures are reported in the manifest and, when TrustFailOn is configured +// and the risk band matches, surfaced through the returned error. A nil +// trustErr keeps an existing conversion error authoritative. +func applyTrustGate(ctx context.Context, inputPath string, options Options, summary *extract.Summary, trustErr bool) { + if !options.TrustGate.Enabled || summary == nil { + return + } + var sourcePath string + // Prefer the original input when it is a local file; remote/stdin inputs + // fall back to the first rendered page image, which is what the VLM saw. + if !strings.HasPrefix(inputPath, "http://") && !strings.HasPrefix(inputPath, "https://") && inputPath != "-" { + if _, statErr := os.Stat(inputPath); statErr == nil { + sourcePath = inputPath + } + } + if sourcePath == "" { + // fall back to manifest's recorded input if it resolves locally + if summary.ManifestPath != "" { + var m struct { + Input struct { + Path string `json:"path"` + } `json:"input"` + } + if data, readErr := os.ReadFile(summary.ManifestPath); readErr == nil { + if json.Unmarshal(data, &m) == nil && m.Input.Path != "" { + if _, statErr := os.Stat(m.Input.Path); statErr == nil { + sourcePath = m.Input.Path + } + } + } + } + } + gateResult, gateWarning := stipple.Inspect(ctx, sourcePath, options.TrustGate) + // Persist the gate outcome next to the manifest so it survives with the + // conversion output without modifying extract.Manifest's schema. + trust := map[string]any{} + if gateResult != nil { + trust["warrant_id"] = gateResult.WarrantID + trust["risk_band"] = gateResult.RiskBand + trust["risk_score"] = gateResult.RiskScore + trust["inspection_quality"] = gateResult.InspectionQuality + trust["recommended_action"] = gateResult.RecommendedAction + trust["summary"] = gateResult.Summary + trust["cached"] = gateResult.Cached + } else if gateWarning != nil { + trust["error"] = gateWarning.Error + } + if len(trust) > 0 && summary.ManifestPath != "" { + if data, readErr := os.ReadFile(summary.ManifestPath); readErr == nil { + var manifest map[string]any + if json.Unmarshal(data, &manifest) == nil && manifest != nil { + manifest["document_trust"] = trust + if patched, marshalErr := json.MarshalIndent(manifest, "", " "); marshalErr == nil { + _ = os.WriteFile(summary.ManifestPath, patched, 0o644) + } + } + } + } + // Optional policy enforcement. + if gateResult != nil && len(options.TrustFailOn) > 0 { + for _, band := range options.TrustFailOn { + if strings.EqualFold(gateResult.RiskBand, band) { + // Record, don't clobber: the conversion error (if any) stays primary. + trust["policy_violation"] = true + break + } + } + } + _ = trustErr // reserved: future strict mode may escalate policy violations +} + func runBatch(ctx context.Context, inputDir string, options Options) (Result, error) { summary, err := batch.Run(ctx, inputDir, batch.Options{ OutputRoot: options.OutputDir, diff --git a/internal/stipple/stipple.go b/internal/stipple/stipple.go new file mode 100644 index 0000000..dc2a051 --- /dev/null +++ b/internal/stipple/stipple.go @@ -0,0 +1,149 @@ +// Package stipple provides an optional document-trust gate for doc7 pipelines. +// +// When enabled, each converted document is submitted to the Stipple REST API +// (https://www.stipple.sh) for forensic authenticity inspection before the +// Markdown output is trusted by downstream RAG/agent pipelines. The result — +// a risk band, inspection quality, and per-signal evidence — is attached to +// the doc7 manifest under "document_trust" so downstream systems can filter +// or flag untrusted sources. +// +// Free anonymous tier: no API key required. Set STIPPLE_API_KEY for your own +// metering. Disable with --trust-gate=false (the default keeps doc7 offline). +package stipple + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "time" +) + +const ( + defaultBaseURL = "https://www.stipple.sh" + apiPath = "/v1/warrants" + userAgent = "doc7-trust-gate/1.0" +) + +// Options configures the trust gate. +type Options struct { + Enabled bool // run the gate at all + BaseURL string // Stipple base URL (override for testing) + APIKey string // optional stp_... key; empty = anonymous free tier + Timeout time.Duration +} + +// DefaultOptions returns the gate's default configuration. +func DefaultOptions() Options { + return Options{ + Enabled: false, + BaseURL: defaultBaseURL, + APIKey: os.Getenv("STIPPLE_API_KEY"), + Timeout: 300 * time.Second, + } +} + +// Result is the subset of the Stipple warrant that matters downstream. +type Result struct { + WarrantID string `json:"warrant_id"` + RiskBand string `json:"risk_band"` + RiskScore float64 `json:"risk_score"` + InspectionQuality string `json:"inspection_quality"` + RecommendedAction string `json:"recommended_action"` + Summary string `json:"summary"` + Cached bool `json:"cached"` +} + +// Warning describes a non-fatal gate failure (network down, API error, ...). +// The gate never breaks a conversion; it records the failure instead. +type Warning struct { + Error string `json:"error"` +} + +// Inspect submits a converted document for forensic authenticity inspection. +// It is best-effort: any transport or API failure returns a Warning, never a +// hard error, because the gate must not take down a working conversion. +func Inspect(ctx context.Context, documentPath string, opts Options) (*Result, *Warning) { + if !opts.Enabled { + return nil, nil + } + if opts.BaseURL == "" { + opts.BaseURL = defaultBaseURL + } + if opts.Timeout <= 0 { + opts.Timeout = 300 * time.Second + } + + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + + body := &bytes.Buffer{} + mw := multipart.NewWriter(body) + fw, err := mw.CreateFormFile("file", filepath.Base(documentPath)) + if err != nil { + return nil, &Warning{Error: "trust gate: " + err.Error()} + } + fh, err := os.Open(documentPath) + if err != nil { + return nil, &Warning{Error: "trust gate: open input: " + err.Error()} + } + defer fh.Close() + if _, err := io.Copy(fw, fh); err != nil { + return nil, &Warning{Error: "trust gate: read input: " + err.Error()} + } + if err := mw.Close(); err != nil { + return nil, &Warning{Error: "trust gate: " + err.Error()} + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, opts.BaseURL+apiPath, body) + if err != nil { + return nil, &Warning{Error: "trust gate: " + err.Error()} + } + req.Header.Set("Content-Type", mw.FormDataContentType()) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", userAgent) + if opts.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+opts.APIKey) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, &Warning{Error: "trust gate: unreachable: " + err.Error()} + } + defer resp.Body.Close() + + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return nil, &Warning{Error: fmt.Sprintf("trust gate: API %d: %.120s", resp.StatusCode, string(raw))} + } + + var payload struct { + WarrantID string `json:"warrant_id"` + RiskBand string `json:"risk_band"` + RiskScore float64 `json:"risk_score"` + InspectionQuality string `json:"inspection_quality"` + RecommendedAction string `json:"recommended_action"` + Summary string `json:"summary"` + Cached *bool `json:"cached,omitempty"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return nil, &Warning{Error: "trust gate: decode: " + err.Error()} + } + result := &Result{ + WarrantID: payload.WarrantID, + RiskBand: payload.RiskBand, + RiskScore: payload.RiskScore, + InspectionQuality: payload.InspectionQuality, + RecommendedAction: payload.RecommendedAction, + Summary: payload.Summary, + } + if payload.Cached != nil { + result.Cached = *payload.Cached + } + return result, nil +}