Little by little

CONTRIBUTING.md

@@ -13,13 +13,15 @@ Thank you for helping improve this project. All contributions are welcome under
 ## Development setup
 
 ```bash
-./local-run setup   # once: Postgres, Python venv, migrations, npm deps
-./local-run         # dev server → http://localhost:3000/home
+./local-run         # installs/syncs deps + dev server → http://localhost:3000/home
+./local-run setup   # optional: deps + migrations only (no dev server)
 ```
 
+`./local-run` automatically installs missing project dependencies (Python pip, npm, NuGet, Playwright/Chromium) and, when Homebrew/winget is available, missing system tools (Docker, Python 3.12+, Node 20+, .NET SDK 10+). Skip auto-install with `WP_SKIP_SYSTEM_INSTALL=1` or skip dependency sync with `WP_SKIP_DEPS_SYNC=1`.
+
 Details: [README.md](README.md), [AGENT.md](AGENT.md), [docs/README.md](docs/README.md).
 
-JavaScript/auto crawl needs Playwright (from `requirements.txt`, installed by `./local-run setup`) and Chromium on `PATH` or `CHROME_PATH`. Unit tests mock the browser fetcher; integration tests use `@pytest.mark.browser` and run in the Docker CI job (`tests/test_crawl_fetchers.py`, `tests/test_crawler_browser_e2e.py`). Locally: `./local-test browser` (skips gracefully if Chromium is missing).
+JavaScript/auto crawl needs Playwright (from `requirements.txt`, installed automatically by `./local-run`) and Chromium on `PATH` or `CHROME_PATH`. Unit tests mock the browser fetcher; integration tests use `@pytest.mark.browser` and run in the Docker CI job (`tests/test_crawl_fetchers.py`, `tests/test_crawler_browser_e2e.py`). Locally: `./local-test browser` (skips gracefully if Chromium is missing).
 
 ## Running tests
 

README.md

@@ -279,6 +279,8 @@ For layout details and common development patterns, see [AGENT.md](AGENT.md).
 | **Node 20+** | Vite + React SPA |
 | **.NET SDK 10+** | BFF, IntegrationsService, AiService, Data, and ReportService (required for `./local-run`; optional if you only use Docker) |
 
+When using `./local-run`, missing tools above are auto-installed via **Homebrew** (macOS) or **winget** (Windows) when available. On Linux, `apt-get` is used when supported. Set `WP_SKIP_SYSTEM_INSTALL=1` to disable system tool auto-install.
+
 ### Docker
 
 Build and run the full dev stack from source:
@@ -296,14 +298,16 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG
 ### Local development
 
 ```bash
-./local-run setup   # First time: Postgres, Python venv, Playwright/Chromium, migrations, npm deps
-./local-run         # Start full dev stack → http://localhost:3000/home
+./local-run         # Auto-install/sync deps + start full dev stack → http://localhost:3000/home
+./local-run setup   # Optional: deps + migrations only (no dev server)
 ./local-run db      # Postgres only (no app)
 ./local-run migrate # Apply EF Core migrations only
 ./local-run stop    # Stop Postgres container
 ./local-prod        # Same DB, Vite production build + preview (no hot reload)
 ```
 
+Optional env flags: `WP_SKIP_SYSTEM_INSTALL=1` (do not auto-install Docker/Node/Python/.NET), `WP_SKIP_DEPS_SYNC=1` (skip pip/npm/dotnet restore on start).
+
 `./local-run` starts (in order): **Data** `:8091` (reports, portfolio, issues, typed config, PDF/Excel export), **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **FastAPI** `:8096` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin.
 
 Default local `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling` (Docker Compose dev stack uses `profiling:profiling`).

config/typed_config_manifest.json

@@ -115,7 +115,8 @@
     "crawl_settings": [
       "start_url", "crawl_discovery_mode", "crawl_url_list", "crawl_user_agent_preset", "crawl_user_agent_custom",
       "compare_mobile_desktop", "crawl_auth_username", "crawl_extra_headers", "crawl_robots_txt_override",
-      "custom_extractors", "max_pages", "concurrency", "timeout", "max_depth", "polite_delay", "ignore_robots",
+      "custom_extractors", "main_content_selectors", "boilerplate_selectors", "pipeline_graph_json",
+      "max_pages", "concurrency", "timeout", "max_depth", "polite_delay", "ignore_robots",
       "allow_external", "store_outlinks", "store_content_excerpt", "content_excerpt_max_chars", "store_page_html",
       "max_stored_html_bytes", "run_content_analysis", "content_analysis_strategy", "content_analysis_workers",
       "custom_extraction_regex", "crawl_path_segments", "crawl_ignore_params", "competitor_domains", "export_logo_url",

docker-compose.prod.yml

@@ -148,7 +148,7 @@ services:
       ASPNETCORE_URLS: http://+:8094
       FASTAPI_URL: http://fastapi:8096
       INTEGRATIONS_SERVICE_URL: http://integrations:8093
-      AISERVICE_URL: http://ai:8092
+      AI_SERVICE_URL: http://ai:8092
       REPORT_SERVICE_USE_PYTHON_BRIDGE: "0"
       USE_FASTAPI_PYTHON_BRIDGE: "1"
       REPORT_SERVICE_WORKER_ENABLED: "1"
@@ -192,8 +192,8 @@ services:
       AI_SERVICE_URL: http://ai:8092
       INTEGRATIONS_SERVICE_URL: http://integrations:8093
       REPORT_SERVICE_URL: http://report:8094
-      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl"
-      DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters,/api/properties,/api/content-drafts,/api/content/score,/api/keywords,/api/page-markdown,/api/alerts,/api/logs,/api/backlinks"
+      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl,/api/pipeline-preview"
+      DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters,/api/properties,/api/content-drafts,/api/content/score,/api/keywords,/api/page-markdown,/api/alerts,/api/logs,/api/backlinks,/api/pipeline-settings,/api/ui-preferences,/api/client-preferences"
       AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-settings,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"
       INTEGRATIONS_ROUTES: "/api/integrations/google,/api/integrations/bing"
       AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET}

docker-compose.pull.yml

@@ -180,9 +180,9 @@ services:
       AI_SERVICE_URL: http://ai:8092
       INTEGRATIONS_SERVICE_URL: http://integrations:8093
       REPORT_SERVICE_URL: http://report:8094
-      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl"
+      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl,/api/pipeline-preview"
       BFF_ALLOWED_ORIGINS: "http://localhost:3000"
-      DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters,/api/properties,/api/content-drafts,/api/content/score,/api/keywords,/api/page-markdown,/api/alerts,/api/logs,/api/backlinks"
+      DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters,/api/properties,/api/content-drafts,/api/content/score,/api/keywords,/api/page-markdown,/api/alerts,/api/logs,/api/backlinks,/api/pipeline-settings,/api/ui-preferences,/api/client-preferences"
       AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-settings,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"
       INTEGRATIONS_ROUTES: "/api/integrations/google,/api/integrations/bing"
     depends_on:

docker-compose.yml

@@ -150,7 +150,7 @@ services:
       ASPNETCORE_URLS: http://+:8094
       FASTAPI_URL: http://fastapi:8096
       INTEGRATIONS_SERVICE_URL: http://integrations:8093
-      AISERVICE_URL: http://ai:8092
+      AI_SERVICE_URL: http://ai:8092
       REPORT_SERVICE_USE_PYTHON_BRIDGE: "0"
       USE_FASTAPI_PYTHON_BRIDGE: "1"
       REPORT_SERVICE_WORKER_ENABLED: "1"
@@ -185,7 +185,7 @@ services:
       AI_SERVICE_URL: http://ai:8092
       INTEGRATIONS_SERVICE_URL: http://integrations:8093
       REPORT_SERVICE_URL: http://report:8094
-      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl"
+      REPORT_ROUTES: "/api/compare,/api/dashboards,/api/run,/api/jobs,/api/schedule,/api/crawl,/api/pipeline-preview"
       AUTH_SECRET: ${AUTH_SECRET:-}
       DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters,/api/properties,/api/content-drafts,/api/content/score,/api/keywords,/api/page-markdown,/api/alerts,/api/logs,/api/backlinks,/api/pipeline-settings,/api/ui-preferences,/api/client-preferences"
       AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-settings,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"

docs/MCP.md

@@ -160,7 +160,7 @@ The MCP endpoint is **`http://<host>:8092/mcp`** locally (`8092` is AiService's
 | `WP_MCP_DOMAIN` | `core` | Tool bundle (same as stdio) |
 | `WP_PROPERTY_ID` | unset | Default property (same as stdio) |
 
-**Security:** `WP_MCP_TOKEN` is required when AiService is reachable outside localhost. Tools are read-only but expose audit, GSC, and GA4 data — treat the token like a database credential.
+**Security:** `mcp_token` (Risk Settings → Remote MCP, or `WP_MCP_TOKEN`) is **required** for HTTP MCP — requests to `/mcp` without a configured token receive **401 Unauthorized**. When AiService is reachable outside localhost, treat the token like a database credential.
 
 **DNS rebinding protection:** When a token **and** allowed hosts are configured — via the UI **or** environment variables — AiService enforces the bearer token plus the Host/Origin allowlist. Set `WP_MCP_ALLOWED_HOSTS` to the public hostname clients use (e.g. `audit.example.com`).
 

requirements.txt

@@ -14,6 +14,9 @@ markdownify>=0.13,<1
 rapidfuzz==3.14.5
 langdetect==1.0.9
 
+# PDF text extraction (content analysis coverage for crawled PDFs)
+pypdf==6.14.2
+
 # Google Search Console + GA4 integration (optional; required for `python -m src google`)
 google-auth==2.53.0
 google-auth-oauthlib==1.4.0

scripts/ensure-deps.ps1

@@ -0,0 +1,231 @@
+# Shared dependency ensure helpers for local-run.ps1 and local-test.ps1.
+# Dot-source after $ROOT, $VENV, $WEB are set.
+
+if (-not $ROOT) {
+    $ROOT = Split-Path -Parent $PSScriptRoot
+}
+
+function Write-EnsureLog([string]$Message) {
+    if (Get-Command Write-Log -ErrorAction SilentlyContinue) {
+        Write-Log $Message
+    } else {
+        Write-Host "-> $Message" -ForegroundColor Cyan
+    }
+}
+
+function Write-EnsureWarn([string]$Message) {
+    if (Get-Command Write-Warn -ErrorAction SilentlyContinue) {
+        Write-Warn $Message
+    } else {
+        Write-Host "! $Message" -ForegroundColor Yellow
+    }
+}
+
+function Write-EnsureDie([string]$Message) {
+    if (Get-Command Write-Die -ErrorAction SilentlyContinue) {
+        Write-Die $Message
+    } else {
+        Write-Host "X $Message" -ForegroundColor Red
+        exit 1
+    }
+}
+
+function Assert-EnsureExitCode([string]$Message) {
+    if (Get-Command Assert-LastExitCode -ErrorAction SilentlyContinue) {
+        Assert-LastExitCode $Message
+        return
+    }
+    $failed = $false
+    if ($PSVersionTable.PSVersion.Major -ge 7) {
+        $failed = ($LASTEXITCODE -ne 0)
+    } else {
+        $failed = (-not $?)
+    }
+    if ($failed) {
+        Write-EnsureDie $Message
+    }
+}
+
+function Test-PythonVersionOk {
+    param([string]$PythonPath)
+
+    if (-not $PythonPath) { return $false }
+    & $PythonPath -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)" 2>$null
+    if ($PSVersionTable.PSVersion.Major -ge 7) {
+        return ($LASTEXITCODE -eq 0)
+    }
+    return $?
+}
+
+function Get-EnsurePythonLauncher {
+    foreach ($cmd in @("python", "python3", "py")) {
+        if (-not (Get-Command $cmd -ErrorAction SilentlyContinue)) { continue }
+        if ($cmd -eq "py") {
+            $candidate = @("py", "-3")
+            & py -3 -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)" 2>$null
+        } else {
+            $candidate = @($cmd)
+            & $cmd -c "import sys; sys.exit(0 if sys.version_info >= (3, 12) else 1)" 2>$null
+        }
+        $ok = if ($PSVersionTable.PSVersion.Major -ge 7) { $LASTEXITCODE -eq 0 } else { $? }
+        if ($ok) { return ,$candidate }
+    }
+    return $null
+}
+
+function Invoke-EnsurePythonLauncher {
+    param(
+        [Parameter(Mandatory = $true)]
+        [string[]]$Launcher,
+        [Parameter(ValueFromRemainingArguments = $true)]
+        [string[]]$PythonArgs
+    )
+
+    if ($Launcher.Count -gt 1) {
+        & $Launcher[0] $Launcher[1] @PythonArgs
+    } else {
+        & $Launcher[0] @PythonArgs
+    }
+    Assert-EnsureExitCode "Python command failed: $($Launcher -join ' ') $($PythonArgs -join ' ')"
+}
+
+function Test-DotnetVersionOk {
+    if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { return $false }
+    $version = (& dotnet --version 2>$null)
+    if (-not $version) { return $false }
+    $major = [int]($version.Split('.')[0])
+    return ($major -ge 10)
+}
+
+function Install-WingetPackage {
+    param(
+        [string]$CommandName,
+        [string]$PackageId
+    )
+
+    if (Get-Command $CommandName -ErrorAction SilentlyContinue) {
+        return
+    }
+    if ($env:WP_SKIP_SYSTEM_INSTALL -eq "1") {
+        Write-EnsureDie "Missing required command: $CommandName (WP_SKIP_SYSTEM_INSTALL=1)"
+    }
+    if (-not (Get-Command winget -ErrorAction SilentlyContinue)) {
+        Write-EnsureDie "Missing required command: $CommandName. Install winget package $PackageId manually."
+    }
+    Write-EnsureLog "Installing $PackageId via winget"
+    & winget install --id $PackageId -e --accept-source-agreements --accept-package-agreements
+    Assert-EnsureExitCode "winget install failed for $PackageId"
+    if (-not (Get-Command $CommandName -ErrorAction SilentlyContinue)) {
+        Write-EnsureWarn "$CommandName not on PATH yet — open a new terminal after winget install"
+        Write-EnsureDie "Still missing required command: $CommandName"
+    }
+}
+
+function Ensure-SystemTools {
+    Install-WingetPackage -CommandName "docker" -PackageId "Docker.DockerDesktop"
+    Install-WingetPackage -CommandName "python" -PackageId "Python.Python.3.12"
+    if (-not (Get-EnsurePythonLauncher)) {
+        Install-WingetPackage -CommandName "python3" -PackageId "Python.Python.3.12"
+    }
+    Install-WingetPackage -CommandName "npm" -PackageId "OpenJS.NodeJS.LTS"
+    if (-not (Test-DotnetVersionOk)) {
+        Install-WingetPackage -CommandName "dotnet" -PackageId "Microsoft.DotNet.SDK.10"
+    }
+    if (-not (Test-DotnetVersionOk)) {
+        Write-EnsureDie ".NET SDK 10+ required (see README.md prerequisites)"
+    }
+}
+
+function Ensure-PythonDeps {
+    if ($env:WP_SKIP_DEPS_SYNC -eq "1") { return }
+
+    $venv = if ($VENV) { $VENV } else { Join-Path $ROOT ".venv" }
+    $venvPython = Join-Path $venv "Scripts\python.exe"
+    $venvPip = Join-Path $venv "Scripts\pip.exe"
+
+    $pyLauncher = Get-EnsurePythonLauncher
+    if (-not $pyLauncher) {
+        Write-EnsureDie "Python 3.12+ required (see README.md prerequisites)"
+    }
+
+    if (-not (Test-Path $venvPython)) {
+        Write-EnsureLog "Creating Python venv at .venv"
+        Invoke-EnsurePythonLauncher -Launcher $pyLauncher -PythonArgs @("-m", "venv", $venv)
+    }
+    Write-EnsureLog "Installing Python dependencies"
+    & $venvPip install -q -r (Join-Path $ROOT "requirements.txt")
+    Assert-EnsureExitCode "Failed to install requirements.txt"
+    $env:PYTHON = $venvPython
+}
+
+function Ensure-WebDeps {
+    if ($env:WP_SKIP_DEPS_SYNC -eq "1") { return }
+
+    $web = if ($WEB) { $WEB } else { Join-Path $ROOT "web" }
+    $lock = Join-Path $web "package-lock.json"
+    $stamp = Join-Path $web ".deps-installed"
+    $nodeModules = Join-Path $web "node_modules"
+
+    if (-not (Get-Command npm -ErrorAction SilentlyContinue)) {
+        Write-EnsureDie "Missing required command: npm"
+    }
+
+    $needsInstall = -not (Test-Path $nodeModules) -or -not (Test-Path $stamp)
+    if (-not $needsInstall -and (Test-Path $lock) -and (Test-Path $stamp)) {
+        $needsInstall = (Get-Item $lock).LastWriteTimeUtc -gt (Get-Item $stamp).LastWriteTimeUtc
+    }
+    if ($needsInstall) {
+        Write-EnsureLog "Installing/updating web dependencies (npm ci)"
+        Push-Location $web
+        try {
+            & npm ci
+            Assert-EnsureExitCode "Failed to install web dependencies (npm ci)"
+            Set-Content -Path $stamp -Value ((Get-Date).ToUniversalTime().ToString("o"))
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
+function Ensure-DotnetDeps {
+    if ($env:WP_SKIP_DEPS_SYNC -eq "1") { return }
+    if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { return }
+
+    Write-EnsureLog "Restoring .NET packages"
+    Push-Location (Join-Path $ROOT "services")
+    try {
+        & dotnet restore WebsiteProfiling.slnx
+        Assert-EnsureExitCode "dotnet restore failed"
+    } finally {
+        Pop-Location
+    }
+}
+
+function Ensure-BrowserDeps {
+    if ($env:WP_SKIP_DEPS_SYNC -eq "1") { return }
+
+    $venv = if ($VENV) { $VENV } else { Join-Path $ROOT ".venv" }
+    $venvPython = Join-Path $venv "Scripts\python.exe"
+
+    Ensure-PythonDeps
+    Write-EnsureLog "Ensuring Playwright + Chromium for JS crawl"
+    $script = @"
+from website_profiling.crawl.fetchers import ensure_browser_deps
+import json, sys
+status = ensure_browser_deps()
+print(json.dumps(status))
+sys.exit(0 if status.get('ok') else 1)
+"@
+    & $venvPython -c $script
+    $failed = if ($PSVersionTable.PSVersion.Major -ge 7) { $LASTEXITCODE -ne 0 } else { -not $? }
+    if ($failed) {
+        Write-EnsureWarn "Browser deps unavailable - JS/auto crawl disabled until Playwright + Chromium install successfully"
+    }
+}
+
+function Ensure-AllProjectDeps {
+    Ensure-PythonDeps
+    Ensure-DotnetDeps
+    Ensure-BrowserDeps
+    Ensure-WebDeps
+}