embed_docsite: download docsite via git clone instead of GitHub contents API

.gitignore

@@ -74,6 +74,7 @@ services/inference/data/
 
 app.log
 adaptors_cache.json
+services/embed_docsite/docsite_cache/
 
 .context
 

pyproject.toml

@@ -50,6 +50,7 @@ testpaths = [
     "services/job_chat/tests",
     "services/latest_adaptors/tests",
     "services/search_docsite/tests",
+    "services/embed_docsite/tests",
     "services/tools",
 ]
 

services/embed_docsite/README.md

@@ -26,6 +26,15 @@ The service uses the DocsiteProcessor to download the documentation and chunk it
 
 The chunked texts can be viewed in `tmp/split_sections`.
 
+## Docs checkout
+
+`general_docs` and `adaptor_docs` are read from a shallow `git clone` of
+[OpenFn/docs](https://github.com/OpenFn/docs), kept at `services/embed_docsite/docsite_cache/` (gitignored). The first call in a process clones; every call after `fetch`es and `reset --hard`s onto the latest `main`.
+
+The clone is blobless and sparse (`--filter=blob:none --sparse`, checked out to only `docs/` and `adaptors/`). If the refresh fails and a checkout already exists, the existing copy is served and a warning is logged — the run only fails if there is no checkout to fall back on. `git` must be on `PATH`.
+
+`adaptor_functions` is a single JSON file from a different repo (`OpenFn/adaptors`), fetched over plain HTTP.
+
 ## Payload Reference
 The input payload is a JSON object. All parameters are optional:
 

services/embed_docsite/docs_repo.py

@@ -0,0 +1,127 @@
+"""A shallow, blobless, sparse git checkout of OpenFn/docs, kept on disk.
+
+Uses a single `git clone`/`fetch`.
+
+`sync_docs_repo` clones on the first call in a process and `fetch`+`reset
+--hard`s on every call after, memoized. A failed refresh serves the
+existing checkout rather than failing the run, same as
+`services/latest_adaptors/latest_adaptors.py`'s cache.
+
+The clone is blobless and sparse-checked-out to only `DOCS_TYPE_PREFIXES`'
+directories, so the missing blobs are fetched lazily for the checkout.
+"""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+from util import ApolloError, create_logger
+
+logger = create_logger("DocsRepo")
+
+DOCS_REPO_URL = "https://github.com/OpenFn/docs.git"
+DOCS_REF = "main"
+CLONE_DIR = Path(__file__).parent / "docsite_cache"
+
+# Which top-level directory in OpenFn/docs backs each docs_type.
+DOCS_TYPE_PREFIXES = {"general_docs": "docs", "adaptor_docs": "adaptors"}
+
+GIT_TIMEOUT_SECONDS = 300
+
+# Memoized per process: a full run reads two docs_types, and they should
+# cost one sync, not two.
+_synced_sha = None
+
+
+def _run_git(args, cwd=None):
+    """Subprocess seam."""
+    return subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        check=True,
+        capture_output=True,
+        text=True,
+        timeout=GIT_TIMEOUT_SECONDS,
+    )
+
+
+def _has_checkout():
+    return (CLONE_DIR / ".git").exists()
+
+
+def _clone():
+    """Wipe and clone. Also the recovery path for a corrupt or partial checkout.
+
+    Blobless (`--filter=blob:none`) and sparse, checked out to only the
+    directories `DOCS_TYPE_PREFIXES` names: the rest of OpenFn/docs is images
+    and static assets we never index, and this way their blobs are never even
+    transferred. `sparse-checkout set` fetches the blobs it needs to
+    materialize those paths as part of running; nothing later in this module
+    reads outside them, so no further blob fetch happens after this call.
+    """
+    if CLONE_DIR.exists():
+        shutil.rmtree(CLONE_DIR)
+    _run_git(["clone", "--depth", "1", "--filter=blob:none", "--sparse", DOCS_REPO_URL, str(CLONE_DIR)])
+    _run_git(["sparse-checkout", "set", *DOCS_TYPE_PREFIXES.values()], cwd=CLONE_DIR)
+    logger.info(f"Cloned {DOCS_REPO_URL} to {CLONE_DIR}")
+
+
+def _update():
+    _run_git(["fetch", "--depth", "1", "origin", DOCS_REF], cwd=CLONE_DIR)
+    _run_git(["reset", "--hard", "FETCH_HEAD"], cwd=CLONE_DIR)
+    logger.info("Docs checkout updated")
+
+
+def _head_sha():
+    result = _run_git(["rev-parse", "HEAD"], cwd=CLONE_DIR)
+    return result.stdout.strip()
+
+
+def sync_docs_repo():
+    """Bring the checkout up to date. Call once per run, before reading.
+
+    :return: the checkout's HEAD sha
+    """
+    global _synced_sha  # noqa: PLW0603 - per-process memo, same pattern as util.py's apollo_port
+
+    if _synced_sha is not None:
+        return _synced_sha
+
+    have_checkout = _has_checkout()
+
+    try:
+        _update() if have_checkout else _clone()
+    except FileNotFoundError as exc:
+        raise ApolloError(
+            500,
+            f"git is required to fetch the docs corpus but is not on PATH: {exc}",
+            type="MISCONFIGURED",
+        ) from exc
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+        if have_checkout:
+            logger.warning(f"Could not refresh the docs checkout, serving existing copy: {exc}")
+            _synced_sha = _head_sha()
+            return _synced_sha
+        raise ApolloError(
+            503,
+            f"Could not clone the docs repo and no cached checkout exists: {exc}",
+            type="UPSTREAM_ERROR",
+        ) from exc
+
+    _synced_sha = _head_sha()
+    return _synced_sha
+
+
+def read_markdown_docs(docs_type):
+    """Markdown docs for one docs_type, read from the checkout on disk.
+
+    Calls `sync_docs_repo` first.
+
+    :return: [{"name": <basename>, "docs": <text>}], sorted by path
+    """
+    if docs_type not in DOCS_TYPE_PREFIXES:
+        raise ApolloError(400, f"Unknown docs_type '{docs_type}'", type="BAD_REQUEST")
+
+    prefix_dir = CLONE_DIR / DOCS_TYPE_PREFIXES[docs_type]
+    paths = sorted(prefix_dir.rglob("*.md"))
+    return [{"name": path.name, "docs": path.read_text(encoding="utf-8")} for path in paths]

services/embed_docsite/github_utils.py

@@ -1,69 +1,9 @@
 import requests
+from embed_docsite.docs_repo import read_markdown_docs, sync_docs_repo
 from util import create_logger
 
 logger = create_logger("GitHubUtils")
 
-def download_main_docs(github_info):
-    """Downloads and processes the main docs from GitHub API info."""
-    output = []
-
-    for file_info in github_info:
-        try:
-            response = requests.get(file_info["download_url"])
-            response.raise_for_status()
-
-            output.append({
-                "name": file_info["name"],
-                "docs": response.text
-            })
-
-        except requests.RequestException as e:
-            logger.info(f"Failed to fetch content for {file_info['name']}: {e}")
-    logger.info(f"Downloaded and processed {len(output)} files from GitHub")
-    logger.info(f'{output[0]}')
-    return output
-
-def get_github_urls(repo, path="", owner="OpenFn", file_type=".md"):
-    """"
-    Get the download URLs for a GitHub repository.
-
-    :param repo: The repository (e.g. "docs")
-    :param path: The path from root (e.g. "")
-    :param owner: The repository owner (default="OpenFn")
-    :return: List of dictionaries {name, path, download_url}
-    """
-    files = []
-
-    def fetch_contents(current_path):
-        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{current_path}"
-        response = requests.get(url)
-        contents = response.json()
-
-        # Handle single file response
-        if not isinstance(contents, list):
-            contents = [contents]
-
-        msg = contents[0].get("message") 
-        if msg and msg.startswith("API rate limit exceeded"):
-                logger.error("GitHub API limit exceeded. Check limits here: \
-                             https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28")
-                return
-
-        for item in contents:
-            if item.get("type") == "file" and item.get("name").endswith(file_type):
-                files.append({
-                    "name": item["name"],
-                    "path": item["path"],
-                    "download_url": item["download_url"]
-                })
-            elif item.get("type") == "dir":
-                fetch_contents(item["path"])
-
-    fetch_contents(path)
-    logger.info(f'Fetched {len(files)} URLs from GitHub for https://api.github.com/repos/{owner}/{repo}/contents/{path}')
-
-    return files
-
 def get_adaptor_function_docs(data_url="https://raw.githubusercontent.com/OpenFn/adaptors/docs/docs/docs.json"):
     """Fetches adaptor data from the preprocessed adaptor docs url."""
     try:
@@ -75,28 +15,8 @@ def get_adaptor_function_docs(data_url="https://raw.githubusercontent.com/OpenFn
     except requests.RequestException as e:
         logger.error(f"Failed to fetch data: {e}")
 
-def get_github_path_contents(repo, path="", owner="OpenFn", file_type=".md"):
-    """
-    Get the contents of files from a GitHub repository.
-
-    :param repo: The repository (e.g. "docs")
-    :param path: The path from root (e.g. "")
-    :param owner: The repository owner (default="OpenFn")
-    :param file_type: File extension to filter by (default=".md")
-    :return: List of dictionaries {name, docs}
-    """
-    # Get list of files
-    github_files = get_github_urls(repo, path, owner, file_type)
-
-    # Download and process each file
-    files_data = download_main_docs(github_files)
-
-    return files_data
-
 def get_docs(docs_type):
     if docs_type == "adaptor_functions":
         return get_adaptor_function_docs()
-    if docs_type == "general_docs":
-        return get_github_path_contents(repo="docs", path="docs")
-    if docs_type == "adaptor_docs": 
-        return get_github_path_contents(repo="docs", path="adaptors")
+    sync_docs_repo()
+    return read_markdown_docs(docs_type)

services/embed_docsite/tests/integration/test_docs_repo_clone.py

@@ -0,0 +1,45 @@
+"""Real-network tests for the git-backed docs checkout."""
+import re
+
+import embed_docsite.docs_repo as m
+import pytest
+
+GENERAL_DOC_COUNT = 93
+ADAPTOR_DOC_COUNT = 98
+GIT_SHA_LENGTH = 40
+
+
+@pytest.fixture(autouse=True)
+def _reset_memo(monkeypatch):
+    monkeypatch.setattr(m, "_synced_sha", None)
+
+
+def test_cold_clone_yields_the_full_markdown_corpus():
+    m.sync_docs_repo()
+
+    general = m.read_markdown_docs("general_docs")
+    adaptors = m.read_markdown_docs("adaptor_docs")
+
+    assert len(general) == GENERAL_DOC_COUNT
+    assert len(adaptors) == ADAPTOR_DOC_COUNT
+
+
+def test_second_sync_is_a_noop_and_corpus_is_unchanged(monkeypatch):
+    m.sync_docs_repo()
+    before = m.read_markdown_docs("general_docs")
+
+    # Bypass the per-process memo to force a real second sync, as a fresh
+    # process would perform. An already-current checkout should fetch
+    # nothing and leave the corpus untouched.
+    monkeypatch.setattr(m, "_synced_sha", None)
+    m.sync_docs_repo()
+    after = m.read_markdown_docs("general_docs")
+
+    assert before == after
+
+
+def test_head_sha_is_a_git_sha():
+    sha = m.sync_docs_repo()
+
+    assert len(sha) == GIT_SHA_LENGTH
+    assert re.fullmatch(r"[0-9a-f]+", sha)