diff --git a/.gitignore b/.gitignore index de7319d0..d8ca6734 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ services/inference/data/ app.log adaptors_cache.json +services/embed_docsite/docsite_cache/ .context diff --git a/pyproject.toml b/pyproject.toml index f67ccac0..ab2ab61f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ testpaths = [ "services/job_chat/tests", "services/latest_adaptors/tests", "services/search_docsite/tests", + "services/embed_docsite/tests", "services/tools", ] diff --git a/services/embed_docsite/README.md b/services/embed_docsite/README.md index 6129f303..3437116c 100644 --- a/services/embed_docsite/README.md +++ b/services/embed_docsite/README.md @@ -26,6 +26,15 @@ The service uses the DocsiteProcessor to download the documentation and chunk it The chunked texts can be viewed in `tmp/split_sections`. +## Docs checkout + +`general_docs` and `adaptor_docs` are read from a shallow `git clone` of +[OpenFn/docs](https://github.com/OpenFn/docs), kept at `services/embed_docsite/docsite_cache/` (gitignored). The first call in a process clones; every call after `fetch`es and `reset --hard`s onto the latest `main`. + +The clone is blobless and sparse (`--filter=blob:none --sparse`, checked out to only `docs/` and `adaptors/`). If the refresh fails and a checkout already exists, the existing copy is served and a warning is logged — the run only fails if there is no checkout to fall back on. `git` must be on `PATH`. + +`adaptor_functions` is a single JSON file from a different repo (`OpenFn/adaptors`), fetched over plain HTTP. + ## Payload Reference The input payload is a JSON object. All parameters are optional: diff --git a/services/embed_docsite/__init__.py b/services/embed_docsite/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/embed_docsite/docs_repo.py b/services/embed_docsite/docs_repo.py new file mode 100644 index 00000000..0f70d737 --- /dev/null +++ b/services/embed_docsite/docs_repo.py @@ -0,0 +1,127 @@ +"""A shallow, blobless, sparse git checkout of OpenFn/docs, kept on disk. + +Uses a single `git clone`/`fetch`. + +`sync_docs_repo` clones on the first call in a process and `fetch`+`reset +--hard`s on every call after, memoized. A failed refresh serves the +existing checkout rather than failing the run, same as +`services/latest_adaptors/latest_adaptors.py`'s cache. + +The clone is blobless and sparse-checked-out to only `DOCS_TYPE_PREFIXES`' +directories, so the missing blobs are fetched lazily for the checkout. +""" + +import shutil +import subprocess +from pathlib import Path + +from util import ApolloError, create_logger + +logger = create_logger("DocsRepo") + +DOCS_REPO_URL = "https://github.com/OpenFn/docs.git" +DOCS_REF = "main" +CLONE_DIR = Path(__file__).parent / "docsite_cache" + +# Which top-level directory in OpenFn/docs backs each docs_type. +DOCS_TYPE_PREFIXES = {"general_docs": "docs", "adaptor_docs": "adaptors"} + +GIT_TIMEOUT_SECONDS = 300 + +# Memoized per process: a full run reads two docs_types, and they should +# cost one sync, not two. +_synced_sha = None + + +def _run_git(args, cwd=None): + """Subprocess seam.""" + return subprocess.run( + ["git", *args], + cwd=cwd, + check=True, + capture_output=True, + text=True, + timeout=GIT_TIMEOUT_SECONDS, + ) + + +def _has_checkout(): + return (CLONE_DIR / ".git").exists() + + +def _clone(): + """Wipe and clone. Also the recovery path for a corrupt or partial checkout. + + Blobless (`--filter=blob:none`) and sparse, checked out to only the + directories `DOCS_TYPE_PREFIXES` names: the rest of OpenFn/docs is images + and static assets we never index, and this way their blobs are never even + transferred. `sparse-checkout set` fetches the blobs it needs to + materialize those paths as part of running; nothing later in this module + reads outside them, so no further blob fetch happens after this call. + """ + if CLONE_DIR.exists(): + shutil.rmtree(CLONE_DIR) + _run_git(["clone", "--depth", "1", "--filter=blob:none", "--sparse", DOCS_REPO_URL, str(CLONE_DIR)]) + _run_git(["sparse-checkout", "set", *DOCS_TYPE_PREFIXES.values()], cwd=CLONE_DIR) + logger.info(f"Cloned {DOCS_REPO_URL} to {CLONE_DIR}") + + +def _update(): + _run_git(["fetch", "--depth", "1", "origin", DOCS_REF], cwd=CLONE_DIR) + _run_git(["reset", "--hard", "FETCH_HEAD"], cwd=CLONE_DIR) + logger.info("Docs checkout updated") + + +def _head_sha(): + result = _run_git(["rev-parse", "HEAD"], cwd=CLONE_DIR) + return result.stdout.strip() + + +def sync_docs_repo(): + """Bring the checkout up to date. Call once per run, before reading. + + :return: the checkout's HEAD sha + """ + global _synced_sha # noqa: PLW0603 - per-process memo, same pattern as util.py's apollo_port + + if _synced_sha is not None: + return _synced_sha + + have_checkout = _has_checkout() + + try: + _update() if have_checkout else _clone() + except FileNotFoundError as exc: + raise ApolloError( + 500, + f"git is required to fetch the docs corpus but is not on PATH: {exc}", + type="MISCONFIGURED", + ) from exc + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc: + if have_checkout: + logger.warning(f"Could not refresh the docs checkout, serving existing copy: {exc}") + _synced_sha = _head_sha() + return _synced_sha + raise ApolloError( + 503, + f"Could not clone the docs repo and no cached checkout exists: {exc}", + type="UPSTREAM_ERROR", + ) from exc + + _synced_sha = _head_sha() + return _synced_sha + + +def read_markdown_docs(docs_type): + """Markdown docs for one docs_type, read from the checkout on disk. + + Calls `sync_docs_repo` first. + + :return: [{"name": , "docs": }], sorted by path + """ + if docs_type not in DOCS_TYPE_PREFIXES: + raise ApolloError(400, f"Unknown docs_type '{docs_type}'", type="BAD_REQUEST") + + prefix_dir = CLONE_DIR / DOCS_TYPE_PREFIXES[docs_type] + paths = sorted(prefix_dir.rglob("*.md")) + return [{"name": path.name, "docs": path.read_text(encoding="utf-8")} for path in paths] diff --git a/services/embed_docsite/github_utils.py b/services/embed_docsite/github_utils.py index a9a6149d..3b0a0c71 100644 --- a/services/embed_docsite/github_utils.py +++ b/services/embed_docsite/github_utils.py @@ -1,69 +1,9 @@ import requests +from embed_docsite.docs_repo import read_markdown_docs, sync_docs_repo from util import create_logger logger = create_logger("GitHubUtils") -def download_main_docs(github_info): - """Downloads and processes the main docs from GitHub API info.""" - output = [] - - for file_info in github_info: - try: - response = requests.get(file_info["download_url"]) - response.raise_for_status() - - output.append({ - "name": file_info["name"], - "docs": response.text - }) - - except requests.RequestException as e: - logger.info(f"Failed to fetch content for {file_info['name']}: {e}") - logger.info(f"Downloaded and processed {len(output)} files from GitHub") - logger.info(f'{output[0]}') - return output - -def get_github_urls(repo, path="", owner="OpenFn", file_type=".md"): - """" - Get the download URLs for a GitHub repository. - - :param repo: The repository (e.g. "docs") - :param path: The path from root (e.g. "") - :param owner: The repository owner (default="OpenFn") - :return: List of dictionaries {name, path, download_url} - """ - files = [] - - def fetch_contents(current_path): - url = f"https://api.github.com/repos/{owner}/{repo}/contents/{current_path}" - response = requests.get(url) - contents = response.json() - - # Handle single file response - if not isinstance(contents, list): - contents = [contents] - - msg = contents[0].get("message") - if msg and msg.startswith("API rate limit exceeded"): - logger.error("GitHub API limit exceeded. Check limits here: \ - https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28") - return - - for item in contents: - if item.get("type") == "file" and item.get("name").endswith(file_type): - files.append({ - "name": item["name"], - "path": item["path"], - "download_url": item["download_url"] - }) - elif item.get("type") == "dir": - fetch_contents(item["path"]) - - fetch_contents(path) - logger.info(f'Fetched {len(files)} URLs from GitHub for https://api.github.com/repos/{owner}/{repo}/contents/{path}') - - return files - def get_adaptor_function_docs(data_url="https://raw.githubusercontent.com/OpenFn/adaptors/docs/docs/docs.json"): """Fetches adaptor data from the preprocessed adaptor docs url.""" try: @@ -75,28 +15,8 @@ def get_adaptor_function_docs(data_url="https://raw.githubusercontent.com/OpenFn except requests.RequestException as e: logger.error(f"Failed to fetch data: {e}") -def get_github_path_contents(repo, path="", owner="OpenFn", file_type=".md"): - """ - Get the contents of files from a GitHub repository. - - :param repo: The repository (e.g. "docs") - :param path: The path from root (e.g. "") - :param owner: The repository owner (default="OpenFn") - :param file_type: File extension to filter by (default=".md") - :return: List of dictionaries {name, docs} - """ - # Get list of files - github_files = get_github_urls(repo, path, owner, file_type) - - # Download and process each file - files_data = download_main_docs(github_files) - - return files_data - def get_docs(docs_type): if docs_type == "adaptor_functions": return get_adaptor_function_docs() - if docs_type == "general_docs": - return get_github_path_contents(repo="docs", path="docs") - if docs_type == "adaptor_docs": - return get_github_path_contents(repo="docs", path="adaptors") \ No newline at end of file + sync_docs_repo() + return read_markdown_docs(docs_type) \ No newline at end of file diff --git a/services/embed_docsite/tests/__init__.py b/services/embed_docsite/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/embed_docsite/tests/integration/__init__.py b/services/embed_docsite/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/embed_docsite/tests/integration/test_docs_repo_clone.py b/services/embed_docsite/tests/integration/test_docs_repo_clone.py new file mode 100644 index 00000000..56c7c1a1 --- /dev/null +++ b/services/embed_docsite/tests/integration/test_docs_repo_clone.py @@ -0,0 +1,45 @@ +"""Real-network tests for the git-backed docs checkout.""" +import re + +import embed_docsite.docs_repo as m +import pytest + +GENERAL_DOC_COUNT = 93 +ADAPTOR_DOC_COUNT = 98 +GIT_SHA_LENGTH = 40 + + +@pytest.fixture(autouse=True) +def _reset_memo(monkeypatch): + monkeypatch.setattr(m, "_synced_sha", None) + + +def test_cold_clone_yields_the_full_markdown_corpus(): + m.sync_docs_repo() + + general = m.read_markdown_docs("general_docs") + adaptors = m.read_markdown_docs("adaptor_docs") + + assert len(general) == GENERAL_DOC_COUNT + assert len(adaptors) == ADAPTOR_DOC_COUNT + + +def test_second_sync_is_a_noop_and_corpus_is_unchanged(monkeypatch): + m.sync_docs_repo() + before = m.read_markdown_docs("general_docs") + + # Bypass the per-process memo to force a real second sync, as a fresh + # process would perform. An already-current checkout should fetch + # nothing and leave the corpus untouched. + monkeypatch.setattr(m, "_synced_sha", None) + m.sync_docs_repo() + after = m.read_markdown_docs("general_docs") + + assert before == after + + +def test_head_sha_is_a_git_sha(): + sha = m.sync_docs_repo() + + assert len(sha) == GIT_SHA_LENGTH + assert re.fullmatch(r"[0-9a-f]+", sha) diff --git a/services/embed_docsite/tests/unit/__init__.py b/services/embed_docsite/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/embed_docsite/tests/unit/test_docs_repo.py b/services/embed_docsite/tests/unit/test_docs_repo.py new file mode 100644 index 00000000..72eb4cc5 --- /dev/null +++ b/services/embed_docsite/tests/unit/test_docs_repo.py @@ -0,0 +1,165 @@ +"""Unit tests for the git-backed docs checkout. + +`_run_git` is replaced with a fake that records argv +and never spawns `git`. +""" +import subprocess + +import embed_docsite.docs_repo as m +import pytest +from util import ApolloError + +FAKE_SHA = "deadbeefcafefeed0000111122223333deadbee" + +HTTP_BAD_REQUEST = 400 +HTTP_MISCONFIGURED = 500 +HTTP_UPSTREAM_ERROR = 503 + + +class FakeCompletedProcess: + def __init__(self, stdout=""): + self.stdout = stdout + + +def make_git(rev_sha=FAKE_SHA, fail_on=None): + """A fake `_run_git`. `fail_on(args) -> bool` marks which calls raise.""" + calls = [] + + def fake(args, cwd=None): # noqa: ARG001 - cwd recorded implicitly via calls, not asserted + calls.append(args) + if fail_on and fail_on(args): + raise subprocess.CalledProcessError(1, ["git", *args]) + if args[:2] == ["rev-parse", "HEAD"]: + return FakeCompletedProcess(stdout=f"{rev_sha}\n") + return FakeCompletedProcess() + + fake.calls = calls + return fake + + +@pytest.fixture(autouse=True) +def _isolate(tmp_path, monkeypatch): + monkeypatch.setattr(m, "CLONE_DIR", tmp_path / "docsite_cache") + monkeypatch.setattr(m, "_synced_sha", None) + + +def mark_checkout_present(): + (m.CLONE_DIR / ".git").mkdir(parents=True) + + +def test_cold_clone_when_no_checkout_exists(monkeypatch): + git = make_git() + monkeypatch.setattr(m, "_run_git", git) + + sha = m.sync_docs_repo() + + assert git.calls[0] == [ + "clone", + "--depth", + "1", + "--filter=blob:none", + "--sparse", + m.DOCS_REPO_URL, + str(m.CLONE_DIR), + ] + assert git.calls[1] == ["sparse-checkout", "set", *m.DOCS_TYPE_PREFIXES.values()] + assert sha == FAKE_SHA + + +def test_sparse_checkout_paths_are_derived_not_hardcoded(monkeypatch): + monkeypatch.setattr(m, "DOCS_TYPE_PREFIXES", {"made_up_type": "somewhere"}) + git = make_git() + monkeypatch.setattr(m, "_run_git", git) + + m.sync_docs_repo() + + assert ["sparse-checkout", "set", "somewhere"] in git.calls + + +def test_warm_update_when_checkout_exists(monkeypatch): + mark_checkout_present() + git = make_git() + monkeypatch.setattr(m, "_run_git", git) + + m.sync_docs_repo() + + assert not any(call[0] == "clone" for call in git.calls) + assert ["fetch", "--depth", "1", "origin", m.DOCS_REF] in git.calls + assert ["reset", "--hard", "FETCH_HEAD"] in git.calls + + +def test_clone_dir_without_git_is_wiped_then_cloned(monkeypatch): + m.CLONE_DIR.mkdir(parents=True) + (m.CLONE_DIR / "stale.txt").write_text("leftover") + git = make_git() + monkeypatch.setattr(m, "_run_git", git) + + m.sync_docs_repo() + + assert git.calls[0][0] == "clone" + assert not (m.CLONE_DIR / "stale.txt").exists() + + +def test_update_failure_serves_stale_checkout(monkeypatch, caplog): + mark_checkout_present() + git = make_git(fail_on=lambda args: args[0] == "fetch") + monkeypatch.setattr(m, "_run_git", git) + + sha = m.sync_docs_repo() + + assert sha == FAKE_SHA + assert "serving existing" in caplog.text.lower() or "stale" in caplog.text.lower() + + +def test_clone_failure_with_no_checkout_raises(monkeypatch): + git = make_git(fail_on=lambda args: args[0] == "clone") + monkeypatch.setattr(m, "_run_git", git) + + with pytest.raises(ApolloError) as exc_info: + m.sync_docs_repo() + + assert exc_info.value.code == HTTP_UPSTREAM_ERROR + + +def test_git_not_on_path_raises_apollo_error(monkeypatch): + def fake(args, cwd=None): # noqa: ARG001 + raise FileNotFoundError("git") + + monkeypatch.setattr(m, "_run_git", fake) + + with pytest.raises(ApolloError) as exc_info: + m.sync_docs_repo() + + assert exc_info.value.code == HTTP_MISCONFIGURED + assert "git" in exc_info.value.message.lower() + + +def test_read_markdown_docs_returns_sorted_basenames_and_contents(): + general_dir = m.CLONE_DIR / "docs" / "build" + general_dir.mkdir(parents=True) + (general_dir / "zebra.md").write_text("z content") + (general_dir / "alpha.md").write_text("a content") + (general_dir / "notes.txt").write_text("ignored, not markdown") + + docs = m.read_markdown_docs("general_docs") + + assert [d["name"] for d in docs] == ["alpha.md", "zebra.md"] + assert docs[0]["docs"] == "a content" + + +def test_read_markdown_docs_unknown_type_raises(): + with pytest.raises(ApolloError) as exc_info: + m.read_markdown_docs("not_a_real_type") + + assert exc_info.value.code == HTTP_BAD_REQUEST + + +def test_sync_docs_repo_memoizes_within_a_process(monkeypatch): + git = make_git() + monkeypatch.setattr(m, "_run_git", git) + + m.sync_docs_repo() + calls_after_first = len(git.calls) + m.sync_docs_repo() + + assert len(git.calls) == calls_after_first