diff --git a/.chronus/changes/python-post-processing-scope.md b/.chronus/changes/python-post-processing-scope.md new file mode 100644 index 00000000000..48727770487 --- /dev/null +++ b/.chronus/changes/python-post-processing-scope.md @@ -0,0 +1,7 @@ +--- +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +Limit Black formatting and pylint suppression insertion to Python files written during the current generation, leaving pre-existing user-managed tests, samples, and other Python files unchanged. diff --git a/packages/http-client-python/ARCHITECTURE.md b/packages/http-client-python/ARCHITECTURE.md index 8f1b78d4ac9..79b28bfe92e 100644 --- a/packages/http-client-python/ARCHITECTURE.md +++ b/packages/http-client-python/ARCHITECTURE.md @@ -154,6 +154,22 @@ def {{ operation.name }}(self, {{ operation.parameters }}): {% endfor %} ``` +#### Post-processing + +Native Python and Pyodide use the shared `pygen.generate.generate` pipeline. The +`ReaderAndWriter.write_file` method records successful writes as normalized absolute paths. +`CodeGenerator` shares its per-run `written_files` set with `JinjaSerializer` and resets it on +each generation. `BlackScriptPlugin` receives those paths explicitly and formats only `.py` +files, adding pylint suppression comments where needed without invoking pylint. + +Batch workers return their recorded paths to the parent process for post-processing after +generation. Formatting never scans the output directory, including when the recorded set is +empty. Absolute paths also cover generated packaging files, tests, and samples outside the +immediate generation directory. + +The standalone `python -m pygen.black --output-folder ` command +also requires an explicit file list. + ## Code Generation Flow ### Detailed Step-by-Step Process diff --git a/packages/http-client-python/README.md b/packages/http-client-python/README.md index 0c3335df5fa..1d86a3bed82 100644 --- a/packages/http-client-python/README.md +++ b/packages/http-client-python/README.md @@ -33,6 +33,11 @@ options: option: value ``` +After generation, Black formatting and pylint suppression comments are applied only to Python +files written during that generation, including regenerated files and generated tests and samples +when enabled. Other existing files, such as hand-written `tests/` and `samples/`, are not +post-processed. This step does not run pylint. + ## Emitter options ### `emitter-output-dir` diff --git a/packages/http-client-python/emitter/src/constants.ts b/packages/http-client-python/emitter/src/constants.ts index 15bec92c979..92422c73156 100644 --- a/packages/http-client-python/emitter/src/constants.ts +++ b/packages/http-client-python/emitter/src/constants.ts @@ -2,26 +2,3 @@ export const PYODIDE_VERSION = "0.26.2"; export const PYGEN_WHEEL_FILENAME = "pygen-0.1.0-py3-none-any.whl"; export const BLOB_STORAGE_BASE_URL = "https://typespec.blob.core.windows.net/pkgs"; export const PACKAGE_NAME = "@typespec/http-client-python"; - -export const blackExcludeDirs = [ - "__pycache__/", - "node_modules/", - "venv/", - "env/", - ".direnv", - ".eggs", - ".git", - ".hg", - ".tox", - ".venv", - ".eggs", - ".mypy_cache", - ".pytest_cache", - ".vscode", - ".*_build/", - "/build/", - "dist", - ".nox", - ".svn", - "TempTypeSpecFiles/", -]; diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index c68517f99f4..859df88585a 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -106,10 +106,8 @@ const pyodideGenerationCode = ` async def main(): import warnings with warnings.catch_warnings(): - from pygen import preprocess, codegen, black - preprocess.PreProcessPlugin(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() - codegen.CodeGenerator(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process() - black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process() + from pygen.generate import generate + generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs) await main()`; diff --git a/packages/http-client-python/emitter/src/node-runner.ts b/packages/http-client-python/emitter/src/node-runner.ts index 399ebcfd593..0e23fa19f9c 100644 --- a/packages/http-client-python/emitter/src/node-runner.ts +++ b/packages/http-client-python/emitter/src/node-runner.ts @@ -9,12 +9,11 @@ import type { EmitContext } from "@typespec/compiler"; import { NoTarget } from "@typespec/compiler"; import { execSync } from "child_process"; import fs from "fs"; -import os from "os"; import path, { dirname } from "path"; import type { PyodideInterface } from "pyodide"; import { loadPyodide } from "pyodide"; import { fileURLToPath } from "url"; -import { blackExcludeDirs, PYGEN_WHEEL_FILENAME } from "./constants.js"; +import { PYGEN_WHEEL_FILENAME } from "./constants.js"; import { saveCodeModelAsYaml } from "./external-process.js"; import type { PythonEmitterOptions } from "./lib.js"; import { reportDiagnostic } from "./lib.js"; @@ -112,14 +111,8 @@ export async function runNodeEmit({ const commandFlags = Object.entries(commandArgs) .map(([key, value]) => `--${key}=${quoteShellArg(String(value))}`) .join(" "); - const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`; + const command = `${quoteShellArg(venvPath)} ${quoteShellArg(path.join(root, "eng", "scripts", "setup", "run_tsp.py"))} ${commandFlags}`; execSync(command); - - const excludePattern = blackExcludeDirs.join("|"); - execSync( - `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`, - ); - await checkForPylintIssues(outputDir, excludePattern); } async function setupPyodideCall(root: string): Promise { @@ -161,64 +154,3 @@ async function setupPyodideCall(root: string): Promise { } return pyodide; } - -async function checkForPylintIssues(outputDir: string, excludePattern: string) { - const excludeRegex = new RegExp(excludePattern); - - const shouldExcludePath = (filePath: string): boolean => { - const relativePath = path.relative(outputDir, filePath); - const normalizedPath = relativePath.replace(/\\/g, "/"); - return excludeRegex.test(normalizedPath); - }; - - const processFile = async (filePath: string) => { - let fileContent = await fs.promises.readFile(filePath, "utf-8"); - const pylintDisables: string[] = []; - const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n"; - const lines: string[] = fileContent.split(lineEnding); - if (lines.length > 0) { - if (!lines[0].includes("line-too-long") && lines.some((line) => line.length > 120)) { - pylintDisables.push("line-too-long", "useless-suppression"); - } - if (!lines[0].includes("too-many-lines") && lines.length > 1000) { - pylintDisables.push("too-many-lines"); - } - if (pylintDisables.length > 0) { - fileContent = lines[0].includes("pylint: disable=") - ? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding) - : `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent; - await fs.promises.writeFile(filePath, fileContent); - } - } - }; - - const collectPythonFiles = async (dir: string): Promise => { - if (shouldExcludePath(dir)) { - return []; - } - - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - - const promises = entries.map(async (entry) => { - const filePath = path.join(dir, entry.name); - - if (shouldExcludePath(filePath)) { - return []; - } - - if (entry.isDirectory()) { - return collectPythonFiles(filePath); - } else if (entry.name.endsWith(".py")) { - return [filePath]; - } - return []; - }); - - const results = await Promise.all(promises); - return results.flat(); - }; - - // Collect all Python files first, then process in parallel - const pythonFiles = await collectPythonFiles(outputDir); - await Promise.all(pythonFiles.map(processFile)); -} diff --git a/packages/http-client-python/emitter/test/post-processing.test.ts b/packages/http-client-python/emitter/test/post-processing.test.ts new file mode 100644 index 00000000000..db8b3c97179 --- /dev/null +++ b/packages/http-client-python/emitter/test/post-processing.test.ts @@ -0,0 +1,141 @@ +import fs from "fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { basename, join, sep } from "path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EmitterTester } from "./test-host.js"; + +const { execSync, saveCodeModelAsYaml, loadPyodide } = vi.hoisted(() => ({ + execSync: vi.fn(), + saveCodeModelAsYaml: vi.fn(), + loadPyodide: vi.fn(), +})); + +vi.mock("child_process", () => ({ execSync })); +vi.mock("../src/external-process.js", () => ({ saveCodeModelAsYaml })); +vi.mock("pyodide", () => ({ loadPyodide })); + +describe("typespec-python: post-processing", () => { + let outputDir: string; + let yamlPath: string; + const userFiles = ["tests/custom.py", "samples/custom.py", "custom.py"]; + const content = Buffer.from( + "\uFEFFvalue={'key':1}\r\n# " + "x".repeat(121) + "\r\n" + "# filler\r\n".repeat(1000), + ); + + beforeEach(async () => { + outputDir = await mkdtemp(join(tmpdir(), "python post-processing ")); + yamlPath = join(outputDir, "code-model.yaml"); + saveCodeModelAsYaml.mockResolvedValue(yamlPath); + for (const file of userFiles) { + const filePath = join(outputDir, file); + await mkdir(join(filePath, ".."), { recursive: true }); + await writeFile(filePath, content); + } + }); + + afterEach(async () => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + await rm(outputDir, { recursive: true, force: true }); + }); + + async function emit(usePyodide: boolean) { + const [, diagnostics] = await EmitterTester.compileAndDiagnose( + "@service namespace Test; op ping(): void;", + { + compilerOptions: { + options: { + "@typespec/http-client-python": { + "emitter-output-dir": outputDir, + "use-pyodide": usePyodide, + }, + }, + }, + }, + ); + expect(diagnostics).toEqual([]); + } + + it("does not scan output files after native Python generation", async () => { + const existsSync = fs.existsSync.bind(fs); + vi.spyOn(fs, "existsSync").mockImplementation( + (file) => + String(file).endsWith(`${sep}venv`) || + String(file).endsWith(`${sep}venv${sep}Scripts`) || + existsSync(file), + ); + + // The native pipeline wrote no files. Post-processing must not fall back to a directory scan. + await emit(false); + + expect(execSync).toHaveBeenCalledTimes(1); + expect(execSync.mock.calls[0][0]).toContain("run_tsp.py"); + expect(execSync.mock.calls[0][0]).not.toContain("-m black"); + for (const file of userFiles) { + expect(await readFile(join(outputDir, file))).toEqual(content); + } + }); + + it("uses the shared scoped generation pipeline in Node Pyodide", async () => { + const toPy = vi.fn((globals) => globals); + const runPythonAsync = vi.fn(); + loadPyodide.mockResolvedValue({ + FS: { + mkdirTree: vi.fn(), + mount: vi.fn(), + filesystems: { NODEFS: {} }, + }, + toPy, + runPythonAsync, + loadPackage: vi.fn(), + pyimport: vi.fn(() => ({ install: vi.fn() })), + }); + + await emit(true); + + expect(execSync).not.toHaveBeenCalled(); + expect(toPy).toHaveBeenCalledWith({ + outputFolder: "/output", + yamlFile: `/yaml/${basename(yamlPath)}`, + commandArgs: expect.any(Object), + }); + expect(runPythonAsync).toHaveBeenCalledTimes(1); + expect(runPythonAsync.mock.calls[0][0]).toContain("from pygen.generate import generate"); + expect(runPythonAsync.mock.calls[0][0]).toContain( + "generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs)", + ); + }); + + it("uses the same scoped generation pipeline in browser Pyodide", async () => { + const toPy = vi.fn((globals) => globals); + const runPythonAsync = vi.fn(); + loadPyodide.mockResolvedValue({ + FS: { + mkdirTree: vi.fn(), + readdir: vi.fn(() => [".", ".."]), + writeFile: vi.fn(), + }, + toPy, + runPythonAsync, + loadPackage: vi.fn(), + pyimport: vi.fn(() => ({ install: vi.fn() })), + }); + vi.stubGlobal("window", globalThis); + + await emit(true); + + expect(execSync).not.toHaveBeenCalled(); + expect(toPy).toHaveBeenCalledWith({ + outputFolder: "/output", + yamlFile: "/yaml/python-yaml-path.yaml", + commandArgs: expect.any(Object), + }); + expect(runPythonAsync).toHaveBeenCalledTimes(1); + expect(runPythonAsync.mock.calls[0][0]).toContain("from pygen.generate import generate"); + expect(runPythonAsync.mock.calls[0][0]).toContain( + "generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs)", + ); + }); +}); diff --git a/packages/http-client-python/eng/scripts/setup/run_batch.py b/packages/http-client-python/eng/scripts/setup/run_batch.py index 9b978ec0319..fcf6b7e5f50 100644 --- a/packages/http-client-python/eng/scripts/setup/run_batch.py +++ b/packages/http-client-python/eng/scripts/setup/run_batch.py @@ -15,16 +15,17 @@ from pathlib import Path from concurrent.futures import ProcessPoolExecutor, as_completed from multiprocessing import freeze_support +from typing import Any # Add the generator to the path _ROOT_DIR = Path(__file__).parent.parent.parent.parent sys.path.insert(0, str(_ROOT_DIR / "generator")) -def process_single_spec(config_path_str: str) -> tuple[str, bool, str]: +def process_single_spec(config_path_str: str) -> tuple[str, bool, str, list[Path]]: """Process a single spec from its config file. - Returns: (output_dir, success, error_message) + Returns: (output_dir, success, error_message, written_files) """ # Import inside function for multiprocessing compatibility from pygen import preprocess, codegen @@ -44,26 +45,27 @@ def process_single_spec(config_path_str: str) -> tuple[str, bool, str]: # of pygen.utils.parse_args (the CLI path). Without this, the emitter passes string # "false" for flags like keep-setup-py, which is truthy in Python and causes pygen to # take the wrong branch (e.g. generating setup.py instead of pyproject.toml). - def _coerce(value): + def _coerce(value: Any) -> Any: if value == "true": return True if value == "false": return False return value - pygen_args = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]} + pygen_args: dict[str, Any] = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]} # Run preprocess and codegen (black is batched at the end for performance) preprocess.PreProcessPlugin(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process() - codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process() + generator = codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args) + generator.process() # Clean up the config file config_path.unlink() - return (output_dir, True, "") + return (output_dir, True, "", sorted(generator.written_files)) except Exception as e: - return (str(config_path), False, str(e)) + return (str(config_path), False, str(e), []) def render_progress_bar(completed: int, failed: int, total: int, width: int = 40) -> str: @@ -152,10 +154,10 @@ def update_progress(): futures = {executor.submit(process_single_spec, cf): cf for cf in config_files} for future in as_completed(futures): - output_dir, success, error = future.result() + output_dir, success, error, written_files = future.result() if success: succeeded += 1 - output_dirs.append(output_dir) + output_dirs.append((output_dir, written_files)) else: failed += 1 failed_specs.append(f"{output_dir}: {error}") @@ -188,8 +190,8 @@ def update_progress(): from pygen.black import BlackScriptPlugin print(f"Formatting {len(output_dirs)} packages with black...") - for d in output_dirs: - BlackScriptPlugin(output_folder=d).process() + for output_dir, written_files in output_dirs: + BlackScriptPlugin(output_folder=output_dir, files=written_files).process() if __name__ == "__main__": diff --git a/packages/http-client-python/eng/scripts/setup/run_tsp.py b/packages/http-client-python/eng/scripts/setup/run_tsp.py index 6d5d3247042..d2d347dd4f8 100644 --- a/packages/http-client-python/eng/scripts/setup/run_tsp.py +++ b/packages/http-client-python/eng/scripts/setup/run_tsp.py @@ -6,7 +6,7 @@ import sys import logging from pathlib import Path -from pygen import preprocess, codegen +from pygen.generate import generate from pygen.utils import parse_args # eng/scripts/setup/run_tsp.py -> need to go up 4 levels to get to package root @@ -14,7 +14,8 @@ _LOGGER = logging.getLogger(__name__) -if __name__ == "__main__": + +def main() -> None: venv_path = _ROOT_DIR / "venv" venv_preexists = venv_path.exists() @@ -34,7 +35,9 @@ debugpy.wait_for_client() breakpoint() # pylint: disable=undefined-variable - # pre-process args, unknown_args = parse_args() - preprocess.PreProcessPlugin(output_folder=args.output_folder, tsp_file=args.tsp_file, **unknown_args).process() - codegen.CodeGenerator(output_folder=args.output_folder, tsp_file=args.tsp_file, **unknown_args).process() + generate(output_folder=args.output_folder, tsp_file=args.tsp_file, **unknown_args) + + +if __name__ == "__main__": + main() diff --git a/packages/http-client-python/generator/pygen/__init__.py b/packages/http-client-python/generator/pygen/__init__.py index 5b77d715d8d..c407fa35079 100644 --- a/packages/http-client-python/generator/pygen/__init__.py +++ b/packages/http-client-python/generator/pygen/__init__.py @@ -228,8 +228,11 @@ def items(self) -> ItemsView[str, Any]: class ReaderAndWriter: - def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: + def __init__( + self, *, output_folder: Union[str, Path], written_files: Optional[set[Path]] = None, **kwargs: Any + ) -> None: self.output_folder = Path(output_folder) + self.written_files = written_files if written_files is not None else set() self._list_file: list[str] = [] try: with open( @@ -261,6 +264,7 @@ def write_file(self, filename: Union[str, Path], file_content: str) -> None: Path.mkdir(self.output_folder / file_folder, parents=True, exist_ok=True) with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: fd.write(file_content) + self.written_files.add((self.output_folder / Path(filename)).resolve()) def remove_file(self, filename: Union[str, Path]) -> None: try: diff --git a/packages/http-client-python/generator/pygen/black.py b/packages/http-client-python/generator/pygen/black.py index c8dd4d5bd65..50e1ba89c96 100644 --- a/packages/http-client-python/generator/pygen/black.py +++ b/packages/http-client-python/generator/pygen/black.py @@ -3,14 +3,16 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import argparse import logging +from collections.abc import Iterable from pathlib import Path import os +from typing import Any, Union import black from black.report import NothingChanged from . import Plugin -from .utils import parse_args _LOGGER = logging.getLogger("blib2to3") @@ -19,8 +21,9 @@ class BlackScriptPlugin(Plugin): - def __init__(self, **kwargs): + def __init__(self, *, files: Iterable[Union[str, Path]], **kwargs: Any): super().__init__(**kwargs) + self._files = tuple(Path(file) for file in files) output_folder = self.options.get("output-folder", str(self.output_folder)) if output_folder.startswith("file:"): output_folder = output_folder[5:] @@ -29,31 +32,15 @@ def __init__(self, **kwargs): self.output_folder = Path(output_folder) def process(self) -> bool: - # apply format_file on every .py file in the output folder - list( - map( - self.format_file, - [ - Path(f) - for f in self.list_file() - if Path(f).parts[0] - not in ( - "__pycache__", - "node_modules", - "venv", - "env", - ) - # we shall also format generated files like "../../../generated_tests/test_xxx.py" - and (not Path(f).parts[0].startswith(".") or Path(f).parts[0] == "..") and Path(f).suffix == ".py" - ], - ) - ) + for file in sorted(set(self._files)): + if file.suffix == ".py": + self.format_file(file) return True def format_file(self, file: Path) -> None: file_content = "" try: - file_content = self.read_file(file) + file_content = (self.output_folder / file).read_text(encoding="utf-8-sig") file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE) except NothingChanged: pass @@ -69,14 +56,16 @@ def format_file(self, file: Path) -> None: pylint_disables.append("too-many-lines") if pylint_disables: file_content = ( - os.linesep.join([lines[0] + ",".join([""] + pylint_disables)] + lines[1:]) + file_content.replace(lines[0], lines[0] + "," + ",".join(pylint_disables), 1) if "pylint: disable=" in lines[0] - else f"# pylint: disable={','.join(pylint_disables)}{os.linesep}" + file_content + else f"# pylint: disable={','.join(pylint_disables)}\n" + file_content ) self.write_file(file, file_content) if __name__ == "__main__": - # TSP pipeline will call this - args, unknown_args = parse_args(need_tsp_file=False) - BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process() + parser = argparse.ArgumentParser(description="Format explicitly listed generated Python files") + parser.add_argument("--output-folder", required=True) + parser.add_argument("files", nargs="+", type=Path) + args = parser.parse_args() + BlackScriptPlugin(output_folder=args.output_folder, files=args.files).process() diff --git a/packages/http-client-python/generator/pygen/codegen/__init__.py b/packages/http-client-python/generator/pygen/codegen/__init__.py index 74e3031729c..c0fdadd0716 100644 --- a/packages/http-client-python/generator/pygen/codegen/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/__init__.py @@ -65,9 +65,10 @@ def get_yaml(self) -> dict[str, Any]: return yaml.safe_load(fd.read()) def get_serializer(self, code_model: CodeModel): - return JinjaSerializer(code_model, output_folder=self.output_folder) + return JinjaSerializer(code_model, output_folder=self.output_folder, written_files=self.written_files) def process(self) -> bool: + self.written_files.clear() # List the input file, should be only one yaml_data = self.get_yaml() diff --git a/packages/http-client-python/generator/pygen/generate.py b/packages/http-client-python/generator/pygen/generate.py new file mode 100644 index 00000000000..bfe53bddcd4 --- /dev/null +++ b/packages/http-client-python/generator/pygen/generate.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from pathlib import Path +from typing import Any, Union + +from . import codegen, preprocess +from .black import BlackScriptPlugin + + +def generate(*, output_folder: Union[str, Path], tsp_file: str, **kwargs: Any) -> None: + preprocess.PreProcessPlugin(output_folder=output_folder, tsp_file=tsp_file, **kwargs).process() + generator = codegen.CodeGenerator(output_folder=output_folder, tsp_file=tsp_file, **kwargs) + generator.process() + BlackScriptPlugin(output_folder=output_folder, files=generator.written_files, **kwargs).process() diff --git a/packages/http-client-python/tests/unit/test_post_processing.py b/packages/http-client-python/tests/unit/test_post_processing.py new file mode 100644 index 00000000000..6b5704c1e5b --- /dev/null +++ b/packages/http-client-python/tests/unit/test_post_processing.py @@ -0,0 +1,463 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""Post-processing must only touch files written by the current generation.""" + +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import black +import pytest +import yaml + +from pygen import ReaderAndWriter +from pygen.black import BlackScriptPlugin +from pygen.codegen import CodeGenerator + +_PACKAGE_ROOT = Path(__file__).resolve().parents[2] +_UNFORMATTED = "value={'key':1}\n# " + "x" * 121 + "\n" + "# filler\n" * 1000 +_USER_FILES = ( + "tests/test_custom.py", + "samples/custom.py", + "custom.py", + "contoso/widget/custom.py", + "generated_tests/test_custom.py", + "generated_samples/custom.py", +) + + +def _seed_user_files(output_folder): + contents = {} + for name in _USER_FILES: + path = output_folder / name + path.parent.mkdir(parents=True, exist_ok=True) + content = b"\xef\xbb\xbf" + _UNFORMATTED.replace("\n", "\r\n").encode() + path.write_bytes(content) + contents[path] = content + return contents + + +def _assert_user_files_unchanged(contents): + for path, content in contents.items(): + assert path.read_bytes() == content, path + + +def _assert_formatted(path): + content = path.read_text(encoding="utf-8") + assert 'value = {"key": 1}' in content + assert content.splitlines()[0] == "# pylint: disable=line-too-long,useless-suppression,too-many-lines" + + +def _load_setup_script(name): + spec = importlib.util.spec_from_file_location(name, _PACKAGE_ROOT / "eng/scripts/setup" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _write_code_model(path, has_operations=False): + operation_groups = [] + if has_operations: + operation_groups.append( + { + "className": "WidgetOperations", + "identifyName": "widgets", + "propertyName": "widgets", + "operations": [ + { + "name": "ping", + "description": "Ping.", + "groupName": "widgets", + "url": "/ping", + "method": "GET", + "parameters": [], + "responses": [{"statusCodes": [204], "headers": []}], + "exceptions": [], + "overloads": [], + "isOverload": False, + "discriminator": "operation", + "apiVersions": [], + "samples": { + "ping": { + "x-ms-original-file": "ping.json", + "parameters": {}, + "responses": {"204": {}}, + } + }, + } + ], + } + ) + path.write_text( + yaml.safe_dump( + { + "namespace": "contoso.widget", + "types": [], + "clients": [ + { + "name": "WidgetClient", + "description": "A widget client.", + "namespace": "contoso.widget", + "moduleName": "widget", + "parameters": [], + "url": "https://example.org", + "operationGroups": operation_groups, + } + ], + } + ), + encoding="utf-8", + ) + + +def test_only_explicit_python_files_are_post_processed(tmp_path): + contents = _seed_user_files(tmp_path) + generated = tmp_path / "contoso/widget/_generated/client.py" + regenerated = tmp_path / "setup.py" + outside = tmp_path.parent / f"{tmp_path.name}_generated_sample.py" + non_python = tmp_path / "README.md" + files = [generated, regenerated, outside, non_python] + for path in files: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_UNFORMATTED if path.suffix == ".py" else "Not Python!", encoding="utf-8") + + try: + BlackScriptPlugin(output_folder=tmp_path, files=files).process() + + _assert_user_files_unchanged(contents) + for path in files[:-1]: + _assert_formatted(path) + assert non_python.read_text(encoding="utf-8") == "Not Python!" + finally: + outside.unlink() + + +@pytest.mark.parametrize("files", [[], ()]) +def test_empty_file_set_does_not_scan_output(tmp_path, files): + contents = _seed_user_files(tmp_path) + + BlackScriptPlugin(output_folder=tmp_path, files=files).process() + + _assert_user_files_unchanged(contents) + + +def test_missing_file_list_is_not_a_directory_scan(tmp_path): + with pytest.raises(TypeError): + BlackScriptPlugin(output_folder=tmp_path) + + +def test_missing_emitted_file_is_reported(tmp_path, caplog): + path = tmp_path / "missing.py" + with pytest.raises(FileNotFoundError): + BlackScriptPlugin(output_folder=tmp_path, files=[path]).process() + assert "failed to format" in caplog.text + assert not path.exists() + + +@pytest.mark.parametrize("include_files", [False, True]) +def test_black_cli_requires_explicit_files(tmp_path, include_files): + contents = _seed_user_files(tmp_path) + path = tmp_path / "emitted.py" + path.write_text(_UNFORMATTED, encoding="utf-8") + command = [sys.executable, "-m", "pygen.black", "--output-folder", str(tmp_path)] + if include_files: + command.append(str(path)) + + result = subprocess.run(command, capture_output=True, text=True, check=False, timeout=30) + + assert result.returncode == (0 if include_files else 2), result.stderr + _assert_user_files_unchanged(contents) + if include_files: + _assert_formatted(path) + else: + assert "files" in result.stderr + assert path.read_text(encoding="utf-8") == _UNFORMATTED + + +@pytest.mark.parametrize("line_length", [120, 121]) +@pytest.mark.parametrize("line_count", [1000, 1001]) +def test_pylint_suppression_thresholds(tmp_path, line_length, line_count): + path = tmp_path / "generated.py" + path.write_text("#" * line_length + "\n" + "# filler\n" * (line_count - 1), encoding="utf-8") + + BlackScriptPlugin(output_folder=tmp_path, files=[path]).process() + + content = path.read_text(encoding="utf-8") + assert ("line-too-long" in content) == (line_length > 120) + assert ("useless-suppression" in content) == (line_length > 120) + assert ("too-many-lines" in content) == (line_count > 1000) + + +def test_existing_suppressions_are_not_duplicated(tmp_path): + path = tmp_path / "generated.py" + path.write_text("# pylint: disable=unused-import\n" + _UNFORMATTED, encoding="utf-8") + formatter = BlackScriptPlugin(output_folder=tmp_path, files=[path]) + formatter.process() + content = path.read_bytes() + + formatter.process() + + assert path.read_bytes() == content + assert path.read_text(encoding="utf-8").splitlines()[0] == ( + "# pylint: disable=unused-import,line-too-long,useless-suppression,too-many-lines" + ) + + +def test_writer_tracks_successful_new_and_existing_writes(tmp_path): + output_folder = tmp_path / "output" + writer = ReaderAndWriter(output_folder=output_folder) + path = output_folder / "sdk/client.py" + writer.write_file("sdk/client.py", "value=1\n") + writer.write_file(path, "value=2\n") + writer.write_file(Path("sdk") / ".." / "sdk" / "client.py", "value=3\n") + writer.write_file("../setup.py", "value=4\n") + + assert writer.written_files == {path.resolve(), (tmp_path / "setup.py").resolve()} + assert ReaderAndWriter(output_folder=output_folder).written_files == set() + + with pytest.raises(OSError): + writer.write_file("sdk", "cannot write a directory") + assert (output_folder / "sdk").resolve() not in writer.written_files + + +@pytest.mark.parametrize("generation_subdir", [None, "_generated"]) +@pytest.mark.parametrize("no_namespace_folders", [False, True]) +def test_generator_tracks_all_serializer_writes_and_resets_on_rerun( + tmp_path, monkeypatch, generation_subdir, no_namespace_folders +): + output_folder = tmp_path / "contoso/widget" if no_namespace_folders else tmp_path + output_folder.mkdir(parents=True, exist_ok=True) + contents = _seed_user_files(output_folder) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path) + generator = CodeGenerator( + output_folder=output_folder, + tsp_file=str(yaml_path), + **{ + "flavor": "unbranded", + "from-typespec": True, + "generation-subdir": generation_subdir, + "no-namespace-folders": no_namespace_folders, + "package-mode": "azure-dataplane", + "keep-setup-py": True, + "package-name": "contoso-widget", + "package-version": "1.0.0", + }, + ) + generator.process() + + assert generator.written_files + assert (tmp_path / "setup.py").resolve() in generator.written_files + assert not set(contents).intersection(generator.written_files) + assert all(path.is_absolute() and path.is_file() for path in generator.written_files) + BlackScriptPlugin(output_folder=output_folder, files=generator.written_files).process() + _assert_user_files_unchanged(contents) + for path in generator.written_files: + if path.suffix == ".py": + content = path.read_text(encoding="utf-8") + assert black.format_str(content, mode=black.Mode(line_length=120)) == content + + monkeypatch.setattr("pygen.codegen.JinjaSerializer.serialize", lambda self: None) + generator.process() + assert generator.written_files == set() + + +def test_shared_native_and_pyodide_pipeline_preserves_user_files(tmp_path): + from pygen.generate import generate + + contents = _seed_user_files(tmp_path) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path) + templates = tmp_path / "templates" + templates.mkdir() + (templates / "setup.py.jinja2").write_text(_UNFORMATTED, encoding="utf-8") + options = { + "from-typespec": True, + "package-mode": str(templates), + "keep-setup-py": True, + "package-version": "1.0.0", + } + + generate(output_folder=tmp_path, tsp_file=str(yaml_path), flavor="unbranded", **options) + + _assert_user_files_unchanged(contents) + assert (tmp_path / "contoso/widget/__init__.py").is_file() + _assert_formatted(tmp_path / "setup.py") + (tmp_path / "setup.py").write_text("old=True\n", encoding="utf-8") + + generate(output_folder=tmp_path, tsp_file=str(yaml_path), flavor="unbranded", **options) + + _assert_formatted(tmp_path / "setup.py") + _assert_user_files_unchanged(contents) + + +@pytest.mark.parametrize("generate_test", [False, True]) +@pytest.mark.parametrize("generate_sample", [False, True]) +@pytest.mark.parametrize("no_namespace_folders", [False, True]) +def test_generated_tests_and_samples_are_formatted_only_when_written( + tmp_path, caplog, generate_test, generate_sample, no_namespace_folders +): + from pygen.generate import generate + + output_folder = tmp_path / "contoso/widget" if no_namespace_folders else tmp_path + output_folder.mkdir(parents=True, exist_ok=True) + contents = _seed_user_files(tmp_path) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path, has_operations=True) + + generate( + output_folder=output_folder, + tsp_file=str(yaml_path), + flavor="unbranded", + **{ + "from-typespec": True, + "generate-test": generate_test, + "generate-sample": generate_sample, + "generation-subdir": "_generated", + "no-namespace-folders": no_namespace_folders, + }, + ) + + assert not [record for record in caplog.records if record.levelname == "ERROR"] + _assert_user_files_unchanged(contents) + for folder, enabled in (("generated_tests", generate_test), ("generated_samples", generate_sample)): + generated_files = set((tmp_path / folder).rglob("*.py")) - set(contents) + assert bool(generated_files) == enabled + for path in generated_files: + content = path.read_text(encoding="utf-8") + assert black.format_str(content, mode=black.Mode(line_length=120)) == content + + +@pytest.mark.parametrize("has_operations,show_operations", [(False, True), (True, False)]) +def test_tests_and_samples_require_visible_operations(tmp_path, has_operations, show_operations): + from pygen.generate import generate + + contents = _seed_user_files(tmp_path) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path, has_operations=has_operations) + + generate( + output_folder=tmp_path, + tsp_file=str(yaml_path), + flavor="unbranded", + **{ + "from-typespec": True, + "generate-test": True, + "generate-sample": True, + "show-operations": show_operations, + "combine-operation-files": show_operations, + "builders-visibility": "public", + }, + ) + + _assert_user_files_unchanged(contents) + for folder in ("generated_tests", "generated_samples"): + assert not set((tmp_path / folder).rglob("*.py")) - set(contents) + + +def test_native_entry_point_preserves_user_files(tmp_path, monkeypatch): + run_tsp = _load_setup_script("run_tsp") + monkeypatch.setattr(run_tsp, "_ROOT_DIR", tmp_path) + (tmp_path / "venv").mkdir() + output_folder = tmp_path / "output" + contents = _seed_user_files(output_folder) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path) + monkeypatch.setattr( + sys, + "argv", + [ + "run_tsp.py", + f"--output-folder={output_folder}", + f"--tsp-file={yaml_path}", + "--flavor=unbranded", + "--from-typespec=true", + ], + ) + + run_tsp.main() + + _assert_user_files_unchanged(contents) + assert (output_folder / "contoso/widget/__init__.py").is_file() + + +def test_batch_worker_returns_only_files_written_this_run(tmp_path): + run_batch = _load_setup_script("run_batch") + contents = _seed_user_files(tmp_path) + yaml_path = tmp_path / "code-model.yaml" + _write_code_model(yaml_path) + config_path = tmp_path / ".tsp-codegen-test.json" + config_path.write_text( + json.dumps( + { + "yamlPath": str(yaml_path), + "outputDir": str(tmp_path), + "commandArgs": {"flavor": "unbranded", "from-typespec": "true"}, + } + ), + encoding="utf-8", + ) + + output_dir, success, error, files = run_batch.process_single_spec(str(config_path)) + + assert success, error + assert output_dir == str(tmp_path) + assert files and not set(contents).intersection(files) + BlackScriptPlugin(output_folder=output_dir, files=files).process() + _assert_user_files_unchanged(contents) + assert not config_path.exists() + + +def test_batch_parent_formats_only_worker_writes(tmp_path): + contents = {} + for name in ("first", "second"): + output_folder = tmp_path / "tests/generated/unbranded" / name + contents.update(_seed_user_files(output_folder)) + yaml_path = tmp_path / f"{name}.yaml" + _write_code_model(yaml_path) + templates = output_folder / "templates" + templates.mkdir() + (templates / "setup.py.jinja2").write_text(_UNFORMATTED, encoding="utf-8") + (output_folder / ".tsp-codegen-test.json").write_text( + json.dumps( + { + "yamlPath": str(yaml_path), + "outputDir": str(output_folder), + "commandArgs": { + "flavor": "unbranded", + "from-typespec": "true", + "package-mode": str(templates), + "keep-setup-py": "true", + "package-version": "1.0.0", + }, + } + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(_PACKAGE_ROOT / "eng/scripts/setup/run_batch.py"), + "--generated-dir", + str(tmp_path), + "--flavor", + "unbranded", + "--jobs", + "2", + ], + capture_output=True, + text=True, + check=False, + timeout=120, + ) + + assert result.returncode == 0, result.stdout + result.stderr + _assert_user_files_unchanged(contents) + for name in ("first", "second"): + _assert_formatted(tmp_path / "tests/generated/unbranded" / name / "setup.py")