Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .chronus/changes/python-post-processing-scope.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
changeKind: fix
packages:
- "@typespec/http-client-python"
---

Limit Black formatting and pylint suppression insertion to Python files written during the current generation, leaving pre-existing user-managed tests, samples, and other Python files unchanged.
16 changes: 16 additions & 0 deletions packages/http-client-python/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,22 @@ def {{ operation.name }}(self, {{ operation.parameters }}):
{% endfor %}
```

#### Post-processing

Native Python and Pyodide use the shared `pygen.generate.generate` pipeline. The
`ReaderAndWriter.write_file` method records successful writes as normalized absolute paths.
`CodeGenerator` shares its per-run `written_files` set with `JinjaSerializer` and resets it on
each generation. `BlackScriptPlugin` receives those paths explicitly and formats only `.py`
files, adding pylint suppression comments where needed without invoking pylint.

Batch workers return their recorded paths to the parent process for post-processing after
generation. Formatting never scans the output directory, including when the recorded set is
empty. Absolute paths also cover generated packaging files, tests, and samples outside the
immediate generation directory.

The standalone `python -m pygen.black --output-folder <folder> <files...>` command
also requires an explicit file list.

## Code Generation Flow

### Detailed Step-by-Step Process
Expand Down
5 changes: 5 additions & 0 deletions packages/http-client-python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ options:
option: value
```

After generation, Black formatting and pylint suppression comments are applied only to Python
files written during that generation, including regenerated files and generated tests and samples
when enabled. Other existing files, such as hand-written `tests/` and `samples/`, are not
post-processed. This step does not run pylint.

## Emitter options

### `emitter-output-dir`
Expand Down
23 changes: 0 additions & 23 deletions packages/http-client-python/emitter/src/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,3 @@ export const PYODIDE_VERSION = "0.26.2";
export const PYGEN_WHEEL_FILENAME = "pygen-0.1.0-py3-none-any.whl";
export const BLOB_STORAGE_BASE_URL = "https://typespec.blob.core.windows.net/pkgs";
export const PACKAGE_NAME = "@typespec/http-client-python";

export const blackExcludeDirs = [
"__pycache__/",
"node_modules/",
"venv/",
"env/",
".direnv",
".eggs",
".git",
".hg",
".tox",
".venv",
".eggs",
".mypy_cache",
".pytest_cache",
".vscode",
".*_build/",
"/build/",
"dist",
".nox",
".svn",
"TempTypeSpecFiles/",
];
6 changes: 2 additions & 4 deletions packages/http-client-python/emitter/src/emitter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,8 @@ const pyodideGenerationCode = `
async def main():
import warnings
with warnings.catch_warnings():
from pygen import preprocess, codegen, black
preprocess.PreProcessPlugin(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process()
codegen.CodeGenerator(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs).process()
black.BlackScriptPlugin(output_folder=outputFolder, **commandArgs).process()
from pygen.generate import generate
generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs)

await main()`;

Expand Down
72 changes: 2 additions & 70 deletions packages/http-client-python/emitter/src/node-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,11 @@ import type { EmitContext } from "@typespec/compiler";
import { NoTarget } from "@typespec/compiler";
import { execSync } from "child_process";
import fs from "fs";
import os from "os";
import path, { dirname } from "path";
import type { PyodideInterface } from "pyodide";
import { loadPyodide } from "pyodide";
import { fileURLToPath } from "url";
import { blackExcludeDirs, PYGEN_WHEEL_FILENAME } from "./constants.js";
import { PYGEN_WHEEL_FILENAME } from "./constants.js";
import { saveCodeModelAsYaml } from "./external-process.js";
import type { PythonEmitterOptions } from "./lib.js";
import { reportDiagnostic } from "./lib.js";
Expand Down Expand Up @@ -112,14 +111,8 @@ export async function runNodeEmit({
const commandFlags = Object.entries(commandArgs)
.map(([key, value]) => `--${key}=${quoteShellArg(String(value))}`)
.join(" ");
const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`;
const command = `${quoteShellArg(venvPath)} ${quoteShellArg(path.join(root, "eng", "scripts", "setup", "run_tsp.py"))} ${commandFlags}`;
execSync(command);

const excludePattern = blackExcludeDirs.join("|");
execSync(
`${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`,
);
await checkForPylintIssues(outputDir, excludePattern);
}

async function setupPyodideCall(root: string): Promise<PyodideInterface> {
Expand Down Expand Up @@ -161,64 +154,3 @@ async function setupPyodideCall(root: string): Promise<PyodideInterface> {
}
return pyodide;
}

async function checkForPylintIssues(outputDir: string, excludePattern: string) {
const excludeRegex = new RegExp(excludePattern);

const shouldExcludePath = (filePath: string): boolean => {
const relativePath = path.relative(outputDir, filePath);
const normalizedPath = relativePath.replace(/\\/g, "/");
return excludeRegex.test(normalizedPath);
};

const processFile = async (filePath: string) => {
let fileContent = await fs.promises.readFile(filePath, "utf-8");
const pylintDisables: string[] = [];
const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n";
const lines: string[] = fileContent.split(lineEnding);
if (lines.length > 0) {
if (!lines[0].includes("line-too-long") && lines.some((line) => line.length > 120)) {
pylintDisables.push("line-too-long", "useless-suppression");
}
if (!lines[0].includes("too-many-lines") && lines.length > 1000) {
pylintDisables.push("too-many-lines");
}
if (pylintDisables.length > 0) {
fileContent = lines[0].includes("pylint: disable=")
? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding)
: `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent;
await fs.promises.writeFile(filePath, fileContent);
}
}
};

const collectPythonFiles = async (dir: string): Promise<string[]> => {
if (shouldExcludePath(dir)) {
return [];
}

const entries = await fs.promises.readdir(dir, { withFileTypes: true });

const promises = entries.map(async (entry) => {
const filePath = path.join(dir, entry.name);

if (shouldExcludePath(filePath)) {
return [];
}

if (entry.isDirectory()) {
return collectPythonFiles(filePath);
} else if (entry.name.endsWith(".py")) {
return [filePath];
}
return [];
});

const results = await Promise.all(promises);
return results.flat();
};

// Collect all Python files first, then process in parallel
const pythonFiles = await collectPythonFiles(outputDir);
await Promise.all(pythonFiles.map(processFile));
}
141 changes: 141 additions & 0 deletions packages/http-client-python/emitter/test/post-processing.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import fs from "fs";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "fs/promises";
import { tmpdir } from "os";
import { basename, join, sep } from "path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { EmitterTester } from "./test-host.js";

const { execSync, saveCodeModelAsYaml, loadPyodide } = vi.hoisted(() => ({
execSync: vi.fn(),
saveCodeModelAsYaml: vi.fn(),
loadPyodide: vi.fn(),
}));

vi.mock("child_process", () => ({ execSync }));
vi.mock("../src/external-process.js", () => ({ saveCodeModelAsYaml }));
vi.mock("pyodide", () => ({ loadPyodide }));

describe("typespec-python: post-processing", () => {
let outputDir: string;
let yamlPath: string;
const userFiles = ["tests/custom.py", "samples/custom.py", "custom.py"];
const content = Buffer.from(
"\uFEFFvalue={'key':1}\r\n# " + "x".repeat(121) + "\r\n" + "# filler\r\n".repeat(1000),
);

beforeEach(async () => {
outputDir = await mkdtemp(join(tmpdir(), "python post-processing "));
yamlPath = join(outputDir, "code-model.yaml");
saveCodeModelAsYaml.mockResolvedValue(yamlPath);
for (const file of userFiles) {
const filePath = join(outputDir, file);
await mkdir(join(filePath, ".."), { recursive: true });
await writeFile(filePath, content);
}
});

afterEach(async () => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
vi.clearAllMocks();
await rm(outputDir, { recursive: true, force: true });
});

async function emit(usePyodide: boolean) {
const [, diagnostics] = await EmitterTester.compileAndDiagnose(
"@service namespace Test; op ping(): void;",
{
compilerOptions: {
options: {
"@typespec/http-client-python": {
"emitter-output-dir": outputDir,
"use-pyodide": usePyodide,
},
},
},
},
);
expect(diagnostics).toEqual([]);
}

it("does not scan output files after native Python generation", async () => {
const existsSync = fs.existsSync.bind(fs);
vi.spyOn(fs, "existsSync").mockImplementation(
(file) =>
String(file).endsWith(`${sep}venv`) ||
String(file).endsWith(`${sep}venv${sep}Scripts`) ||
existsSync(file),
);

// The native pipeline wrote no files. Post-processing must not fall back to a directory scan.
await emit(false);

expect(execSync).toHaveBeenCalledTimes(1);
expect(execSync.mock.calls[0][0]).toContain("run_tsp.py");
expect(execSync.mock.calls[0][0]).not.toContain("-m black");
for (const file of userFiles) {
expect(await readFile(join(outputDir, file))).toEqual(content);
}
});

it("uses the shared scoped generation pipeline in Node Pyodide", async () => {
const toPy = vi.fn((globals) => globals);
const runPythonAsync = vi.fn();
loadPyodide.mockResolvedValue({
FS: {
mkdirTree: vi.fn(),
mount: vi.fn(),
filesystems: { NODEFS: {} },
},
toPy,
runPythonAsync,
loadPackage: vi.fn(),
pyimport: vi.fn(() => ({ install: vi.fn() })),
});

await emit(true);

expect(execSync).not.toHaveBeenCalled();
expect(toPy).toHaveBeenCalledWith({
outputFolder: "/output",
yamlFile: `/yaml/${basename(yamlPath)}`,
commandArgs: expect.any(Object),
});
expect(runPythonAsync).toHaveBeenCalledTimes(1);
expect(runPythonAsync.mock.calls[0][0]).toContain("from pygen.generate import generate");
expect(runPythonAsync.mock.calls[0][0]).toContain(
"generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs)",
);
});

it("uses the same scoped generation pipeline in browser Pyodide", async () => {
const toPy = vi.fn((globals) => globals);
const runPythonAsync = vi.fn();
loadPyodide.mockResolvedValue({
FS: {
mkdirTree: vi.fn(),
readdir: vi.fn(() => [".", ".."]),
writeFile: vi.fn(),
},
toPy,
runPythonAsync,
loadPackage: vi.fn(),
pyimport: vi.fn(() => ({ install: vi.fn() })),
});
vi.stubGlobal("window", globalThis);

await emit(true);

expect(execSync).not.toHaveBeenCalled();
expect(toPy).toHaveBeenCalledWith({
outputFolder: "/output",
yamlFile: "/yaml/python-yaml-path.yaml",
commandArgs: expect.any(Object),
});
expect(runPythonAsync).toHaveBeenCalledTimes(1);
expect(runPythonAsync.mock.calls[0][0]).toContain("from pygen.generate import generate");
expect(runPythonAsync.mock.calls[0][0]).toContain(
"generate(output_folder=outputFolder, tsp_file=yamlFile, **commandArgs)",
);
});
});
24 changes: 13 additions & 11 deletions packages/http-client-python/eng/scripts/setup/run_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,17 @@
from pathlib import Path
from concurrent.futures import ProcessPoolExecutor, as_completed
from multiprocessing import freeze_support
from typing import Any

# Add the generator to the path
_ROOT_DIR = Path(__file__).parent.parent.parent.parent
sys.path.insert(0, str(_ROOT_DIR / "generator"))


def process_single_spec(config_path_str: str) -> tuple[str, bool, str]:
def process_single_spec(config_path_str: str) -> tuple[str, bool, str, list[Path]]:
"""Process a single spec from its config file.

Returns: (output_dir, success, error_message)
Returns: (output_dir, success, error_message, written_files)
"""
# Import inside function for multiprocessing compatibility
from pygen import preprocess, codegen
Expand All @@ -44,26 +45,27 @@ def process_single_spec(config_path_str: str) -> tuple[str, bool, str]:
# of pygen.utils.parse_args (the CLI path). Without this, the emitter passes string
# "false" for flags like keep-setup-py, which is truthy in Python and causes pygen to
# take the wrong branch (e.g. generating setup.py instead of pyproject.toml).
def _coerce(value):
def _coerce(value: Any) -> Any:
if value == "true":
return True
if value == "false":
return False
return value

pygen_args = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]}
pygen_args: dict[str, Any] = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]}

# Run preprocess and codegen (black is batched at the end for performance)
preprocess.PreProcessPlugin(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process()

codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process()
generator = codegen.CodeGenerator(output_folder=output_dir, tsp_file=yaml_path, **pygen_args)
generator.process()

# Clean up the config file
config_path.unlink()

return (output_dir, True, "")
return (output_dir, True, "", sorted(generator.written_files))
except Exception as e:
return (str(config_path), False, str(e))
return (str(config_path), False, str(e), [])


def render_progress_bar(completed: int, failed: int, total: int, width: int = 40) -> str:
Expand Down Expand Up @@ -152,10 +154,10 @@ def update_progress():
futures = {executor.submit(process_single_spec, cf): cf for cf in config_files}

for future in as_completed(futures):
output_dir, success, error = future.result()
output_dir, success, error, written_files = future.result()
if success:
succeeded += 1
output_dirs.append(output_dir)
output_dirs.append((output_dir, written_files))
else:
failed += 1
failed_specs.append(f"{output_dir}: {error}")
Expand Down Expand Up @@ -188,8 +190,8 @@ def update_progress():
from pygen.black import BlackScriptPlugin

print(f"Formatting {len(output_dirs)} packages with black...")
for d in output_dirs:
BlackScriptPlugin(output_folder=d).process()
for output_dir, written_files in output_dirs:
BlackScriptPlugin(output_folder=output_dir, files=written_files).process()


if __name__ == "__main__":
Expand Down
Loading
Loading