diff --git a/docs/list.md b/docs/list.md index a5c2f14..68cfaf3 100644 --- a/docs/list.md +++ b/docs/list.md @@ -11,6 +11,7 @@ Options: -v, --verbose Verbosity: v=INFO, vv=DEBUG. -q, --quiet Only errors shown in logs. --write Write the events to file. + --json Output as JSON. --help Show this message and exit. ``` @@ -100,3 +101,62 @@ Within Datatrail, there are two types of datasets: Please see the CLI reference page for more information on the `list` command: [datatrail list](../cli/#datatrail-list) + +## 🤖 Machine-readable JSON output + +The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: + +```bash +# Get all scopes as JSON +$ datatrail ls --json +{ + "scopes": [ + "chime.event.baseband.raw", + "chime.event.intensity.raw", + "kko.acquisition.processed", + ... + ] +} + +# Get larger datasets as JSON +$ datatrail ls kko.scheduled.baseband.raw --json +{ + "scope": "kko.scheduled.baseband.raw", + "larger_datasets": [ + "20230804095251", + "scheduled.commissioning.steady" + ] +} + +# Get child datasets as JSON +$ datatrail ls kko.scheduled.baseband.raw scheduled.commissioning.steady --json +{ + "datasets": [ + "20230616150511", + "20230604135840", + "20230604134842", + ... + ] +} +``` + +### Usage in scripts + +```python +import json +import subprocess + +# Get all scopes +result = subprocess.run(["datatrail", "ls", "--json"], capture_output=True, text=True) +data = json.loads(result.stdout) +scopes = data["scopes"] + +# Get datasets for a scope +result = subprocess.run( + ["datatrail", "ls", "chime.event.baseband.raw", "--json"], + capture_output=True, + text=True, +) +data = json.loads(result.stdout) +larger_datasets = data["larger_datasets"] +``` diff --git a/docs/ps.md b/docs/ps.md index 38ecbf5..efc6b68 100644 --- a/docs/ps.md +++ b/docs/ps.md @@ -11,6 +11,7 @@ Options: -s, --show-files Show file names. -v, --verbose Verbosity: v=INFO, vv=DEBUG. -q, --quiet Set log level to ERROR. + --json Output as JSON. --help Show this message and exit. ``` @@ -80,3 +81,72 @@ if the `--show-files` or `-s` flag is passed. │ - baseband_308892599_111.h5 │ : ``` + +## 🤖 Machine-readable JSON output + +The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse dataset information in scripts and pipelines: + +```bash +$ datatrail ps kko.event.baseband.raw 308892599 --json +{ + "dataset": "308892599", + "scope": "kko.event.baseband.raw", + "files": { + "contains_datasets": 0, + "datasets_contained": [], + "file_replica_locations": { + "minoc": [ + "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_129.h5", + "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_1013.h5", + ... + ] + } + }, + "policies": { + "replication_policy": { + "preferred_storage_elements": ["chime"], + "priority": "low", + "default": true + }, + "deletion_policy": [ + { + "storage_element": "minoc", + "priority": "low", + "default": true, + "delete_after_days": 36500 + }, + ... + ], + "belongs_to": [ + { + "scope": "kko.event.baseband.raw", + "name": "B0531+21.commissioning.pulsar.temp" + } + ] + } +} +``` + +### Usage in scripts + +```python +import json +import subprocess + +# Get dataset information +result = subprocess.run( + ["datatrail", "ps", "kko.event.baseband.raw", "308892599", "--json"], + capture_output=True, + text=True, +) +data = json.loads(result.stdout) + +# Access file locations +file_locations = data["files"]["file_replica_locations"] +minoc_files = file_locations.get("minoc", []) + +# Access policies +replication_policy = data["policies"]["replication_policy"] +deletion_policy = data["policies"]["deletion_policy"] +belongs_to = data["policies"]["belongs_to"] +``` diff --git a/dtcli/ls.py b/dtcli/ls.py index 857bc77..d6335da 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -34,14 +34,16 @@ @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.") @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") @click.option("--write", is_flag=True, help="Write the events to file.") +@click.option("--json", "output_json", is_flag=True, help="Output as JSON.") @click.pass_context -def list( +def list( # noqa: C901 ctx: click.Context, scope: Optional[str] = None, datasets: Optional[str] = None, verbose: int = 0, quiet: bool = False, write: bool = False, + output_json: bool = False, ): """List Datatrail Scopes & Datasets. @@ -52,6 +54,7 @@ def list( verbose (int): Verbosity: v=INFO, vv=DEBUG. quiet (bool): Only errors shown in logs. write (bool): Write the events to file. + output_json (bool): Output as JSON. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -73,6 +76,13 @@ def list( return None results = functions.list(scope, datasets, verbose, quiet) + # Output JSON if requested. + if output_json: + print(json.dumps(results, indent=2)) + if "error" in results: + ctx.exit(1) + return + # Display scopes. if "scopes" in results.keys(): table = Table( diff --git a/dtcli/ps.py b/dtcli/ps.py index b67ab85..1d7e225 100644 --- a/dtcli/ps.py +++ b/dtcli/ps.py @@ -26,14 +26,16 @@ @click.option("-s", "--show-files", is_flag=True, help="Show file names.") @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.") @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.") +@click.option("--json", "output_json", is_flag=True, help="Output as JSON.") @click.pass_context -def ps( +def ps( # noqa: C901 ctx: click.Context, scope: str, dataset: str, show_files: bool, verbose: int, quiet: bool, + output_json: bool, ): """Detailed status of a dataset. @@ -44,6 +46,7 @@ def ps( show_files (bool): Show list of files. verbose (int): Verbosity: v=INFO, vv=DUBUG. quiet (bool): Set log level to ERROR. + output_json (bool): Output as JSON. Returns: None @@ -73,13 +76,41 @@ def ps( try: files, policies = functions.ps(scope, dataset, verbose, quiet) if isinstance(files, str) or isinstance(policies, str): + if output_json: + import json + + print( + json.dumps( + {"error": {"files": str(files), "policies": str(policies)}}, + indent=2, + ) + ) + ctx.exit(1) error_console.print("Error: files = ", files) error_console.print("Error: policies = ", policies) return None except Exception as e: + if output_json: + import json + + print(json.dumps({"error": str(e)}, indent=2)) + ctx.exit(1) error_console.print(e) return None + # Handle JSON output + if output_json: + import json + + result = { + "dataset": dataset, + "scope": scope, + "files": files, + "policies": policies, + } + print(json.dumps(result, indent=2)) + return None + if show_files and files: # Files table file_table = create_files_table(dataset, scope, files) diff --git a/tests/test_cli.py b/tests/test_cli.py index aea606b..1074ec6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -97,18 +97,15 @@ def test_cli_list_help(runner: CliRunner) -> None: runner (CliRunner) -> None: Click runner. """ result = runner.invoke(datatrail, ["ls", "--help"]) - expected_response = """Usage: cli list [OPTIONS] [SCOPE] [DATASETS] - - List scopes & datasets - -Options: - -v, --verbose Verbosity: v=INFO, vv=DEBUG. - -q, --quiet Only errors shown in logs. - --write Write the events to file. - --help Show this message and exit. -""" assert result.exit_code == 0 - assert result.output == expected_response + # Check that all the expected elements are present + assert "Usage: cli list [OPTIONS] [SCOPE] [DATASETS]" in result.output + assert "List scopes & datasets" in result.output + assert "--verbose" in result.output + assert "--quiet" in result.output + assert "--write" in result.output + assert "--json" in result.output + assert "Output as JSON" in result.output def test_cli_ps_help(runner: CliRunner) -> None: @@ -118,18 +115,15 @@ def test_cli_ps_help(runner: CliRunner) -> None: runner (CliRunner) -> None: Click runner. """ result = runner.invoke(datatrail, ["ps", "--help"]) - expected_response = """Usage: cli ps [OPTIONS] SCOPE DATASET - - Details of a dataset. - -Options: - -s, --show-files Show file names. - -v, --verbose Verbosity: v=INFO, vv=DEBUG. - -q, --quiet Set log level to ERROR. - --help Show this message and exit. -""" assert result.exit_code == 0 - assert result.output == expected_response + # Check that all the expected elements are present + assert "Usage: cli ps [OPTIONS] SCOPE DATASET" in result.output + assert "Details of a dataset" in result.output + assert "--show-files" in result.output + assert "--verbose" in result.output + assert "--quiet" in result.output + assert "--json" in result.output + assert "Output as JSON" in result.output def test_cli_pull_help(runner: CliRunner) -> None: @@ -580,3 +574,94 @@ def test_cli_version(runner: CliRunner) -> None: result = runner.invoke(datatrail, ["version"]) assert result.exit_code == 0 assert "Datatrail Versions" in result.output + + +def test_cli_list_scopes_json(runner: CliRunner) -> None: + """Test for CLI list to output scopes as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke(datatrail, ["ls", "--json"]) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have a 'scopes' key with a list of scopes + assert "scopes" in output_data + assert isinstance(output_data["scopes"], list) + assert "chime.event.intensity.raw" in output_data["scopes"] + + +def test_cli_list_larger_json(runner: CliRunner) -> None: + """Test for CLI to list larger datasets as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke(datatrail, ["ls", "chime.event.baseband.raw", "--json"]) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have a 'larger_datasets' key with a list of datasets + assert "larger_datasets" in output_data + assert isinstance(output_data["larger_datasets"], list) + assert "classified.FRB" in output_data["larger_datasets"] + + +def test_cli_list_children_json(runner: CliRunner) -> None: + """Test for CLI list to show child datasets as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, ["ls", "chime.event.baseband.raw", "classified.FRB", "--json"] + ) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have a 'datasets' key with a list of datasets + assert "datasets" in output_data + assert isinstance(output_data["datasets"], list) + assert "289007650" in output_data["datasets"] + + +def test_cli_ps_json(runner: CliRunner) -> None: + """Test for CLI ps command with JSON output. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, ["ps", "chime.event.baseband.raw", "289007650", "--json"] + ) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have dataset, scope, files, and policies keys + assert "dataset" in output_data + assert "scope" in output_data + assert "files" in output_data + assert "policies" in output_data + assert output_data["dataset"] == "289007650" + assert output_data["scope"] == "chime.event.baseband.raw"