diff --git a/docs/list.md b/docs/list.md index 68cfaf3..a5c2f14 100644 --- a/docs/list.md +++ b/docs/list.md @@ -11,7 +11,6 @@ Options: -v, --verbose Verbosity: v=INFO, vv=DEBUG. -q, --quiet Only errors shown in logs. --write Write the events to file. - --json Output as JSON. --help Show this message and exit. ``` @@ -101,62 +100,3 @@ Within Datatrail, there are two types of datasets: Please see the CLI reference page for more information on the `list` command: [datatrail list](../cli/#datatrail-list) - -## 🤖 Machine-readable JSON output - -The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: - -```bash -# Get all scopes as JSON -$ datatrail ls --json -{ - "scopes": [ - "chime.event.baseband.raw", - "chime.event.intensity.raw", - "kko.acquisition.processed", - ... - ] -} - -# Get larger datasets as JSON -$ datatrail ls kko.scheduled.baseband.raw --json -{ - "scope": "kko.scheduled.baseband.raw", - "larger_datasets": [ - "20230804095251", - "scheduled.commissioning.steady" - ] -} - -# Get child datasets as JSON -$ datatrail ls kko.scheduled.baseband.raw scheduled.commissioning.steady --json -{ - "datasets": [ - "20230616150511", - "20230604135840", - "20230604134842", - ... - ] -} -``` - -### Usage in scripts - -```python -import json -import subprocess - -# Get all scopes -result = subprocess.run(["datatrail", "ls", "--json"], capture_output=True, text=True) -data = json.loads(result.stdout) -scopes = data["scopes"] - -# Get datasets for a scope -result = subprocess.run( - ["datatrail", "ls", "chime.event.baseband.raw", "--json"], - capture_output=True, - text=True, -) -data = json.loads(result.stdout) -larger_datasets = data["larger_datasets"] -``` diff --git a/docs/ps.md b/docs/ps.md index efc6b68..38ecbf5 100644 --- a/docs/ps.md +++ b/docs/ps.md @@ -11,7 +11,6 @@ Options: -s, --show-files Show file names. -v, --verbose Verbosity: v=INFO, vv=DEBUG. -q, --quiet Set log level to ERROR. - --json Output as JSON. --help Show this message and exit. ``` @@ -81,72 +80,3 @@ if the `--show-files` or `-s` flag is passed. │ - baseband_308892599_111.h5 │ : ``` - -## 🤖 Machine-readable JSON output - -The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse dataset information in scripts and pipelines: - -```bash -$ datatrail ps kko.event.baseband.raw 308892599 --json -{ - "dataset": "308892599", - "scope": "kko.event.baseband.raw", - "files": { - "contains_datasets": 0, - "datasets_contained": [], - "file_replica_locations": { - "minoc": [ - "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_129.h5", - "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_1013.h5", - ... - ] - } - }, - "policies": { - "replication_policy": { - "preferred_storage_elements": ["chime"], - "priority": "low", - "default": true - }, - "deletion_policy": [ - { - "storage_element": "minoc", - "priority": "low", - "default": true, - "delete_after_days": 36500 - }, - ... - ], - "belongs_to": [ - { - "scope": "kko.event.baseband.raw", - "name": "B0531+21.commissioning.pulsar.temp" - } - ] - } -} -``` - -### Usage in scripts - -```python -import json -import subprocess - -# Get dataset information -result = subprocess.run( - ["datatrail", "ps", "kko.event.baseband.raw", "308892599", "--json"], - capture_output=True, - text=True, -) -data = json.loads(result.stdout) - -# Access file locations -file_locations = data["files"]["file_replica_locations"] -minoc_files = file_locations.get("minoc", []) - -# Access policies -replication_policy = data["policies"]["replication_policy"] -deletion_policy = data["policies"]["deletion_policy"] -belongs_to = data["policies"]["belongs_to"] -``` diff --git a/dtcli/ls.py b/dtcli/ls.py index d6335da..857bc77 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -34,16 +34,14 @@ @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.") @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") @click.option("--write", is_flag=True, help="Write the events to file.") -@click.option("--json", "output_json", is_flag=True, help="Output as JSON.") @click.pass_context -def list( # noqa: C901 +def list( ctx: click.Context, scope: Optional[str] = None, datasets: Optional[str] = None, verbose: int = 0, quiet: bool = False, write: bool = False, - output_json: bool = False, ): """List Datatrail Scopes & Datasets. @@ -54,7 +52,6 @@ def list( # noqa: C901 verbose (int): Verbosity: v=INFO, vv=DEBUG. quiet (bool): Only errors shown in logs. write (bool): Write the events to file. - output_json (bool): Output as JSON. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -76,13 +73,6 @@ def list( # noqa: C901 return None results = functions.list(scope, datasets, verbose, quiet) - # Output JSON if requested. - if output_json: - print(json.dumps(results, indent=2)) - if "error" in results: - ctx.exit(1) - return - # Display scopes. if "scopes" in results.keys(): table = Table( diff --git a/dtcli/ps.py b/dtcli/ps.py index 1d7e225..b67ab85 100644 --- a/dtcli/ps.py +++ b/dtcli/ps.py @@ -26,16 +26,14 @@ @click.option("-s", "--show-files", is_flag=True, help="Show file names.") @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.") @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.") -@click.option("--json", "output_json", is_flag=True, help="Output as JSON.") @click.pass_context -def ps( # noqa: C901 +def ps( ctx: click.Context, scope: str, dataset: str, show_files: bool, verbose: int, quiet: bool, - output_json: bool, ): """Detailed status of a dataset. @@ -46,7 +44,6 @@ def ps( # noqa: C901 show_files (bool): Show list of files. verbose (int): Verbosity: v=INFO, vv=DUBUG. quiet (bool): Set log level to ERROR. - output_json (bool): Output as JSON. Returns: None @@ -76,41 +73,13 @@ def ps( # noqa: C901 try: files, policies = functions.ps(scope, dataset, verbose, quiet) if isinstance(files, str) or isinstance(policies, str): - if output_json: - import json - - print( - json.dumps( - {"error": {"files": str(files), "policies": str(policies)}}, - indent=2, - ) - ) - ctx.exit(1) error_console.print("Error: files = ", files) error_console.print("Error: policies = ", policies) return None except Exception as e: - if output_json: - import json - - print(json.dumps({"error": str(e)}, indent=2)) - ctx.exit(1) error_console.print(e) return None - # Handle JSON output - if output_json: - import json - - result = { - "dataset": dataset, - "scope": scope, - "files": files, - "policies": policies, - } - print(json.dumps(result, indent=2)) - return None - if show_files and files: # Files table file_table = create_files_table(dataset, scope, files) diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index 5b70a1d..fa9cb69 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -4,7 +4,7 @@ import os import re import shutil -import subprocess +import stat import time from collections import defaultdict from pathlib import Path @@ -258,6 +258,29 @@ def find_missing_dataset_files( return {"missing": missing_files, "existing": existing_files} +def _apply_chime_frb_rw_permissions(folder: str) -> None: + """Apply chime-frb-rw group and group-write permissions to a folder recursively.""" + for root, dirs, files_in_dir in os.walk(folder): + try: + shutil.chown(root, group="chime-frb-rw") + except OSError: + pass + try: + os.chmod(root, os.stat(root).st_mode | stat.S_IWGRP) + except OSError: + pass + for f in files_in_dir: + path = os.path.join(root, f) + try: + shutil.chown(path, group="chime-frb-rw") + except OSError: + pass + try: + os.chmod(path, os.stat(path).st_mode | stat.S_IWGRP) + except OSError: + pass + + def get_files( files: List[str], site: str, @@ -298,8 +321,7 @@ def get_files( if site == "canfar": for folder in folders: os.makedirs(folder, exist_ok=True) - subprocess.run(["chgrp", "-R", "chime-frb-rw", folder]) - subprocess.run(["chmod", "-R", "g+w", folder]) + _apply_chime_frb_rw_permissions(folder) else: for folder in folders: os.makedirs(folder, exist_ok=True) diff --git a/dtcli/utilities/cadcclient.py b/dtcli/utilities/cadcclient.py index 4e78074..1ba461c 100644 --- a/dtcli/utilities/cadcclient.py +++ b/dtcli/utilities/cadcclient.py @@ -3,7 +3,6 @@ import logging import os import sys -from concurrent.futures import ThreadPoolExecutor from io import StringIO from multiprocessing import Process # Use the standard library only from typing import Any, Dict, List, Optional, Tuple @@ -411,25 +410,23 @@ def status( ] if not certfile: certfile = procure(key="vospace_certfile") - - def check_url(url: str) -> bool: + minoc_status = False + luskan_status = False + for index, url in enumerate(urls): response = requests.get(url, cert=certfile, allow_redirects=True) try: response.raise_for_status() authorised = response.headers.get("x-vo-authenticated") if isinstance(authorised, str): - return True + if index == 0: + minoc_status = True + else: + luskan_status = True else: raise TypeError except HTTPError as error: logger.warning(error) logger.warning(f"{url.split('/')[3]} is down.") - return False except TypeError: logger.error("Canfar certificate is not valid.") - return False - - with ThreadPoolExecutor(max_workers=len(urls)) as executor: - results = list(executor.map(check_url, urls)) - - return results[0], results[1] + return minoc_status, luskan_status diff --git a/dtcli/utilities/utilities.py b/dtcli/utilities/utilities.py index e36ad70..2a1dbd7 100644 --- a/dtcli/utilities/utilities.py +++ b/dtcli/utilities/utilities.py @@ -70,9 +70,6 @@ def split(data: List[Any], count: int) -> List[List[Any]]: Returns: List[List[Any]]: List of batches. """ - if count <= 0: - raise ValueError("count must be greater than 0") - batch_size = len(data) // count remainder = len(data) % count batches: List[Any] = [] diff --git a/tests/test_cli.py b/tests/test_cli.py index 1074ec6..aea606b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -97,15 +97,18 @@ def test_cli_list_help(runner: CliRunner) -> None: runner (CliRunner) -> None: Click runner. """ result = runner.invoke(datatrail, ["ls", "--help"]) + expected_response = """Usage: cli list [OPTIONS] [SCOPE] [DATASETS] + + List scopes & datasets + +Options: + -v, --verbose Verbosity: v=INFO, vv=DEBUG. + -q, --quiet Only errors shown in logs. + --write Write the events to file. + --help Show this message and exit. +""" assert result.exit_code == 0 - # Check that all the expected elements are present - assert "Usage: cli list [OPTIONS] [SCOPE] [DATASETS]" in result.output - assert "List scopes & datasets" in result.output - assert "--verbose" in result.output - assert "--quiet" in result.output - assert "--write" in result.output - assert "--json" in result.output - assert "Output as JSON" in result.output + assert result.output == expected_response def test_cli_ps_help(runner: CliRunner) -> None: @@ -115,15 +118,18 @@ def test_cli_ps_help(runner: CliRunner) -> None: runner (CliRunner) -> None: Click runner. """ result = runner.invoke(datatrail, ["ps", "--help"]) + expected_response = """Usage: cli ps [OPTIONS] SCOPE DATASET + + Details of a dataset. + +Options: + -s, --show-files Show file names. + -v, --verbose Verbosity: v=INFO, vv=DEBUG. + -q, --quiet Set log level to ERROR. + --help Show this message and exit. +""" assert result.exit_code == 0 - # Check that all the expected elements are present - assert "Usage: cli ps [OPTIONS] SCOPE DATASET" in result.output - assert "Details of a dataset" in result.output - assert "--show-files" in result.output - assert "--verbose" in result.output - assert "--quiet" in result.output - assert "--json" in result.output - assert "Output as JSON" in result.output + assert result.output == expected_response def test_cli_pull_help(runner: CliRunner) -> None: @@ -574,94 +580,3 @@ def test_cli_version(runner: CliRunner) -> None: result = runner.invoke(datatrail, ["version"]) assert result.exit_code == 0 assert "Datatrail Versions" in result.output - - -def test_cli_list_scopes_json(runner: CliRunner) -> None: - """Test for CLI list to output scopes as JSON. - - Args: - runner (CliRunner): Click runner. - """ - import json - - result = runner.invoke(datatrail, ["ls", "--json"]) - assert result.exit_code == 0 - # Extract JSON from output (skip version check message if present) - json_start = result.output.find("{") - json_output = result.output[json_start:] - # Parse the output as JSON - output_data = json.loads(json_output) - # Should have a 'scopes' key with a list of scopes - assert "scopes" in output_data - assert isinstance(output_data["scopes"], list) - assert "chime.event.intensity.raw" in output_data["scopes"] - - -def test_cli_list_larger_json(runner: CliRunner) -> None: - """Test for CLI to list larger datasets as JSON. - - Args: - runner (CliRunner): Click runner. - """ - import json - - result = runner.invoke(datatrail, ["ls", "chime.event.baseband.raw", "--json"]) - assert result.exit_code == 0 - # Extract JSON from output (skip version check message if present) - json_start = result.output.find("{") - json_output = result.output[json_start:] - # Parse the output as JSON - output_data = json.loads(json_output) - # Should have a 'larger_datasets' key with a list of datasets - assert "larger_datasets" in output_data - assert isinstance(output_data["larger_datasets"], list) - assert "classified.FRB" in output_data["larger_datasets"] - - -def test_cli_list_children_json(runner: CliRunner) -> None: - """Test for CLI list to show child datasets as JSON. - - Args: - runner (CliRunner): Click runner. - """ - import json - - result = runner.invoke( - datatrail, ["ls", "chime.event.baseband.raw", "classified.FRB", "--json"] - ) - assert result.exit_code == 0 - # Extract JSON from output (skip version check message if present) - json_start = result.output.find("{") - json_output = result.output[json_start:] - # Parse the output as JSON - output_data = json.loads(json_output) - # Should have a 'datasets' key with a list of datasets - assert "datasets" in output_data - assert isinstance(output_data["datasets"], list) - assert "289007650" in output_data["datasets"] - - -def test_cli_ps_json(runner: CliRunner) -> None: - """Test for CLI ps command with JSON output. - - Args: - runner (CliRunner): Click runner. - """ - import json - - result = runner.invoke( - datatrail, ["ps", "chime.event.baseband.raw", "289007650", "--json"] - ) - assert result.exit_code == 0 - # Extract JSON from output (skip version check message if present) - json_start = result.output.find("{") - json_output = result.output[json_start:] - # Parse the output as JSON - output_data = json.loads(json_output) - # Should have dataset, scope, files, and policies keys - assert "dataset" in output_data - assert "scope" in output_data - assert "files" in output_data - assert "policies" in output_data - assert output_data["dataset"] == "289007650" - assert output_data["scope"] == "chime.event.baseband.raw" diff --git a/tests/test_utils.py b/tests/test_utils.py index ab108b6..b12aa35 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,3 @@ -import pytest - from dtcli.utilities import utilities @@ -10,20 +8,6 @@ def test_split(): assert result == [["a"], ["b"]] -def test_split_edges(): - """Test split() with edge cases like empty list and invalid counts.""" - # Empty data - assert utilities.split([], 2) == [] - - # Zero count should raise ValueError - with pytest.raises(ValueError, match="count must be greater than 0"): - utilities.split([1, 2, 3], 0) - - # Negative count should raise ValueError - with pytest.raises(ValueError, match="count must be greater than 0"): - utilities.split([1, 2, 3], -1) - - def test_split_more_batches_than_items(): """split() must not return more batches than there are items.