⚡ Performance Improvement: Remove subshell bottlenecks for chmod and chgrp

docs/list.md

@@ -11,7 +11,6 @@ Options:
   -v, --verbose  Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet    Only errors shown in logs.
   --write        Write the events to file.
-  --json         Output as JSON.
   --help         Show this message and exit.
 
 ```
@@ -101,62 +100,3 @@ Within Datatrail, there are two types of datasets:
 
     Please see the CLI reference page for more information on the `list` command:
     [datatrail list](../cli/#datatrail-list)
-
-## 🤖 Machine-readable JSON output
-
-The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines:
-
-```bash
-# Get all scopes as JSON
-$ datatrail ls --json
-{
-  "scopes": [
-    "chime.event.baseband.raw",
-    "chime.event.intensity.raw",
-    "kko.acquisition.processed",
-    ...
-  ]
-}
-
-# Get larger datasets as JSON
-$ datatrail ls kko.scheduled.baseband.raw --json
-{
-  "scope": "kko.scheduled.baseband.raw",
-  "larger_datasets": [
-    "20230804095251",
-    "scheduled.commissioning.steady"
-  ]
-}
-
-# Get child datasets as JSON
-$ datatrail ls kko.scheduled.baseband.raw scheduled.commissioning.steady --json
-{
-  "datasets": [
-    "20230616150511",
-    "20230604135840",
-    "20230604134842",
-    ...
-  ]
-}
-```
-
-### Usage in scripts
-
-```python
-import json
-import subprocess
-
-# Get all scopes
-result = subprocess.run(["datatrail", "ls", "--json"], capture_output=True, text=True)
-data = json.loads(result.stdout)
-scopes = data["scopes"]
-
-# Get datasets for a scope
-result = subprocess.run(
-    ["datatrail", "ls", "chime.event.baseband.raw", "--json"],
-    capture_output=True,
-    text=True,
-)
-data = json.loads(result.stdout)
-larger_datasets = data["larger_datasets"]
-```

docs/ps.md

@@ -11,7 +11,6 @@ Options:
   -s, --show-files  Show file names.
   -v, --verbose     Verbosity: v=INFO, vv=DEBUG.
   -q, --quiet       Set log level to ERROR.
-  --json            Output as JSON.
   --help            Show this message and exit.
 ```
 
@@ -81,72 +80,3 @@ if the `--show-files` or `-s` flag is passed.
     │ - baseband_308892599_111.h5                                     │
     :
     ```
-
-## 🤖 Machine-readable JSON output
-
-The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse dataset information in scripts and pipelines:
-
-```bash
-$ datatrail ps kko.event.baseband.raw 308892599 --json
-{
-  "dataset": "308892599",
-  "scope": "kko.event.baseband.raw",
-  "files": {
-    "contains_datasets": 0,
-    "datasets_contained": [],
-    "file_replica_locations": {
-      "minoc": [
-        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_129.h5",
-        "data/kko/baseband/raw/2023/08/07/astro_308892599/baseband_308892599_1013.h5",
-        ...
-      ]
-    }
-  },
-  "policies": {
-    "replication_policy": {
-      "preferred_storage_elements": ["chime"],
-      "priority": "low",
-      "default": true
-    },
-    "deletion_policy": [
-      {
-        "storage_element": "minoc",
-        "priority": "low",
-        "default": true,
-        "delete_after_days": 36500
-      },
-      ...
-    ],
-    "belongs_to": [
-      {
-        "scope": "kko.event.baseband.raw",
-        "name": "B0531+21.commissioning.pulsar.temp"
-      }
-    ]
-  }
-}
-```
-
-### Usage in scripts
-
-```python
-import json
-import subprocess
-
-# Get dataset information
-result = subprocess.run(
-    ["datatrail", "ps", "kko.event.baseband.raw", "308892599", "--json"],
-    capture_output=True,
-    text=True,
-)
-data = json.loads(result.stdout)
-
-# Access file locations
-file_locations = data["files"]["file_replica_locations"]
-minoc_files = file_locations.get("minoc", [])
-
-# Access policies
-replication_policy = data["policies"]["replication_policy"]
-deletion_policy = data["policies"]["deletion_policy"]
-belongs_to = data["policies"]["belongs_to"]
-```

dtcli/ls.py

@@ -34,16 +34,14 @@
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
 @click.option("--write", is_flag=True, help="Write the events to file.")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def list(  # noqa: C901
+def list(
     ctx: click.Context,
     scope: Optional[str] = None,
     datasets: Optional[str] = None,
     verbose: int = 0,
     quiet: bool = False,
     write: bool = False,
-    output_json: bool = False,
 ):
     """List Datatrail Scopes & Datasets.
 
@@ -54,7 +52,6 @@ def list(  # noqa: C901
         verbose (int): Verbosity: v=INFO, vv=DEBUG.
         quiet (bool): Only errors shown in logs.
         write (bool): Write the events to file.
-        output_json (bool): Output as JSON.
     """
     # Set logging level.
     set_log_level(logger, verbose, quiet)
@@ -76,13 +73,6 @@ def list(  # noqa: C901
             return None
     results = functions.list(scope, datasets, verbose, quiet)
 
-    # Output JSON if requested.
-    if output_json:
-        print(json.dumps(results, indent=2))
-        if "error" in results:
-            ctx.exit(1)
-        return
-
     # Display scopes.
     if "scopes" in results.keys():
         table = Table(

dtcli/ps.py

@@ -26,16 +26,14 @@
 @click.option("-s", "--show-files", is_flag=True, help="Show file names.")
 @click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.")
 @click.option("-q", "--quiet", is_flag=True, help="Set log level to ERROR.")
-@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
 @click.pass_context
-def ps(  # noqa: C901
+def ps(
     ctx: click.Context,
     scope: str,
     dataset: str,
     show_files: bool,
     verbose: int,
     quiet: bool,
-    output_json: bool,
 ):
     """Detailed status of a dataset.
 
@@ -46,7 +44,6 @@ def ps(  # noqa: C901
         show_files (bool): Show list of files.
         verbose (int): Verbosity: v=INFO, vv=DUBUG.
         quiet (bool): Set log level to ERROR.
-        output_json (bool): Output as JSON.
 
     Returns:
         None
@@ -76,41 +73,13 @@ def ps(  # noqa: C901
     try:
         files, policies = functions.ps(scope, dataset, verbose, quiet)
         if isinstance(files, str) or isinstance(policies, str):
-            if output_json:
-                import json
-
-                print(
-                    json.dumps(
-                        {"error": {"files": str(files), "policies": str(policies)}},
-                        indent=2,
-                    )
-                )
-                ctx.exit(1)
             error_console.print("Error: files = ", files)
             error_console.print("Error: policies = ", policies)
             return None
     except Exception as e:
-        if output_json:
-            import json
-
-            print(json.dumps({"error": str(e)}, indent=2))
-            ctx.exit(1)
         error_console.print(e)
         return None
 
-    # Handle JSON output
-    if output_json:
-        import json
-
-        result = {
-            "dataset": dataset,
-            "scope": scope,
-            "files": files,
-            "policies": policies,
-        }
-        print(json.dumps(result, indent=2))
-        return None
-
     if show_files and files:
         # Files table
         file_table = create_files_table(dataset, scope, files)

dtcli/src/functions.py

@@ -4,7 +4,7 @@
 import os
 import re
 import shutil
-import subprocess
+import stat
 import time
 from collections import defaultdict
 from pathlib import Path
@@ -258,6 +258,29 @@ def find_missing_dataset_files(
     return {"missing": missing_files, "existing": existing_files}
 
 
+def _apply_chime_frb_rw_permissions(folder: str) -> None:
+    """Apply chime-frb-rw group and group-write permissions to a folder recursively."""
+    for root, dirs, files_in_dir in os.walk(folder):
+        try:
+            shutil.chown(root, group="chime-frb-rw")
+        except OSError:
+            pass
+        try:
+            os.chmod(root, os.stat(root).st_mode | stat.S_IWGRP)
+        except OSError:
+            pass
+        for f in files_in_dir:
+            path = os.path.join(root, f)
+            try:
+                shutil.chown(path, group="chime-frb-rw")
+            except OSError:
+                pass
+            try:
+                os.chmod(path, os.stat(path).st_mode | stat.S_IWGRP)
+            except OSError:
+                pass
+
+
 def get_files(
     files: List[str],
     site: str,
@@ -298,8 +321,7 @@ def get_files(
         if site == "canfar":
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)
-                subprocess.run(["chgrp", "-R", "chime-frb-rw", folder])
-                subprocess.run(["chmod", "-R", "g+w", folder])
+                _apply_chime_frb_rw_permissions(folder)
         else:
             for folder in folders:
                 os.makedirs(folder, exist_ok=True)

dtcli/utilities/cadcclient.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import sys
-from concurrent.futures import ThreadPoolExecutor
 from io import StringIO
 from multiprocessing import Process  # Use the standard library only
 from typing import Any, Dict, List, Optional, Tuple
@@ -411,25 +410,23 @@ def status(
     ]
     if not certfile:
         certfile = procure(key="vospace_certfile")
-
-    def check_url(url: str) -> bool:
+    minoc_status = False
+    luskan_status = False
+    for index, url in enumerate(urls):
         response = requests.get(url, cert=certfile, allow_redirects=True)
         try:
             response.raise_for_status()
             authorised = response.headers.get("x-vo-authenticated")
             if isinstance(authorised, str):
-                return True
+                if index == 0:
+                    minoc_status = True
+                else:
+                    luskan_status = True
             else:
                 raise TypeError
         except HTTPError as error:
             logger.warning(error)
             logger.warning(f"{url.split('/')[3]} is down.")
-            return False
         except TypeError:
             logger.error("Canfar certificate is not valid.")
-            return False
-
-    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
-        results = list(executor.map(check_url, urls))
-
-    return results[0], results[1]
+    return minoc_status, luskan_status

dtcli/utilities/utilities.py

@@ -70,9 +70,6 @@ def split(data: List[Any], count: int) -> List[List[Any]]:
     Returns:
         List[List[Any]]: List of batches.
     """
-    if count <= 0:
-        raise ValueError("count must be greater than 0")
-
     batch_size = len(data) // count
     remainder = len(data) % count
     batches: List[Any] = []