fix(config): use api.{realm} CRUD host and add O11y API URL override (SAO-18181)

AGENTS.md

@@ -118,7 +118,7 @@ CI supports Python 3.11–3.14; root CI also spans Linux, macOS, and Windows.
 
 | Deployment | Required authentication |
 |---|---|
-| O11y Cloud | `SPLUNK_AO_REALM` plus at least one token: `SPLUNK_AO_O11Y_TOKEN` for telemetry and, when permitted, CRUD; and/or `SPLUNK_AO_O11Y_API_TOKEN` for CRUD-only or dedicated CRUD authentication |
+| O11y Cloud | `SPLUNK_AO_REALM` plus at least one token: `SPLUNK_AO_O11Y_TOKEN` for telemetry and, when permitted, CRUD; and/or `SPLUNK_AO_O11Y_API_TOKEN` for CRUD-only or dedicated CRUD authentication. CRUD uses `api.<realm>`; optional `SPLUNK_AO_O11Y_API_URL` overrides the CRUD API origin only (never OTLP ingest) |
 | Standalone | `SPLUNK_AO_API_KEY` plus `SPLUNK_AO_CONSOLE_URL`; `SPLUNK_AO_API_URL` is optional |
 
 - Detection lives in `deployment.py::resolve_deployment()`. Never mix O11y and standalone variable sets.

ARCHITECTURE.md

@@ -171,7 +171,7 @@ modes. It also requires attention to when exporters/configuration singletons are
 
 | Mode | CRUD auth | Telemetry auth/endpoints |
 |---|---|---|
-| O11y Cloud | `SPLUNK_AO_O11Y_TOKEN` when permitted, or a dedicated `SPLUNK_AO_O11Y_API_TOKEN` | realm-derived endpoint plus `SPLUNK_AO_O11Y_TOKEN` |
+| O11y Cloud | `SPLUNK_AO_O11Y_TOKEN` when permitted, or a dedicated `SPLUNK_AO_O11Y_API_TOKEN`; host `api.<realm>`, overridable via `SPLUNK_AO_O11Y_API_URL` | realm-derived endpoint plus `SPLUNK_AO_O11Y_TOKEN` |
 | Standalone | `SPLUNK_AO_API_KEY` | console/API configuration and deployment-derived OTLP endpoint |
 
 O11y also requires `SPLUNK_AO_REALM`; standalone requires `SPLUNK_AO_CONSOLE_URL`. The primary O11y token can authorize

CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- O11y CRUD requests now target `https://api.{realm}.observability.splunkcloud.com` instead of
+  `https://app.{realm}.observability.splunkcloud.com`. The OTLP ingest endpoint and the console URL are unchanged.
+
+### Added
+
+- `SPLUNK_AO_O11Y_API_URL` optionally overrides the O11y CRUD API origin (for example a private or custom endpoint).
+  It must be an `https://` origin without a path. OTLP ingest is not affected.
+
 ## [0.7.0]
 
 ### Added

README.md

@@ -76,6 +76,7 @@ export SPLUNK_AO_CONSOLE_URL="https://console.subdomain.yourcompany.com"
 | `SPLUNK_AO_REALM` | Splunk Observability Cloud realm (Required) |
 | `SPLUNK_AO_O11Y_TOKEN` | O11y ingest token used for OTLP export (Required for telemetry) |
 | `SPLUNK_AO_O11Y_API_TOKEN` | Dedicated O11y API token used for CRUD operations (Optional) |
+| `SPLUNK_AO_O11Y_API_URL` | Override the CRUD API origin, e.g. a private or custom endpoint (Optional; origin only, no path) |
 
 For telemetry export, set your realm and O11y ingest token:
 
@@ -100,8 +101,8 @@ When both tokens are set, `SPLUNK_AO_O11Y_API_TOKEN` is preferred for CRUD and
 required until the application exports telemetry. Attempting to construct an
 OTLP exporter without `SPLUNK_AO_O11Y_TOKEN` raises a configuration error.
 
-The SDK derives the console, API and OTLP ingest endpoints from the
-realm. Do not set `SPLUNK_AO_CONSOLE_URL` or `SPLUNK_AO_API_URL` for O11y
+The SDK derives the console, API (`api.<realm>`) and OTLP ingest endpoints from the
+realm. `SPLUNK_AO_O11Y_API_URL` overrides only the CRUD API origin. Do not set `SPLUNK_AO_CONSOLE_URL` or `SPLUNK_AO_API_URL` for O11y
 Cloud.
 
 #### Routing

splunk-ao-adk/tests/conftest.py

@@ -160,8 +160,12 @@ def set_validated_config(
     mock_projects: None,
     mock_log_streams: None,
     mock_sessions: None,
+    monkeypatch: pytest.MonkeyPatch,
 ) -> Generator[None, None, None]:
     """Automatically set up validated config for tests."""
+    # Keep ambient O11y settings from selecting the O11y deployment for these standalone-credential tests.
+    for name in ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN", "SPLUNK_AO_O11Y_API_URL"):
+        monkeypatch.delenv(name, raising=False)
     # Reset any existing config state
     if SplunkAOConfig._instance is not None:
         SplunkAOConfig._instance.reset()

splunk-ao-adk/tests/test_observer.py

@@ -22,7 +22,7 @@ def observer() -> SplunkAOObserver:
 @pytest.mark.asyncio
 async def test_observer_completion_in_copied_task_restores_caller_context(monkeypatch: pytest.MonkeyPatch) -> None:
     # Given: ADK's normal exporter path uses the shared base handler and logger.
-    for name in ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN"):
+    for name in ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN", "SPLUNK_AO_O11Y_API_URL"):
         monkeypatch.delenv(name, raising=False)
     monkeypatch.setenv("SPLUNK_AO_CONSOLE_URL", "http://fake.test:8088")
     monkeypatch.setenv("SPLUNK_AO_API_KEY", "api-1234567890")

src/splunk_ao/config.py

@@ -263,7 +263,8 @@ def _val(kwarg_name: str, env_name: str) -> str | None:
         realm = os.environ.get("SPLUNK_AO_REALM")
         o11y_token = os.environ.get("SPLUNK_AO_O11Y_TOKEN")
         o11y_api_token = os.environ.get("SPLUNK_AO_O11Y_API_TOKEN")
-        if realm or o11y_token or o11y_api_token:
+        o11y_api_url = os.environ.get("SPLUNK_AO_O11Y_API_URL")
+        if realm or o11y_token or o11y_api_token or o11y_api_url:
             if not realm:
                 return "O11y authentication requires SPLUNK_AO_REALM to be set."
             if not o11y_token and not o11y_api_token:

src/splunk_ao/deployment.py

@@ -3,12 +3,13 @@
 import os
 from dataclasses import dataclass
 from enum import StrEnum
+from urllib.parse import urlsplit
 
 from pydantic import SecretStr
 
-from splunk_ao.shared.exceptions import AmbiguousConfigurationError, MissingConfigurationError
+from splunk_ao.shared.exceptions import AmbiguousConfigurationError, ConfigurationError, MissingConfigurationError
 
-_O11Y_ENV_VARS = ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN")
+_O11Y_ENV_VARS = ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN", "SPLUNK_AO_O11Y_API_URL")
 _STANDALONE_ENV_VARS = ("SPLUNK_AO_API_KEY", "SPLUNK_AO_CONSOLE_URL", "SPLUNK_AO_API_URL")
 
 
@@ -46,13 +47,34 @@ def resolve_deployment() -> DeploymentMode:
     )
 
 
+def _normalize_api_origin(value: str) -> str:
+    """Validate an API origin override and return it as ``https://host[:port]``."""
+    try:
+        parts = urlsplit(value.strip())
+        port = parts.port
+    except ValueError:
+        raise ConfigurationError("SPLUNK_AO_O11Y_API_URL is not a valid URL.") from None
+    if parts.scheme != "https" or not parts.hostname:
+        raise ConfigurationError("SPLUNK_AO_O11Y_API_URL must be an https:// URL.")
+    if "@" in parts.netloc or parts.query or parts.fragment or parts.path.strip("/"):
+        raise ConfigurationError(
+            "SPLUNK_AO_O11Y_API_URL must be an origin only (scheme://host[:port]) without credentials, path, "
+            "query, or fragment."
+        )
+    if any(ch.isspace() for ch in parts.netloc):
+        raise ConfigurationError("SPLUNK_AO_O11Y_API_URL has an invalid host.")
+    host = f"[{parts.hostname}]" if ":" in parts.hostname else parts.hostname
+    return f"https://{host}:{port}" if port is not None else f"https://{host}"
+
+
 @dataclass
 class O11yConfig:
     """Configuration for a Splunk Observability Cloud deployment."""
 
     realm: str
     o11y_token: SecretStr | None = None
     o11y_api_token: SecretStr | None = None
+    api_url: str | None = None
 
     def __post_init__(self) -> None:
         missing = []
@@ -63,6 +85,11 @@ def __post_init__(self) -> None:
         if missing:
             raise MissingConfigurationError(f"O11y deployment requires {' and '.join(missing)} to be set.")
 
+        if self.api_url:
+            self.api_url = _normalize_api_origin(self.api_url)
+        else:
+            self.api_url = None
+
         if self.o11y_token is not None and not isinstance(self.o11y_token, SecretStr):
             self.o11y_token = SecretStr(self.o11y_token)
         if self.o11y_api_token is not None and not isinstance(self.o11y_api_token, SecretStr):
@@ -78,6 +105,7 @@ def from_env(cls) -> "O11yConfig":
             realm=realm or "",
             o11y_token=SecretStr(o11y_token) if o11y_token else None,
             o11y_api_token=SecretStr(o11y_api_token) if o11y_api_token else None,
+            api_url=_env("SPLUNK_AO_O11Y_API_URL"),
         )
 
     @property
@@ -107,11 +135,13 @@ def require_ingest_token(self) -> SecretStr:
 
     @property
     def api_root(self) -> str:
-        """Return the realm-derived application origin used by the AO API."""
-        return f"https://app.{self.realm}.observability.splunkcloud.com"
+        """Return the explicit API origin override, or the realm-derived origin used by the AO API."""
+        if self.api_url:
+            return self.api_url
+        return f"https://api.{self.realm}.observability.splunkcloud.com"
 
     def require_api_url(self) -> str:
-        """Return the realm-derived AO CRUD API URL."""
+        """Return the AO CRUD API URL from the override origin or the realm-derived origin."""
         return f"{self.api_root}/ao/api/"
 
     def require_console_url(self) -> str:

tests/conftest.py

@@ -174,7 +174,7 @@ def set_validated_config(
 ) -> Generator[None, None, None]:
     """Automatically set up validated config for tests."""
     SplunkAOLoggerSingleton().reset_all()
-    for name in ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN"):
+    for name in ("SPLUNK_AO_REALM", "SPLUNK_AO_O11Y_TOKEN", "SPLUNK_AO_O11Y_API_TOKEN", "SPLUNK_AO_O11Y_API_URL"):
         monkeypatch.delenv(name, raising=False)
     monkeypatch.setenv("SPLUNK_AO_CONSOLE_URL", "http://fake.test:8088")
     monkeypatch.setenv("SPLUNK_AO_API_KEY", "api-1234567890")

tests/test_deployment.py

@@ -7,13 +7,14 @@
 import pytest
 
 from splunk_ao.config import SplunkAOConfig
-from splunk_ao.deployment import DeploymentMode, O11yConfig, StandaloneConfig
-from splunk_ao.shared.exceptions import AmbiguousConfigurationError, MissingConfigurationError
+from splunk_ao.deployment import DeploymentMode, O11yConfig, StandaloneConfig, resolve_deployment
+from splunk_ao.shared.exceptions import AmbiguousConfigurationError, ConfigurationError, MissingConfigurationError
 
 _DETECTION_ENV_VARS = (
     "SPLUNK_AO_REALM",
     "SPLUNK_AO_O11Y_TOKEN",
     "SPLUNK_AO_O11Y_API_TOKEN",
+    "SPLUNK_AO_O11Y_API_URL",
     "SPLUNK_AO_API_KEY",
     "SPLUNK_AO_CONSOLE_URL",
     "SPLUNK_AO_API_URL",
@@ -176,14 +177,79 @@ def test_require_ingest_token_rejects_crud_only_config() -> None:
 
 def test_api_root_derives_from_realm() -> None:
     cfg = O11yConfig(realm="lab0", o11y_token="tok")
-    assert cfg.api_root == "https://app.lab0.observability.splunkcloud.com"
+    assert cfg.api_root == "https://api.lab0.observability.splunkcloud.com"
 
 
 def test_require_api_url_derives_from_realm() -> None:
     cfg = O11yConfig(realm="lab0", o11y_token="tok")
-    assert cfg.require_api_url() == "https://app.lab0.observability.splunkcloud.com/ao/api/"
+    assert cfg.require_api_url() == "https://api.lab0.observability.splunkcloud.com/ao/api/"
     assert cfg.require_api_url() == f"{cfg.api_root}/ao/api/"
-    assert cfg.require_api_url() == f"{cfg.require_console_url()}ao/api/"
+
+
+def test_api_url_override_replaces_api_root_only() -> None:
+    # Given: an explicit API origin override
+    cfg = O11yConfig(realm="lab0", o11y_token="tok", api_url="https://api.example.com/")
+    # Then: CRUD uses the override while ingest and console stay realm-derived
+    assert cfg.api_root == "https://api.example.com"
+    assert cfg.require_api_url() == "https://api.example.com/ao/api/"
+    assert cfg.otlp_endpoint == "https://ingest.lab0.observability.splunkcloud.com/v2/trace/otlp"
+    assert cfg.require_console_url() == "https://app.lab0.observability.splunkcloud.com/"
+
+
+def test_api_url_override_read_from_env() -> None:
+    with env(
+        SPLUNK_AO_REALM="lab0", SPLUNK_AO_O11Y_TOKEN="tok", SPLUNK_AO_O11Y_API_URL="https://api.lab0.signalfx.com"
+    ):
+        assert O11yConfig.from_env().api_root == "https://api.lab0.signalfx.com"
+
+
+def test_empty_api_url_override_uses_realm_default() -> None:
+    cfg = O11yConfig(realm="lab0", o11y_token="tok", api_url="")
+    assert cfg.api_root == "https://api.lab0.observability.splunkcloud.com"
+
+
+def test_api_url_override_counts_as_o11y_and_conflicts_with_standalone() -> None:
+    with env(SPLUNK_AO_O11Y_API_URL="https://api.lab0.signalfx.com"):
+        assert resolve_deployment() == DeploymentMode.O11Y
+    with env(SPLUNK_AO_O11Y_API_URL="https://api.lab0.signalfx.com", SPLUNK_AO_API_KEY="key"):
+        with pytest.raises(AmbiguousConfigurationError):
+            resolve_deployment()
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        "http://api.lab0.signalfx.com",
+        "http://localhost:8080",
+        "api.lab0.signalfx.com",
+        "https://user:secret@api.lab0.signalfx.com",
+        "https://api.lab0.signalfx.com/ao/api/",
+        "https://api.lab0.signalfx.com?x=1",
+        "https://api.lab0.signalfx.com#frag",
+        "https://[::1",
+        "https://@api.lab0.signalfx.com",
+        "https://api.lab0.signalfx.com:abc",
+        "https://api.lab0.signalfx.com:99999",
+        "https://api.lab0 .signalfx.com",
+    ],
+)
+def test_api_url_override_rejects_invalid_values(value: str) -> None:
+    with pytest.raises(ConfigurationError) as exc_info:
+        O11yConfig(realm="lab0", o11y_token="tok", api_url=value)
+    assert "secret" not in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("https://api.lab0.signalfx.com:", "https://api.lab0.signalfx.com"),
+        ("https://api.lab0.signalfx.com:8443/", "https://api.lab0.signalfx.com:8443"),
+        ("  https://API.lab0.signalfx.com  ", "https://api.lab0.signalfx.com"),
+        ("https://[::1]:8443", "https://[::1]:8443"),
+    ],
+)
+def test_api_url_override_is_rebuilt_from_host_and_port(value: str, expected: str) -> None:
+    assert O11yConfig(realm="lab0", o11y_token="tok", api_url=value).api_root == expected
 
 
 def test_require_console_url_derives_from_realm() -> None:

tests/test_o11y_config.py

@@ -21,6 +21,7 @@
     "SPLUNK_AO_REALM",
     "SPLUNK_AO_O11Y_TOKEN",
     "SPLUNK_AO_O11Y_API_TOKEN",
+    "SPLUNK_AO_O11Y_API_URL",
     "SPLUNK_AO_API_KEY",
     "SPLUNK_AO_API_URL",
     "SPLUNK_AO_CONSOLE_URL",
@@ -57,7 +58,7 @@ def config_env(**overrides: str) -> Iterator[None]:
 
 def _o11y_client(token: str = "tok") -> O11yApiClient:
     client = O11yApiClient(
-        host="https://app.lab0.observability.splunkcloud.com", o11y_token=SecretStr(token), jwt_token=SecretStr("")
+        host="https://api.lab0.observability.splunkcloud.com", o11y_token=SecretStr(token), jwt_token=SecretStr("")
     )
     client.thread_local.client = None
     return client
@@ -99,7 +100,7 @@ async def fake_make_request(
 
     assert captured == {
         "method": RequestMethod.GET,
-        "url": "https://app.lab0.observability.splunkcloud.com/ao/api/projects",
+        "url": "https://api.lab0.observability.splunkcloud.com/ao/api/projects",
         "headers": {"accept": "application/json", "Content-Type": "application/json", "X-SF-Token": "tok"},
         "instrumentation_enabled": False,
     }
@@ -119,7 +120,7 @@ async def fake_make_request(
     monkeypatch.setattr(ApiClient, "make_request", staticmethod(fake_make_request))
     await _o11y_client().arequest(RequestMethod.GET, "/ao/api/projects", {"X-Custom": "value"})
 
-    assert captured["url"] == "https://app.lab0.observability.splunkcloud.com/ao/api/projects"
+    assert captured["url"] == "https://api.lab0.observability.splunkcloud.com/ao/api/projects"
     assert captured["headers"] == {"X-Custom": "value", "X-SF-Token": "tok"}
 
 
@@ -160,6 +161,11 @@ def test_o11y_get_with_api_token_but_no_realm_fails_clearly(monkeypatch: pytest.
             SplunkAOConfig.get()
 
 
+def test_o11y_auth_guard_reports_missing_realm_when_only_api_url_set() -> None:
+    with config_env(SPLUNK_AO_O11Y_API_URL="https://api.lab0.signalfx.com"):
+        assert "SPLUNK_AO_REALM" in (SplunkAOConfig._check_auth_config({}) or "")
+
+
 def test_o11y_auth_guard_requires_at_least_one_token() -> None:
     with config_env(SPLUNK_AO_REALM="us1"):
         error = SplunkAOConfig._check_auth_config({}) or ""
@@ -227,9 +233,9 @@ async def async_fail(*args: object, **kwargs: object) -> None:
 
         assert cfg.jwt_token is None
         assert str(cfg.console_url) == "https://app.lab0.observability.splunkcloud.com/"
-        assert str(cfg.api_url) == "https://app.lab0.observability.splunkcloud.com/ao/api/"
+        assert str(cfg.api_url) == "https://api.lab0.observability.splunkcloud.com/ao/api/"
         assert isinstance(client, O11yApiClient)
-        assert str(client.host) == "https://app.lab0.observability.splunkcloud.com/"
+        assert str(client.host) == "https://api.lab0.observability.splunkcloud.com/"
         assert client.auth_header == {"X-SF-Token": expected_token}
         assert client.ssl_context is False
 
@@ -255,6 +261,27 @@ async def async_fail(*args: object, **kwargs: object) -> None:
     assert client.auth_header == {"X-SF-Token": "api-token"}
 
 
+def test_o11y_get_uses_api_url_override_for_crud_only(monkeypatch: pytest.MonkeyPatch) -> None:
+    def fail(*args: object, **kwargs: object) -> None:
+        raise AssertionError("o11y configuration must not use standalone validation")
+
+    monkeypatch.setattr(SplunkAOConfig, "_instance", None)
+    monkeypatch.setattr(GalileoConfig, "get_jwt_token", staticmethod(fail))
+
+    # Given: an explicit API origin override
+    with config_env(
+        SPLUNK_AO_REALM="lab0", SPLUNK_AO_O11Y_TOKEN="tok", SPLUNK_AO_O11Y_API_URL="https://api.lab0.signalfx.com/"
+    ):
+        cfg = SplunkAOConfig.get(ssl_context=False)
+        client = cfg.api_client
+
+        # Then: CRUD follows the override while the console link stays realm-derived
+        assert str(cfg.api_url) == "https://api.lab0.signalfx.com/ao/api/"
+        assert isinstance(client, O11yApiClient)
+        assert str(client.host) == "https://api.lab0.signalfx.com/"
+        assert str(cfg.console_url) == "https://app.lab0.observability.splunkcloud.com/"
+
+
 def test_ambiguous_environment_fails_before_config_construction(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(SplunkAOConfig, "_instance", None)
     with config_env(SPLUNK_AO_REALM="us1", SPLUNK_AO_O11Y_TOKEN="tok", SPLUNK_AO_API_KEY="key"):