Skip to content

fix(mode): Ingest Mode report query metadata [Backport #32533 - 1.13] #33147

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
harshsoni2024 merged 3 commits into from
Sep 11, 2026
Merged

fix(mode): Ingest Mode report query metadata [Backport #32533 - 1.13] #33147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fc7b12b
fix(mode): Ingest Mode report query metadata (#32533)
harshsoni2024 Sep 8, 2026
ad466b8
Address Mode report pagination review feedback
harshsoni2024 Sep 10, 2026
052b430
Make the Mode unit test directory a package
harshsoni2024 Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 0 additions & 128 deletions ingestion/.basedpyright/baseline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24813,30 +24813,6 @@
"lineCount": 1
}
},
{
"code": "reportIndexIssue",
"range": {
"startColumn": 22,
"endColumn": 42,
"lineCount": 1
}
},
{
"code": "reportOptionalSubscript",
"range": {
"startColumn": 22,
"endColumn": 42,
"lineCount": 1
}
},
{
"code": "reportReturnType",
"range": {
"startColumn": 19,
"endColumn": 27,
"lineCount": 1
}
},
{
"code": "reportReturnType",
"range": {
Expand Down Expand Up @@ -27049,30 +27025,6 @@
"lineCount": 3
}
},
{
"code": "reportReturnType",
"range": {
"startColumn": 15,
"endColumn": 41,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 28,
"endColumn": 63,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 25,
"endColumn": 66,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
Expand All @@ -27081,38 +27033,6 @@
"lineCount": 6
}
},
{
"code": "reportAttributeAccessIssue",
"range": {
"startColumn": 56,
"endColumn": 73,
"lineCount": 1
}
},
{
"code": "reportAttributeAccessIssue",
"range": {
"startColumn": 48,
"endColumn": 54,
"lineCount": 1
}
},
{
"code": "reportAttributeAccessIssue",
"range": {
"startColumn": 39,
"endColumn": 56,
"lineCount": 1
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 14,
"endColumn": 45,
"lineCount": 1
}
},
{
"code": "reportGeneralTypeIssues",
"range": {
Expand All @@ -27121,54 +27041,6 @@
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 28,
"endColumn": 25,
"lineCount": 6
}
},
{
"code": "reportReturnType",
"range": {
"startColumn": 30,
"endColumn": 25,
"lineCount": 3
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 38,
"endColumn": 47,
"lineCount": 1
}
},
{
"code": "reportArgumentType",
"range": {
"startColumn": 61,
"endColumn": 72,
"lineCount": 1
}
},
{
"code": "reportCallIssue",
"range": {
"startColumn": 18,
"endColumn": 13,
"lineCount": 7
}
},
{
"code": "reportAttributeAccessIssue",
"range": {
"startColumn": 51,
"endColumn": 68,
"lineCount": 1
}
},
{
"code": "reportCallIssue",
"range": {
Expand Down
93 changes: 60 additions & 33 deletions ingestion/src/metadata/ingestion/source/dashboard/mode/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@
"""
REST Auth & Client for Mode
"""

import json
import traceback
from base64 import b64encode
from typing import Optional
from typing import Any, Dict, List, Optional, cast

from metadata.ingestion.connections.source_api_client import TrackedREST
from metadata.ingestion.ometa.client import ClientConfig
Expand All @@ -24,7 +26,7 @@


EMBEDDED = "_embedded"
COLLECTIONS = "collections"
SPACES = "spaces"
TOKEN = "token"
REPORTS = "reports"
QUERIES = "queries"
Expand All @@ -40,6 +42,14 @@
HREF = "href"


def _report_key(report: Dict[str, Any]) -> str:
"""Identify a report for de-duplication, tolerating a missing token."""
token = report.get(TOKEN)
if token:
return str(token)
return json.dumps(report, sort_keys=True, default=str)


class ModeApiClient:
"""
REST Auth & Client for Mode
Expand All @@ -66,56 +76,73 @@ def __init__(self, config):

def fetch_all_reports(
self, workspace_name: str, filter: Optional[str] = "all"
) -> Optional[list]:
) -> List[Dict[str, Any]]:
"""Method to fetch all reports for Mode

Mode neither documents a stable report page size nor guarantees an empty page
past the last one: an out-of-range page may be clamped back to the last page,
and a page parameter the API ignores repeats page one indefinitely. Pagination
therefore stops once a page carries no unseen report, which terminates in all
of those cases without assuming how many records a full page holds.

Args:
workspace_name:
filter:
Returns:
dict
the report records of every visible space
"""
if filter not in ["custom", "all"]:
logger.warning(
"Invalid value for filter. Should be one of ['custom', 'all']"
raise ValueError(
f"Invalid Mode filter [{filter}]. Expected one of ['custom', 'all']"
)
return

all_reports = []
all_reports: List[Dict[str, Any]] = []
filter_param = f"?filter={filter}"
response_collections = self.client.get(
f"/{workspace_name}/{COLLECTIONS}{filter_param}"
response_spaces = cast(
Dict[str, Any],
self.client.get(f"/{workspace_name}/{SPACES}{filter_param}"),
)
collections = response_collections[EMBEDDED]["spaces"]
for collection in collections:
response_reports = self.get_all_reports_for_collection(
workspace_name=workspace_name,
collection_token=collection.get(TOKEN),
)
if response_reports:
spaces = response_spaces[EMBEDDED][SPACES]
for space in spaces:
seen_reports = set()
page = 1
while True:
response_reports = self.get_reports_for_space(
workspace_name=workspace_name,
space_token=space[TOKEN],
page=page,
)
reports = response_reports[EMBEDDED][REPORTS]
all_reports.extend(reports)
new_reports = [
report
for report in reports
if _report_key(report) not in seen_reports
]
if not new_reports:
break
seen_reports.update(_report_key(report) for report in new_reports)
all_reports.extend(new_reports)
page += 1
Comment thread
gitar-bot[bot] marked this conversation as resolved.
return all_reports

def get_all_reports_for_collection(
self, workspace_name: str, collection_token: str
) -> Optional[dict]:
"""Method to fetch all reports for a collection
def get_reports_for_space(
self, workspace_name: str, space_token: str, page: int
) -> Dict[str, Any]:
"""Fetch one page of reports for a space.

Args:
workspace_name:
collection_token:
space_token:
page:
Returns:
dict
"""
try:
response = self.client.get(
f"/{workspace_name}/{COLLECTIONS}/{collection_token}/{REPORTS}"
)
return response
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(f"Error fetching charts: {exc}")

return None
return cast(
Dict[str, Any],
self.client.get(
f"/{workspace_name}/{SPACES}/{space_token}/{REPORTS}?page={page}"
),
)

def get_all_queries(self, workspace_name: str, report_token: str) -> Optional[dict]:
"""Method to fetch all queries
Expand Down
Loading
Loading