Skip to content

Commit bc84abe

Browse files
committed
feat(bigquery): add bigquery_create_manifest_biglake_table sample to create a BigLake table backed by a Symfoni/Hive manifest file
Problem: The BigQuery documentation (https://cloud.google.com/bigquery/docs/query-open-table-format-using-manifest-files) needs a standalone, tested Python code sample demonstrating how to create a BigLake table backed by a Symfoni/Hive manifest file (region tag `bigquery_create_manifest_biglake_table`). Solution: - Add `create_manifest_biglake_table(project_id: str, dataset_id: str, connection_id: str, manifest_uri: str) -> None` in `bigquery/biglake/bigquery_create_manifest_biglake_table.py` (Create a BigLake external table over open-table-format Parquet files using a manifest file URI and Cloud Resource connection.) - Add unit tests (`bigquery/biglake/bigquery_create_manifest_biglake_table_test.py`) covering both the success path and API error handling (`GoogleAPICallError`)
1 parent 186be98 commit bc84abe

5 files changed

Lines changed: 176 additions & 0 deletions

File tree

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START bigquery_create_manifest_biglake_table]
16+
import sys
17+
18+
from google.api_core import exceptions
19+
from google.cloud import bigquery
20+
21+
22+
def create_manifest_biglake_table(
23+
project_id: str,
24+
dataset_id: str,
25+
connection_id: str,
26+
manifest_uri: str,
27+
) -> None:
28+
"""Creates a BigLake external table backed by a newline-delimited manifest file.
29+
30+
Args:
31+
project_id: Google Cloud project ID (for example, 'example-project').
32+
dataset_id: BigQuery dataset ID where the BigLake table is created
33+
(for example, 'example_dataset').
34+
connection_id: Fully qualified BigQuery Cloud Resource connection ID
35+
(for example, 'example-project.us.example_biglake_conn').
36+
manifest_uri: Cloud Storage URI of the SymLinkTextInputFormat manifest file
37+
(for example, 'gs://example-bucket/manifests/symlink.manifest').
38+
"""
39+
client = bigquery.Client(project=project_id)
40+
# Use a fixed table identifier inside the caller-supplied dataset.
41+
table_id = "example_manifest_biglake_table"
42+
table_ref = f"{project_id}.{dataset_id}.{table_id}"
43+
# Setting file_set_spec_type to NEW_LINE_DELIMITED_MANIFEST tells BigQuery to resolve the
44+
# Parquet data files listed inside the manifest URI via the BigLake connection.
45+
sql = f"""
46+
CREATE EXTERNAL TABLE `{table_ref}`
47+
WITH CONNECTION `{connection_id}`
48+
OPTIONS (
49+
format = 'PARQUET',
50+
uris = ['{manifest_uri}'],
51+
file_set_spec_type = 'NEW_LINE_DELIMITED_MANIFEST'
52+
);
53+
"""
54+
55+
try:
56+
print(f"Creating manifest-backed BigLake table: {table_ref}")
57+
query_job = client.query(sql)
58+
query_job.result()
59+
except exceptions.AlreadyExists as err:
60+
print(
61+
f"error: BigLake table '{table_id}' already exists in dataset '{dataset_id}' "
62+
f"for project '{project_id}' (connection '{connection_id}', "
63+
f"manifest '{manifest_uri}'): {err.message}",
64+
file=sys.stderr,
65+
)
66+
return
67+
except exceptions.GoogleAPICallError as err:
68+
print(
69+
f"error: failed to create BigLake table '{table_id}' in dataset '{dataset_id}' "
70+
f"for project '{project_id}' (connection '{connection_id}', "
71+
f"manifest '{manifest_uri}'): {err.message}",
72+
file=sys.stderr,
73+
)
74+
return
75+
76+
print(f"Resource ID: {table_ref}")
77+
print(f"Connection ID: {connection_id}")
78+
print("File Set Spec Type: NEW_LINE_DELIMITED_MANIFEST")
79+
print(f"Job ID: {query_job.job_id}")
80+
print(f"Manifest URI: {manifest_uri}")
81+
print(f"Statement Type: {query_job.statement_type}")
82+
83+
84+
# [END bigquery_create_manifest_biglake_table]
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from unittest import mock
16+
17+
import bigquery_create_manifest_biglake_table
18+
from google.api_core import exceptions
19+
import pytest
20+
21+
22+
def test_create_manifest_biglake_table_success(
23+
capsys: pytest.CaptureFixture[str],
24+
) -> None:
25+
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
26+
mock_client = mock_client_cls.return_value
27+
mock_job = mock.MagicMock()
28+
mock_job.job_id = "example-job-123"
29+
mock_job.statement_type = "CREATE_EXTERNAL_TABLE"
30+
mock_job.result.return_value = []
31+
mock_client.query.return_value = mock_job
32+
33+
bigquery_create_manifest_biglake_table.create_manifest_biglake_table(
34+
project_id="example-project",
35+
dataset_id="example_dataset",
36+
connection_id="example-project.us.example_biglake_conn",
37+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
38+
)
39+
40+
mock_client.query.assert_called_once()
41+
captured = capsys.readouterr()
42+
assert (
43+
"Resource ID: example-project.example_dataset.example_manifest_biglake_table"
44+
in captured.out
45+
)
46+
assert "File Set Spec Type: NEW_LINE_DELIMITED_MANIFEST" in captured.out
47+
assert "Statement Type: CREATE_EXTERNAL_TABLE" in captured.out
48+
49+
50+
def test_create_manifest_biglake_table_already_exists(
51+
capsys: pytest.CaptureFixture[str],
52+
) -> None:
53+
with mock.patch("google.cloud.bigquery.Client") as mock_client_cls:
54+
mock_client = mock_client_cls.return_value
55+
mock_client.query.side_effect = exceptions.AlreadyExists("Table exists")
56+
57+
bigquery_create_manifest_biglake_table.create_manifest_biglake_table(
58+
project_id="example-project",
59+
dataset_id="example_dataset",
60+
connection_id="example-project.us.example_biglake_conn",
61+
manifest_uri="gs://example-bucket/manifests/symlink.manifest",
62+
)
63+
64+
captured = capsys.readouterr()
65+
assert "error:" in captured.err
66+
assert "example_dataset" in captured.err

‎bigquery/biglake/noxfile_config.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2026 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
TEST_CONFIG_OVERRIDE = {
16+
# You can opt out from the test for specific Python versions.
17+
"ignored_versions": ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"],
18+
# Enforce type hints on new samples.
19+
"enforce_type_hints": True,
20+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
21+
"pip_version_override": None,
22+
"envs": {},
23+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pytest==9.0.3; python_version >= "3.10"

‎bigquery/biglake/requirements.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
google-cloud-bigquery==3.47.0
2+
google-cloud-bigquery-storage==2.43.0

0 commit comments

Comments
 (0)