From 6e46f2119fb29bbaf1ef2a51a0d25137fbf62aff Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 6 Nov 2026 11:36:42 +0000 Subject: [PATCH] fix(django-google-spanner): support JSONField read and write operations --- .../django_spanner/__init__.py | 13 - .../django_spanner/compiler.py | 23 ++ .../django_spanner/features.py | 36 ++- .../django_spanner/lookups.py | 44 ++++ .../django_spanner/operations.py | 68 ++++++ .../tests/mockserver_tests/models.py | 5 + .../tests/mockserver_tests/test_basics.py | 223 +++++++++++++++++- .../system/django_spanner/test_json_field.py | 39 ++- .../unit/django_spanner/test_compiler.py | 31 +++ .../unit/django_spanner/test_features.py | 4 + .../tests/unit/django_spanner/test_lookups.py | 73 ++++++ .../unit/django_spanner/test_operations.py | 139 +++++++++++ 12 files changed, 673 insertions(+), 25 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/__init__.py b/packages/django-google-spanner/django_spanner/__init__.py index fc8cb56b2104..c85b9a2e00de 100644 --- a/packages/django-google-spanner/django_spanner/__init__.py +++ b/packages/django-google-spanner/django_spanner/__init__.py @@ -13,7 +13,6 @@ import django from django.db import DEFAULT_DB_ALIAS -from django.db.models import JSONField from django.db.models.fields import ( NOT_PROVIDED, AutoField, @@ -27,7 +26,6 @@ from google.api_core.datetime_helpers import ( DatetimeWithNanoseconds, ) -from google.cloud.spanner_v1 import JsonObject from .functions import register_functions from .lookups import register_lookups @@ -84,17 +82,6 @@ def autofield_init(self, *args, **kwargs): SmallAutoField.__init__ = autofield_init BigAutoField.__init__ = autofield_init - -def get_prep_value(self, value): - # Json encoding and decoding for spanner is done in python-spanner. - if not isinstance(value, JsonObject) and isinstance(value, dict): - return JsonObject(value) - - return value - - -JSONField.get_prep_value = get_prep_value - old_datetimewithnanoseconds_eq = getattr(DatetimeWithNanoseconds, "__eq__", None) diff --git a/packages/django-google-spanner/django_spanner/compiler.py b/packages/django-google-spanner/django_spanner/compiler.py index 6a01a37f0483..f0c0b5ff88c9 100644 --- a/packages/django-google-spanner/django_spanner/compiler.py +++ b/packages/django-google-spanner/django_spanner/compiler.py @@ -5,6 +5,7 @@ # https://developers.google.com/open-source/licenses/bsd from django.core.exceptions import EmptyResultSet +from django.db.models.fields.json import JSONField from django.db.models.sql.compiler import ( SQLAggregateCompiler as BaseSQLAggregateCompiler, ) @@ -31,6 +32,28 @@ class SQLCompiler(BaseSQLCompiler): functionality. """ + def get_converters(self, expressions): + converters = {} + for i, expression in enumerate(expressions): + if expression: + backend_converters = self.connection.ops.get_db_converters(expression) + field_converters = expression.get_db_converters(self.connection) + output_field = getattr(expression, "output_field", None) + if ( + self.connection.ops.convert_jsonfield_value in backend_converters + and isinstance(output_field, JSONField) + and type(output_field).from_db_value is JSONField.from_db_value + ): + field_converters = [ + converter + for converter in field_converters + if converter != output_field.from_db_value + ] + convs = backend_converters + field_converters + if convs: + converters[i] = (convs, expression) + return converters + def get_combinator_sql(self, combinator, all): """Override the native Django method. diff --git a/packages/django-google-spanner/django_spanner/features.py b/packages/django-google-spanner/django_spanner/features.py index 8b182ebdb4e0..4d81724cc42d 100644 --- a/packages/django-google-spanner/django_spanner/features.py +++ b/packages/django-google-spanner/django_spanner/features.py @@ -57,16 +57,16 @@ def introspected_field_types(self): supports_timezones = False supports_transactions = True if USE_EMULATOR: - # Emulator does not support json. - supports_json_field = False # Emulator does not support check constraints. supports_column_check_constraints = False supports_table_check_constraints = False else: supports_column_check_constraints = True supports_table_check_constraints = True - supports_json_field = True - supports_primitives_in_json_field = False + has_native_json_field = True + supports_json_field = True + supports_primitives_in_json_field = True + supports_json_field_contains = False supports_composite_primary_keys = True # Spanner does not support order by null modifiers. supports_order_by_nulls_modifier = False @@ -107,6 +107,7 @@ def introspected_field_types(self): "constraints.tests.CheckConstraintTests.test_validate_boolean_expressions", "constraints.tests.CheckConstraintTests.test_validate_custom_error", "constraints.tests.CheckConstraintTests.test_validate_generated_field_stored", + "constraints.tests.CheckConstraintTests.test_validate_jsonfield_exact", "constraints.tests.CheckConstraintTests.test_validate_pk_field", "constraints.tests.UniqueConstraintTests.test_validate_case_when", "constraints.tests.UniqueConstraintTests.test_validate_expression_condition", @@ -132,6 +133,17 @@ def introspected_field_types(self): "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_lookup_name_sql_injection", "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_time_comparison", "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_trunc_timezone_applied_before_truncation", + "db_functions.json.test_json_array.JSONArrayObjectTests.test_nested_json_array_object", + "db_functions.json.test_json_array.JSONArrayObjectTests.test_nested_json_object_array", + "db_functions.json.test_json_array.JSONArrayObjectTests.test_order_by_nested_key", + "db_functions.json.test_json_array.JSONArrayTests.test_basic", + "db_functions.json.test_json_array.JSONArrayTests.test_empty", + "db_functions.json.test_json_array.JSONArrayTests.test_expressions", + "db_functions.json.test_json_array.JSONArrayTests.test_nested_empty_json_array", + "db_functions.json.test_json_array.JSONArrayTests.test_nested_json_array", + "db_functions.json.test_json_array.JSONArrayTests.test_order_by_key", + "db_functions.json.test_json_array.JSONArrayTests.test_order_by_nested_key", + "db_functions.json.test_json_array.JSONArrayTests.test_textfield", "db_functions.json.test_json_object.JSONObjectTests.test_basic", "db_functions.json.test_json_object.JSONObjectTests.test_empty", "db_functions.json.test_json_object.JSONObjectTests.test_expressions", @@ -1512,8 +1524,19 @@ def introspected_field_types(self): "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", + "model_fields.test_jsonfield.TestQuerying.test_deep_values", + "model_fields.test_jsonfield.TestQuerying.test_expression_wrapper_key_transform", + "model_fields.test_jsonfield.TestQuerying.test_has_key_literal_lookup", + "model_fields.test_jsonfield.TestQuerying.test_has_key_number", + "model_fields.test_jsonfield.TestQuerying.test_icontains", + "model_fields.test_jsonfield.TestQuerying.test_key_sql_injection_escape", + "model_fields.test_jsonfield.TestQuerying.test_key_transform_raw_expression", + "model_fields.test_jsonfield.TestQuerying.test_lookups_special_chars", + "model_fields.test_jsonfield.TestQuerying.test_lookups_special_chars_double_quotes", + "model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_raw_expression", "model_fields.test_uuid.TestSerialization.test_dumping", "queries.test_bulk_update.BulkUpdateNoteTests.test_unsaved_models", + "queries.test_bulk_update.BulkUpdateTests.test_json_field_sql_null", "queries.test_bulk_update.BulkUpdateTests.test_large_batch", "queries.test_bulk_update.BulkUpdateTests.test_unsaved_parent", "queries.test_qs_combinators.QuerySetSetOperationTests.test_difference_with_values", @@ -2283,6 +2306,7 @@ def introspected_field_types(self): django_6_0_skip_tests = ( # Spanner uses random int64 IDs; test assumes monotonic ordering matching insertion order. + "prefetch_related.tests.PrefetchRelatedMTICacheTests.test_grandparent_m2m_available_in_child", "prefetch_related.tests.PrefetchRelatedMTICacheTests.test_parent_m2m_available_in_child", # Client-side AutoField ID generation sets pk before save; _is_pk_set() triggers refresh_from_db instead of AttributeError. "defer_regress.tests.DeferCopyInstanceTests.test_bulk_create", @@ -2303,6 +2327,10 @@ def introspected_field_types(self): "bulk_create.tests.BulkCreateTransactionTests.test_objs_with_and_without_pk", # Tie-breaker ordering on pk assumes sequential integer IDs; Spanner uses random IDs. "ordering.tests.OrderingTests.test_order_by_case_when_constant_value", + # Spanner GoogleSQL JSONPath does not support negative array indices (e.g. $[-1]). + "model_fields.test_jsonfield.TestQuerying.test_deep_negative_lookup_array", + "model_fields.test_jsonfield.TestQuerying.test_deep_negative_lookup_mixed", + "model_fields.test_jsonfield.TestQuerying.test_shallow_list_negative_lookup", ) if django.VERSION >= (6, 0): diff --git a/packages/django-google-spanner/django_spanner/lookups.py b/packages/django-google-spanner/django_spanner/lookups.py index 7346ec83bb17..3c44a6869a1e 100644 --- a/packages/django-google-spanner/django_spanner/lookups.py +++ b/packages/django-google-spanner/django_spanner/lookups.py @@ -4,6 +4,11 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +from django.db.models.fields.json import ( + HasKeyLookup, + KeyTextTransform, + KeyTransform, +) from django.db.models.lookups import ( Contains, EndsWith, @@ -263,6 +268,42 @@ def cast_param_to_float(self, compiler, connection): return sql, tuple(params) +def _compile_json_path(connection, key_transforms): + if hasattr(connection.ops, "compile_json_path"): + return connection.ops.compile_json_path(key_transforms) + from django.db.models.fields.json import compile_json_path + + return compile_json_path(key_transforms) + + +def _quote_json_path(json_path): + """Inline a compiled JSONPath as a safely escaped GoogleSQL string literal.""" + escaped = json_path.replace("\\", "\\\\").replace("'", "\\'").replace("%", "%%") + return f"'{escaped}'" + + +def key_transform_as_spanner(self, compiler, connection): + lhs, params, key_transforms = self.preprocess_lhs(compiler, connection) + json_path = _quote_json_path(_compile_json_path(connection, key_transforms)) + return f"JSON_QUERY({lhs}, {json_path})", tuple(params) + + +def key_text_transform_as_spanner(self, compiler, connection): + lhs, params, key_transforms = self.preprocess_lhs(compiler, connection) + json_path = _quote_json_path(_compile_json_path(connection, key_transforms)) + return f"JSON_VALUE({lhs}, {json_path})", tuple(params) + + +def has_key_lookup_as_spanner(self, compiler, connection): + sql_parts = [] + params = [] + for lhs_sql, lhs_params, rhs_json_path in self._as_sql_parts(compiler, connection): + json_path = _quote_json_path(rhs_json_path) + sql_parts.append(f"JSON_QUERY({lhs_sql}, {json_path}) IS NOT NULL") + params.extend(lhs_params) + return self._combine_sql_parts(sql_parts), tuple(params) + + def register_lookups(): """Registers the above methods with the corersponding Django classes.""" Contains.as_spanner = contains @@ -279,3 +320,6 @@ def register_lookups(): GreaterThanOrEqual.as_spanner = cast_param_to_float LessThan.as_spanner = cast_param_to_float LessThanOrEqual.as_spanner = cast_param_to_float + KeyTransform.as_spanner = key_transform_as_spanner + KeyTextTransform.as_spanner = key_text_transform_as_spanner + HasKeyLookup.as_spanner = has_key_lookup_as_spanner diff --git a/packages/django-google-spanner/django_spanner/operations.py b/packages/django-google-spanner/django_spanner/operations.py index 5a5b8e3e4612..e7e6a8f1fe2b 100644 --- a/packages/django-google-spanner/django_spanner/operations.py +++ b/packages/django-google-spanner/django_spanner/operations.py @@ -4,6 +4,7 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +import json import os import re from base64 import b64decode @@ -12,14 +13,43 @@ from django.conf import settings from django.db.backends.base.operations import BaseDatabaseOperations +from django.db.models.fields.json import JSONField from django.db.utils import DatabaseError from django.utils import timezone from django.utils.duration import duration_microseconds from google.cloud.spanner_dbapi.parse_utils import ( + TYPES_MAP, DateStr, TimestampStr, escape_name, ) +from google.cloud.spanner_v1 import JsonObject, param_types + + +class _JsonString(JsonObject): + """Wrap a pre-serialized JSON string for spanner_dbapi without re-parsing.""" + + def __init__(self, raw_json): + self._is_null = False + self._is_array = False + self._is_scalar_value = False + self._raw_json = raw_json + + def serialize(self): + return self._raw_json + + def __repr__(self): + return f"_JsonString({self._raw_json!r})" + + def __eq__(self, other): + if isinstance(other, _JsonString): + return self._raw_json == other._raw_json + if isinstance(other, JsonObject): + return self._raw_json == other.serialize() + return False + + +TYPES_MAP[_JsonString] = param_types.JSON class DatabaseOperations(BaseDatabaseOperations): @@ -291,6 +321,13 @@ def adapt_timefield_value(self, value): "0001-01-01T" + value.isoformat(timespec="microseconds") + "Z" ) + def adapt_json_value(self, value, encoder): + """Wrap Django's serialized JSON string for Spanner JSON parameter binding.""" + raw_json = super().adapt_json_value(value, encoder) + if raw_json is None: + return None + return _JsonString(raw_json) + def get_db_converters(self, expression): """Get a list of functions needed to convert field data. @@ -310,6 +347,8 @@ def get_db_converters(self, expression): converters.append(self.convert_binaryfield_value) elif internal_type == "UUIDField": converters.append(self.convert_uuidfield_value) + elif internal_type == "JSONField": + converters.append(self.convert_jsonfield_value) return converters def convert_binaryfield_value(self, value, expression, connection): @@ -406,6 +445,35 @@ def convert_uuidfield_value(self, value, expression, connection): value = UUID(value) return value + def convert_jsonfield_value(self, value, expression, connection): + """Convert Spanner JsonObject received from DB to a Python value for Django.""" + if value is None: + return None + output_field = getattr(expression, "output_field", None) + has_custom_from_db_value = ( + isinstance(output_field, JSONField) + and type(output_field).from_db_value is not JSONField.from_db_value + ) + decoder = getattr(output_field, "decoder", None) + if isinstance(value, JsonObject): + if value._is_null: + return None + if has_custom_from_db_value: + return value.serialize() + if decoder is not None: + return json.loads(value.serialize(), cls=decoder) + if value._is_array: + return value._array_value + if value._is_scalar_value: + return value._simple_value + return dict(value) + if isinstance(value, str) and not has_custom_from_db_value: + try: + return json.loads(value, cls=decoder) + except json.JSONDecodeError: + return value + return value + def date_extract_sql(self, lookup_type, field_name, params=None): """Extract date from the lookup. diff --git a/packages/django-google-spanner/tests/mockserver_tests/models.py b/packages/django-google-spanner/tests/mockserver_tests/models.py index 66cafc60412f..ac2de7d9ae4a 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/models.py +++ b/packages/django-google-spanner/tests/mockserver_tests/models.py @@ -12,9 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from django.core.serializers.json import DjangoJSONEncoder from django.db import models class Singer(models.Model): first_name = models.CharField(max_length=200) last_name = models.CharField(max_length=200) + + +class Detail(models.Model): + value = models.JSONField(null=True, encoder=DjangoJSONEncoder) diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index f435cad54b53..e728c1e3570e 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -11,8 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import uuid +from decimal import Decimal from unittest import mock +import google.cloud.spanner_v1.types.result_set as result_set +import google.cloud.spanner_v1.types.type as spanner_type from django.db import connection, models from google.cloud.spanner_v1 import ( BatchCreateSessionsRequest, @@ -24,12 +29,13 @@ from tests.mockserver_tests.mock_server_test_base import ( MockServerTestBase, + add_result, add_select1_result, add_singer_query_result, add_single_result, add_update_count, ) -from tests.mockserver_tests.models import Singer +from tests.mockserver_tests.models import Detail, Singer from tests.settings import DATABASES @@ -183,3 +189,218 @@ def test_delete_all_without_dbapi_intervention(self): requests[0].sql, "DELETE FROM tests_singer WHERE 1=1", ) + + def _make_detail_result_set(self, rows): + select_result = result_set.ResultSet( + dict( + metadata=result_set.ResultSetMetadata( + dict( + row_type=spanner_type.StructType( + dict( + fields=[ + spanner_type.StructType.Field( + dict( + name="id", + type=spanner_type.Type( + dict(code=TypeCode.INT64) + ), + ) + ), + spanner_type.StructType.Field( + dict( + name="value", + type=spanner_type.Type( + dict(code=TypeCode.JSON) + ), + ) + ), + ] + ) + ) + ) + ), + ) + ) + select_result.rows.extend(rows) + return select_result + + def test_insert_and_fetch_json_field(self): + add_single_result( + "INSERT INTO tests_detail (id, value) VALUES (@a0, @a1) THEN RETURN id", + "id", + TypeCode.INT64, + [("1",)], + ) + add_result( + "INSERT INTO tests_detail (id, value) VALUES (@a0, @a1) THEN RETURN id, value", + self._make_detail_result_set([("1", "null")]), + ) + add_single_result( + "INSERT INTO tests_detail (id, value) VALUES (@a0, @a1), (@a2, @a3)", + "id", + TypeCode.INT64, + [("10",), ("11",)], + ) + add_single_result( + "UPDATE tests_detail SET value = @a0 WHERE tests_detail.id = @a1", + "id", + TypeCode.INT64, + [("1",)], + ) + add_result( + "SELECT tests_detail.id, tests_detail.value FROM tests_detail", + self._make_detail_result_set( + [ + ("1", '{"age": "26", "name": "Jakob"}'), + ("2", "[1, 2, 3]"), + ("3", None), + ("4", '"hello"'), + ("5", "42"), + ("6", "true"), + ("7", "3.14"), + ("8", "null"), + ] + ), + ) + add_result( + "SELECT tests_detail.id, tests_detail.value FROM tests_detail WHERE JSON_QUERY(tests_detail.value, '$.\"name\"') = @a0", + self._make_detail_result_set([("1", '{"age": "26", "name": "Jakob"}')]), + ) + add_result( + "SELECT tests_detail.id, tests_detail.value FROM tests_detail WHERE JSON_QUERY(tests_detail.value, '$.\"name\"') IS NOT NULL", + self._make_detail_result_set([("1", '{"age": "26", "name": "Jakob"}')]), + ) + add_result( + "SELECT tests_detail.id, tests_detail.value FROM tests_detail WHERE tests_detail.value = @a0", + self._make_detail_result_set([("8", "null")]), + ) + add_result( + "SELECT tests_detail.id, tests_detail.value FROM tests_detail WHERE tests_detail.value IS NULL", + self._make_detail_result_set([("3", None)]), + ) + + # 1. Dict insert + detail = Detail(value={"name": "Jakob", "age": "26"}) + detail.save() + self.assertEqual(1, detail.id) + + # 2. SQL NULL insert + detail_sql_null = Detail(value=None) + detail_sql_null.save() + + # 3. JSON null insert via Value(None, JSONField()) + detail_json_null = Detail(value=models.Value(None, models.JSONField())) + detail_json_null.save() + + # 4. Primitives insert (str, int, bool, float) + for primitive in ("hello", 42, True, 3.14): + Detail(value=primitive).save() + + # 5. Custom encoder (DjangoJSONEncoder) insert with UUID, Decimal, date + custom_uuid = uuid.UUID("12345678-1234-5678-1234-567812345678") + Detail( + value={ + "uuid": custom_uuid, + "amount": Decimal("19.99"), + "date": datetime.date(2026, 4, 8), + } + ).save() + + # 6. bulk_create and QuerySet.update + Detail.objects.bulk_create( + [ + Detail(id=10, value={"bulk": 1}), + Detail(id=11, value=None), + ] + ) + Detail.objects.filter(id=1).update(value={"name": "Updated"}) + + # 7. Fetch all (dict, list, SQL NULL, primitives, JSON null) + details = list(Detail.objects.all()) + self.assertEqual(len(details), 8) + self.assertEqual(details[0].value, {"name": "Jakob", "age": "26"}) + self.assertIs(type(details[0].value), dict) + self.assertEqual(details[1].value, [1, 2, 3]) + self.assertIs(type(details[1].value), list) + self.assertIsNone(details[2].value) + self.assertEqual(details[3].value, "hello") + self.assertEqual(details[4].value, 42) + self.assertIs(details[5].value, True) + self.assertEqual(details[6].value, 3.14) + self.assertIsNone(details[7].value) + + # 8. Filter by JSON key, has_key, JSON null (Value(None, JSONField()) & None), and SQL NULL (value__isnull=True) + by_key = list(Detail.objects.filter(value__name="Jakob")) + self.assertEqual(len(by_key), 1) + self.assertEqual(by_key[0].value, {"name": "Jakob", "age": "26"}) + + by_has_key = list(Detail.objects.filter(value__has_key="name")) + self.assertEqual(len(by_has_key), 1) + + by_json_null_expr = list( + Detail.objects.filter(value=models.Value(None, models.JSONField())) + ) + self.assertEqual(len(by_json_null_expr), 1) + self.assertIsNone(by_json_null_expr[0].value) + + by_json_null_direct = list(Detail.objects.filter(value=None)) + self.assertEqual(len(by_json_null_direct), 1) + self.assertIsNone(by_json_null_direct[0].value) + + by_sql_null = list(Detail.objects.filter(value__isnull=True)) + self.assertEqual(len(by_sql_null), 1) + self.assertIsNone(by_sql_null[0].value) + + exec_requests = [ + req + for req in self.spanner_service.requests + if isinstance(req, ExecuteSqlRequest) + ] + + # Dict insert -> JSON string + self.assertEqual(exec_requests[0].param_types["a1"].code, TypeCode.JSON) + self.assertEqual( + exec_requests[0].params["a1"], + '{"name": "Jakob", "age": "26"}', + ) + + # SQL NULL insert -> None (NULL_VALUE in proto Struct) + self.assertIsNone(exec_requests[1].params["a1"]) + + # JSON null insert -> "null" with TypeCode.JSON + self.assertEqual(exec_requests[2].param_types["a1"].code, TypeCode.JSON) + self.assertEqual(exec_requests[2].params["a1"], "null") + + # Primitives insert + self.assertEqual(exec_requests[3].params["a1"], '"hello"') + self.assertEqual(exec_requests[4].params["a1"], "42") + self.assertEqual(exec_requests[5].params["a1"], "true") + self.assertEqual(exec_requests[6].params["a1"], "3.14") + + # Custom encoder insert + self.assertEqual( + exec_requests[7].params["a1"], + '{"uuid": "12345678-1234-5678-1234-567812345678", "amount": "19.99", "date": "2026-04-08"}', + ) + + # bulk_create request + self.assertEqual(exec_requests[8].params["a1"], '{"bulk": 1}') + self.assertIsNone(exec_requests[8].params["a3"]) + + # update request + self.assertEqual( + exec_requests[9].params["a0"], + '{"name": "Updated"}', + ) + + # filter(value__name="Jakob") request (JSONPath is inlined as a literal, @a0 is JSON "Jakob") + self.assertEqual(exec_requests[11].param_types["a0"].code, TypeCode.JSON) + self.assertEqual(exec_requests[11].params["a0"], '"Jakob"') + + # filter(value=Value(None, JSONField())) request + self.assertEqual(exec_requests[13].param_types["a0"].code, TypeCode.JSON) + self.assertEqual(exec_requests[13].params["a0"], "null") + + # filter(value=None) request -> also queries JSON null ("null", TypeCode.JSON) + self.assertEqual(exec_requests[14].param_types["a0"].code, TypeCode.JSON) + self.assertEqual(exec_requests[14].params["a0"], "null") diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py index 396bd8c4cfb1..be3afd02a1ea 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py @@ -4,12 +4,9 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd -import unittest - -from django.db import connection +from django.db import connection, models from django.test import TransactionTestCase -from django_spanner import USE_EMULATOR from tests.system.django_spanner.utils import ( setup_database, setup_instance, @@ -20,7 +17,6 @@ from .models import Detail -@unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") class TestJsonField(TransactionTestCase): @classmethod def setUpClass(cls): @@ -45,7 +41,36 @@ def test_insert_and_fetch_value(self): """ json_data = Detail(value={"name": "Jakob", "age": "26"}) json_data.save() - qs1 = Detail.objects.all() - self.assertEqual(qs1[0].value, {"name": "Jakob", "age": "26"}) + json_null = Detail(value=models.Value(None, models.JSONField())) + json_null.save() + json_prim = Detail(value="hello") + json_prim.save() + + by_key = list(Detail.objects.filter(value__name="Jakob")) + self.assertEqual(len(by_key), 1) + self.assertEqual(by_key[0].value, {"name": "Jakob", "age": "26"}) + + by_has_key = list(Detail.objects.filter(value__has_key="name")) + self.assertEqual(len(by_has_key), 1) + self.assertEqual(by_has_key[0].value, {"name": "Jakob", "age": "26"}) + + by_json_null = list( + Detail.objects.filter(value=models.Value(None, models.JSONField())) + ) + self.assertEqual(len(by_json_null), 1) + self.assertIsNone(by_json_null[0].value) + + by_none_exact = list(Detail.objects.filter(value=None)) + self.assertEqual(len(by_none_exact), 1) + self.assertEqual(by_none_exact[0].id, json_null.id) + self.assertIsNone(by_none_exact[0].value) + + by_sql_null = list(Detail.objects.filter(value__isnull=True)) + self.assertEqual(len(by_sql_null), 0) + + by_prim = list(Detail.objects.filter(value="hello")) + self.assertEqual(len(by_prim), 1) + self.assertEqual(by_prim[0].value, "hello") + # Delete data from Detail table. Detail.objects.all().delete() diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_compiler.py b/packages/django-google-spanner/tests/unit/django_spanner/test_compiler.py index 4a618a071832..260272e715bf 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_compiler.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_compiler.py @@ -268,3 +268,34 @@ def test_update_with_subquery_does_not_add_where_to_subquery(self): end_index = sql.index(")") subquery_sql = sql[start_index:end_index] self.assertNotIn("WHERE", subquery_sql) + + def test_get_converters_deduplicates_jsonfield_from_db_value(self): + from django.db.models import JSONField + from django.db.models.expressions import Col + from django.db.models.sql import Query + + from django_spanner.compiler import SQLCompiler + from tests.unit.django_spanner.models import Number + + field = JSONField() + field.set_attributes_from_name("value") + expr = Col("tests_detail", field) + query = Query(Number) + compiler = SQLCompiler(query, self.connection, "default") + converters = compiler.get_converters([expr]) + self.assertIn(0, converters) + convs, _ = converters[0] + self.assertIn(self.connection.ops.convert_jsonfield_value, convs) + self.assertNotIn(field.from_db_value, convs) + + class TaggedJSONField(JSONField): + def from_db_value(self, value, expression, connection): + return value + + sub_field = TaggedJSONField() + sub_field.set_attributes_from_name("value") + sub_expr = Col("tests_detail", sub_field) + sub_converters = compiler.get_converters([sub_expr]) + sub_convs, _ = sub_converters[0] + self.assertIn(self.connection.ops.convert_jsonfield_value, sub_convs) + self.assertIn(sub_field.from_db_value, sub_convs) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_features.py b/packages/django-google-spanner/tests/unit/django_spanner/test_features.py index 00023c45ce31..2329b4b25a9d 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_features.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_features.py @@ -34,3 +34,7 @@ def test_spanner_specific_feature_flags(self): self.assertFalse(features.can_rollback_tests) self.assertEqual(features.max_query_params, 900) self.assertTrue(features.requires_literal_defaults) + self.assertTrue(features.has_native_json_field) + self.assertTrue(features.supports_json_field) + self.assertTrue(features.supports_primitives_in_json_field) + self.assertFalse(features.supports_json_field_contains) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_lookups.py b/packages/django-google-spanner/tests/unit/django_spanner/test_lookups.py index 7b8045d7f806..3d53acc0dcc7 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_lookups.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_lookups.py @@ -322,3 +322,76 @@ def test_iregex_with_lhs_params(self): compiler = SQLCompiler(qs.query, self.connection, "default") sql, params = compiler.as_sql() self.assertEqual(params, ("User: ", "", "", "(?i)^john")) + + def test_json_key_transforms_and_has_key(self): + from unittest.mock import MagicMock + + from django.db.models import JSONField + from django.db.models.expressions import Col + from django.db.models.fields.json import ( + HasKey, + KeyTextTransform, + KeyTransform, + ) + + field = JSONField() + field.set_attributes_from_name("value") + col = Col("tests_detail", field) + compiler = MagicMock() + compiler.compile.return_value = ("tests_detail.value", []) + + kt = KeyTransform("name", col) + sql, params = kt.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + "JSON_QUERY(tests_detail.value, '$.\"name\"')", + ) + self.assertEqual(params, ()) + + ktt = KeyTextTransform("name", col) + sql, params = ktt.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + "JSON_VALUE(tests_detail.value, '$.\"name\"')", + ) + self.assertEqual(params, ()) + + kt_nested = KeyTransform("b", KeyTransform("0", KeyTransform("a", col))) + sql, params = kt_nested.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + 'JSON_QUERY(tests_detail.value, \'$."a"[0]."b"\')', + ) + self.assertEqual(params, ()) + + kt_escaped = KeyTransform("a'b%s", col) + sql, params = kt_escaped.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + "JSON_QUERY(tests_detail.value, '$.\"a\\'b%%s\"')", + ) + self.assertEqual(params, ()) + + hk = HasKey(col, "name") + sql, params = hk.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + "JSON_QUERY(tests_detail.value, '$.\"name\"') IS NOT NULL", + ) + self.assertEqual(params, ()) + + hk_nested_int = HasKey(KeyTransform("nested", col), "0") + sql, params = hk_nested_int.as_spanner(compiler, self.connection) + self.assertEqual( + sql, + 'JSON_QUERY(tests_detail.value, \'$."nested"."0"\') IS NOT NULL', + ) + self.assertEqual(params, ()) + + def test_json_field_get_prep_value(self): + from django.db.models import JSONField + + field = JSONField() + val = {"key": "value"} + res = field.get_prep_value(val) + self.assertEqual(res, val) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py index 520d268bce10..eac1054f9c76 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py @@ -12,9 +12,13 @@ from django.conf import settings from django.core.management.color import no_style +from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import JSONField from django.db.utils import DatabaseError from google.cloud.spanner_dbapi.types import DateStr +from google.cloud.spanner_v1 import JsonObject +from django_spanner.operations import _JsonString from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass @@ -111,6 +115,141 @@ def test_convert_uuidfield_value_none(self): ), ) + def test_json_field_feature_flags(self): + self.assertTrue(self.connection.features.has_native_json_field) + self.assertTrue(self.connection.features.supports_json_field) + self.assertTrue(self.connection.features.supports_primitives_in_json_field) + self.assertFalse(self.connection.features.supports_json_field_contains) + + def test_adapt_json_value_dict(self): + value = {"a": 1, "b": "test"} + res = self.db_operations.adapt_json_value(value, None) + self.assertIsInstance(res, _JsonString) + self.assertIsInstance(res, JsonObject) + self.assertFalse(res._is_null) + self.assertFalse(res._is_array) + self.assertFalse(res._is_scalar_value) + self.assertEqual(dict(res), {}) + self.assertEqual(repr(res), f"_JsonString({res.serialize()!r})") + self.assertEqual(res.serialize(), '{"a": 1, "b": "test"}') + self.assertNotEqual(res, None) + + def test_adapt_json_value_list(self): + value = [1, 2, 3] + res = self.db_operations.adapt_json_value(value, None) + self.assertIsInstance(res, _JsonString) + self.assertEqual(res.serialize(), "[1, 2, 3]") + + def test_adapt_json_value_none_becomes_json_null(self): + res = self.db_operations.adapt_json_value(None, None) + self.assertIsInstance(res, _JsonString) + self.assertFalse(res._is_null) + self.assertEqual(dict(res), {}) + self.assertEqual(res.serialize(), "null") + self.assertNotEqual(res, None) + + def test_adapt_json_value_primitives(self): + for primitive, expected in [ + ("hello", '"hello"'), + (42, "42"), + (True, "true"), + (3.14, "3.14"), + ]: + res = self.db_operations.adapt_json_value(primitive, None) + self.assertIsInstance(res, _JsonString) + self.assertEqual(res.serialize(), expected) + + def test_adapt_json_value_custom_encoder(self): + u = uuid.UUID("12345678-1234-5678-1234-567812345678") + res = self.db_operations.adapt_json_value( + {"id": u, "amount": Decimal("12.34")}, DjangoJSONEncoder + ) + self.assertIsInstance(res, _JsonString) + self.assertEqual( + res.serialize(), + '{"id": "12345678-1234-5678-1234-567812345678", "amount": "12.34"}', + ) + + def test_get_db_converters_jsonfield(self): + expression = mock.MagicMock() + expression.output_field = JSONField() + converters = self.db_operations.get_db_converters(expression) + self.assertIn(self.db_operations.convert_jsonfield_value, converters) + + def test_convert_jsonfield_value_dict(self): + obj = JsonObject({"a": 1}) + res = self.db_operations.convert_jsonfield_value(obj, None, None) + self.assertEqual(res, {"a": 1}) + + def test_convert_jsonfield_value_list(self): + obj = JsonObject([1, "two", {"three": 3}]) + res = self.db_operations.convert_jsonfield_value(obj, None, None) + self.assertEqual(res, [1, "two", {"three": 3}]) + + def test_convert_jsonfield_value_primitives(self): + for raw, expected in [ + ('"hello"', "hello"), + ("42", 42), + ("true", True), + ("3.14", 3.14), + ]: + obj = JsonObject.from_str(raw) + res = self.db_operations.convert_jsonfield_value(obj, None, None) + self.assertEqual(res, expected) + + def test_convert_jsonfield_value_null_json_object(self): + obj = JsonObject.from_str("null") + res = self.db_operations.convert_jsonfield_value(obj, None, None) + self.assertIsNone(res) + + def test_convert_jsonfield_value_custom_decoder(self): + import json + + class CustomDecoder(json.JSONDecoder): + def decode(self, s): + data = super().decode(s) + if isinstance(data, dict): + data["decoded"] = True + return data + + expr = mock.MagicMock() + expr.output_field = JSONField(decoder=CustomDecoder) + obj = JsonObject({"a": 1}) + res = self.db_operations.convert_jsonfield_value(obj, expr, None) + self.assertEqual(res, {"a": 1, "decoded": True}) + + null_obj = JsonObject.from_str("null") + self.assertIsNone( + self.db_operations.convert_jsonfield_value(null_obj, expr, None) + ) + + def test_convert_jsonfield_value_subclass_with_custom_from_db_value(self): + class TaggedJSONField(JSONField): + def from_db_value(self, value, expression, connection): + return {"wrapped": super().from_db_value(value, expression, connection)} + + expr = mock.MagicMock() + expr.output_field = TaggedJSONField() + obj = JsonObject({"a": 1}) + raw = self.db_operations.convert_jsonfield_value(obj, expr, None) + self.assertEqual(raw, '{"a":1}') + self.assertEqual( + expr.output_field.from_db_value(raw, expr, None), + {"wrapped": {"a": 1}}, + ) + + def test_convert_jsonfield_value_none(self): + res = self.db_operations.convert_jsonfield_value(None, None, None) + self.assertIsNone(res) + + def test_convert_jsonfield_value_other(self): + res = self.db_operations.convert_jsonfield_value('{"a": 1}', None, None) + self.assertEqual(res, {"a": 1}) + invalid_res = self.db_operations.convert_jsonfield_value( + "{invalid json", None, None + ) + self.assertEqual(invalid_res, "{invalid json") + def test_date_extract_sql(self): self.assertEqual( self.db_operations.date_extract_sql("week", "dummy_field"),