From 5d372bb91afa7308e5186a7b5f9dbb521f7fd8ae Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Wed, 29 Jul 2026 17:05:21 +0200 Subject: [PATCH 1/8] Add dry run to check query syntax --- django_ormql/engine.py | 9 +++++++-- django_ormql/query.py | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/django_ormql/engine.py b/django_ormql/engine.py index d4eb9cd..d5807c6 100644 --- a/django_ormql/engine.py +++ b/django_ormql/engine.py @@ -16,11 +16,16 @@ def query( placeholders=None, timezone=datetime.timezone.utc, default_limit=None, + dry_run=False, ): - return Query( + query = Query( query, self.tables, placeholders, timezone, default_limit, - ).evaluate() + ) + if dry_run: + return query.parse() + else: + return query.evaluate() diff --git a/django_ormql/query.py b/django_ormql/query.py index 3bcb9db..ad009c3 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -838,7 +838,7 @@ def _select_to_qs(self, root, parent_table_stack): return qs, values_names - def evaluate(self): + def parse(self): try: ast = parse_one(self.sql, dialect=OrmqlDialect) except ParseError as e: @@ -860,6 +860,11 @@ def evaluate(self): except Exception as e: raise QueryError("Query parsing failed") from e + return qs, values_names + + def evaluate(self): + qs, values_names = self.parse() + if isinstance(qs, dict): yield {values_names[k]: v for k, v in qs.items()} else: From 70d72850b48aa9c4eb7e16a10bc78db357e292f6 Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 13:52:56 +0200 Subject: [PATCH 2/8] WIP: add UNION query support (doesn't work yet, as _select_to_qs often returns different number of columns due to grpNN columns added) --- django_ormql/query.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/django_ormql/query.py b/django_ormql/query.py index ad009c3..ab21814 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -97,6 +97,7 @@ class Tokenizer(Tokenizer): "TIME": TokenType.TIME, "DATE": TokenType.DATE, "DATETIME": TokenType.DATETIME, + "UNION": TokenType.UNION, } class Generator(Generator): @@ -657,6 +658,9 @@ def _where_to_django(self, node, **kwargs): return self._expression_to_django(node, **kwargs) def _select_to_qs(self, root, parent_table_stack): + if not isinstance(root, expressions.Select): + raise QueryNotSupported("Only SELECT queries are supported") + table = root.args["from_"].this if not isinstance(table, expressions.Table): raise QueryNotSupported("Unsupported FROM statement") @@ -838,6 +842,23 @@ def _select_to_qs(self, root, parent_table_stack): return qs, values_names + def _union_to_qs(self, root: expressions.Union): + if isinstance(root.left, expressions.Union): + qs_left, values_names = self._union_to_qs(root.left) + elif isinstance(root.left, expressions.Subquery) and isinstance(root.left.this, expressions.Select): + qs_left, values_names = self._select_to_qs(root.left.this, []) + else: + qs_left, values_names = self._select_to_qs(root.left, []) + if isinstance(root.right, expressions.Union): + qs_right, _ = self._union_to_qs(root.right) + elif isinstance(root.right, expressions.Subquery) and isinstance(root.right.this, expressions.Select): + qs_right, _ = self._select_to_qs(root.right.this, []) + else: + qs_right, _ = self._select_to_qs(root.right, []) + if isinstance(qs_left, dict) or isinstance(qs_right, dict): + raise QueryError("Unsupported UNION components") + return qs_left.union(qs_right), values_names + def parse(self): try: ast = parse_one(self.sql, dialect=OrmqlDialect) @@ -848,11 +869,11 @@ def parse(self): if settings.DEBUG: print(f"Parsed statement: {ast!r}") - if not isinstance(ast, expressions.Select): - raise QueryNotSupported("Only SELECT queries are supported") - try: - qs, values_names = self._select_to_qs(ast, []) + if isinstance(ast, expressions.Union): + qs, values_names = self._union_to_qs(ast) + else: + qs, values_names = self._select_to_qs(ast, []) except QueryError: raise except FieldError as e: From 8dee3962729102b16599a5628c517b7c6e3fe20d Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 15:01:15 +0200 Subject: [PATCH 3/8] add UNION query support by concatenating results in python --- django_ormql/query.py | 58 +++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/django_ormql/query.py b/django_ormql/query.py index ab21814..8aff655 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -842,22 +842,15 @@ def _select_to_qs(self, root, parent_table_stack): return qs, values_names - def _union_to_qs(self, root: expressions.Union): - if isinstance(root.left, expressions.Union): - qs_left, values_names = self._union_to_qs(root.left) - elif isinstance(root.left, expressions.Subquery) and isinstance(root.left.this, expressions.Select): - qs_left, values_names = self._select_to_qs(root.left.this, []) + def _flatten_unions(self, root): + if isinstance(root, expressions.Select): + return [root] + elif isinstance(root, expressions.Subquery) and isinstance(root.this, expressions.Select): + return [root.this] + elif isinstance(root, expressions.Union): + return self._flatten_unions(root.left) + self._flatten_unions(root.right) else: - qs_left, values_names = self._select_to_qs(root.left, []) - if isinstance(root.right, expressions.Union): - qs_right, _ = self._union_to_qs(root.right) - elif isinstance(root.right, expressions.Subquery) and isinstance(root.right.this, expressions.Select): - qs_right, _ = self._select_to_qs(root.right.this, []) - else: - qs_right, _ = self._select_to_qs(root.right, []) - if isinstance(qs_left, dict) or isinstance(qs_right, dict): - raise QueryError("Unsupported UNION components") - return qs_left.union(qs_right), values_names + raise QueryNotSupported("Only SELECT and SELECT ... UNION queries are supported") def parse(self): try: @@ -870,10 +863,8 @@ def parse(self): print(f"Parsed statement: {ast!r}") try: - if isinstance(ast, expressions.Union): - qs, values_names = self._union_to_qs(ast) - else: - qs, values_names = self._select_to_qs(ast, []) + queries = self._flatten_unions(ast) + results = [self._select_to_qs(query, []) for query in queries] except QueryError: raise except FieldError as e: @@ -881,20 +872,21 @@ def parse(self): except Exception as e: raise QueryError("Query parsing failed") from e - return qs, values_names + return [qs for qs, values_names in results], results[0][1] def evaluate(self): - qs, values_names = self.parse() + querysets, values_names = self.parse() - if isinstance(qs, dict): - yield {values_names[k]: v for k, v in qs.items()} - else: - try: - if settings.DEBUG: - print(f"Generated statement: {qs.query!s}") - for row in qs: - yield { - values_names[k]: v for k, v in row.items() if k in values_names - } - except (FieldError, ValueError) as e: - raise QueryError("Invalid combination of types") from e + for qs in querysets: + if isinstance(qs, dict): + yield {values_names[k]: v for k, v in qs.items()} + else: + try: + if settings.DEBUG: + print(f"Generated statement: {qs.query!s}") + for row in qs: + yield { + values_names[k]: v for k, v in row.items() if k in values_names + } + except (FieldError, ValueError) as e: + raise QueryError("Invalid combination of types") from e From 77cae1d7238e569dad25d14d4028d60548ff51ee Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 15:38:24 +0200 Subject: [PATCH 4/8] check unsupported UNION modifers --- django_ormql/query.py | 11 ++++++++--- doc/syntax.md | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/django_ormql/query.py b/django_ormql/query.py index 8aff655..c640b8e 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -34,6 +34,8 @@ class OrmqlDialect(Dialect): QUOTE_END = "'" IDENTIFIER_START = "`" IDENTIFIER_END = "`" + MODIFIERS_ATTACHED_TO_SET_OP = False + SET_OP_MODIFIERS = {} class Tokenizer(Tokenizer): QUOTES = ["'", '"'] @@ -48,6 +50,7 @@ class Tokenizer(Tokenizer): "!=": TokenType.NEQ, "||": TokenType.DPIPE, "->": TokenType.ARROW, + "ALL": TokenType.ALL, "AND": TokenType.AND, "ASC": TokenType.ASC, "AS": TokenType.ALIAS, @@ -82,6 +85,7 @@ class Tokenizer(Tokenizer): "SELECT": TokenType.SELECT, "THEN": TokenType.THEN, "TRUE": TokenType.TRUE, + "UNION": TokenType.UNION, "WHEN": TokenType.WHEN, "WHERE": TokenType.WHERE, # TYPES @@ -97,7 +101,6 @@ class Tokenizer(Tokenizer): "TIME": TokenType.TIME, "DATE": TokenType.DATE, "DATETIME": TokenType.DATETIME, - "UNION": TokenType.UNION, } class Generator(Generator): @@ -847,10 +850,12 @@ def _flatten_unions(self, root): return [root] elif isinstance(root, expressions.Subquery) and isinstance(root.this, expressions.Select): return [root.this] - elif isinstance(root, expressions.Union): + elif isinstance(root, expressions.Union) and not root.args['distinct']: + if root.args.get('limit') or root.args.get('order') or root.args.get('offset'): + raise QueryError("ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries") return self._flatten_unions(root.left) + self._flatten_unions(root.right) else: - raise QueryNotSupported("Only SELECT and SELECT ... UNION queries are supported") + raise QueryNotSupported("Only SELECT and SELECT ... UNION ALL queries are supported") def parse(self): try: diff --git a/doc/syntax.md b/doc/syntax.md index d11a5e3..eb6e7fd 100644 --- a/doc/syntax.md +++ b/doc/syntax.md @@ -17,7 +17,8 @@ - Bitwise operators - `IS DISTINCT` - `LIMIT` and `OFFSET` with complex expressions - - `UNION`, `INTERSECT`, `EXCEPT` + - `UNION` queries without `ALL`, `UNION` with `ORDER BY`, `LIMIT`, `OFFSET` modifiers + - `INTERSECT`, `EXCEPT` This list of differences is not complete. From 6ee643e815b35ed15160e8bdb3ee8b963556d0b6 Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 15:38:46 +0200 Subject: [PATCH 5/8] remove options that seem to have no effect --- django_ormql/query.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django_ormql/query.py b/django_ormql/query.py index c640b8e..4a57fc6 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -34,8 +34,6 @@ class OrmqlDialect(Dialect): QUOTE_END = "'" IDENTIFIER_START = "`" IDENTIFIER_END = "`" - MODIFIERS_ATTACHED_TO_SET_OP = False - SET_OP_MODIFIERS = {} class Tokenizer(Tokenizer): QUOTES = ["'", '"'] From bf56e80d2b98802d66df592ab5f5d89d9defb0a6 Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 16:00:33 +0200 Subject: [PATCH 6/8] Enforce same column count for UNION parts --- django_ormql/query.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django_ormql/query.py b/django_ormql/query.py index 4a57fc6..72bc504 100644 --- a/django_ormql/query.py +++ b/django_ormql/query.py @@ -868,6 +868,8 @@ def parse(self): try: queries = self._flatten_unions(ast) results = [self._select_to_qs(query, []) for query in queries] + if len(set(len(values_names.keys()) for qs, values_names in results)) != 1: + raise QueryError("All parts of UNION query must return same number of columns") except QueryError: raise except FieldError as e: From 11cd83c38ddf7b1207179edea7a6bfb4b9dc84fc Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 16:30:41 +0200 Subject: [PATCH 7/8] Add test cases for UNION --- tests/test_select_col_from.py | 82 ++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/tests/test_select_col_from.py b/tests/test_select_col_from.py index 9e1839d..34afa00 100644 --- a/tests/test_select_col_from.py +++ b/tests/test_select_col_from.py @@ -234,7 +234,7 @@ def test_undeclared_field(engine_t1): @pytest.mark.django_db def test_compound_not_allowed(engine_t1): - with pytest.raises(QueryError, match="Invalid expression / Unexpected token"): + with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): list( engine_t1.query( """ @@ -272,9 +272,87 @@ def test_compound_not_allowed(engine_t1): ) +@pytest.mark.django_db +def test_union_modifiers_not_allowed(engine_t1): + with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION + SELECT title + FROM products + """ + ) + ) + with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT title + FROM products + LIMIT 1 + """ + ) + ) + with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT title + FROM products + ORDER BY title + """ + ) + ) + + +@pytest.mark.django_db +def test_union(engine_t1): + res = engine_t1.query( + """ + SELECT 'Category' type, title + FROM categories + UNION ALL + SELECT 'Product' type, title + FROM products + """ + ) + assert list(res) == [ + {"type": "Category", "title": "Books"}, + {"type": "Category", "title": "DVDs"}, + {"type": "Product", "title": "Lord of the rings"}, + {"type": "Product", "title": "SQL for Dummies"}, + {"type": "Product", "title": "Lord of the rings DVD"}, + ] + res = engine_t1.query( + """ + (SELECT 'Category' type, title + FROM categories ORDER BY title DESC) + UNION ALL + (SELECT 'Product' type, title + FROM products ORDER BY title DESC) + """ + ) + assert list(res) == [ + {"type": "Category", "title": "DVDs"}, + {"type": "Category", "title": "Books"}, + {"type": "Product", "title": "SQL for Dummies"}, + {"type": "Product", "title": "Lord of the rings DVD"}, + {"type": "Product", "title": "Lord of the rings"}, + ] + + @pytest.mark.django_db def test_values_query_not_allowed(engine_t1): - with pytest.raises(QueryError, match="Only SELECT queries are supported"): + with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): list( engine_t1.query( """ From a4224ba4c07194fba7f36f16ace3beca5a7e9556 Mon Sep 17 00:00:00 2001 From: Mira Weller Date: Thu, 30 Jul 2026 16:48:57 +0200 Subject: [PATCH 8/8] Add more test cases --- tests/test_select_col_from.py | 78 ------------------ tests/test_subqueries.py | 14 ++++ tests/test_union.py | 151 ++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 78 deletions(-) create mode 100644 tests/test_union.py diff --git a/tests/test_select_col_from.py b/tests/test_select_col_from.py index 34afa00..e41981c 100644 --- a/tests/test_select_col_from.py +++ b/tests/test_select_col_from.py @@ -272,84 +272,6 @@ def test_compound_not_allowed(engine_t1): ) -@pytest.mark.django_db -def test_union_modifiers_not_allowed(engine_t1): - with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): - list( - engine_t1.query( - """ - SELECT title - FROM categories - UNION - SELECT title - FROM products - """ - ) - ) - with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): - list( - engine_t1.query( - """ - SELECT title - FROM categories - UNION ALL - SELECT title - FROM products - LIMIT 1 - """ - ) - ) - with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): - list( - engine_t1.query( - """ - SELECT title - FROM categories - UNION ALL - SELECT title - FROM products - ORDER BY title - """ - ) - ) - - -@pytest.mark.django_db -def test_union(engine_t1): - res = engine_t1.query( - """ - SELECT 'Category' type, title - FROM categories - UNION ALL - SELECT 'Product' type, title - FROM products - """ - ) - assert list(res) == [ - {"type": "Category", "title": "Books"}, - {"type": "Category", "title": "DVDs"}, - {"type": "Product", "title": "Lord of the rings"}, - {"type": "Product", "title": "SQL for Dummies"}, - {"type": "Product", "title": "Lord of the rings DVD"}, - ] - res = engine_t1.query( - """ - (SELECT 'Category' type, title - FROM categories ORDER BY title DESC) - UNION ALL - (SELECT 'Product' type, title - FROM products ORDER BY title DESC) - """ - ) - assert list(res) == [ - {"type": "Category", "title": "DVDs"}, - {"type": "Category", "title": "Books"}, - {"type": "Product", "title": "SQL for Dummies"}, - {"type": "Product", "title": "Lord of the rings DVD"}, - {"type": "Product", "title": "Lord of the rings"}, - ] - - @pytest.mark.django_db def test_values_query_not_allowed(engine_t1): with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): diff --git a/tests/test_subqueries.py b/tests/test_subqueries.py index d35446f..12d483f 100644 --- a/tests/test_subqueries.py +++ b/tests/test_subqueries.py @@ -202,3 +202,17 @@ def test_invalid_outerref(engine_t1): """ ) ) + + +@pytest.mark.django_db +def test_union_subquery_not_allowed(engine_t1): + with pytest.raises(QueryError, match="Only SELECT subqueries are supported"): + list( + engine_t1.query( + """ + SELECT title + FROM products + WHERE EXISTS(SELECT 1 FROM orderpositions WHERE product = OUTER(id) AND order.status = "paid" UNION SELECT 1 FROM orderpositions WHERE product = OUTER(id) AND order.status = "canceled") + """ + ) + ) diff --git a/tests/test_union.py b/tests/test_union.py new file mode 100644 index 0000000..7b2e89c --- /dev/null +++ b/tests/test_union.py @@ -0,0 +1,151 @@ +from decimal import Decimal + +import pytest + +from django_ormql.exceptions import QueryNotSupported, QueryError + + +@pytest.mark.django_db +def test_union_modifiers_not_allowed(engine_t1): + with pytest.raises(QueryError, match="Only SELECT and SELECT ... UNION ALL queries are supported"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION + SELECT title + FROM products + """ + ) + ) + with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT title + FROM products + LIMIT 1 + """ + ) + ) + with pytest.raises(QueryError, match="ORDER, LIMIT and OFFSET modifiers are not supported on UNION queries"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT title + FROM products + ORDER BY title + """ + ) + ) + + +@pytest.mark.django_db +def test_union(engine_t1): + res = engine_t1.query( + """ + SELECT 'Category' type, title + FROM categories + UNION ALL + SELECT 'Product' type, title + FROM products + """ + ) + assert list(res) == [ + {"type": "Category", "title": "Books"}, + {"type": "Category", "title": "DVDs"}, + {"type": "Product", "title": "Lord of the rings"}, + {"type": "Product", "title": "SQL for Dummies"}, + {"type": "Product", "title": "Lord of the rings DVD"}, + ] + res = engine_t1.query( + """ + (SELECT 'Category' type, title + FROM categories ORDER BY title DESC) + UNION ALL + (SELECT 'Product' type, title + FROM products ORDER BY title DESC) + """ + ) + assert list(res) == [ + {"type": "Category", "title": "DVDs"}, + {"type": "Category", "title": "Books"}, + {"type": "Product", "title": "SQL for Dummies"}, + {"type": "Product", "title": "Lord of the rings DVD"}, + {"type": "Product", "title": "Lord of the rings"}, + ] + + +@pytest.mark.django_db +def test_union_col_count(engine_t1): + with pytest.raises(QueryError, match="All parts of UNION query must return same number of columns"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT category.title, SUM(price) + FROM products + GROUP BY category.title + """ + ) + ) + with pytest.raises(QueryError, match="All parts of UNION query must return same number of columns"): + list( + engine_t1.query( + """ + SELECT title + FROM categories + UNION ALL + SELECT title, price + FROM products + """ + ) + ) + + +@pytest.mark.django_db +def test_union_grouped(engine_t1): + res = engine_t1.query( + """ + SELECT 'Category' type, title, 0 sales, 0 total + FROM categories + UNION ALL + SELECT 'Product' type, product.title, SUM(quantity) sales, SUM(single_price * quantity) total + FROM orderpositions + GROUP BY product.id + """ + ) + assert list(res) == [ + {"type": "Category", "title": "Books", "sales": 0, "total": 0}, + {"type": "Category", "title": "DVDs", "sales": 0, "total": 0}, + {"type": "Product", "title": "Lord of the rings", "sales": 3, "total": Decimal('32.10')}, + {"type": "Product", "title": "SQL for Dummies", "sales": 1, "total": Decimal('21.40')}, + {"type": "Product", "title": "Lord of the rings DVD", "sales": 4, "total": Decimal('76.00')}, + ] + res = engine_t1.query( + """ + SELECT 'Category' type, product.category.title title, SUM(quantity) sales, SUM(single_price * quantity) total + FROM orderpositions + GROUP BY product.category.id + UNION ALL + SELECT 'Product' type, product.title, SUM(quantity) sales, SUM(single_price * quantity) total + FROM orderpositions + GROUP BY product.id + """ + ) + assert list(res) == [ + {"type": "Category", "title": "Books", "sales": 4, "total": Decimal('53.50')}, + {"type": "Category", "title": "DVDs", "sales": 4, "total": Decimal('76.00')}, + {"type": "Product", "title": "Lord of the rings", "sales": 3, "total": Decimal('32.10')}, + {"type": "Product", "title": "SQL for Dummies", "sales": 1, "total": Decimal('21.40')}, + {"type": "Product", "title": "Lord of the rings DVD", "sales": 4, "total": Decimal('76.00')}, + ] \ No newline at end of file