diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index f6ff5b20..bdc2692c 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -4,6 +4,7 @@ * xref:quickstart.adoc[Quickstart] * How-to ** xref:how-to/databases-and-queries.adoc[Databases and queries] +** xref:how-to/transactions.adoc[Transactions] ** xref:how-to/concurrency.adoc[Concurrency, processes and crashes] ** xref:how-to/logs.adoc[Where the logs go] ** xref:how-to/bulk-loads.adoc[Bulk loads and commit size] diff --git a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc index 1b28f105..ace65a4c 100644 --- a/docs/modules/ROOT/pages/how-to/bulk-loads.adoc +++ b/docs/modules/ROOT/pages/how-to/bulk-loads.adoc @@ -24,7 +24,7 @@ The JVM defaults are the same as before. They stay until someone measures them on the JVM build. -An explicit value always wins. `commit_every=0` gives one transaction for `insert_many` and one commit for each flush for `graph_batch`, on both packages. In a transaction that you open yourself, `insert_many` commits nothing. +An explicit value always wins. `commit_every=0` gives one transaction for `insert_many` and one commit for each flush for `graph_batch`, on both packages. In a transaction that you open yourself, `insert_many` commits nothing. `graph_batch` refuses to run in one: see xref:how-to/transactions.adoc#graph-batch[GraphBatch inside a transaction]. == Why the native defaults are smaller diff --git a/docs/modules/ROOT/pages/how-to/concurrency.adoc b/docs/modules/ROOT/pages/how-to/concurrency.adoc index 7d1d35a6..c49fb616 100644 --- a/docs/modules/ROOT/pages/how-to/concurrency.adoc +++ b/docs/modules/ROOT/pages/how-to/concurrency.adoc @@ -29,6 +29,7 @@ Every number on this page, except the ratios in <>. + +== begin, commit and rollback + +`db.begin()`, `db.commit()` and `db.rollback()` do the same by hand. Use them when the start and the end of the transaction are not in one block, and put the rollback in a `finally` or an `except` clause, because nothing rolls back for you. + +[source,python] +---- +db.begin() +db.command("sql", "INSERT INTO Account SET name = 'bob'") +print(db.is_transaction_active()) +db.rollback() +print(names()) + +db.begin() +db.command("sql", "INSERT INTO Account SET name = 'bob'") +db.commit() +print(names()) +---- + +[source,text] +---- +True +['alice'] +['alice', 'bob'] +---- + +[[outside]] +== Outside a transaction + +A write outside a transaction raises `ArcadeDBError` with "Transaction not begun" in the message, and writes nothing. So does `commit()` when no transaction is active. `rollback()` with nothing to roll back does nothing and raises nothing. + +[source,python] +---- +try: + db.command("sql", "INSERT INTO Account SET name = 'carol'") +except ArcadeDBError as error: + print("write:", "Transaction not begun" in str(error)) +try: + db.commit() +except ArcadeDBError as error: + print("commit:", "Transaction not begun" in str(error)) +db.rollback() +print("rollback: no error") +---- + +[source,text] +---- +write: True +commit: True +rollback: no error +---- + +The same holds for the context manager. If the block calls `db.commit()` itself, no transaction is left when the block ends, and the commit at the end raises `ArcadeDBError` ("Transaction not begun"). The rows of the first commit stay. Leave the commit to the `with` statement. + +Schema statements such as `CREATE TYPE` apply at once and need no transaction, as on xref:quickstart.adoc[Quickstart]. + +== Auto-transaction + +`db.set_auto_transaction(True)` makes each statement outside a transaction run in a transaction of its own, which commits when the statement ends. It is off by default and is not stored: a reopened database starts with it off. Prefer an explicit transaction when several statements belong together, because with auto-transaction each one commits alone. + +[source,python] +---- +db.set_auto_transaction(True) +db.command("sql", "INSERT INTO Account SET name = 'carol'") +print("carol" in names(), db.is_transaction_active()) +db.set_auto_transaction(False) +---- + +[source,text] +---- +True False +---- + +== What a transaction sees + +A transaction sees its own rows at once. Another thread sees them only after the commit. This block starts a thread for each count: + +[source,python] +---- +import threading + + +def count_in_other_thread(): + seen = [] + + def read(): + seen.append(len(db.query("sql", "SELECT FROM Account").to_list())) + + thread = threading.Thread(target=read) + thread.start() + thread.join() + return seen[0] + + +with db.transaction(): + db.command("sql", "INSERT INTO Account SET name = 'dave'") + print("this thread:", len(names()), "other thread:", count_in_other_thread()) +print("after the commit, other thread:", count_in_other_thread()) +---- + +[source,text] +---- +this thread: 4 other thread: 3 +after the commit, other thread: 4 +---- + +Share one `Database` between threads, as on xref:how-to/concurrency.adoc#threads[Concurrency, processes and crashes]. That page also covers what happens when two transactions change the same record. + +== A failed statement inside a transaction + +A statement that fails inside your transaction is undone on its own, and the transaction goes on. This is the behavior of ArcadeDB 26.11.1 and later. Before it, a statement that had written part of its rows left them in the transaction, and the transaction could no longer be committed. + +[source,python] +---- +with db.transaction(): + db.command("sql", "INSERT INTO Account SET name = 'erin'") + try: + db.command("sql", "INSERT INTO Account (name) VALUES ('frank'), ('frank')") + except ArcadeDBError as error: + print("refused:", "DuplicatedKeyException" in str(error)) + print("still active:", db.is_transaction_active()) + db.command("sql", "INSERT INTO Account SET name = 'grace'") +print(names()) +---- + +[source,text] +---- +refused: True +still active: True +['alice', 'bob', 'carol', 'dave', 'erin', 'grace'] +---- + +The second statement writes `frank` once, and then the unique index refuses the second `frank`. The first `frank` is gone with it, while `erin` from the statement before and `grace` from the statement after are committed. The engine keeps a journal of what each statement changes in order to undo it. When a statement changes more entries than `arcadedb.txStatementSavepointMaxEntries` (100,000 by default), the engine does not undo it, and the transaction can no longer be committed: roll it back. The setting is listed in the main manual: {arcadedb-docs-url}/reference/settings[Settings]. + +A failed statement is one thing, and an exception that you let leave the `with` block is another: that one rolls the whole transaction back, as above. + +== A second begin + +`db.begin()` while a transaction is active does not join it. It opens a nested transaction of its own, and the next `commit()` or `rollback()` ends that nested one only. The two do not share their uncommitted rows: + +* The nested transaction does not see the rows that the outer one has not committed. +* Committing the nested transaction makes its rows durable at once. A later rollback of the outer transaction does not take them back. +* The outer transaction sees what the nested one committed, because it is committed data. + +[source,python] +---- +db.begin() +db.command("sql", "INSERT INTO Account SET name = 'outer'") +db.begin() +print("nested sees outer:", "outer" in names()) +db.command("sql", "INSERT INTO Account SET name = 'inner'") +db.commit() +print("still active:", db.is_transaction_active()) +db.rollback() +print("outer kept:", "outer" in names(), "inner kept:", "inner" in names()) +---- + +[source,text] +---- +nested sees outer: False +still active: True +outer kept: False inner kept: True +---- + +Nested `with db.transaction():` blocks behave the same way, one transaction for each block. An exception in the inner block rolls back the inner one, and if you catch it inside the outer block, the outer transaction goes on. Because the nested transaction commits on its own, it is not a way to make one part of the work undoable: use one transaction for work that has to succeed or fail together. + +== run_in_transaction + +`db.run_in_transaction(fn, retries=12, backoff_s=0.005)` calls `fn` in a transaction, commits, and returns what `fn` returned. When `fn` or the commit loses a conflict with another writer, it rolls back and calls `fn` again, up to `retries` more times, with a wait that grows by `backoff_s` each time. `fn` takes no arguments and must do all of its reading and writing itself, so that a repeat starts from the committed state. The loop, and what it cannot repeat, are on xref:how-to/concurrency.adoc#retry-loop[Concurrency, processes and crashes]. + +[source,python] +---- +def add_account(): + db.command("sql", "INSERT INTO Account SET name = 'heidi'") + return "added" + + +print(db.run_in_transaction(add_account)) +---- + +[source,text] +---- +added +---- + +[[graph-batch]] +== GraphBatch inside a transaction + +`GraphBatch` (`db.graph_batch()`) manages its own transactions. From engine 26.10.1 on, its `create_vertices`, `flush` and `close`, and `new_edge` and `new_edges` when the buffer is full, refuse to run inside a transaction that you opened. They raise `ArcadeDBError` and leave your transaction as it was. Engines before 26.10.1 committed your transaction instead. Commit or roll back your own work, then call the batch. The commit sizes of a batch are on xref:how-to/bulk-loads.adoc[Bulk loads and commit size]. + +[source,python] +---- +db.command("sql", "CREATE VERTEX TYPE City") +with db.graph_batch(parallel_flush=False) as batch: + db.begin() + try: + batch.create_vertices("City", 2) + except ArcadeDBError: + print("refused, transaction still active:", db.is_transaction_active()) + db.rollback() +---- + +[source,text] +---- +refused, transaction still active: True +---- diff --git a/docs/modules/ROOT/pages/quickstart.adoc b/docs/modules/ROOT/pages/quickstart.adoc index 50b74046..1ed57336 100644 --- a/docs/modules/ROOT/pages/quickstart.adoc +++ b/docs/modules/ROOT/pages/quickstart.adoc @@ -33,7 +33,7 @@ The database runs in your Python process. You use SQL for the schema and for CRU Two rules cover most first mistakes: * Use a `with` block. It closes the database when the block ends, also when an exception leaves it. -* *Writes need a transaction.* `INSERT`, `UPDATE` and `DELETE` go inside `with db.transaction():`. Schema statements apply immediately. Reads need no transaction. +* *Writes need a transaction.* `INSERT`, `UPDATE` and `DELETE` go inside `with db.transaction():`. Schema statements apply immediately. Reads need no transaction. What a transaction does on an error, and how to roll back by hand, is on xref:how-to/transactions.adoc[Transactions]. == A complete example diff --git a/tests/test_docs_transactions_page.py b/tests/test_docs_transactions_page.py new file mode 100644 index 00000000..319ad04d --- /dev/null +++ b/tests/test_docs_transactions_page.py @@ -0,0 +1,60 @@ +"""Every Python block of the Antora page ``docs/modules/ROOT/pages/how-to/transactions.adoc`` runs, and prints what the page says. + +Same pattern as ``test_docs_vectors_page.py``: the blocks are chained on purpose, they run in the order of the page, in one +namespace, against one fresh temporary database that the namespace holds as ``db``. A block that is followed by a +``[source,text]`` block must print exactly that text. What each statement of the page claims is pinned on its own in +``test_transactions_behavior.py``. The test uses no Java object, so it runs on the JVM build and the native build alike. +""" + +import contextlib +import io +import re +from pathlib import Path + +PAGE = ( + Path(__file__).resolve().parents[1] + / "docs" + / "modules" + / "ROOT" + / "pages" + / "how-to" + / "transactions.adoc" +) + +# a Python listing, then (optionally, directly after it) the text listing with its output +BLOCK_RE = re.compile( + r"\[source,python[^\]\n]*\]\n----\n(?P.*?)\n----\n" + r"(?:\n\[source,text\]\n----\n(?P.*?)\n----\n)?", + re.DOTALL, +) + + +def _blocks(): + text = PAGE.read_text(encoding="utf-8") + found = [(m.group("code"), m.group("out")) for m in BLOCK_RE.finditer(text)] + # no Python listing may escape the extraction (a different spelling of the block header would run nowhere) + assert text.count("[source,python") == len(found) + return found + + +def test_the_page_has_its_blocks(): + blocks = _blocks() + assert len(blocks) >= 9 + assert sum(1 for _, out in blocks if out is not None) >= 8 + + +def test_every_block_runs_in_order_and_prints_what_the_page_says(temp_db): + namespace = {"db": temp_db, "__name__": "transactions_page"} + for number, (code, expected) in enumerate(_blocks(), start=1): + captured = io.StringIO() + compiled = compile(code, f"transactions.adoc python block {number}", "exec") + with contextlib.redirect_stdout(captured): + exec(compiled, namespace) # nosec B102 - the page of this repository + if expected is not None: + assert ( + captured.getvalue().strip() == expected.strip() + ), f"block {number} printed other text than the page shows:\n{code}" + else: + assert ( + captured.getvalue() == "" + ), f"block {number} prints, the page shows no output" diff --git a/tests/test_transactions_behavior.py b/tests/test_transactions_behavior.py new file mode 100644 index 00000000..ff7bfa00 --- /dev/null +++ b/tests/test_transactions_behavior.py @@ -0,0 +1,203 @@ +"""What a transaction does, as the how-to page ``docs/modules/ROOT/pages/how-to/transactions.adoc`` says it (ArcadeData/arcadedb-python#267). + +Each statement on that page is pinned here first: commit, rollback, the context manager, an exception inside it, a failed statement +inside a transaction, a second ``begin()``, the visibility of uncommitted rows, ``set_auto_transaction``, the defaults of +``run_in_transaction`` and the refusal of ``GraphBatch``. The tests use SQL and no Java object, so they run on the JVM build and the +native build alike. +""" + +import inspect +import threading + +import pytest + +from arcadedb import ArcadeDBError + + +@pytest.fixture +def db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Account") + temp_db.command("sql", "CREATE PROPERTY Account.name STRING") + temp_db.command("sql", "CREATE INDEX ON Account (name) UNIQUE") + return temp_db + + +def _names(db): + rows = db.query("sql", "SELECT name FROM Account ORDER BY name").to_list() + return [row["name"] for row in rows] + + +def _insert(db, name): + db.command("sql", "INSERT INTO Account SET name = ?", name) + + +def _count_in_another_thread(db): + seen = [] + + def read(): + seen.append(len(db.query("sql", "SELECT FROM Account").to_list())) + + thread = threading.Thread(target=read) + thread.start() + thread.join() + return seen[0] + + +def test_the_context_manager_commits_on_exit(db): + with db.transaction(): + assert db.is_transaction_active() is True + _insert(db, "alice") + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + + +def test_an_exception_in_the_block_rolls_back_and_is_raised_unchanged(db): + with db.transaction(): + _insert(db, "alice") + + with pytest.raises(ValueError, match="stop"): + with db.transaction(): + _insert(db, "bob") + raise ValueError("stop") + + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + + +def test_begin_then_rollback_or_commit(db): + db.begin() + _insert(db, "alice") + assert db.is_transaction_active() is True + db.rollback() + assert db.is_transaction_active() is False + assert _names(db) == [] + + db.begin() + _insert(db, "alice") + db.commit() + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + + +def test_a_write_commit_or_rollback_outside_a_transaction(db): + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + _insert(db, "alice") + assert _names(db) == [] + + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + db.commit() + + # a rollback with nothing to roll back is not an error + db.rollback() + assert db.is_transaction_active() is False + + +def test_a_commit_inside_the_block_makes_the_commit_at_exit_fail(db): + # the block committed, so the transaction is gone when the context manager commits again + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + with db.transaction(): + _insert(db, "alice") + db.commit() + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + + +def test_a_transaction_sees_its_own_rows_and_other_threads_see_them_after_the_commit( + db, +): + with db.transaction(): + _insert(db, "alice") + assert _names(db) == ["alice"] + assert _count_in_another_thread(db) == 0 + assert _count_in_another_thread(db) == 1 + + +def test_a_failed_statement_is_undone_and_the_transaction_goes_on(db): + with db.transaction(): + _insert(db, "alice") + with pytest.raises(ArcadeDBError, match="DuplicatedKeyException"): + # the first row is written before the second is refused + db.command("sql", "INSERT INTO Account (name) VALUES ('bob'), ('bob')") + assert db.is_transaction_active() is True + _insert(db, "carol") + # 26.11.1: the rows of the failed statement are gone, the others are committed + assert _names(db) == ["alice", "carol"] + + +def test_a_second_begin_opens_a_nested_transaction_of_its_own(db): + db.begin() + _insert(db, "outer") + db.begin() + # the nested transaction does not see what the outer one has not committed + assert _names(db) == [] + _insert(db, "inner") + db.commit() + # that commit ended the nested transaction only, and what it committed is visible to the outer one + assert db.is_transaction_active() is True + assert _names(db) == ["inner", "outer"] + db.rollback() + assert db.is_transaction_active() is False + # the nested commit stands, the outer rollback took back only the outer rows + assert _names(db) == ["inner"] + + +def test_nested_with_blocks_end_one_at_a_time(db): + with pytest.raises(ValueError, match="stop"): + with db.transaction(): + _insert(db, "outer") + with db.transaction(): + _insert(db, "inner") + raise ValueError("stop") + assert db.is_transaction_active() is False + assert _names(db) == ["inner"] + + +def test_an_exception_in_a_nested_block_rolls_back_only_that_block(db): + with db.transaction(): + _insert(db, "outer") + with pytest.raises(ValueError, match="stop"): + with db.transaction(): + _insert(db, "inner") + raise ValueError("stop") + assert db.is_transaction_active() is True + assert _names(db) == ["outer"] + + +def test_set_auto_transaction_commits_each_bare_write_alone(db): + db.set_auto_transaction(True) + try: + _insert(db, "alice") + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + finally: + db.set_auto_transaction(False) + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + _insert(db, "bob") + + +def test_run_in_transaction_returns_the_value_and_has_these_defaults(db): + def work(): + _insert(db, "alice") + return "done" + + assert db.run_in_transaction(work) == "done" + assert db.is_transaction_active() is False + assert _names(db) == ["alice"] + + parameters = inspect.signature(db.run_in_transaction).parameters + assert parameters["retries"].default == 12 + assert parameters["backoff_s"].default == 0.005 + + +def test_graph_batch_refuses_to_run_inside_your_transaction(db): + db.command("sql", "CREATE VERTEX TYPE City") + with db.graph_batch(parallel_flush=False) as batch: + db.begin() + try: + with pytest.raises(ArcadeDBError): + batch.create_vertices("City", 2) + # the refusal leaves the caller's transaction as it was + assert db.is_transaction_active() is True + finally: + db.rollback() + assert db.count_type("City") == 0