From d2490a695ed007de2697b8cc6dad5c0d34e22fc7 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 24 Oct 2026 10:37:44 +0900 Subject: [PATCH 1/3] Python runtime fixes in src: serialized start_jvm(), heap default, log directory, XML limits, read-only open, async docs Ports the runtime-fix group of #19 from humemai/arcadedb-embedded-python bindings/python, with its tests: - start_jvm() holds a lock, so threads whose first call is create_database() or open_database() no longer race into jpype.startJVM() (humemai #454) - default heap is 50 percent of the RAM the JVM sees (container limit inside a container) instead of 4g; an -Xms above the maximum heap raises ValueError before the JVM starts (humemai #420) - engine log, crash log and a rendered logging.properties go to a per-user log directory (ARCADEDB_LOG_DIR) instead of ./log (humemai #407) - the JDK XML entity limits stay on by default; import_documents(disable_xml_limits=True) lifts them for one call, and a limit error names the option (humemai #407) - open_database(read_only=...), DatabaseFactory.open(read_only=...), Database.is_read_only() (humemai #385) - AsyncExecutor docstrings say what each completion signal means, with tests (humemai #405) Not included: SIGINT and SIGTERM handling (#26), Python functions and record events, the polyglot engine default, the AOT cache. Refs #19 Co-Authored-By: Claude Opus 5.5 --- src/arcadedb_embedded/async_executor.py | 27 +- src/arcadedb_embedded/core.py | 48 ++- src/arcadedb_embedded/importer.py | 70 ++++- src/arcadedb_embedded/jvm.py | 299 +++++++++++++++--- tests/test_async_executor.py | 101 +++++++ tests/test_free_threading.py | 384 ++++++++++++++++++++++++ tests/test_jvm.py | 143 ++++++++- tests/test_jvm_args.py | 4 +- tests/test_log_dir.py | 206 +++++++++++++ tests/test_open_read_only.py | 78 +++++ tests/test_xml_limits.py | 98 ++++++ 11 files changed, 1394 insertions(+), 64 deletions(-) create mode 100644 tests/test_free_threading.py create mode 100644 tests/test_log_dir.py create mode 100644 tests/test_open_read_only.py create mode 100644 tests/test_xml_limits.py diff --git a/src/arcadedb_embedded/async_executor.py b/src/arcadedb_embedded/async_executor.py index 333d893a..98267362 100644 --- a/src/arcadedb_embedded/async_executor.py +++ b/src/arcadedb_embedded/async_executor.py @@ -187,7 +187,8 @@ def set_transaction_use_wal(self, use_wal: bool) -> "AsyncExecutor": Enable/disable Write-Ahead Log for async operations. Disabling WAL is faster but less durable (data loss on crash). - Only disable for bulk imports where data can be re-imported. + Only disable for bulk imports where data can be re-imported: with the + WAL off a batch that ``on_ok`` reported can still be lost in a crash. Args: use_wal: True to use WAL (default), False to disable @@ -806,8 +807,10 @@ def wait_completion(self, timeout_ms: Optional[int] = None): """ Wait for all async operations to complete. - Blocks until all pending operations finish. If timeout is reached, - raises TimeoutError. + Blocks until every operation queued before the call is processed: its + batch is committed, or the failure went to an error callback. This is + the point at which async writes are stored (with the WAL on); a + per-record callback is not. If timeout is reached, raises TimeoutError. Args: timeout_ms: Optional timeout in milliseconds. @@ -897,14 +900,16 @@ def close(self): def on_ok(self, callback: Callable[[], None]) -> "AsyncExecutor": """ - Set global success callback for all operations. - - **Note:** Global callbacks have JPype proxy compatibility issues. - Prefer per-operation callbacks on async SQL/Cypher commands: - - async_exec.command( - "sql", "INSERT INTO Log SET id = :id", callback=on_success, id=1 - ) + Set a global callback that runs after a batch commits. + + This is the signal that the records applied so far are stored. The + per-record ``callback`` of ``command()`` or ``create_record()`` is not: + it runs when the record is applied to the current batch, before the + commit. With the WAL off (``set_transaction_use_wal(False)``) a batch + that ``on_ok`` reported can still be lost in a crash. The callback takes + no arguments, does not say how many records the batch held and can fire + for an empty commit: count what you submitted, and use + ``wait_completion()`` to know that all of it is done. Args: callback: Success callback, no args diff --git a/src/arcadedb_embedded/core.py b/src/arcadedb_embedded/core.py index eed4ed0e..2a85037a 100644 --- a/src/arcadedb_embedded/core.py +++ b/src/arcadedb_embedded/core.py @@ -1034,6 +1034,20 @@ def get_name(self) -> str: except Exception as e: raise ArcadeDBError(f"Failed to get database name: {e}") from e + def is_read_only(self) -> bool: + """Return True if the database was opened read-only. + + Example: + >>> db = arcadedb.open_database("./mydb", read_only=True) + >>> db.is_read_only() + True + """ + self._check_not_closed() + try: + return str(self._java_db.getMode()) == "READ_ONLY" + except Exception as e: + raise ArcadeDBError(f"Failed to read the database mode: {e}") from e + def get_database_path(self) -> str: """Get the database path.""" self._check_not_closed() @@ -1675,6 +1689,7 @@ def import_documents( trim_text: Optional[bool] = None, on_row_error: Optional[str] = None, extra_settings: Optional[Mapping[str, Any]] = None, + disable_xml_limits: bool = False, ) -> ImportResult: """ Import document-shaped data through ArcadeDB's Java importer framework. @@ -1706,6 +1721,10 @@ def import_documents( value raises `ValueError`, because the engine treats everything that is not `"skip"` as `"abort"`. extra_settings: Additional raw importer settings. + disable_xml_limits: Lift the JDK's XML entity limits for this call only, for an + XML file with a very large text node or many entity references. Default + False. The limits are restored when the call ends. An import that hits one + raises ``ArcadeDBError`` whose message names this option. Returns: ImportResult with normalized metadata and importer statistics. @@ -1729,6 +1748,7 @@ def import_documents( trim_text=trim_text, on_row_error=on_row_error, extra_settings=extra_settings, + disable_xml_limits=disable_xml_limits, ) @property @@ -1921,10 +1941,22 @@ def create(self) -> Database: except Exception as e: raise ArcadeDBError(f"Failed to create database: {e}") from e - def open(self) -> Database: - """Open an existing database.""" + def open(self, read_only: bool = False) -> Database: + """Open an existing database. + + Args: + read_only: Open in ``ComponentFile.MODE.READ_ONLY``. Reads, queries + and scans work; every write and schema change raises + ``ArcadeDBError``. ``begin()`` does not raise: the first write + does, and so does the commit. Use it to inspect a database that + must not change, such as a backup restored for audit. + """ try: - java_db = self._java_factory.open() + if read_only: + mode = _java_class("com.arcadedb.engine.ComponentFile$MODE").READ_ONLY + java_db = self._java_factory.open(mode) + else: + java_db = self._java_factory.open() return Database(java_db) except Exception as e: raise ArcadeDBError(f"Failed to open database: {e}") from e @@ -1959,6 +1991,7 @@ def create_database( def open_database( path: str, jvm_kwargs: Optional[dict] = None, + read_only: bool = False, ) -> Database: """Open an existing database at the given path. @@ -1966,12 +1999,19 @@ def open_database( path: Database path jvm_kwargs: Optional JVM args passed to start_jvm() Example: {"heap_size": "8g"} + read_only: Open the database read-only. Writes, schema changes and + transactions raise ``ArcadeDBError``. + + Example: + >>> db = arcadedb.open_database("./mydb", read_only=True) + >>> db.is_read_only() + True """ factory = DatabaseFactory( path, jvm_kwargs=jvm_kwargs, ) - return factory.open() + return factory.open(read_only=read_only) def database_exists(path: str) -> bool: diff --git a/src/arcadedb_embedded/importer.py b/src/arcadedb_embedded/importer.py index c9bf07f2..ded66346 100644 --- a/src/arcadedb_embedded/importer.py +++ b/src/arcadedb_embedded/importer.py @@ -1,5 +1,8 @@ from __future__ import annotations +import re +import threading +from contextlib import contextmanager from dataclasses import dataclass, field from os import PathLike, fspath from pathlib import Path @@ -14,6 +17,52 @@ # means "abort". Keep this list in step with that method, not with the docs. _VALID_ON_ROW_ERROR = frozenset({"abort", "skip"}) +# The JDK's XML entity limits, as system properties. The parser factory reads +# them when it is created, which the XML importer does once per import, so +# setting them around one import relaxes the limits for that import only. +_XML_LIMIT_PROPERTIES = ( + "jdk.xml.maxGeneralEntitySizeLimit", + "jdk.xml.entityExpansionLimit", + "jdk.xml.totalEntitySizeLimit", +) +_xml_limits_lock = threading.Lock() + +# JAXP00010001 entity expansion, JAXP00010003 general entity size, JAXP00010004 +# total entity size: the three limits the properties above lift. +_XML_LIMIT_ERROR = re.compile(r"JAXP0001000[134]") +_XML_LIMIT_HINT = ( + " The JDK's XML entity limits stopped this import. Pass" + " disable_xml_limits=True to import_documents to lift them for this call" + " only, or to start_jvm to lift them for the whole process (needed for SQL" + " IMPORT DATABASE of XML)." +) + + +@contextmanager +def _xml_limits_lifted(enabled: bool): + """Lift the JDK XML entity limits for the duration of the block. + + The previous values are restored afterwards. The lock serializes lifted + imports with each other; an XML parse started by another thread inside the + block sees the lifted limits too, because JDK system properties are global. + """ + if not enabled: + yield + return + System = jpype.JClass("java.lang.System") + with _xml_limits_lock: + saved = {name: System.getProperty(name) for name in _XML_LIMIT_PROPERTIES} + for name in _XML_LIMIT_PROPERTIES: + System.setProperty(name, "0") + try: + yield + finally: + for name, value in saved.items(): + if value is None: + System.clearProperty(name) + else: + System.setProperty(name, str(value)) + @dataclass(slots=True) class ImportResult: @@ -188,6 +237,7 @@ def import_documents( trim_text: bool | None = None, on_row_error: str | None = None, extra_settings: Mapping[str, Any] | None = None, + disable_xml_limits: bool = False, ) -> ImportResult: source_url = normalize_import_source(source) settings = _build_import_settings( @@ -214,13 +264,14 @@ def import_documents( importer = Importer(java_database, None) importer.setSettings(_to_java_settings_map(settings)) - java_stats = _run_import_with_runtime_settings( - java_database, - importer, - parallel=parallel, - commit_every=commit_every, - wal=wal, - ) + with _xml_limits_lifted(disable_xml_limits): + java_stats = _run_import_with_runtime_settings( + java_database, + importer, + parallel=parallel, + commit_every=commit_every, + wal=wal, + ) statistics = _java_value_to_python(java_stats) if java_stats is not None else {} return ImportResult( @@ -230,7 +281,10 @@ def import_documents( statistics=statistics, ) except Exception as e: - raise ArcadeDBError(f"Document import failed: {e}") from e + message = f"Document import failed: {e}" + if not disable_xml_limits and _XML_LIMIT_ERROR.search(str(e)): + message += _XML_LIMIT_HINT + raise ArcadeDBError(message) from e def _run_import_with_runtime_settings( diff --git a/src/arcadedb_embedded/jvm.py b/src/arcadedb_embedded/jvm.py index 75a43907..72d5e412 100644 --- a/src/arcadedb_embedded/jvm.py +++ b/src/arcadedb_embedded/jvm.py @@ -4,12 +4,16 @@ Handles JVM initialization and JAR file management. """ +import functools import glob import os import platform +import re import shlex import shutil import sys +import tempfile +import threading import zipfile from pathlib import Path from typing import Iterable, Optional, Union @@ -17,10 +21,34 @@ import jpype import jpype.imports +from ._logging import get_logger from .exceptions import ArcadeDBError +_LOGGER = get_logger(__name__) + _JVM_CONFIG = None +# One start at a time: JPype's startJVM is not safe to call twice at once, and a free-threaded CPython (no GIL) lets +# threads whose first call is create_database() or open_database() reach it together. The thread that loses the race +# waits here, then takes the "JVM already started" branch of start_jvm(). +_START_LOCK = threading.RLock() +# Default heap when neither heap_size nor a user flag sets one: this share of the +# RAM the JVM sees (the container limit inside a container, the machine's RAM +# otherwise). Chosen from memory probes. +_DEFAULT_MAX_RAM_PERCENTAGE = 50 +# Flags that already fix the maximum heap; the default is added only without them. +_MAX_HEAP_FLAG_PREFIXES = ( + "-Xmx", + "-XX:MaxRAMPercentage=", + "-XX:MaxRAMFraction=", + "-XX:MaxRAM=", + "-XX:MaxHeapSize=", +) _JVM_INTERRUPT = None # the `interrupt` the running JVM was started with +_LOG_DIR_USED = None # the log directory the running JVM was started with + +_LOG_DIR_ENV = "ARCADEDB_LOG_DIR" +_LOGGING_CONFIG_NAME = "logging.properties" +_ENGINE_LOG_RESOURCE = "arcadedb-log.properties" def _project_dir() -> Path: @@ -264,9 +292,19 @@ def get_bundled_jre_lib_path() -> str: return str(jvm_lib_path) +def _serialized(function): + @functools.wraps(function) + def locked(*args, **kwargs): + with _START_LOCK: + return function(*args, **kwargs) + + return locked + + +@_serialized def start_jvm( - heap_size: Optional[str] = "4g", - disable_xml_limits: bool = True, + heap_size: Optional[str] = None, + disable_xml_limits: bool = False, jvm_args: Optional[Union[Iterable[str], str]] = None, common_pool_parallelism: Optional[int] = None, interrupt: Optional[bool] = None, @@ -277,15 +315,35 @@ def start_jvm( JVM Configuration (programmatic preferred): ------------------------------------------- heap_size (optional) - Max heap size (e.g. "8g", "4096m"). Defaults to "4g". - A value other than "4g" replaces every -Xmx from jvm_args or the - environment. With "4g" or None, an -Xmx given there is kept (the - largest wins if there are several), and -Xmx4g is added only when - none is given. + Max heap size (e.g. "8g", "4096m"). Defaults to None: the heap is then + a share of the RAM the JVM sees (50 percent: -XX:MaxRAMPercentage=50), + which is the container limit inside a container, so no sizing is + needed. A value given here always wins: it replaces every -Xmx from + jvm_args or the environment. With None, an -Xmx or other max-heap flag + (-XX:MaxRAMPercentage=, -XX:MaxRAM=) given there is kept (the largest + -Xmx wins if there are several), and the percentage is added only when + none is given. ArcadeDB's page cache (arcadedb.maxPageRAM) is a quarter + of the resolved heap unless you set it. + + Initial heap (-Xms): leave it unset in a Python process. The JVM then + starts small (1/64 of RAM) and grows as needed; pinning -Xms equal to + -Xmx commits the whole cap at start, with the Python side still to fit + beside it. No -Xms and no -XX:InitialRAMPercentage is added here. + Pinning is the usual practice for a dedicated ArcadeDB server outside + Python. In containers, size with percentages and leave headroom for + direct memory, metaspace and thread stacks. An -Xms larger than an + explicit heap_size (or -Xmx) raises ValueError before the JVM starts, + naming both values, because the JVM would otherwise end the process. + With only a percentage as the maximum the check is skipped: the JVM + raises the maximum to the -Xms value. disable_xml_limits (optional) - If True, relaxes JDK XML entity limits to support large XML - dumps (adds jdk.xml.* limits). + Default False: the JDK's XML entity limits stay on, which is the safe + setting for XML you did not write. If True, lifts them for the whole + process (adds the jdk.xml.* flags) so SQL ``IMPORT DATABASE`` of a + large XML file works. For ``Database.import_documents`` prefer its own + ``disable_xml_limits=True``, which lifts them for that one call only. + Earlier releases defaulted to True. jvm_args (optional) Additional JVM flags to pass through (e.g. "-XX:MaxDirectMemorySize=8g", @@ -330,8 +388,11 @@ def start_jvm( -Djava.util.concurrent.ForkJoinPool.common.parallelism= Limit common pool threads if you want fewer threads for graph builds - XML import may exceed default JDK entity limits. - Configure these BEFORE import to avoid JAXP limit errors: + A large XML import may exceed the default JDK entity limits (a JAXP + limit error). Prefer ``disable_xml_limits=True`` on + ``Database.import_documents`` (one call) or on ``start_jvm`` (whole + process); these flags do the same by hand and must be set BEFORE the + JVM starts: -Djdk.xml.maxGeneralEntitySizeLimit=0 -Djdk.xml.entityExpansionLimit=0 -Djdk.xml.totalEntitySizeLimit=0 @@ -343,16 +404,28 @@ def start_jvm( # Development/testing (smaller memory) start_jvm(heap_size="2g") + # Another share of RAM instead of a fixed size + start_jvm(jvm_args="-XX:MaxRAMPercentage=70") + # Explicit thread cap for reproducible benchmarks start_jvm(heap_size="8g", common_pool_parallelism=8) + ARCADEDB_LOG_DIR (optional) + Directory for the engine log (arcadedb.log.*), the JVM crash log and the + generated logging.properties. Default: a per-user state directory, + never the working directory (Linux ``$XDG_STATE_HOME`` or + ``~/.local/state``, macOS ``~/Library/Logs``, Windows ``%LOCALAPPDATA%``, + then ``arcadedb-embedded/log``). A ``-Darcadedb.server.logsDirectory=`` or + ``-Djava.util.logging.config.file=`` in ``jvm_args`` wins over it. + ARCADEDB_JVM_ERROR_FILE (optional) - Path for JVM crash logs (default: ./log/hs_err_pid%p.log) + Path for JVM crash logs (default: ``hs_err_pid%p.log`` in the log + directory above). ``-XX:ErrorFile=`` in ``jvm_args`` wins over it. Note: JVM options must be set BEFORE the first JVM start, as the JVM can only be configured once per Python process. """ - global _JVM_CONFIG, _JVM_INTERRUPT + global _JVM_CONFIG, _JVM_INTERRUPT, _LOG_DIR_USED if jpype.isJVMStarted(): candidate_args = tuple( _build_jvm_args( @@ -360,12 +433,13 @@ def start_jvm( disable_xml_limits=disable_xml_limits, jvm_args=jvm_args, common_pool_parallelism=common_pool_parallelism, + log_dir=_LOG_DIR_USED, ) ) has_overrides = ( jvm_args is not None - or (heap_size not in (None, "4g")) - or (disable_xml_limits is not True) + or (heap_size is not None) + or (disable_xml_limits is not False) or (common_pool_parallelism is not None) ) if ( @@ -405,12 +479,16 @@ def start_jvm( # Get bundled JRE's JVM library path jvm_path = get_bundled_jre_lib_path() + log_dir = _prepare_log_dir(_default_log_dir(), jar_files) + jvm_args = _build_jvm_args( heap_size=heap_size, disable_xml_limits=disable_xml_limits, jvm_args=jvm_args, common_pool_parallelism=common_pool_parallelism, + log_dir=log_dir, ) + _check_initial_heap(jvm_args) try: # Always use bundled JRE @@ -422,6 +500,7 @@ def start_jvm( jvm_path, *jvm_args, classpath=classpath, interrupt=_JVM_INTERRUPT ) _JVM_CONFIG = tuple(jvm_args) + _LOG_DIR_USED = log_dir except Exception as e: raise ArcadeDBError(f"Failed to start JVM: {e}") from e @@ -471,25 +550,46 @@ def _normalize_jvm_args(jvm_args: Optional[Union[Iterable[str], str]]) -> list[s def _parse_memory_size(value: str) -> Optional[int]: - """Parse JVM memory size to bytes. Supports k, m, g (case-insensitive).""" + """Parse a JVM memory size to bytes: k, m, g, t (case-insensitive), or bytes.""" if not value: return None + units = {"k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4} try: unit = value[-1].lower() - number = value[:-1] - if unit in {"k", "m", "g"}: - base = float(number) - if unit == "k": - return int(base * 1024) - if unit == "m": - return int(base * 1024 * 1024) - if unit == "g": - return int(base * 1024 * 1024 * 1024) + if unit in units: + return int(float(value[:-1]) * units[unit]) return int(value) - except Exception: + except ValueError: return None +def _check_initial_heap(args: list[str]) -> None: + """Refuse an -Xms larger than the maximum heap before the JVM starts. + + The JVM prints an error and ends the whole process for that mix, which a + Python caller cannot catch. The maximum is known only when -Xmx (or + -XX:MaxHeapSize=) is in the arguments. With only a percentage + (-XX:MaxRAMPercentage=) the check is skipped: the JVM then raises the + maximum to the -Xms value instead of aborting (measured), so nothing + needs preventing. + """ + initial = [a[4:] for a in args if a.startswith("-Xms")] + maxima = [a[4:] for a in args if a.startswith("-Xmx")] or [ + a[len("-XX:MaxHeapSize=") :] for a in args if a.startswith("-XX:MaxHeapSize=") + ] + if not initial or not maxima: + return + xms_text, xmx_text = initial[-1], maxima[-1] + xms, xmx = _parse_memory_size(xms_text), _parse_memory_size(xmx_text) + if xms is not None and xmx is not None and xms > xmx: + raise ValueError( + f"-Xms{xms_text} is larger than the maximum heap ({xmx_text}); the " + "JVM would refuse to start and end the process. Leave -Xms unset " + "(the heap then grows as needed), or make it no larger than " + "heap_size / -Xmx." + ) + + def _dedupe_max_heap(args: list[str]) -> list[str]: """Deduplicate -Xmx args keeping the maximum value when possible.""" heap_values = [] @@ -514,11 +614,119 @@ def _dedupe_max_heap(args: list[str]) -> list[str]: return filtered +def _state_base_dir() -> Path: + """Per-user base directory for state and logs, by platform (stdlib only). + + The home directory is read only where the platform needs it, and when it + cannot be determined (no HOME or USERPROFILE, as in a stripped-down service + environment) the system temp directory is used, so this never raises. + """ + if sys.platform == "win32": + local = os.environ.get("LOCALAPPDATA") + if local: + return Path(local) + elif sys.platform != "darwin": + xdg = os.environ.get("XDG_STATE_HOME") + # The XDG spec ignores a relative value. + if xdg and os.path.isabs(xdg): + return Path(xdg) + try: + home = Path.home() + except (RuntimeError, KeyError, OSError): + return Path(tempfile.gettempdir()) + if sys.platform == "win32": + return home / "AppData" / "Local" + if sys.platform == "darwin": + return home / "Library" / "Logs" + return home / ".local" / "state" + + +def _default_log_dir() -> Path: + """Where the engine log, the crash log and logging.properties go. + + ``ARCADEDB_LOG_DIR`` if set, else ``arcadedb-embedded/log`` under the + per-user state directory. Never the working directory. + """ + override = os.environ.get(_LOG_DIR_ENV) + if override: + return Path(os.path.abspath(os.path.expanduser(override))) + return _state_base_dir() / "arcadedb-embedded" / "log" + + +def _read_engine_log_properties(jar_files: Iterable[str]) -> Optional[str]: + """Read the engine's bundled ``arcadedb-log.properties`` from its jar.""" + for jar in sorted(jar_files, key=lambda j: "engine" not in os.path.basename(j)): + try: + with zipfile.ZipFile(jar) as zf: + return zf.read(_ENGINE_LOG_RESOURCE).decode("iso-8859-1") + except (KeyError, OSError, zipfile.BadZipFile): + continue + return None + + +def _render_logging_config(engine_text: str) -> str: + """The engine's logging config with the file log in ``logsDirectory``. + + The jar's own file hard-codes ``./log/arcadedb.log``, which lands in the + working directory. The engine resolves ``${arcadedb.server.logsDirectory}`` + in the pattern, and the JVM argument that sets it is added in + ``_build_jvm_args``. + """ + lines = [ + line + for line in engine_text.splitlines() + if not re.match(r"\s*java\.util\.logging\.FileHandler\.pattern\s*[=:]", line) + ] + lines.append( + "java.util.logging.FileHandler.pattern=" + "${arcadedb.server.logsDirectory}/arcadedb.log" + ) + return "\n".join(lines) + "\n" + + +def _prepare_log_dir(log_dir: Path, jar_files: Iterable[str]) -> Path: + """Create the log directory and write its logging.properties. + + Returns the directory to use: ``log_dir``, or a fresh directory under the + system temp directory when ``log_dir`` cannot be created or written (a + read-only home, for example), so that nothing is ever written to the + working directory. + """ + try: + log_dir.mkdir(parents=True, exist_ok=True) + if not os.access(log_dir, os.W_OK | os.X_OK): + raise OSError(f"{log_dir} is not writable") + except OSError as exc: + fallback = Path(tempfile.mkdtemp(prefix="arcadedb-embedded-log-")) + _LOGGER.warning( + "Cannot use %s for ArcadeDB logs (%s); using %s. Set %s to choose one.", + log_dir, + exc, + fallback, + _LOG_DIR_ENV, + ) + log_dir = fallback + + engine_text = _read_engine_log_properties(jar_files) + if engine_text is not None: + config = log_dir / _LOGGING_CONFIG_NAME + rendered = _render_logging_config(engine_text) + try: + if not config.exists() or config.read_text("iso-8859-1") != rendered: + tmp = log_dir / f"{_LOGGING_CONFIG_NAME}.{os.getpid()}.tmp" + tmp.write_text(rendered, encoding="iso-8859-1", newline="\n") + os.replace(tmp, config) + except OSError as exc: + _LOGGER.warning("Cannot write %s: %s", config, exc) + return log_dir + + def _build_jvm_args( heap_size: Optional[str], disable_xml_limits: bool, jvm_args: Optional[Union[Iterable[str], str]], common_pool_parallelism: Optional[int] = None, + log_dir: Optional[Path] = None, ) -> list[str]: """Helper to construct JVM arguments from params, env vars, and defaults.""" if common_pool_parallelism is not None and common_pool_parallelism < 1: @@ -597,23 +805,40 @@ def _build_jvm_args( merged_args.append("-XX:+UseCompactObjectHeaders") # Heap handling (single place): - # - If heap_size is explicitly set to non-default, override. - # - If heap_size is default or None, keep env/user -Xmx and dedupe (or add default). - has_xmx = any(arg.startswith("-Xmx") for arg in merged_args) - if heap_size is not None and heap_size != "4g": + # - An explicit heap_size wins over every -Xmx from the environment or jvm_args. + # - With heap_size None, a user's -Xmx (deduped to the largest) or other + # max-heap flag is kept, and a share of RAM is added only when there is none. + if heap_size is not None: merged_args = [arg for arg in merged_args if not arg.startswith("-Xmx")] merged_args.append(f"-Xmx{heap_size}") + elif not any(arg.startswith(_MAX_HEAP_FLAG_PREFIXES) for arg in merged_args): + merged_args.append(f"-XX:MaxRAMPercentage={_DEFAULT_MAX_RAM_PERCENTAGE}") else: - if not has_xmx: - merged_args.append("-Xmx4g") merged_args = _dedupe_max_heap(merged_args) + # Logs and the crash log go to a per-user directory, never the working + # directory. An argument the caller gave wins over each default. + if log_dir is None: + log_dir = _default_log_dir() + log_dir_posix = log_dir.as_posix() + if not any( + arg.startswith("-Darcadedb.server.logsDirectory=") for arg in merged_args + ): + merged_args.append(f"-Darcadedb.server.logsDirectory={log_dir_posix}") + if not any( + arg.startswith("-Djava.util.logging.config.file=") for arg in merged_args + ): + merged_args.append( + f"-Djava.util.logging.config.file={log_dir_posix}/{_LOGGING_CONFIG_NAME}" + ) + # Configure JVM crash log location (hs_err_pid*.log files) - error_file = os.environ.get("ARCADEDB_JVM_ERROR_FILE") - if error_file: - merged_args.append(f"-XX:ErrorFile={error_file}") - else: - merged_args.append("-XX:ErrorFile=./log/hs_err_pid%p.log") + if not any(arg.startswith("-XX:ErrorFile=") for arg in merged_args): + error_file = os.environ.get("ARCADEDB_JVM_ERROR_FILE") + if error_file: + merged_args.append(f"-XX:ErrorFile={error_file}") + else: + merged_args.append(f"-XX:ErrorFile={log_dir_posix}/hs_err_pid%p.log") return merged_args diff --git a/tests/test_async_executor.py b/tests/test_async_executor.py index be27bd71..336c2092 100644 --- a/tests/test_async_executor.py +++ b/tests/test_async_executor.py @@ -471,3 +471,104 @@ def test_async_executor_wait_completion_positive_timeout_still_times_out(temp_db assert async_exec.wait_completion(30000) is None finally: async_exec.close() + + +# What each async signal means (docs/api/async_executor.md, "When is async work +# stored?"): a per-record callback runs when the record is applied to the writer's +# current batch, on_ok runs after a batch commits, and wait_completion() returns +# when everything queued is committed. A crash test (kill -9) showed what that +# means for durability; these tests pin the ordering without killing anything. + + +def test_per_record_callback_runs_before_the_batch_commits_and_on_ok_after(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Signal") + + ok_visible_counts = [] + applied = threading.Event() + release = threading.Event() + + def on_ok(): + # on_ok runs on the writer thread after the commit: the rows are visible + ok_visible_counts.append(db.count_type("Signal")) + + # one worker, a batch far larger than the load: nothing commits until the + # queue drains, so the first record sits in an uncommitted batch + ex = db.async_executor().set_parallel_level(1).set_commit_every(1000) + ex.on_ok(on_ok) + + def on_applied(_record): + applied.set() + release.wait(30) + + doc = db.new_document("Signal") + doc.set("n", 0) + ex.create_record(doc, callback=on_applied) + assert applied.wait(30), "the per-record callback never ran" + + try: + # the callback has fired, so the record is applied; it is not committed + assert ex.is_processing() or ex.is_pending() + assert ok_visible_counts == [], "on_ok fired before the batch committed" + assert db.count_type("Signal") == 0, "an applied record was already visible" + finally: + release.set() + ex.wait_completion() + ex.close() + + assert db.count_type("Signal") == 1 + assert ok_visible_counts, "on_ok never fired after the commit" + assert ok_visible_counts[-1] == 1, "on_ok ran before the commit was visible" + + +def test_wait_completion_commits_everything_queued_including_the_tail(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Tail") + + errors = [] + callbacks = [] + # 250 records at a commit every 100: two full batches and a tail of 50 that only + # the end of the queue commits + ex = db.async_executor().set_parallel_level(1).set_commit_every(100) + ex.on_error(errors.append) + for i in range(250): + doc = db.new_document("Tail") + doc.set("n", i) + ex.create_record(doc, callback=callbacks.append) + + ex.wait_completion() + + assert not ex.is_pending() + assert not ex.is_processing() + assert errors == [] + # nothing left to apply and nothing left uncommitted + assert len(callbacks) == 250 + assert db.count_type("Tail") == 250 + ex.close() + + +@pytest.mark.parametrize("use_wal", [True, False]) +def test_wait_completion_then_reopen_has_every_record(temp_db_path, use_wal): + # After wait_completion() the load is stored, with the WAL on or off. The WAL + # setting only matters for a crash (docs/api/async_executor.md). + db = arcadedb.create_database(temp_db_path) + try: + db.command("sql", "CREATE DOCUMENT TYPE Stored") + ex = db.async_executor().set_parallel_level(1).set_commit_every(100) + ex.set_transaction_use_wal(use_wal) + assert ex.is_transaction_use_wal() is use_wal + for i in range(250): + doc = db.new_document("Stored") + doc.set("n", i) + ex.create_record(doc) + ex.wait_completion() + ex.close() + finally: + db.close() + + db = arcadedb.open_database(temp_db_path) + try: + assert db.count_type("Stored") == 250 + assert db.query("sql", "SELECT max(n) AS m FROM Stored").first().get("m") == 249 + finally: + db.close() diff --git a/tests/test_free_threading.py b/tests/test_free_threading.py new file mode 100644 index 00000000..33fe68b1 --- /dev/null +++ b/tests/test_free_threading.py @@ -0,0 +1,384 @@ +"""Many Python threads against one database and against one database each, on either backend. + +Built for a free-threaded CPython (3.13t, 3.14t, no GIL) where Python threads run in parallel, but it also runs, and must +pass, with the GIL. Every thread runs the same deterministic workload (transactional inserts, numpy vector inserts, queries, +vector searches and, on the JPype backend, Python functions and record events), and the per-thread results and the final +database state are compared with a sequential run of the same workload. + + FT_THREADS=16 FT_REPEAT=50 python -X dev tests/test_free_threading.py # as a script: repeat count, one line per scenario + FT_THREADS=16 FT_REPEAT=3 pytest tests/test_free_threading.py # as a test +""" + +import os +import shutil +import subprocess # nosec B404 - test-controlled child process +import sys +import tempfile +import textwrap +import threading + +import numpy as np +import pytest + +import arcadedb_embedded as arcadedb + +THREADS = int(os.environ.get("FT_THREADS", "16")) +REPEAT = int(os.environ.get("FT_REPEAT", "1")) +ROWS = int( + os.environ.get("FT_ROWS", "40") +) # documents (and vertices with a vector) per thread +DIM = 8 +BASE = 64 # vertices in the indexed type the searches run against +NATIVE = os.environ.get("ARCADEDB_BACKEND", "jpype").lower() == "native" +NN = "SELECT bid FROM (SELECT expand(vectorNeighbors('Base[embedding]', ?, ?)))" + + +def _val(tid, seq): + return tid * 1000 + seq * 3 + 1 + + +def _vec(tid, seq): + """A float32 row far from the Base vectors (offset 1000), so a search for a Base vector never meets it.""" + return (np.random.default_rng(tid * 100003 + seq).random(DIM) + 1000.0).astype( + np.float32 + ) + + +def _base_vec(i): + return np.random.default_rng(900001 + i).random(DIM).astype(np.float32) + + +def _mix(val, seq): + return val * 31 + seq % 7 + + +class Probes: + """What the callbacks saw; the lock guards the call counter, the event list is appended to without one.""" + + def __init__(self): + self.lock = threading.Lock() + self.calls = 0 + self.events = [] # (tid, seq) per after_create, appended without a lock + + def udf(self, val, seq): + with self.lock: + self.calls += 1 + return _mix(val, seq) + + def on_create(self, record): + self.events.append((int(record.get("tid")), int(record.get("seq")))) + + +def _schema(db): + db.command("sql", "CREATE DOCUMENT TYPE Doc") + db.command("sql", "CREATE VERTEX TYPE Vec") + db.command("sql", "CREATE VERTEX TYPE Base") + db.command("sql", "CREATE PROPERTY Base.embedding ARRAY_OF_FLOATS") + db.command("sql", "CREATE PROPERTY Vec.embedding ARRAY_OF_FLOATS") + with db.transaction(): + for i in range(BASE): + db.command( + "sql", "CREATE VERTEX Base SET bid = ?, embedding = ?", i, _base_vec(i) + ) + db.command( + "sql", + 'CREATE INDEX ON Base (embedding) LSM_VECTOR METADATA {"dimensions": %d, "similarity": "EUCLIDEAN"}' + % DIM, + ) + + +def _callbacks(db, probes, suffix): + """Python function and event callbacks on this database (JPype backend only); returns the function name or None.""" + if NATIVE or not hasattr(db, "register_function"): + return None + name = f"py_mix{suffix}" + db.register_function(name, probes.udf) + db.on("after_create", probes.on_create, "Doc") + return name + + +def _work(db, tid, udf_name): + out = {"counts": [], "nn": []} + for b in range(0, ROWS, 10): + + def batch(b=b): + for s in range(b, min(b + 10, ROWS)): + db.command( + "sql", + "INSERT INTO Doc SET tid = ?, seq = ?, val = ?", + tid, + s, + _val(tid, s), + ) + db.command( + "sql", + "CREATE VERTEX Vec SET tid = ?, seq = ?, embedding = ?", + tid, + s, + _vec(tid, s), + ) + + # two threads that write the same page conflict at commit (ConcurrentModificationException): the engine's answer is a retry + db.run_in_transaction(batch, retries=50) + out["counts"].append( + db.query("sql", "SELECT count(*) AS c FROM Doc WHERE tid = ?", tid) + .first() + .get("c") + ) + out["rows"] = [ + (r["seq"], r["val"]) + for r in db.query( + "sql", "SELECT seq, val FROM Doc WHERE tid = ? ORDER BY seq", tid + ).to_list() + ] + out["vecs"] = [ + tuple(round(float(x), 3) for x in r["embedding"]) + for r in db.query( + "sql", "SELECT embedding FROM Vec WHERE tid = ? ORDER BY seq", tid + ).to_list() + ] + for j in range(6): + i = (tid * 6 + j) % BASE + out["nn"].append( + (i, [int(r["bid"]) for r in db.query("sql", NN, _base_vec(i), 1).to_list()]) + ) + if udf_name: + out["udf"] = [ + r["m"] + for r in db.query( + "sql", + f"SELECT {udf_name}(val, seq) AS m FROM Doc WHERE tid = ? ORDER BY seq", # nosec B608 - test-owned UDF name + tid, + ).to_list() + ] + return out + + +def _state(db): + return ( + [ + tuple(r.values()) + for r in db.query( + "sql", + "SELECT tid, count(*) AS c, sum(val) AS s FROM Doc GROUP BY tid ORDER BY tid", + ).to_list() + ], + db.query("sql", "SELECT count(*) AS c FROM Vec").first().get("c"), + ) + + +def _expect(tid): + return { + "counts": [min(b + 10, ROWS) for b in range(0, ROWS, 10)], + "rows": [(s, _val(tid, s)) for s in range(ROWS)], + "vecs": [tuple(round(float(x), 3) for x in _vec(tid, s)) for s in range(ROWS)], + "nn": [((tid * 6 + j) % BASE, [(tid * 6 + j) % BASE]) for j in range(6)], + } + + +def _check(tid, out, udf): + for key, want in _expect(tid).items(): + assert ( + out[key] == want + ), f"thread {tid}: {key} differs from the sequential result" + if udf: + assert out["udf"] == [ + _mix(_val(tid, s), s) for s in range(ROWS) + ], f"thread {tid}: function results differ" + + +def _run_threads(n, target): + errors, results = [], {} + barrier = threading.Barrier(n) + + def run(tid): + try: + barrier.wait() + results[tid] = target(tid) + except BaseException as e: # reported below with its thread + errors.append((tid, repr(e))) + + threads = [threading.Thread(target=run, args=(t,)) for t in range(n)] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errors, errors + return results + + +def shared_database(tmp, n=THREADS): + """One database, n threads; the final state equals the sequential run of the same workloads.""" + with arcadedb.create_database(os.path.join(tmp, "seq")) as db: + _schema(db) + probes = Probes() + name = _callbacks(db, probes, "_seq") + for tid in range(n): + _check(tid, _work(db, tid, name), bool(name)) + seq_state = _state(db) + seq_events = sorted(probes.events) + if name: + db.unregister_function(name) + with arcadedb.create_database(os.path.join(tmp, "par")) as db: + _schema(db) + probes = Probes() + name = _callbacks(db, probes, "_par") + results = _run_threads(n, lambda tid: _work(db, tid, name)) + for tid, out in results.items(): + _check(tid, out, bool(name)) + assert _state(db) == seq_state + if name: + # an event fires per created record, also in an attempt that a commit conflict rolled back and a retry repeated + assert ( + sorted(set(probes.events)) == seq_events + ), "record events differ from the sequential run" + assert probes.calls == n * ROWS, probes.calls + db.unregister_function(name) + + +def database_per_thread(tmp, n=THREADS): + """One database per thread (each with its own function name and event handler).""" + + def target(tid): + with arcadedb.create_database(os.path.join(tmp, f"own{tid}")) as db: + _schema(db) + probes = Probes() + name = _callbacks(db, probes, f"_own{tid}") + out = _work(db, tid, name) + _check(tid, out, bool(name)) + if name: + assert sorted(set(probes.events)) == [ + (tid, s) for s in range(ROWS) + ], f"thread {tid}: events differ" + assert probes.calls == ROWS + db.unregister_function(name) + assert _state(db)[1] == ROWS + return out + + _run_threads(n, target) + + +def reopen_and_close_race(tmp, n=THREADS): + """Threads open, read and close the same database path one after the other under contention; each read is exact.""" + path = os.path.join(tmp, "shared_open") + with arcadedb.create_database(path) as db: + _schema(db) + with db.transaction(): + for s in range(ROWS): + db.command( + "sql", + "INSERT INTO Doc SET tid = 0, seq = ?, val = ?", + s, + _val(0, s), + ) + lock = ( + threading.Lock() + ) # one database object per path in a process is the engine's rule: open one at a time + + def target(tid): + for _ in range(3): + with lock: + with arcadedb.open_database(path) as db: + assert db.count_type("Doc") == ROWS + return None + + _run_threads(n, target) + + +FIRST_START = textwrap.dedent(""" + import os, sys, threading + import arcadedb_embedded as arcadedb + n = int(sys.argv[1]); tmp = sys.argv[2] + barrier = threading.Barrier(n); errors = [] + def run(tid): + try: + barrier.wait() + with arcadedb.create_database(os.path.join(tmp, f"d{tid}")) as db: + db.command("sql", "CREATE DOCUMENT TYPE T") + with db.transaction(): + db.command("sql", "INSERT INTO T SET a = ?", tid) + assert db.query("sql", "SELECT a FROM T").first().get("a") == tid + except BaseException as e: + errors.append((tid, repr(e))) + ts = [threading.Thread(target=run, args=(t,)) for t in range(n)] + [t.start() for t in ts]; [t.join() for t in ts] + print("FIRST_START", "ERRORS" if errors else "OK", errors[:3], flush=True) + os._exit(1 if errors else 0) # JPype 1.7.1 hangs at exit once a non-main thread has called Java + """) + + +def first_start_race(tmp, n=THREADS): + """The first call of every thread starts the JVM (or loads the library) in a fresh process: exactly one start wins.""" + script = os.path.join(tmp, "first_start.py") + with open(script, "w") as f: + f.write(FIRST_START) + work = os.path.join(tmp, "fs") + os.makedirs(work) + r = subprocess.run( # nosec B603 + [sys.executable, script, str(n), work], + capture_output=True, + text=True, + timeout=300, + ) + assert r.returncode == 0, r.stdout[-2000:] + r.stderr[-3000:] + + +SCENARIOS = [ + shared_database, + database_per_thread, + reopen_and_close_race, + first_start_race, +] + + +@pytest.fixture +def tmp(): + d = tempfile.mkdtemp(prefix="arcadedb_ft_") + yield d + shutil.rmtree(d, ignore_errors=True) + + +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda s: s.__name__) +def test_threads(scenario, tmp): + for rep in range(REPEAT): + d = os.path.join(tmp, f"r{rep}") + os.makedirs(d) + scenario(d) + + +if __name__ == "__main__": + gil = lambda: getattr(sys, "_is_gil_enabled", lambda: True)() + print( + f"FTSTRESS python={sys.version.split()[0]} gil={gil()} backend={'native' if NATIVE else 'jpype'} " + f"threads={THREADS} repeat={REPEAT} rows={ROWS}", + flush=True, + ) + failed = 0 + only = os.environ.get("FT_ONLY") + for scenario in SCENARIOS: + if only and scenario.__name__ not in only.split(","): + continue + ok = 0 + for rep in range(REPEAT): + d = tempfile.mkdtemp(prefix="arcadedb_ft_") + try: + scenario(d) + ok += 1 + except ( + BaseException + ) as e: # keep going: the count of failing repeats is the result + failed += 1 + print( + f"FTSTRESS FAIL {scenario.__name__} rep={rep}: {type(e).__name__}: {str(e)[:600]}", + flush=True, + ) + finally: + shutil.rmtree(d, ignore_errors=True) + print( + f"FTSTRESS {scenario.__name__}: {ok}/{REPEAT} repeats ok (gil={gil()})", + flush=True, + ) + print("FTSTRESS DONE failed=%d" % failed, flush=True) + # JPype 1.7.1 hangs at interpreter exit once a non-main Python thread has called Java (also with the GIL, also on 3.12); + # a result is printed, so leave without the shutdown + os._exit(1 if failed else 0) diff --git a/tests/test_jvm.py b/tests/test_jvm.py index 266deb79..eb39ce42 100644 --- a/tests/test_jvm.py +++ b/tests/test_jvm.py @@ -1,5 +1,7 @@ """Tests for start_jvm() re-entry behavior once the JVM is running.""" +import os + import jpype import pytest @@ -30,9 +32,10 @@ def test_identical_config_is_idempotent(monkeypatch): args = tuple( jvm._build_jvm_args( heap_size="6g", - disable_xml_limits=True, + disable_xml_limits=False, jvm_args="-Xms6g", common_pool_parallelism=None, + log_dir=jvm._LOG_DIR_USED, ) ) monkeypatch.setattr(jvm, "_JVM_CONFIG", args) @@ -40,7 +43,7 @@ def test_identical_config_is_idempotent(monkeypatch): def test_conflicting_override_raises(monkeypatch): - monkeypatch.setattr(jvm, "_JVM_CONFIG", ("-Xmx4g",)) + monkeypatch.setattr(jvm, "_JVM_CONFIG", ("-XX:MaxRAMPercentage=50",)) with pytest.raises(ArcadeDBError, match="already started"): jvm.start_jvm(heap_size="99g") @@ -82,3 +85,139 @@ def test_interpreter_exits_with_unclosed_database(tmp_path): ) assert "OK" in proc.stdout assert proc.returncode == 0 + + +def _args(monkeypatch, env=None, **kwargs): + if env is None: + monkeypatch.delenv("ARCADEDB_JVM_ARGS", raising=False) + else: + monkeypatch.setenv("ARCADEDB_JVM_ARGS", env) + kwargs.setdefault("heap_size", None) + kwargs.setdefault("disable_xml_limits", True) + kwargs.setdefault("jvm_args", None) + return jvm._build_jvm_args(**kwargs) + + +def test_default_heap_is_a_share_of_ram(monkeypatch): + args = _args(monkeypatch) + assert f"-XX:MaxRAMPercentage={jvm._DEFAULT_MAX_RAM_PERCENTAGE}" in args + assert not any(a.startswith("-Xmx") for a in args) + + +def test_explicit_heap_size_wins_over_everything(monkeypatch): + args = _args(monkeypatch, env="-Xmx16g", heap_size="8g", jvm_args="-Xmx2g") + assert [a for a in args if a.startswith("-Xmx")] == ["-Xmx8g"] + assert not any("MaxRAMPercentage" in a for a in args) + # "4g" is an ordinary explicit value now, not a stand-in for "default" + args = _args(monkeypatch, env="-Xmx16g", heap_size="4g") + assert [a for a in args if a.startswith("-Xmx")] == ["-Xmx4g"] + + +def test_user_xmx_is_kept_and_no_default_is_added(monkeypatch): + args = _args(monkeypatch, jvm_args="-Xmx3g") + assert [a for a in args if a.startswith("-Xmx")] == ["-Xmx3g"] + assert not any("MaxRAMPercentage" in a for a in args) + args = _args(monkeypatch, env="-Xmx3g", jvm_args="-Xmx6g") + assert [a for a in args if a.startswith("-Xmx")] == ["-Xmx6g"] # largest wins + + +@pytest.mark.parametrize( + "flag", ["-XX:MaxRAMPercentage=70", "-XX:MaxRAM=6g", "-XX:MaxHeapSize=5g"] +) +def test_user_max_heap_flag_suppresses_the_default(monkeypatch, flag): + args = _args(monkeypatch, env=flag) + assert flag in args + assert not any(a.startswith("-Xmx") for a in args) + assert not any(a.startswith("-XX:MaxRAMPercentage=") and a != flag for a in args) + + +def test_default_heap_resolves_in_a_fresh_jvm(tmp_path): + """A JVM started with no heap setting gets the share of RAM it sees. + + Reads the RAM the JVM itself reports (container limit in a container) and + the page cache default that follows the heap. + """ + import subprocess # nosec B404 - test-controlled child process + import sys as _sys + + script = ( + "import arcadedb_embedded as a\n" + f"db = a.create_database({str(tmp_path / 'heap_db')!r})\n" + "import jpype\n" + "Runtime = jpype.JClass('java.lang.Runtime')\n" + "MX = jpype.JClass('java.lang.management.ManagementFactory')\n" + "GC = jpype.JClass('com.arcadedb.GlobalConfiguration')\n" + "ram = int(MX.getOperatingSystemMXBean().getTotalMemorySize())\n" + "heap = int(Runtime.getRuntime().maxMemory())\n" + "page_mb = int(GC.MAX_PAGE_RAM.getValueAsLong())\n" + "print('R', ram, heap, page_mb, flush=True)\n" + "db.close()\n" + ) + env = {k: v for k, v in os.environ.items() if k != "ARCADEDB_JVM_ARGS"} + proc = subprocess.run( # nosec B603 - fixed argv, no shell, test-owned + [_sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + lines = [ln for ln in proc.stdout.splitlines() if ln.startswith("R ")] + assert lines, proc.stderr[-1500:] + proc.stdout[-500:] + line = lines[0] + ram, heap, page_mb = (int(x) for x in line.split()[1:]) + expected = ram * jvm._DEFAULT_MAX_RAM_PERCENTAGE / 100 + assert 0.9 * expected <= heap <= 1.1 * expected + assert page_mb == pytest.approx(heap / 4 / 1024 / 1024, abs=2) + + +def test_no_initial_heap_flag_is_added(monkeypatch): + args = _args(monkeypatch) + assert not any(a.startswith(("-Xms", "-XX:InitialRAMPercentage")) for a in args) + args = _args(monkeypatch, heap_size="8g") + assert not any(a.startswith(("-Xms", "-XX:InitialRAMPercentage")) for a in args) + args = _args(monkeypatch, jvm_args="-Xms1g") # a user's own -Xms is kept + assert "-Xms1g" in args + + +@pytest.mark.parametrize( + "kwargs", + [ + {"heap_size": "1g", "jvm_args": "-Xms2g"}, + {"jvm_args": "-Xmx512M -Xms1G"}, + {"heap_size": "1g", "jvm_args": "-Xms1100m"}, + {"heap_size": "1T", "jvm_args": "-Xms2t"}, + {"heap_size": "1048576", "jvm_args": "-Xms2m"}, # bytes, no suffix + ], +) +def test_xms_larger_than_max_heap_raises_before_start(tmp_path, kwargs): + """The JVM would end the process; start_jvm raises ValueError first.""" + import subprocess # nosec B404 - test-controlled child process + import sys as _sys + + script = ( + "import arcadedb_embedded as a\n" + "try:\n" + f" a.create_database({str(tmp_path / 'xms_db')!r}, jvm_kwargs={kwargs!r})\n" + "except ValueError as e:\n" + " print('VALUEERROR', e, flush=True)\n" + "print('ALIVE', flush=True)\n" + ) + proc = subprocess.run( # nosec B603 - fixed argv, no shell, test-owned + [_sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=120, + ) + assert "VALUEERROR" in proc.stdout and "-Xms" in proc.stdout, proc.stderr[-800:] + assert "ALIVE" in proc.stdout # the Python process survived + assert proc.returncode == 0 + + +def test_check_initial_heap_accepts_valid_and_percentage_only(): + check = jvm._check_initial_heap + check(["-Xmx2g", "-Xms2g"]) # equal is fine + check(["-Xmx2g", "-Xms512m"]) + check(["-Xms64g", "-XX:MaxRAMPercentage=50"]) # percentage only: skipped + check(["-Xms2g", "-Xmx4g"]) + with pytest.raises(ValueError, match="2g.*1g"): + check(["-Xmx1g", "-Xms2g"]) diff --git a/tests/test_jvm_args.py b/tests/test_jvm_args.py index 570fa76b..842b1783 100644 --- a/tests/test_jvm_args.py +++ b/tests/test_jvm_args.py @@ -30,7 +30,7 @@ def test_custom_jvm_args_merging(): """Test merging critical flags when user provides custom JVM args.""" with patch.dict(os.environ, {"ARCADEDB_JVM_ARGS": "-Xmx8g -Dfoo=bar"}, clear=True): args = _build_jvm_args( - heap_size="4g", + heap_size=None, # the default: an -Xmx from the environment is kept (an explicit heap_size wins, see test_jvm.py) disable_xml_limits=True, jvm_args=None, ) @@ -57,7 +57,7 @@ def test_dedupe_heap_keeps_max(): os.environ, {"ARCADEDB_JVM_ARGS": "-Xmx2g -Xmx4096m -Xmx1g"}, clear=True ): args = _build_jvm_args( - heap_size="4g", + heap_size=None, # the default: an -Xmx from the environment is kept (an explicit heap_size wins, see test_jvm.py) disable_xml_limits=True, jvm_args=None, ) diff --git a/tests/test_log_dir.py b/tests/test_log_dir.py new file mode 100644 index 00000000..39999187 --- /dev/null +++ b/tests/test_log_dir.py @@ -0,0 +1,206 @@ +"""Logs and the JVM crash file go to a per-user directory, never the working directory.""" + +import os +import subprocess # nosec B404 - test-controlled child process +import sys +import tempfile +from pathlib import Path +from unittest.mock import patch + +import pytest + +from arcadedb_embedded import jvm +from arcadedb_embedded.jvm import _build_jvm_args, _default_log_dir + +CHILD = """ +import sys +import arcadedb_embedded as arcadedb +from arcadedb_embedded import jvm + +jvm_args = sys.argv[2:] +db = arcadedb.create_database(sys.argv[1], jvm_kwargs={"jvm_args": jvm_args}) +db.command("sql", "CREATE DOCUMENT TYPE Doc") +db.close() +print("LOGDIR=" + str(jvm._LOG_DIR_USED)) +""" + + +def _run_child(tmp_path, env_overrides, jvm_args=()): + """Start a JVM in a child process whose working directory is empty and read-only.""" + cwd = tmp_path / "cwd" + cwd.mkdir() + home = tmp_path / "home" + home.mkdir() + cwd.chmod(0o555) # not enforced everywhere; the listing below holds regardless + env = {k: v for k, v in os.environ.items() if k != "ARCADEDB_JVM_ARGS"} + env.pop("ARCADEDB_LOG_DIR", None) + env.pop("ARCADEDB_JVM_ERROR_FILE", None) + env.update( + HOME=str(home), + USERPROFILE=str(home), + XDG_STATE_HOME=str(home / "state"), + LOCALAPPDATA=str(home / "localappdata"), + ) + env.update(env_overrides) + try: + result = subprocess.run( # nosec B603 - fixed argument list, no shell + [ + sys.executable, + "-c", + CHILD, + str(tmp_path / "db"), + *jvm_args, + ], + cwd=cwd, + env=env, + capture_output=True, + text=True, + timeout=300, + ) + finally: + cwd.chmod(0o755) + assert result.returncode == 0, result.stdout + result.stderr + line = next(l for l in result.stdout.splitlines() if l.startswith("LOGDIR=")) + return cwd, Path(line[len("LOGDIR=") :]), home + + +def test_default_logs_land_in_the_state_dir_not_the_cwd(tmp_path): + cwd, log_dir, home = _run_child(tmp_path, {}) + assert list(cwd.iterdir()) == [] + assert home in log_dir.parents + assert log_dir.name == "log" and log_dir.parent.name == "arcadedb-embedded" + assert list(log_dir.glob("arcadedb.log*")), list(log_dir.iterdir()) + config = (log_dir / "logging.properties").read_text("iso-8859-1") + assert "${arcadedb.server.logsDirectory}/arcadedb.log" in config + assert "./log" not in config + + +def test_env_var_sets_the_log_dir(tmp_path): + target = tmp_path / "chosen logs" + cwd, log_dir, _ = _run_child(tmp_path, {"ARCADEDB_LOG_DIR": str(target)}) + assert list(cwd.iterdir()) == [] + assert log_dir == target + assert list(target.glob("arcadedb.log*")) + + +def test_jvm_args_can_move_the_engine_log(tmp_path): + target = tmp_path / "from jvm args" + cwd, _, home = _run_child( + tmp_path, {}, jvm_args=[f"-Darcadedb.server.logsDirectory={target.as_posix()}"] + ) + assert list(cwd.iterdir()) == [] + assert list(target.glob("arcadedb.log*")) + + +def test_unusable_log_dir_falls_back_to_a_temp_dir(tmp_path): + blocker = tmp_path / "a file" + blocker.write_text("not a directory") + cwd, log_dir, _ = _run_child(tmp_path, {"ARCADEDB_LOG_DIR": str(blocker / "log")}) + assert list(cwd.iterdir()) == [] + assert log_dir != blocker / "log" + assert list(log_dir.glob("arcadedb.log*")) + + +def test_args_never_name_the_working_directory(): + with patch.dict(os.environ, {}, clear=True): + args = _build_jvm_args(heap_size="4g", disable_xml_limits=False, jvm_args=None) + # the expected directory comes from the same (cleared) environment the args were built in: on Windows the real + # environment has LOCALAPPDATA and the cleared one falls back, so computing it outside the block would differ + log_dir = _default_log_dir().as_posix() + assert not any("./log" in a for a in args) + assert f"-Darcadedb.server.logsDirectory={log_dir}" in args + assert f"-Djava.util.logging.config.file={log_dir}/logging.properties" in args + assert f"-XX:ErrorFile={log_dir}/hs_err_pid%p.log" in args + + +def test_state_dir_by_platform(tmp_path): + env = {"HOME": str(tmp_path), "USERPROFILE": str(tmp_path)} + with patch.dict(os.environ, {**env, "XDG_STATE_HOME": str(tmp_path / "xdg")}): + with patch.object(jvm.sys, "platform", "linux"): + assert _default_log_dir() == tmp_path / "xdg" / "arcadedb-embedded" / "log" + with patch.dict(os.environ, {**env, "XDG_STATE_HOME": "relative"}): + with patch.object(jvm.sys, "platform", "linux"): + expected = tmp_path / ".local" / "state" / "arcadedb-embedded" / "log" + assert _default_log_dir() == expected + with patch.dict(os.environ, env): + with patch.object(jvm.sys, "platform", "darwin"): + expected = tmp_path / "Library" / "Logs" / "arcadedb-embedded" / "log" + assert _default_log_dir() == expected + with patch.dict(os.environ, {**env, "LOCALAPPDATA": str(tmp_path / "lad")}): + with patch.object(jvm.sys, "platform", "win32"): + assert _default_log_dir() == tmp_path / "lad" / "arcadedb-embedded" / "log" + + +def test_user_arguments_win_over_the_log_defaults(tmp_path): + user = [ + "-Darcadedb.server.logsDirectory=/x/logs", + "-Djava.util.logging.config.file=/x/my.properties", + "-XX:ErrorFile=/x/crash.log", + ] + with patch.dict(os.environ, {"ARCADEDB_JVM_ERROR_FILE": "/y/env.log"}, clear=True): + args = _build_jvm_args( + heap_size="4g", disable_xml_limits=False, jvm_args=user, log_dir=tmp_path + ) + for arg in user: + assert args.count(arg) == 1 + assert sum(a.startswith("-Darcadedb.server.logsDirectory=") for a in args) == 1 + assert sum(a.startswith("-Djava.util.logging.config.file=") for a in args) == 1 + assert sum(a.startswith("-XX:ErrorFile=") for a in args) == 1 + + +def test_env_error_file_still_overrides_the_default(tmp_path): + with patch.dict(os.environ, {"ARCADEDB_JVM_ERROR_FILE": "/y/env.log"}, clear=True): + args = _build_jvm_args( + heap_size="4g", disable_xml_limits=False, jvm_args=None, log_dir=tmp_path + ) + assert "-XX:ErrorFile=/y/env.log" in args + assert sum(a.startswith("-XX:ErrorFile=") for a in args) == 1 + + +def test_rendered_config_keeps_the_engine_settings_and_moves_the_file_log(): + engine = ( + "handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler\n" + "java.util.logging.FileHandler.pattern=./log/arcadedb.log\n" + "java.util.logging.FileHandler.count=10\n" + ) + rendered = jvm._render_logging_config(engine) + assert "./log" not in rendered + assert "FileHandler.count=10" in rendered + assert rendered.count("FileHandler.pattern") == 1 + assert "${arcadedb.server.logsDirectory}/arcadedb.log" in rendered + + +def test_the_engine_jar_carries_the_log_config_the_render_starts_from(): + import glob + + jars = glob.glob(os.path.join(jvm.get_jar_path(), "*.jar")) + text = jvm._read_engine_log_properties(jars) + assert text is not None + assert "java.util.logging.FileHandler.pattern" in text + + +def test_windows_branch_needs_no_home_directory(tmp_path): + # The Windows runner clears the environment in the unit tests above; the base + # directory must come from LOCALAPPDATA, or the temp dir, without a home directory. + with patch.object(jvm.sys, "platform", "win32"): + with patch.dict(os.environ, {"LOCALAPPDATA": str(tmp_path)}, clear=True): + with patch.object(Path, "home", side_effect=RuntimeError("no home")): + assert _default_log_dir() == tmp_path / "arcadedb-embedded" / "log" + with patch.dict(os.environ, {}, clear=True): + with patch.object(Path, "home", side_effect=RuntimeError("no home")): + expected = Path(tempfile.gettempdir()) / "arcadedb-embedded" / "log" + assert _default_log_dir() == expected + with patch.dict(os.environ, {}, clear=True): + with patch.object(Path, "home", return_value=tmp_path): + expected = tmp_path / "AppData" / "Local" / "arcadedb-embedded" / "log" + assert _default_log_dir() == expected + + +def test_arg_values_use_forward_slashes_on_every_platform(tmp_path): + # as_posix() keeps a Windows drive path usable in -D and -XX values. + with patch.dict(os.environ, {}, clear=True): + args = _build_jvm_args( + heap_size="4g", disable_xml_limits=False, jvm_args=None, log_dir=tmp_path + ) + assert f"-Darcadedb.server.logsDirectory={tmp_path.as_posix()}" in args + assert f"-XX:ErrorFile={tmp_path.as_posix()}/hs_err_pid%p.log" in args diff --git a/tests/test_open_read_only.py b/tests/test_open_read_only.py new file mode 100644 index 00000000..51c4da0d --- /dev/null +++ b/tests/test_open_read_only.py @@ -0,0 +1,78 @@ +"""Opening a database read-only (#383). + +`open_database(path, read_only=True)` and `DatabaseFactory.open(read_only=True)` +open the files in `ComponentFile.MODE.READ_ONLY`. A mode is not a query, so it +is the one part of #383 the bindings wrap: every other item there is a SQL +statement (see `test_sql_constraints_and_maintenance.py`). +""" + +import jpype +import pytest + +import arcadedb_embedded as arcadedb +from arcadedb_embedded import ArcadeDBError + + +@pytest.fixture +def populated_path(temp_db_path): + """A closed database holding two `Note` documents.""" + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE DOCUMENT TYPE Note") + with db.transaction(): + db.command("sql", "INSERT INTO Note SET n = 1") + db.command("sql", "INSERT INTO Note SET n = 2") + return temp_db_path + + +def test_read_only_open_reads_and_refuses_writes(populated_path): + with arcadedb.open_database(populated_path, read_only=True) as db: + assert db.is_read_only() + assert db.count_type("Note") == 2 + rows = db.query("sql", "SELECT n FROM Note ORDER BY n").to_list() + assert [r["n"] for r in rows] == [1, 2] + + with pytest.raises(ArcadeDBError, match="read only"): + with db.transaction(): + db.command("sql", "INSERT INTO Note SET n = 3") + with pytest.raises(ArcadeDBError, match="read only"): + db.command("sql", "CREATE DOCUMENT TYPE Other") + + # The refused writes left the files as they were. + with arcadedb.open_database(populated_path) as db: + assert not db.is_read_only() + assert db.count_type("Note") == 2 + assert not db.schema.exists_type("Other") + + +def test_read_only_begin_is_not_refused_the_write_and_the_commit_are(populated_path): + """begin() returns on a read-only database; the first write raises, and so does the commit + (the engine starts the transaction then). A write through Document.save() reaches + the engine's own exception. Reads need no transaction.""" + with arcadedb.open_database(populated_path, read_only=True) as db: + db.begin() # not refused + assert db.count_type("Note") == 2 + with pytest.raises(jpype.JException, match="(?i)read.?only"): + db.new_document("Note").set("n", 3).save() + with pytest.raises(ArcadeDBError, match="(?i)read.?only"): + db.commit() + assert db.count_type("Note") == 2 + assert db.query("sql", "SELECT n FROM Note").to_list() + + +def test_factory_open_read_only_and_default_is_writable(populated_path): + factory = arcadedb.DatabaseFactory(populated_path) + + db = factory.open(read_only=True) + try: + assert db.is_read_only() + finally: + db.close() + + db = factory.open() + try: + assert not db.is_read_only() + with db.transaction(): + db.command("sql", "INSERT INTO Note SET n = 3") + assert db.count_type("Note") == 3 + finally: + db.close() diff --git a/tests/test_xml_limits.py b/tests/test_xml_limits.py new file mode 100644 index 00000000..8acd4c9c --- /dev/null +++ b/tests/test_xml_limits.py @@ -0,0 +1,98 @@ +"""The JDK's XML entity limits stay on unless the caller lifts them.""" + +import inspect + +import jpype +import pytest + +import arcadedb_embedded as arcadedb +from arcadedb_embedded import jvm +from arcadedb_embedded.exceptions import ArcadeDBError +from arcadedb_embedded.jvm import _build_jvm_args + +LIMIT_PROPERTIES = ( + "jdk.xml.maxGeneralEntitySizeLimit", + "jdk.xml.entityExpansionLimit", + "jdk.xml.totalEntitySizeLimit", +) + + +@pytest.fixture +def db(tmp_path): + database = arcadedb.create_database(str(tmp_path / "xml_limits_db")) + database.command("sql", "CREATE DOCUMENT TYPE Item") + yield database + if database.is_transaction_active(): + database.rollback() + database.drop() + + +@pytest.fixture +def big_xml(tmp_path): + """One text node of 1,000,000 characters (200,000 entity references).""" + path = tmp_path / "big.xml" + path.write_text( + "" + "&" * 200_000 + "" + "small" + ) + return path + + +def _system_property(name): + value = jpype.JClass("java.lang.System").getProperty(name) + return None if value is None else str(value) + + +def test_start_jvm_default_keeps_the_limits(): + assert ( + inspect.signature(jvm.start_jvm).parameters["disable_xml_limits"].default + is False + ) + args = _build_jvm_args(heap_size="4g", disable_xml_limits=False, jvm_args=None) + assert not any(a.startswith("-Djdk.xml.") for a in args) + lifted = _build_jvm_args(heap_size="4g", disable_xml_limits=True, jvm_args=None) + for name in LIMIT_PROPERTIES: + assert f"-D{name}=0" in lifted + + +def test_import_hitting_a_limit_names_the_option(db, big_xml): + # The session JVM must have been started with the default (limits on). + assert [_system_property(n) for n in LIMIT_PROPERTIES] == [None] * 3 + with pytest.raises(ArcadeDBError) as excinfo: + db.import_documents(big_xml, document_type="Item", file_type="xml") + assert "JAXP0001000" in str(excinfo.value) + assert "disable_xml_limits=True" in str(excinfo.value) + + +def test_per_call_option_lifts_the_limits_for_that_call_only(db, big_xml): + assert [_system_property(n) for n in LIMIT_PROPERTIES] == [None] * 3 + result = db.import_documents( + big_xml, document_type="Item", file_type="xml", disable_xml_limits=True + ) + assert result.result == "OK" + assert result.get("parsedRecords") == 2 + assert [_system_property(n) for n in LIMIT_PROPERTIES] == [None] * 3 + + +def test_per_call_option_restores_a_value_that_was_already_set(): + from arcadedb_embedded.importer import _xml_limits_lifted + + System = jpype.JClass("java.lang.System") + name = LIMIT_PROPERTIES[0] + System.setProperty(name, "12345") + try: + with _xml_limits_lifted(True): + assert _system_property(name) == "0" + assert _system_property(name) == "12345" + finally: + System.clearProperty(name) + + +def test_limits_are_restored_when_the_import_fails(db, tmp_path): + missing = tmp_path / "missing.xml" + with pytest.raises(ArcadeDBError) as excinfo: + db.import_documents( + missing, document_type="Item", file_type="xml", disable_xml_limits=True + ) + assert "disable_xml_limits=True" not in str(excinfo.value) + assert [_system_property(n) for n in LIMIT_PROPERTIES] == [None] * 3 From a1ae1ef9963a221f600eaf934e4679bc038e9062 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 24 Oct 2026 10:38:11 +0900 Subject: [PATCH 2/3] examples: say the new heap and log defaults Co-Authored-By: Claude Opus 5.5 --- examples/01_simple_document_store.py | 4 +++- examples/04_csv_import_documents.py | 6 +++--- examples/06_vector_search_recommendations.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/01_simple_document_store.py b/examples/01_simple_document_store.py index 60396de4..04e93d02 100644 --- a/examples/01_simple_document_store.py +++ b/examples/01_simple_document_store.py @@ -508,7 +508,9 @@ print(" - Task_0.1.65536.v0.bucket - Data bucket with Task documents") print(" - dictionary.0.327680.v0.dict - String compression dictionary") print(" - statistics.json - Database statistics") -print(" • ./log/ - ArcadeDB and JVM log files") +print( + " • ARCADEDB_LOG_DIR (default: a per-user state directory) - ArcadeDB and JVM log files" +) print() print("💡 Next steps:") print(" • See docs/examples/ for detailed explanations") diff --git a/examples/04_csv_import_documents.py b/examples/04_csv_import_documents.py index 8985cb33..655855b1 100644 --- a/examples/04_csv_import_documents.py +++ b/examples/04_csv_import_documents.py @@ -963,7 +963,7 @@ def check_dataset_exists(data_dir): "--heap-size", type=str, default=None, - help="Set JVM max heap size (e.g. 8g, 4096m). Overrides default 4g.", + help="Set JVM max heap size (e.g. 8g, 4096m). Default: 50 percent of the RAM the JVM sees.", ) parser.add_argument( "--db-name", @@ -1017,8 +1017,8 @@ def check_dataset_exists(data_dir): if args.heap_size: print(f"💡 JVM Max Heap: {args.heap_size} (from --heap-size)") else: - print("💡 JVM Max Heap: 4g (default)") - print(" ℹ️ Using default JVM heap (4g)") + print("💡 JVM Max Heap: 50 percent of RAM (default)") + print(" ℹ️ Using default JVM heap (50 percent of RAM)") if args.dataset == "movielens-large": print(" 💡 For large datasets, you can increase it:") print(" Use --heap-size 8g (or higher)") diff --git a/examples/06_vector_search_recommendations.py b/examples/06_vector_search_recommendations.py index 25ddc07f..932496a3 100644 --- a/examples/06_vector_search_recommendations.py +++ b/examples/06_vector_search_recommendations.py @@ -445,7 +445,7 @@ def main(): "--heap-size", type=str, default=None, - help="Set JVM max heap size (e.g. 8g, 4096m). Overrides default 4g.", + help="Set JVM max heap size (e.g. 8g, 4096m). Default: 50 percent of the RAM the JVM sees.", ) args = parser.parse_args() From 618d45095b36ac67c6d2d7a51e662d735d3c1bb9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 24 Oct 2026 10:43:48 +0900 Subject: [PATCH 3/3] start_jvm(): polyglot engine off by default while the wheel ships no GraalVM language (humemai#409) The wheel excludes js-language, truffle, icu4j and regex (scripts/jar_exclusions.txt), so the polyglot engine only logs 'no languages found' at the first SQL command. -Darcadedb.polyglotEngineEnabled=false is added unless the caller sets the flag; the old WarnInterpreterOnly flag goes. Co-Authored-By: Claude Opus 5.5 --- src/arcadedb_embedded/jvm.py | 10 +++++++--- tests/test_jvm_args.py | 26 ++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/arcadedb_embedded/jvm.py b/src/arcadedb_embedded/jvm.py index 72d5e412..d5252944 100644 --- a/src/arcadedb_embedded/jvm.py +++ b/src/arcadedb_embedded/jvm.py @@ -794,11 +794,15 @@ def _build_jvm_args( if not any("java.base/java.lang=" in arg for arg in merged_args): merged_args.append("--add-opens=java.base/java.lang=ALL-UNNAMED") - # Silence Truffle/GraalVM interpreter warning on standard HotSpot JDKs + # The wheel does not ship GraalJS (js-language, truffle, icu4j and regex are in + # scripts/jar_exclusions.txt), so the polyglot engine would start with no + # language to register and log "GraalPolyglotEngine: no languages found" at the + # first SQL command. Off by default; pass -Darcadedb.polyglotEngineEnabled=true + # in jvm_args or ARCADEDB_JVM_ARGS to turn it back on. if not any( - arg.startswith("-Dpolyglot.engine.WarnInterpreterOnly=") for arg in merged_args + arg.startswith("-Darcadedb.polyglotEngineEnabled=") for arg in merged_args ): - merged_args.append("-Dpolyglot.engine.WarnInterpreterOnly=false") + merged_args.append("-Darcadedb.polyglotEngineEnabled=false") # Enable compact object headers (JDK 25+) to reduce heap footprint if not any("UseCompactObjectHeaders" in arg for arg in merged_args): diff --git a/tests/test_jvm_args.py b/tests/test_jvm_args.py index 842b1783..be577c8f 100644 --- a/tests/test_jvm_args.py +++ b/tests/test_jvm_args.py @@ -20,7 +20,8 @@ def test_defaults_no_env_vars(): assert any("java.base/java.util.concurrent.atomic" in a for a in args) assert any("java.base/java.nio.channels.spi" in a for a in args) assert any("java.base/java.lang=" in a for a in args) - assert "-Dpolyglot.engine.WarnInterpreterOnly=false" in args + assert "-Darcadedb.polyglotEngineEnabled=false" in args + assert not any("WarnInterpreterOnly" in a for a in args) assert "-XX:+UseCompactObjectHeaders" in args # Should have default error log assert any("hs_err_pid" in arg for arg in args) @@ -47,10 +48,31 @@ def test_custom_jvm_args_merging(): assert any("java.base/java.util.concurrent.atomic" in a for a in args) assert any("java.base/java.nio.channels.spi" in a for a in args) assert any("java.base/java.lang=" in a for a in args) - assert "-Dpolyglot.engine.WarnInterpreterOnly=false" in args + assert "-Darcadedb.polyglotEngineEnabled=false" in args + assert not any("WarnInterpreterOnly" in a for a in args) assert "-XX:+UseCompactObjectHeaders" in args +def test_polyglot_engine_default_can_be_overridden(): + """The polyglot engine is off by default and ``jvm_args`` or the env var turn it on. + + The wheel ships no GraalVM language, so an enabled engine only logs + "no languages found" at the first SQL command. The flag follows the other + defaults: whoever sets it themselves wins, and only one copy is passed. + """ + flag = "-Darcadedb.polyglotEngineEnabled=true" + with patch.dict(os.environ, {}, clear=True): + from_jvm_args = _build_jvm_args( + heap_size="4g", disable_xml_limits=True, jvm_args=[flag] + ) + with patch.dict(os.environ, {"ARCADEDB_JVM_ARGS": flag}, clear=True): + from_env = _build_jvm_args( + heap_size="4g", disable_xml_limits=True, jvm_args=None + ) + for args in (from_jvm_args, from_env): + assert [a for a in args if "polyglotEngineEnabled" in a] == [flag] + + def test_dedupe_heap_keeps_max(): """Multiple -Xmx values keep the maximum.""" with patch.dict(