From 46b6a12ed9b806e27710d53b81db9c7a50e9c170 Mon Sep 17 00:00:00 2001 From: radar roark <122068506+xeubie@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:29:53 -0400 Subject: [PATCH 1/5] add compact fn --- README.md | 11 ++++++ deps.edn | 2 +- src/xitdb/db.clj | 76 ++++++++++++++++++++++++++++-------- test/xitdb/database_test.clj | 38 +++++++++++++++++- 4 files changed, 108 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b06e137..00cc7ab 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,17 @@ values of the database, by setting the `*return-history?*` binding to `true`. (println "new value:" new-value))) ``` +## Compaction + +Normally, an immutable database grows forever, because old data is never deleted. To reclaim disk space and clear the history, xitdb supports compaction. This involves completely rebuilding the database file to only contain the data accessible from the latest copy (i.e., "moment") of the database. + +```clojure +(with-open [compacted (xdb/compact db "compacted.xdb")] + (println (xdb/materialize @compacted))) +``` + +The `compact` function takes either a file name or `:memory` and returns a new database rather than modifying the existing one. A file target must not already exist, because in-place compaction is not supported. If you want to delete the original database and replace it with the compacted one, you'll need to do that yourself. + ## Freezing One important distinction from the Clojure atom is that inside a transaction (eg. a `swap!`), the data is temporarily mutable. This is exactly like Clojure's transients, and it is a very important optimization. However, this can lead to a surprising behavior: diff --git a/deps.edn b/deps.edn index 2c831a9..734c957 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,6 @@ {:paths ["src" "test"] :deps {org.clojure/clojure {:mvn/version "1.12.0"} - io.github.radarroark/xitdb {:mvn/version "0.32.0"}} + io.github.radarroark/xitdb {:mvn/version "0.33.0"}} :aliases {:test {:extra-deps {io.github.cognitect-labs/test-runner diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index 495487c..12c268c 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -5,9 +5,11 @@ [xitdb.xitdb-types :as xtypes]) (:import [io.github.radarroark.xitdb - CoreBufferedFile CoreFile CoreMemory Database Database$ContextFunction Hasher + Core CoreBufferedFile CoreMemory Database Database$ContextFunction Hasher RandomAccessBufferedFile RandomAccessMemory ReadArrayList WriteArrayList WriteCursor] - [java.io File RandomAccessFile] + [java.io File] + [java.nio.file Files] + [java.nio.file.attribute FileAttribute] [java.security MessageDigest] [java.util.concurrent.locks ReentrantLock])) @@ -112,9 +114,7 @@ (defn- close-db-internal! "Closes the db file. Does nothing if it's a memory db" [^Database db] - (let [core (-> db .-core)] - (when (instance? CoreFile core) - (.close ^RandomAccessFile (-> db .-core .file))))) + (.close ^Core (.-core db))) (defn ^ReadArrayList read-history @@ -173,6 +173,18 @@ (swap [this f x y args] (apply xitdb-swap-with-lock! (concat [this nil f x y] args)))) +(defn- wrap-db [filename ^Database rwdb] + (if (= :memory filename) + (let [tdbmem (proxy [ThreadLocal] [] + (initialValue [] + rwdb))] + (->XITDBDatabase tdbmem rwdb (ReentrantLock.))) + + (let [tldb (proxy [ThreadLocal] [] + (initialValue [] + (open-database filename "r")))] + (->XITDBDatabase tldb rwdb (ReentrantLock.))))) + (defn xit-db "Returns a new XITDBDatabase which can be used to query and transact data. `filename` can be `:memory` or the name of a file on the filesystem. @@ -180,18 +192,49 @@ The returned database handle can be used from multiple threads. Reads can run in parallel, transactions (eg. `swap!`) will only allow one writer at a time." [filename] - (if (= :memory filename) - (let [memdb (open-database :memory "rw") - tdbmem (proxy [ThreadLocal] [] - (initialValue [] - memdb))] - (->XITDBDatabase tdbmem memdb (ReentrantLock.))) + (wrap-db filename (open-database filename "rw"))) - (let [tldb (proxy [ThreadLocal] [] - (initialValue [] - (open-database filename "r"))) - rwdb (open-database filename "rw")] - (->XITDBDatabase tldb rwdb (ReentrantLock.))))) +(defn- create-compact-target [filename] + (if (= :memory filename) + {:core (CoreMemory. (RandomAccessMemory.))} + (let [file (File. ^String filename)] + (Files/createFile (.toPath file) (make-array FileAttribute 0)) + (try + {:core (CoreBufferedFile. (RandomAccessBufferedFile. file "rw")) + :file file} + (catch Throwable t + (Files/deleteIfExists (.toPath file)) + (throw t)))))) + +(defn compact + "Compacts the latest value of `xdb` into a new database at `target`. + `target` can be `:memory` or the name of a file that does not exist. + Returns an open XITDBDatabase containing at most one history entry. The source + database is unchanged." + [^XITDBDatabase xdb target] + (let [^ReentrantLock lock (.-lock xdb)] + (when (.isHeldByCurrentThread lock) + (throw (IllegalStateException. "compact should not be called from swap! or reset!"))) + (try + (.lock lock) + (let [target-info (create-compact-target target) + ^Core target-core (:core target-info)] + (try + (wrap-db target (.compact ^Database (.-rwdb xdb) target-core)) + (catch Throwable t + ;; Clean up the target without hiding the original error + (try + (.close target-core) + (catch Throwable close-error + (.addSuppressed ^Throwable t close-error))) + (when-let [^File file (:file target-info)] + (try + (Files/deleteIfExists (.toPath file)) + (catch Throwable delete-error + (.addSuppressed ^Throwable t delete-error)))) + (throw t)))) + (finally + (.unlock lock))))) (deftype XITDBCursor [xdb keypath] @@ -243,4 +286,3 @@ (str "freeze! requires a writeable XITDB data structure, got: " (type x))))) (-> x common/-unwrap .cursor .db .freeze) (common/-read-only x)) - diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index ac30deb..ddb2002 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -3,7 +3,10 @@ [clojure.test :refer :all] [xitdb.db :as xdb] [xitdb.common :as common] - [xitdb.test-utils :as tu :refer [with-db]])) + [xitdb.test-utils :as tu :refer [with-db]]) + (:import + [java.nio.file FileAlreadyExistsException Files] + [java.nio.file.attribute FileAttribute])) (deftest DatabaseTest (with-db [db (tu/test-db)] @@ -512,3 +515,36 @@ (swap! db2 assoc :added "new") (is (= (tu/materialize @db1) (tu/materialize (xdb/deref-at db2 0))))))) +(deftest compact-test + (let [dir (Files/createTempDirectory "xitdb-clj-compact" (make-array FileAttribute 0)) + target (.resolve dir "compacted.xdb") + source-file (.resolve dir "source.xdb")] + (try + (with-open [source (xdb/xit-db :memory)] + (reset! source {:version 1}) + (swap! source assoc :version 2) + + (with-open [compacted (xdb/compact source :memory)] + (is (= 1 (count compacted))) + (is (= {:version 2} (tu/materialize @compacted)))) + + (with-open [compacted (xdb/compact source (.toString target))] + (is (= 1 (count compacted))) + (is (= {:version 2} (tu/materialize @compacted))) + (swap! compacted assoc :writable true) + (is (= {:version 2 :writable true} (tu/materialize @compacted)))) + + (is (thrown? FileAlreadyExistsException + (xdb/compact source (.toString target)))) + (is (= 2 (count source))) + (is (= {:version 2} (tu/materialize @source)))) + + (with-open [source (xdb/xit-db (.toString source-file))] + (reset! source {:source true}) + (is (thrown? FileAlreadyExistsException + (xdb/compact source (.toString source-file)))) + (is (= {:source true} (tu/materialize @source)))) + (finally + (Files/deleteIfExists target) + (Files/deleteIfExists source-file) + (Files/deleteIfExists dir))))) From 9ba15fd9a413f9f4370a6cde8c61777913796051 Mon Sep 17 00:00:00 2001 From: radar roark <122068506+xeubie@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:02:06 -0400 Subject: [PATCH 2/5] close Core when opening a database fails --- src/xitdb/db.clj | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index 12c268c..583a207 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -25,11 +25,14 @@ If `filename` is `:memory`, returns a memory based db. open-mode can be `r` or `rw`." [filename ^String open-mode] - (let [core (if (= filename :memory) - (CoreMemory. (RandomAccessMemory.)) - (CoreBufferedFile. (RandomAccessBufferedFile. (File. ^String filename) open-mode))) - hasher (Hasher. (MessageDigest/getInstance "SHA-1"))] - (Database. core hasher))) + (let [^Core core (if (= filename :memory) + (CoreMemory. (RandomAccessMemory.)) + (CoreBufferedFile. (RandomAccessBufferedFile. (File. ^String filename) open-mode)))] + (try + (Database. core (Hasher. (MessageDigest/getInstance "SHA-1"))) + (catch Throwable t + (.close core) + (throw t))))) (defn ^WriteArrayList db-history [^Database db] From b509e4f08c8a157eee14320a7679015a1be7a2b3 Mon Sep 17 00:00:00 2001 From: radar roark <122068506+xeubie@users.noreply.github.com> Date: Sat, 5 Sep 2026 04:02:45 -0400 Subject: [PATCH 3/5] bump xitdb --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 734c957..88cd759 100644 --- a/deps.edn +++ b/deps.edn @@ -1,6 +1,6 @@ {:paths ["src" "test"] :deps {org.clojure/clojure {:mvn/version "1.12.0"} - io.github.radarroark/xitdb {:mvn/version "0.33.0"}} + io.github.radarroark/xitdb {:mvn/version "0.34.0"}} :aliases {:test {:extra-deps {io.github.cognitect-labs/test-runner From bfd4161d90072492279190c63e18afb47e8b7060 Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Sat, 5 Sep 2026 20:23:33 +0200 Subject: [PATCH 4/5] Fix MessageDigest race, add exhaustive tests --- src/xitdb/db.clj | 31 ++-- test/xitdb/compaction_test.clj | 260 +++++++++++++++++++++++++++++++++ test/xitdb/database_test.clj | 39 +---- 3 files changed, 279 insertions(+), 51 deletions(-) create mode 100644 test/xitdb/compaction_test.clj diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index 583a207..c77a44e 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -223,19 +223,24 @@ (let [target-info (create-compact-target target) ^Core target-core (:core target-info)] (try - (wrap-db target (.compact ^Database (.-rwdb xdb) target-core)) - (catch Throwable t - ;; Clean up the target without hiding the original error - (try - (.close target-core) - (catch Throwable close-error - (.addSuppressed ^Throwable t close-error))) - (when-let [^File file (:file target-info)] - (try - (Files/deleteIfExists (.toPath file)) - (catch Throwable delete-error - (.addSuppressed ^Throwable t delete-error)))) - (throw t)))) + (let [compacted (.compact ^Database (.-rwdb xdb) target-core)] + ;; xitdb 0.34.0 shares the source's mutable digest with the copy. + ;; These handles have independent locks, so their digests must too. + (set! (.-md compacted) + (MessageDigest/getInstance (.getAlgorithm (.-md compacted)))) + (wrap-db target compacted)) + (catch Throwable t + ;; Clean up the target without hiding the original error + (try + (.close target-core) + (catch Throwable close-error + (.addSuppressed ^Throwable t close-error))) + (when-let [^File file (:file target-info)] + (try + (Files/deleteIfExists (.toPath file)) + (catch Throwable delete-error + (.addSuppressed ^Throwable t delete-error)))) + (throw t)))) (finally (.unlock lock))))) diff --git a/test/xitdb/compaction_test.clj b/test/xitdb/compaction_test.clj new file mode 100644 index 0000000..183093d --- /dev/null +++ b/test/xitdb/compaction_test.clj @@ -0,0 +1,260 @@ +(ns xitdb.compaction-test + (:require + [clojure.java.io :as io] + [clojure.test :refer :all] + [xitdb.db :as xdb] + [xitdb.sorted :as sorted]) + (:import + [java.nio.file FileAlreadyExistsException Files] + [java.nio.file.attribute FileAttribute] + [java.time Instant] + [java.security MessageDigest] + [java.util Date UUID])) + +(def ^:dynamic *temp-dir* nil) + +(use-fixtures :each + (fn [f] + (let [dir (.toFile (Files/createTempDirectory "xitdb-compaction" (make-array FileAttribute 0)))] + (try + (binding [*temp-dir* dir] (f)) + (finally + (doseq [file (reverse (file-seq dir))] + (io/delete-file file))))))) + +(defn- new-path [] + (.getPath (io/file *temp-dir* (str (UUID/randomUUID) ".xdb")))) + +(defn- location [kind] + (if (= :memory kind) :memory (new-path))) + +(def ^:private storage-pairs + [[:memory :memory] [:memory :file] [:file :memory] [:file :file]]) + +(def ^:private values + [["nil" nil] + ["boolean" false] + ["integer" Long/MIN_VALUE] + ["floating point" -12.5] + ["short string" "hi"] + ["long Unicode string" (apply str (repeat 3000 "λ🌍"))] + ["keyword" :compaction/value] + ["character" \λ] + ["UUID" #uuid "123e4567-e89b-12d3-a456-426614174000"] + ["instant" (Instant/parse "2026-01-02T03:04:05.123456789Z")] + ["date" (Date. 123456789)] + ["empty map" {}] + ["empty vector" []] + ["empty list" '()] + ["empty set" #{}] + ["empty sorted map" (sorted-map)] + ["empty sorted set" (sorted-set)] + ["hash map" {nil nil :a 1 "a" false true :yes \x "char" + #uuid "123e4567-e89b-12d3-a456-426614174000" :uuid + [1 2] {:collection-key true}}] + ["vector" [nil true false 1 -2 3.5 "hello" :ns/key \x]] + ["list" '(nil true false 1 -2 3.5 "hello" :ns/key)] + ["hash set" #{nil true false 1 "one" :one [1 2]}] + ["sorted map" (sorted-map 3 {:nested [1 2]} 1 nil 2 #{:a :b})] + ["sorted set" (sorted-set "z" "a" "λ")] + ["nested collections" + {:vector [{:list (list #{:a :b} (sorted-map 2 [nil] 1 []))}] + :set #{[1 2] {:a [3 4]}} + :sorted (sorted-map "a" (sorted-set 3 1 2) "b" {:empty []})}] + ["deep nesting" (reduce (fn [v i] {i [v]}) :leaf (range 40))]]) + +(deftest compact-preserves-values-test + (doseq [[source-kind target-kind] storage-pairs + [label value] values] + (testing (str label ", " source-kind " -> " target-kind) + (let [target (location target-kind)] + (with-open [source (xdb/xit-db (location source-kind))] + (reset! source {:obsolete "discard this history"}) + (reset! source value) + (let [original-type (type @source)] + (with-open [compacted (xdb/compact source target)] + (is (= value (xdb/materialize @compacted))) + (is (= original-type (type @compacted))) + (is (= 1 (count compacted))) + (is (= value (xdb/materialize (xdb/deref-at compacted 0)))) + (reset! compacted {:new "transaction"}) + (is (= 2 (count compacted))) + (is (= value (xdb/materialize (xdb/deref-at compacted 0))))) + (is (= 2 (count source))) + (is (= value (xdb/materialize @source))) + (is (= {:obsolete "discard this history"} + (xdb/materialize (xdb/deref-at source 0)))))) + (when (= :file target-kind) + (with-open [reopened (xdb/xit-db target)] + (is (= {:new "transaction"} (xdb/materialize @reopened))) + (is (= value (xdb/materialize (xdb/deref-at reopened 0)))))))))) + +(deftest compact-empty-database-test + (doseq [[source-kind target-kind] storage-pairs] + (testing (str source-kind " -> " target-kind) + (let [target (location target-kind)] + (with-open [source (xdb/xit-db (location source-kind))] + (with-open [compacted (xdb/compact source target)] + (is (zero? (count compacted))) + (is (nil? @compacted)) + (reset! compacted {:first true}) + (is (= 1 (count compacted))) + (is (= {:first true} (xdb/materialize @compacted)))) + (is (zero? (count source))) + (is (nil? @source))) + (when (= :file target-kind) + (with-open [reopened (xdb/xit-db target)] + (is (= {:first true} (xdb/materialize @reopened))) + (is (= 1 (count reopened))))))))) + +(deftest compact-large-collections-remain-writable-test + ;; Cross the array-index and B-tree node boundaries, then exercise writes + ;; through the copied pointers and verify the compacted snapshot survives. + (doseq [[source-kind target-kind] storage-pairs + [label value mutate] [["map" (into {} (map (fn [i] [i {:value i}]) (range 300))) + #(assoc (dissoc % 150) 301 {:value :new})] + ["vector" (vec (range 300)) #(conj (assoc % 150 :changed) 300)] + ["list" (apply list (range 300)) #(conj (pop %) :new)] + ["set" (set (range 300)) #(conj (disj % 150) 301)] + ["sorted map" (into (sorted-map) (map (fn [i] [i {:value i}]) (range 300))) + #(assoc (dissoc % 150) 301 {:value :new})] + ["sorted set" (into (sorted-set) (range 300)) #(conj (disj % 150) 301)]]] + (testing (str label ", " source-kind " -> " target-kind) + (let [target (location target-kind) + changed (mutate value)] + (with-open [source (xdb/xit-db (location source-kind))] + (reset! source value) + (swap! source mutate) + (with-open [compacted (xdb/compact source target)] + (is (= changed (xdb/materialize @compacted))) + (is (= (count changed) (count @compacted))) + (when (sorted? value) + (is (sorted? @compacted)) + (is (= (seq changed) (xdb/materialize (seq @compacted)))) + (is (= (rseq changed) (xdb/materialize (rseq @compacted)))) + (is (= (subseq changed >= 145 <= 155) (xdb/materialize (subseq @compacted >= 145 <= 155)))) + (is (= (nth (vec changed) 151) (xdb/materialize (nth @compacted 151)))) + (is (= 150 (sorted/rank @compacted 151))) + (is (= (take 5 (drop 148 changed)) (xdb/materialize (sorted/page @compacted 148 5))))) + (swap! compacted mutate) + (is (= (mutate changed) (xdb/materialize @compacted))) + (is (= changed (xdb/materialize (xdb/deref-at compacted 0))))) + (is (= changed (xdb/materialize @source))) + (is (= value (xdb/materialize (xdb/deref-at source 0))))) + (when (= :file target-kind) + (with-open [reopened (xdb/xit-db target)] + (is (= (mutate changed) (xdb/materialize @reopened))))))))) + +(deftest compact-nested-cursors-and-shared-values-test + (with-open [source (xdb/xit-db :memory)] + (reset! source {:left {:items [1 2] :tags (sorted-set 1 2)}}) + (swap! source #(assoc % :right (xdb/freeze! (:left %)))) + (with-open [compacted (xdb/compact source :memory)] + (let [before (xdb/materialize @source)] + (swap! (xdb/xdb-cursor compacted [:left :items]) conj 3) + (swap! (xdb/xdb-cursor compacted [:left :tags]) conj 3) + (is (= {:items [1 2 3] :tags (sorted-set 1 2 3)} + (xdb/materialize (:left @compacted)))) + (is (= (:right before) (xdb/materialize (:right @compacted)))) + (is (= before (xdb/materialize (xdb/deref-at compacted 0))))) + (swap! (xdb/xdb-cursor source [:right :items]) conj 4) + (is (= [1 2] (xdb/materialize (get-in @compacted [:right :items])))) + (is (= [1 2] (xdb/materialize (get-in @source [:left :items]))))))) + +(deftest compact-reclaims-space-and-survives-source-deletion-test + (let [source-path (new-path) + target (new-path) + expected {:kept (vec (range 100))}] + (with-open [source (xdb/xit-db source-path)] + (dotimes [i 40] + (reset! source {:version i :obsolete (apply str (repeat 10000 (str i)))})) + (reset! source expected) + (with-open [compacted (xdb/compact source target)] + (is (= expected (xdb/materialize @compacted))) + (is (= 1 (count compacted)))) + (is (= 41 (count source)))) + (is (< (.length (io/file target)) (.length (io/file source-path)))) + (io/delete-file source-path) + (with-open [reopened (xdb/xit-db target)] + (is (= expected (xdb/materialize @reopened))) + (with-open [again (xdb/compact reopened :memory)] + (is (= expected (xdb/materialize @again))) + (is (= 1 (count again)))) + (swap! reopened assoc :independent true) + (is (= (assoc expected :independent true) (xdb/materialize @reopened)))))) + +(deftest compact-rejects-existing-targets-test + (let [source-path (new-path) + target (new-path)] + (spit target "do not overwrite") + (with-open [source (xdb/xit-db source-path)] + (reset! source {:source true}) + (doseq [path [source-path target (.getPath *temp-dir*)]] + (is (thrown? FileAlreadyExistsException (xdb/compact source path)))) + (is (= "do not overwrite" (slurp target))) + (is (= {:source true} (xdb/materialize @source))) + ;; A rejected target must release the source's transaction lock. + (is (= {:source true :usable true} (xdb/materialize (swap! source assoc :usable true))))))) + +(deftest compact-rejects-reentrant-transactions-test + (with-open [source (xdb/xit-db :memory)] + (reset! source {:value 1}) + (let [target (new-path)] + (is (thrown-with-msg? IllegalStateException #"compact should not be called" + (swap! source (fn [value] + (xdb/compact source target) + value)))) + (is (not (.exists (io/file target)))) + (is (= {:value 1} (xdb/materialize @source))) + (is (= 1 (count source))) + (with-open [compacted (xdb/compact source target)] + (is (= {:value 1} (xdb/materialize @compacted))))))) + +(deftest compact-source-and-target-hash-independently-test + (with-open [source (xdb/xit-db :memory)] + (reset! source {}) + (let [delegate (MessageDigest/getInstance "SHA-1") + pause-once? (atom true) + hashing-started (promise) + target-written (promise) + pause (fn [] + (when (compare-and-set! pause-once? true false) + (deliver hashing-started true) + (when (= ::timeout (deref target-written 5000 ::timeout)) + (throw (ex-info "Timed out waiting for target write" {}))))) + digest (proxy [MessageDigest] ["SHA-1"] + (engineGetDigestLength [] 20) + (engineUpdate + ([b] (.update delegate (byte b)) (pause)) + ([b offset length] (.update delegate b offset length) (pause))) + (engineDigest [] (.digest delegate)) + (engineReset [] (.reset delegate)))] + ;; Pause a source write midway through hashing its key. A target write + ;; must not consume or reset that partial hash, even though locks differ. + (set! (.-md (.-rwdb source)) digest) + (with-open [compacted (xdb/compact source :memory)] + (let [writer (future (reset! source {:left 1}))] + (try + (is (= true (deref hashing-started 5000 ::timeout))) + (reset! compacted {:right 2}) + (finally + (deliver target-written true))) + (try + (is (not= ::timeout (deref writer 5000 ::timeout))) + (is (= 1 (get @source :left))) + (is (= 2 (get @compacted :right))) + (finally + (future-cancel writer)))))))) + +(deftest compact-cleans-up-failed-copy-test + (let [source-path (new-path) + target (new-path) + source (xdb/xit-db source-path)] + (reset! source {:data "requires reading the source file"}) + (.close source) + (is (thrown? java.io.IOException (xdb/compact source target))) + (is (not (.exists (io/file target)))) + ;; The failed attempt must leave the destination available for a retry. + (with-open [reopened (xdb/xit-db source-path) + compacted (xdb/compact reopened target)] + (is (= {:data "requires reading the source file"} (xdb/materialize @compacted)))))) diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index ddb2002..a8ae84a 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -3,10 +3,7 @@ [clojure.test :refer :all] [xitdb.db :as xdb] [xitdb.common :as common] - [xitdb.test-utils :as tu :refer [with-db]]) - (:import - [java.nio.file FileAlreadyExistsException Files] - [java.nio.file.attribute FileAttribute])) + [xitdb.test-utils :as tu :refer [with-db]])) (deftest DatabaseTest (with-db [db (tu/test-db)] @@ -514,37 +511,3 @@ ;; Verify history works (swap! db2 assoc :added "new") (is (= (tu/materialize @db1) (tu/materialize (xdb/deref-at db2 0))))))) - -(deftest compact-test - (let [dir (Files/createTempDirectory "xitdb-clj-compact" (make-array FileAttribute 0)) - target (.resolve dir "compacted.xdb") - source-file (.resolve dir "source.xdb")] - (try - (with-open [source (xdb/xit-db :memory)] - (reset! source {:version 1}) - (swap! source assoc :version 2) - - (with-open [compacted (xdb/compact source :memory)] - (is (= 1 (count compacted))) - (is (= {:version 2} (tu/materialize @compacted)))) - - (with-open [compacted (xdb/compact source (.toString target))] - (is (= 1 (count compacted))) - (is (= {:version 2} (tu/materialize @compacted))) - (swap! compacted assoc :writable true) - (is (= {:version 2 :writable true} (tu/materialize @compacted)))) - - (is (thrown? FileAlreadyExistsException - (xdb/compact source (.toString target)))) - (is (= 2 (count source))) - (is (= {:version 2} (tu/materialize @source)))) - - (with-open [source (xdb/xit-db (.toString source-file))] - (reset! source {:source true}) - (is (thrown? FileAlreadyExistsException - (xdb/compact source (.toString source-file)))) - (is (= {:source true} (tu/materialize @source)))) - (finally - (Files/deleteIfExists target) - (Files/deleteIfExists source-file) - (Files/deleteIfExists dir))))) From dff76d236918b4727a3ce0cb9afafea26a8b0fcf Mon Sep 17 00:00:00 2001 From: Florin Braghis Date: Sat, 5 Sep 2026 20:54:30 +0200 Subject: [PATCH 5/5] Update docstrings, add test for corrupt database handle leak --- README.md | 2 ++ src/xitdb/db.clj | 9 +++++++-- test/xitdb/database_test.clj | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00cc7ab..18491e8 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ Normally, an immutable database grows forever, because old data is never deleted The `compact` function takes either a file name or `:memory` and returns a new database rather than modifying the existing one. A file target must not already exist, because in-place compaction is not supported. If you want to delete the original database and replace it with the compacted one, you'll need to do that yourself. +Compaction holds the source database's write lock for the duration of the copy: reads continue, but any `swap!` or `reset!` on the source will block until compaction finishes. For the same reason, `compact` must not be called from inside a `swap!` or `reset!` on the database being compacted; it throws an `IllegalStateException` if it is. + ## Freezing One important distinction from the Clojure atom is that inside a transaction (eg. a `swap!`), the data is temporarily mutable. This is exactly like Clojure's transients, and it is a very important optimization. However, this can lead to a surprising behavior: diff --git a/src/xitdb/db.clj b/src/xitdb/db.clj index c77a44e..05f3e55 100644 --- a/src/xitdb/db.clj +++ b/src/xitdb/db.clj @@ -115,7 +115,8 @@ (.unlock lock))))) (defn- close-db-internal! - "Closes the db file. Does nothing if it's a memory db" + "Closes the underlying core of `db` (the file handle for file databases, + the in-memory buffer for memory databases)." [^Database db] (.close ^Core (.-core db))) @@ -213,7 +214,11 @@ "Compacts the latest value of `xdb` into a new database at `target`. `target` can be `:memory` or the name of a file that does not exist. Returns an open XITDBDatabase containing at most one history entry. The source - database is unchanged." + database is unchanged. + + Holds the source's write lock for the whole copy, so `swap!` and `reset!` on + `xdb` block until compaction finishes. Must not be called from inside a + `swap!` or `reset!` on `xdb`; doing so throws IllegalStateException." [^XITDBDatabase xdb target] (let [^ReentrantLock lock (.-lock xdb)] (when (.isHeldByCurrentThread lock) diff --git a/test/xitdb/database_test.clj b/test/xitdb/database_test.clj index a8ae84a..0952fb2 100644 --- a/test/xitdb/database_test.clj +++ b/test/xitdb/database_test.clj @@ -511,3 +511,18 @@ ;; Verify history works (swap! db2 assoc :added "new") (is (= (tu/materialize @db1) (tu/materialize (xdb/deref-at db2 0))))))) + +(deftest open-database-closes-core-on-failure-test + ;; A file with a corrupt header opens fine as a Core but makes the Database + ;; constructor throw. The file handle must not leak when that happens. + (let [os (java.lang.management.ManagementFactory/getOperatingSystemMXBean)] + (when (instance? com.sun.management.UnixOperatingSystemMXBean os) + (let [file (java.io.File/createTempFile "xitdb-corrupt" ".xdb") + fds #(.getOpenFileDescriptorCount ^com.sun.management.UnixOperatingSystemMXBean os)] + (try + (spit file "this is not a valid xitdb header") + (let [before (fds)] + (is (thrown? Throwable (xdb/open-database (.getPath file) "rw"))) + (is (= before (fds)))) + (finally + (.delete file)))))))