diff --git a/.gitignore b/.gitignore
index a824681..6bc3025 100644
--- a/.gitignore
+++ b/.gitignore
@@ -39,3 +39,4 @@ build/
/.idea/
/data-boundaries/
/.env
+/tmp/
diff --git a/.run/PaikkaApplication (Import Schleswig-Holstein).run.xml b/.run/PaikkaApplication (Import Schleswig-Holstein).run.xml
new file mode 100644
index 0000000..6695bbd
--- /dev/null
+++ b/.run/PaikkaApplication (Import Schleswig-Holstein).run.xml
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java
index 88c3d5b..537c1e1 100644
--- a/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java
+++ b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java
@@ -79,9 +79,16 @@ public String toString() {
private volatile long phaseStartTime = System.currentTimeMillis();
private long totalTime;
- private final int TOTAL_STEPS = 2;
+ private final int TOTAL_STEPS = 3;
private int currentStep = 0;
+ private volatile long phase1Duration;
+ private volatile long phase2Duration;
+ private volatile long h3OsmSizeBytes;
+ private volatile long regionMetaSizeBytes;
+ private volatile long regionGeomSizeBytes;
+ private volatile long osmNamesSizeBytes;
+
public long getNodesCached() {
return nodesCached.get();
}
@@ -127,8 +134,14 @@ public String getCurrentPhase() {
}
public void setCurrentPhase(int step, String phase) {
+ long now = System.currentTimeMillis();
+ if (this.currentStep == 1 && step != 1) {
+ this.phase1Duration = now - this.phaseStartTime;
+ } else if (this.currentStep == 2 && step != 2) {
+ this.phase2Duration = now - this.phaseStartTime;
+ }
this.currentPhase = phase;
- this.phaseStartTime = System.currentTimeMillis();
+ this.phaseStartTime = now;
this.currentStep = step;
}
@@ -186,9 +199,16 @@ public long getErrorsTotal() {
public String getMemoryStats() {
Runtime r = Runtime.getRuntime();
- long used = (r.totalMemory() - r.freeMemory()) / 1024 / 1024 / 1024;
- long max = r.maxMemory() / 1024 / 1024 / 1024;
- return String.format("%dGB/%dGB", used, max);
+ long usedBytes = r.totalMemory() - r.freeMemory();
+ long maxBytes = r.maxMemory();
+ return String.format("%.1fG/%.1fG", usedBytes / (1024.0 * 1024.0 * 1024.0), maxBytes / (1024.0 * 1024.0 * 1024.0));
+ }
+
+ public void setOutputSizes(long h3OsmBytes, long regionMetaBytes, long regionGeomBytes, long osmNamesBytes) {
+ this.h3OsmSizeBytes = h3OsmBytes;
+ this.regionMetaSizeBytes = regionMetaBytes;
+ this.regionGeomSizeBytes = regionGeomBytes;
+ this.osmNamesSizeBytes = osmNamesBytes;
}
public void startProgressReporter() {
@@ -210,13 +230,13 @@ public void startProgressReporter() {
sb.append(String.format("\033[1;90m[%d/%d]\033[0m ", currentStep, TOTAL_STEPS));
- if (phase.contains("1.1")) {
+ if (phase.contains("Caching")) {
long nodesPerSec = phaseSeconds > 0 ? (long) (getNodesCached() / phaseSeconds) : 0;
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mCaching Nodes & Ways\033[0m", formatTime(elapsed)));
sb.append(String.format(" │ \033[32mNodes:\033[0m %s \033[33m(%s/s)\033[0m",
formatCompactNumber(getNodesCached()), formatCompactRate(nodesPerSec)));
sb.append(String.format(" │ \033[34mWays:\033[0m %s", formatCompactNumber(getWaysCached())));
- } else if (phase.contains("2.1")) {
+ } else if (phase.contains("Processing")) {
long relsPerSec = phaseSeconds > 0 ? (long) (getRelationsProcessed() / phaseSeconds) : 0;
double percentage = getRelationsFound() > 0 ? (double) getRelationsProcessed() / getRelationsFound() * 100.0 : 0.0;
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mProcessing Relations & H3\033[0m", formatTime(elapsed)));
@@ -224,6 +244,9 @@ public void startProgressReporter() {
formatCompactNumber(getRelationsProcessed()), formatCompactNumber(getRelationsFound()), formatCompactRate(relsPerSec)));
sb.append(String.format(" │ \033[35mProgress:\033[0m %.2f%%", percentage));
sb.append(String.format(" │ \033[36mH3 Cells:\033[0m %s", formatCompactNumber(getH3CellsGenerated())));
+ if (getErrorsTotal() > 0) {
+ sb.append(String.format(" │ \033[31mErrors:\033[0m %d", getErrorsTotal()));
+ }
} else {
sb.append(String.format("\033[1;36m[%s]\033[0m %s", formatTime(elapsed), phase));
}
@@ -248,33 +271,43 @@ public void startProgressReporter() {
}
public void printFinalStatistics() {
- System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("🎯 BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m");
+ System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m");
long totalTime = Math.max(1, getTotalTime());
double totalSeconds = totalTime / 1000.0;
+ double phase1Seconds = Math.max(0.001, phase1Duration / 1000.0);
+ double phase2Seconds = Math.max(0.001, phase2Duration / 1000.0);
- System.out.printf("\n\033[1;37m⏱️ Total Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime()));
+ System.out.printf("\n\033[1;37mTotal Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime()));
- System.out.println("\033[1;37m📊 Processing Summary:\033[0m");
- System.out.println("┌────────────────────┬─────────────────┬─────────────────┐");
- System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │");
- System.out.println("├────────────────────┼─────────────────┼─────────────────┤");
- System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n",
+ System.out.println("\033[1;37mProcessing Summary:\033[0m");
+ System.out.println("┌──────────────────────┬─────────────────┬─────────────────┐");
+ System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │");
+ System.out.println("├──────────────────────┼─────────────────┼─────────────────┤");
+ System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getNodesCached()),
- formatCompactNumber((long) (getNodesCached() / totalSeconds)));
- System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber((long) (getNodesCached() / phase1Seconds)));
+ System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getWaysCached()),
- formatCompactNumber((long) (getWaysCached() / totalSeconds)));
- System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber((long) (getWaysCached() / phase1Seconds)));
+ System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getRelationsFound()),
formatCompactNumber((long) (getRelationsFound() / totalSeconds)));
- System.out.printf("│ \033[36mRelations Processed\033[0m│ %15s │ %13s/s │%n",
+ System.out.printf("│ \033[36mRelations Processed\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getRelationsProcessed()),
- formatCompactNumber((long) (getRelationsProcessed() / totalSeconds)));
- System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber((long) (getRelationsProcessed() / phase2Seconds)));
+ System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n",
formatCompactNumber(getH3CellsGenerated()),
- formatCompactNumber((long) (getH3CellsGenerated() / totalSeconds)));
- System.out.println("└────────────────────┴─────────────────┴─────────────────┘");
+ formatCompactNumber((long) (getH3CellsGenerated() / phase2Seconds)));
+ System.out.println("└──────────────────────┴─────────────────┴─────────────────┘");
+
+ if (h3OsmSizeBytes > 0) {
+ System.out.println("\n\033[1;37mOutput Database Sizes:\033[0m");
+ System.out.printf(" \033[36mh3_to_osm:\033[0m %s%n", formatSize(h3OsmSizeBytes));
+ System.out.printf(" \033[36mregion_metadata:\033[0m %s%n", formatSize(regionMetaSizeBytes));
+ System.out.printf(" \033[36mregion_geometry:\033[0m %s%n", formatSize(regionGeomSizeBytes));
+ System.out.printf(" \033[36mosm_names.tsv:\033[0m %s%n", formatSize(osmNamesSizeBytes));
+ }
System.out.println();
}
@@ -331,6 +364,13 @@ private String formatCompactRate(long n) {
return String.format("%.1fM", n / 1_000_000.0);
}
+ private String formatSize(long bytes) {
+ if (bytes < 1024) return bytes + " B";
+ if (bytes < 1024 * 1024) return String.format("%.1f KB", bytes / 1024.0);
+ if (bytes < 1024L * 1024 * 1024) return String.format("%.1f MB", bytes / (1024.0 * 1024.0));
+ return String.format("%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0));
+ }
+
private String centerText(String text) {
int pad = (80 - text.length()) / 2;
return " ".repeat(Math.max(0, pad)) + text;
diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java
index 2c2c04f..e684ed9 100644
--- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java
+++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java
@@ -45,7 +45,7 @@
* Reads a pre-filtered boundaries_only.pbf (Nodes -> Ways -> Relations ordered)
* and produces three RocksDB databases for offline mobile lookup:
* - h3_to_osm: H3_CELL_ID (uint64) -> List[OSM_ID] (raw byte array)
- * - region_metadata: OSM_ID -> total cell count (int)
+ * - region_metadata: OSM_ID -> [total cell count (long), h3 resolution (int)] (12 bytes)
* - region_geometry: OSM_ID -> simplified WKB (bytes)
*/
@Service
@@ -54,6 +54,7 @@ public class StandaloneBoundaryImporter {
private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory();
private static final double BUFFER_DISTANCE = 0.0001; // ~11m at equator, ensures border cells
+ private static final long H3_THREAD_WRITE_BUFFER = 64L * 1024 * 1024; // per-thread RocksDB write buffer
private final GeometrySimplificationService geometrySimplificationService;
private final PaikkaConfiguration paikkaConfiguration;
@@ -83,7 +84,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc
Path regionGeomPath = out.resolve("region_geometry");
Path nameSql = out.resolve("osm_names.tsv");
- Path tmpH3ToOsmPath = tmp.resolve("tmp_h3_to_osm");
Path tmpRegionMetaPath = tmp.resolve("tmp_region_metadata");
Path tmpRegionGeomPath = tmp.resolve("tmp_region_geometry");
@@ -92,14 +92,13 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc
cleanup(h3ToOsmPath);
cleanup(regionMetaPath);
cleanup(regionGeomPath);
- cleanup(tmpH3ToOsmPath);
cleanup(tmpRegionMetaPath);
cleanup(tmpRegionGeomPath);
- // Shared Rocksoptions (inline with ImportService style)
BlockBasedTableConfig tableCfg = new BlockBasedTableConfig()
.setBlockSize(64 * 1024)
.setFilterPolicy(new BloomFilter(10, false));
+
Options cacheOpts = new Options()
.setCreateIfMissing(true)
.setTableFormatConfig(tableCfg)
@@ -107,6 +106,12 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc
.setWriteBufferSize(512 * 1024 * 1024)
.setMaxWriteBufferNumber(3)
.setLevel0FileNumCompactionTrigger(4);
+
+ // Per-thread H3 DBs use smaller write buffers (64MB each) to control memory
+ Options threadH3Opts = new Options(cacheOpts)
+ .setWriteBufferSize(H3_THREAD_WRITE_BUFFER)
+ .setMaxWriteBufferNumber(2);
+
Options finalOpts = new Options()
.setCreateIfMissing(true)
.setTableFormatConfig(tableCfg)
@@ -125,217 +130,210 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc
stats.startProgressReporter();
- try (
- RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString());
- RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString());
- RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString());
- RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString());
- RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString());
- RocksDB tmpH3ToOsm = RocksDB.open(cacheOpts, tmpH3ToOsmPath.toString());
- RocksDB tmpRegionMeta = RocksDB.open(cacheOpts, tmpRegionMetaPath.toString());
- RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString());
- OsmNameStreamer nameStreamer = new OsmNameStreamer(nameSql.toString())
- ) {
- for (String pbfPath : pbfPaths) {
- stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways");
- try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
-
- // ---------- SINGLE PASS ----------
- PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false);
-
- // Phase 1 & 2: Stream nodes and ways (cached)
- WriteBatch nodeBatch = new WriteBatch();
- WriteBatch wayBatch = new WriteBatch();
- AtomicLong phaseCounter = new AtomicLong();
-
- while (iterator.hasNext()) {
- EntityContainer c = iterator.next();
- if (c.getType() == EntityType.Node) {
- // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair
- OsmNode n = (OsmNode) c.getEntity();
- ByteBuffer bb = ByteBuffer.allocate(16)
- .putDouble(n.getLatitude())
- .putDouble(n.getLongitude());
- nodeBatch.put(longToBytes(n.getId()), bb.array());
- stats.incrementNodesCached();
- if (phaseCounter.incrementAndGet() % 100_000 == 0) {
- nodeCache.write(wo, nodeBatch);
- nodeBatch.clear();
- }
- } else if (c.getType() == EntityType.Way) {
- // PHASE 2: Cache way node-id sequences (long[] as raw bytes)
- OsmWay w = (OsmWay) c.getEntity();
- long[] ids = new long[w.getNumberOfNodes()];
- for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i);
- wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids));
- stats.incrementWaysCached();
- if (phaseCounter.incrementAndGet() % 50_000 == 0) {
- wayCache.write(wo, wayBatch);
- wayBatch.clear();
- }
- } else if (c.getType() == EntityType.Relation) {
- // PHASE 3: Count administrative boundaries for accurate progress tracking
- OsmRelation r = (OsmRelation) c.getEntity();
- if (isAdministrativeBoundary(r)) {
- stats.incrementRelationsFound();
+ // Shared block cache reduces redundant I/O across all RocksDB instances
+ try (Cache sharedCache = new LRUCache(2L * 1024 * 1024 * 1024)) {
+ tableCfg.setBlockCache(sharedCache);
+
+ try (
+ RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString());
+ RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString());
+ RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString());
+ RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString());
+ RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString());
+ RocksDB tmpRegionMeta = RocksDB.open(cacheOpts, tmpRegionMetaPath.toString());
+ RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString());
+ OsmNameStreamer nameStreamer = new OsmNameStreamer(nameSql.toString())
+ ) {
+ int threads = paikkaConfiguration.getImportConfiguration().getThreads();
+
+ // Pre-create per-thread H3 db paths and clean any left-overs
+ Path[] threadH3Paths = new Path[threads];
+ for (int t = 0; t < threads; t++) {
+ threadH3Paths[t] = tmp.resolve("h3_osm_" + t);
+ cleanup(threadH3Paths[t]);
+ }
+
+ for (String pbfPath : pbfPaths) {
+ stats.setCurrentPhase(1, "Phase 1: Caching Nodes & Ways");
+
+ // ------ Single pass: cache nodes/ways and collect relation stubs ------
+ List stubs = new ArrayList<>(500_000);
+
+ try (InputStream is = Files.newInputStream(Paths.get(pbfPath));
+ WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+
+ PbfIterator iterator = new PbfIterator(is, false);
+ WriteBatch nodeBatch = new WriteBatch();
+ WriteBatch wayBatch = new WriteBatch();
+ AtomicLong phaseCounter = new AtomicLong();
+
+ while (iterator.hasNext()) {
+ EntityContainer c = iterator.next();
+ if (c.getType() == EntityType.Node) {
+ OsmNode n = (OsmNode) c.getEntity();
+ ByteBuffer bb = ByteBuffer.allocate(16)
+ .putDouble(n.getLatitude())
+ .putDouble(n.getLongitude());
+ nodeBatch.put(longToBytes(n.getId()), bb.array());
+ stats.incrementNodesCached();
+ if (phaseCounter.incrementAndGet() % 100_000 == 0) {
+ nodeCache.write(wo, nodeBatch);
+ nodeBatch.clear();
+ }
+ } else if (c.getType() == EntityType.Way) {
+ OsmWay w = (OsmWay) c.getEntity();
+ long[] ids = new long[w.getNumberOfNodes()];
+ for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i);
+ wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids));
+ stats.incrementWaysCached();
+ if (phaseCounter.incrementAndGet() % 50_000 == 0) {
+ wayCache.write(wo, wayBatch);
+ wayBatch.clear();
+ }
+ } else if (c.getType() == EntityType.Relation) {
+ OsmRelation r = (OsmRelation) c.getEntity();
+ try {
+ nameStreamer.processEntity(r, "R");
+ } catch (IOException e) {
+ logger.warn("Failed to stream name for relation ID: {}", r.getId(), e);
+ }
+ if (isAdministrativeBoundary(r)) {
+ stats.incrementRelationsFound();
+ stubs.add(buildRelationStub(r));
+ }
}
}
+ nodeCache.write(wo, nodeBatch);
+ wayCache.write(wo, wayBatch);
+ nodeBatch.close();
+ wayBatch.close();
}
- nodeCache.write(wo, nodeBatch);
- wayCache.write(wo, wayBatch);
- nodeBatch.close();
- wayBatch.close();
- }
- stats.setCurrentPhase(2, "2.1: Processing Relations & H3");
- // Re-open iterator for Phase 3 (or use two iterators; here we reuse file)
+ // ------ Process Relations via partitioned thread pool ------
+ stats.setCurrentPhase(2, "Phase 2: Processing Relations & H3");
- // Phase 3: Process Relations (separate iterator pass is fine since PBF is local)
- try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) {
- PbfIterator relIter = new PbfIterator(is, false);
+ if (!stubs.isEmpty()) {
+ ExecutorService executor = Executors.newFixedThreadPool(threads);
+ List> futures = new ArrayList<>();
+ int partitionSize = (stubs.size() + threads - 1) / threads;
- int threads = paikkaConfiguration.getImportConfiguration().getThreads();
- ExecutorService executor = Executors.newFixedThreadPool(threads);
- BlockingQueue> queue = new LinkedBlockingQueue<>(100);
- List POISON_PILL = List.of();
+ for (int t = 0; t < threads; t++) {
+ int from = t * partitionSize;
+ int to = Math.min(from + partitionSize, stubs.size());
+ if (from >= to) break;
- // Producer thread
- Thread producer = new Thread(() -> {
- try {
- List batch = new ArrayList<>(100);
- while (relIter.hasNext()) {
- EntityContainer c = relIter.next();
- if (c.getType() == EntityType.Relation) {
- OsmRelation r = (OsmRelation) c.getEntity();
- try {
- nameStreamer.processEntity(r, "R");
- } catch (IOException e) {
- logger.warn("Failed to stream name for relation ID: {}", r.getId(), e);
- }
- if (isAdministrativeBoundary(r)) {
- batch.add(buildRelationStub(r));
- if (batch.size() >= 100) {
- queue.put(batch);
- batch = new ArrayList<>(100);
- }
- }
- }
- }
- if (!batch.isEmpty()) {
- queue.put(batch);
- }
- } catch (Exception e) {
- stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.READ, null, "producer-thread", e);
- } finally {
- for (int i = 0; i < threads; i++) {
+ List partition = stubs.subList(from, to);
+ final int threadIndex = t;
+
+ futures.add(executor.submit(() -> {
try {
- queue.put(POISON_PILL);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
- });
-
- producer.start();
-
- // Consumer threads
- List> futures = new ArrayList<>();
- for (int i = 0; i < threads; i++) {
- futures.add(executor.submit(() -> {
- try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
- while (true) {
- List batch = queue.take();
- if (batch == POISON_PILL) break;
-
- for (RelationStub stub : batch) {
- if (stub.adminLevel() <= 3) {
- logger.debug("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel());
- }
- try {
- Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache);
- if (geom == null || geom.isEmpty()) {
- continue;
+ try (RocksDB threadH3 = RocksDB.open(threadH3Opts, threadH3Paths[threadIndex].toString());
+ WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+ for (RelationStub stub : partition) {
+ if (stub.adminLevel() <= 3) {
+ logger.debug("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel());
}
-
- // Repair invalid geometries using buffer(0)
- if (!geom.isValid()) {
- logger.debug("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel());
- geom = geom.buffer(0);
- if (geom == null || geom.isEmpty() || !geom.isValid()) {
- logger.error("Relation OSM ID: {} [Admin Level: {}] Geometry repair failed", stub.osmId(), stub.adminLevel());
+ try {
+ Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache);
+ if (geom == null || geom.isEmpty()) {
continue;
}
- }
- // Simplify first to reduce H3 cell count, then buffer
- Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(geom, stub.adminLevel());
- if (simplified == null || simplified.isEmpty()) {
- simplified = geom;
- }
- // Buffer to include border-touching cells
- Geometry buffered = simplified.buffer(BUFFER_DISTANCE);
- if (stub.adminLevel() <= 3) {
- logger.debug("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId());
- }
-
- int resolution = getResolutionForAdminLevel(stub.adminLevel());
-
- // ---- H3 Polyfill ----
- AtomicLong cellCount = new AtomicLong(0);
- long startTime = System.currentTimeMillis();
- processCellsH3Stream(buffered, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution);
- if (stub.adminLevel() <= 3) {
- logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId());
- }
- if (cellCount.get() == 0) continue;
- stats.incrementRelationsProcessed();
- stats.addH3CellsGenerated((int) cellCount.get());
+ // Repair invalid geometries using buffer(0)
+ if (!geom.isValid()) {
+ logger.debug("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel());
+ geom = geom.buffer(0);
+ if (geom == null || geom.isEmpty() || !geom.isValid()) {
+ logger.error("Relation OSM ID: {} [Admin Level: {}] Geometry repair failed", stub.osmId(), stub.adminLevel());
+ continue;
+ }
+ }
+ // Simplify first to reduce H3 cell count, then buffer
+ Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(geom, stub.adminLevel());
+ if (simplified == null || simplified.isEmpty()) {
+ simplified = geom;
+ }
+ // Buffer to include border-touching cells
+ Geometry buffered = simplified.buffer(BUFFER_DISTANCE);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId());
+ }
- startTime = System.currentTimeMillis();
- tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes((int) cellCount.get()));
- if (stub.adminLevel() <= 3) {
- logger.debug("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
- }
- startTime = System.currentTimeMillis();
- byte[] wkb = new WKBWriter().write(simplified);
- tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb);
- if (stub.adminLevel() <= 3) {
- logger.debug("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
+ int resolution = getResolutionForAdminLevel(stub.adminLevel());
+
+ // ---- H3 Polyfill ----
+ AtomicLong cellCount = new AtomicLong(0);
+ long startTime = System.currentTimeMillis();
+ processCellsH3Stream(buffered, stub.osmId(), wo, threadH3, cellCount, resolution);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ long totalCells = cellCount.get();
+ if (totalCells == 0) continue;
+
+ stats.incrementRelationsProcessed();
+ stats.addH3CellsGenerated(totalCells);
+
+ startTime = System.currentTimeMillis();
+ tmpRegionMeta.put(wo, longToBytes(stub.osmId()), cellMetaToBytes(totalCells, resolution));
+ if (stub.adminLevel() <= 3) {
+ logger.debug("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ startTime = System.currentTimeMillis();
+ byte[] wkb = new WKBWriter().write(simplified);
+ tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e);
}
- } catch (Exception e) {
- stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e);
}
}
+ } catch (RocksDBException e) {
+ throw new RuntimeException("Failed to open per-thread H3 DB", e);
}
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }));
- }
+ }));
+ }
- // Wait for consumers to finish
- for (Future f : futures) {
- f.get();
+ for (Future f : futures) {
+ f.get();
+ }
+ executor.shutdown();
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+
+ // Merge per-thread H3 DBs into the final h3_to_osm
+ stats.setCurrentPhase(3, "3.1: Merging H3 thread DBs");
+ for (int t = 0; t < threads; t++) {
+ if (Files.exists(threadH3Paths[t])) {
+ try (RocksDB threadDb = RocksDB.open(cacheOpts, threadH3Paths[t].toString())) {
+ copyH3Db(threadDb, h3ToOsm);
+ }
+ cleanup(threadH3Paths[t]);
+ }
+ }
}
- executor.shutdown();
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
- producer.join();
}
- }
- stats.setCurrentPhase(3, "3.1: Compacting Final Databases");
- // Final Step: Copy from temporary DBs to final DBs in sorted order
- copyDb(tmpRegionMeta, regionMeta);
- copyDb(tmpRegionGeom, regionGeom);
- copyH3Db(tmpH3ToOsm, h3ToOsm);
+ stats.setCurrentPhase(3, "3.2: Compacting Final Databases");
+ // Copy from temporary DBs to final DBs in sorted order
+ copyDb(tmpRegionMeta, regionMeta);
+ copyDb(tmpRegionGeom, regionGeom);
- // Compact finals
- h3ToOsm.compactRange();
- regionMeta.compactRange();
- regionGeom.compactRange();
+ // Compact finals
+ h3ToOsm.compactRange();
+ regionMeta.compactRange();
+ regionGeom.compactRange();
+ }
}
+ stats.setOutputSizes(
+ dirSize(h3ToOsmPath),
+ dirSize(regionMetaPath),
+ dirSize(regionGeomPath),
+ fileSize(nameSql)
+ );
+
stats.stop();
stats.setTotalTime(System.currentTimeMillis() - stats.getStartTime());
stats.printFinalStatistics();
@@ -501,9 +499,11 @@ private int getResolutionForAdminLevel(int adminLevel) {
/**
* Converts a JTS Geometry to H3 cells at the specified resolution.
- * Uses h3.polygonToCellsStream with LatLng vertices. Multipolygons are expanded.
+ * Uses h3.polygonToCells with LatLng vertices. Multipolygons are expanded.
+ * Each thread has its own RocksDB instance (passed via threadH3ToOsm) so no
+ * synchronization is needed on writes.
*/
- private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount, int resolution) {
+ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB threadH3ToOsm, AtomicLong cellCount, int resolution) {
int num = geom.getNumGeometries();
for (int i = 0; i < num; i++) {
Geometry part = geom.getGeometryN(i);
@@ -513,49 +513,45 @@ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, Ro
for (int h = 0; h < poly.getNumInteriorRing(); h++) {
holes.add(toLatLng(poly.getInteriorRingN(h).getCoordinates()));
}
- try {
- List batch = new ArrayList<>(5_000); // Reduced batch size to prevent OOM
- h3.polygonToCells(outer, holes, resolution).forEach(cell -> {
- batch.add(cell);
- if (batch.size() >= 5_000) {
- try {
- processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount);
- } catch (RocksDBException e) {
- throw new RuntimeException(e);
- }
- batch.clear();
- }
- });
- if (!batch.isEmpty()) {
- processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount);
+ List batch = new ArrayList<>(5_000);
+ h3.polygonToCells(outer, holes, resolution).forEach(cell -> {
+ batch.add(cell);
+ if (batch.size() >= 5_000) {
+ processH3Batch(batch, osmId, wo, threadH3ToOsm, cellCount);
+ batch.clear();
}
- } catch (Exception e) {
- stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3Stream", e);
+ });
+ if (!batch.isEmpty()) {
+ processH3Batch(batch, osmId, wo, threadH3ToOsm, cellCount);
}
}
}
- private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) throws RocksDBException {
+ /**
+ * Writes H3 cell -> OSM ID associations to a thread-local RocksDB.
+ * No synchronization needed since each thread owns its RocksDB instance.
+ */
+ private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB h3Db, AtomicLong cellCount) {
List keys = new ArrayList<>(cells.size());
for (long cell : cells) {
keys.add(longToBytes(cell));
}
-
- // Synchronize to prevent race conditions when multiple threads update the same H3 cell
- synchronized (tmpH3ToOsm) {
- List existingValues = tmpH3ToOsm.multiGetAsList(keys);
+ try {
+ List existingValues = h3Db.multiGetAsList(keys);
try (WriteBatch writeBatch = new WriteBatch()) {
for (int i = 0; i < cells.size(); i++) {
cellCount.incrementAndGet();
- byte[] key = keys.get(i);
byte[] existing = existingValues.get(i);
byte[] updated = appendOsmIdToArray(existing, osmId);
- writeBatch.put(key, updated);
+ writeBatch.put(keys.get(i), updated);
}
- tmpH3ToOsm.write(wo, writeBatch);
+ h3Db.write(wo, writeBatch);
}
+ } catch (RocksDBException e) {
+ throw new RuntimeException(e);
}
}
+
private List toLatLng(Coordinate[] coords) {
List list = new ArrayList<>(coords.length);
for (Coordinate c : coords) {
@@ -583,8 +579,11 @@ private long[] bytesToLongArray(byte[] b) {
return arr;
}
- private byte[] intToBytes(int v) {
- return ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(v).array();
+ private byte[] cellMetaToBytes(long cellCount, int resolution) {
+ return ByteBuffer.allocate(12).order(ByteOrder.BIG_ENDIAN)
+ .putLong(cellCount)
+ .putInt(resolution)
+ .array();
}
/**
@@ -661,4 +660,29 @@ private void cleanup(Path p) {
}
}
}
+
+ private long dirSize(Path dir) {
+ try {
+ return Files.walk(dir)
+ .filter(Files::isRegularFile)
+ .mapToLong(p -> {
+ try {
+ return Files.size(p);
+ } catch (IOException e) {
+ return 0L;
+ }
+ })
+ .sum();
+ } catch (IOException e) {
+ return 0L;
+ }
+ }
+
+ private long fileSize(Path file) {
+ try {
+ return Files.exists(file) ? Files.size(file) : 0L;
+ } catch (IOException e) {
+ return 0L;
+ }
+ }
}
diff --git a/src/test/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporterTest.java b/src/test/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporterTest.java
new file mode 100644
index 0000000..f3d1ae2
--- /dev/null
+++ b/src/test/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporterTest.java
@@ -0,0 +1,325 @@
+/*
+ * This file is part of paikka.
+ *
+ * Paikka is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, either version 3 or
+ * any later version.
+ *
+ * Paikka is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Affero General Public License for more details.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Paikka. If not, see .
+ */
+
+package com.dedicatedcode.paikka.service.importer;
+
+import com.dedicatedcode.paikka.config.PaikkaConfiguration;
+import org.junit.jupiter.api.*;
+import org.rocksdb.Options;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Integration tests for StandaloneBoundaryImporter.
+ * Uses the schleswig-holstein-boundaries.osm.pbf test file to verify
+ * the three RocksDB output databases and name streaming.
+ */
+class StandaloneBoundaryImporterTest {
+
+ private static Path tempOutputDir;
+ private static Path tempPbfFile;
+
+ @BeforeAll
+ static void setUp() throws Exception {
+ tempOutputDir = Files.createTempDirectory("paikka-boundary-test");
+ tempPbfFile = Files.createTempFile("paikka-boundary-test", ".pbf");
+
+ try (InputStream is = StandaloneBoundaryImporter.class.getClassLoader()
+ .getResourceAsStream("schleswig-holstein-boundaries.osm.pbf")) {
+ assertNotNull(is, "schleswig-holstein-boundaries.osm.pbf not found in test resources");
+ Files.copy(is, tempPbfFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ PaikkaConfiguration config = new PaikkaConfiguration();
+
+ PaikkaConfiguration.ImportConfiguration importCfg = new PaikkaConfiguration.ImportConfiguration();
+ importCfg.setThreads(4);
+ config.setImportConfiguration(importCfg);
+
+ PaikkaConfiguration.SimplificationConfiguration simplCfg = new PaikkaConfiguration.SimplificationConfiguration();
+ simplCfg.setContinentTolerance(0.005);
+ simplCfg.setCountryTolerance(0.00045);
+ simplCfg.setStateTolerance(0.00009);
+ simplCfg.setPoiTolerance(0.000018);
+ simplCfg.setDefaultTolerance(0.000045);
+ config.setSimplificationConfiguration(simplCfg);
+
+ GeometrySimplificationService simplService = new GeometrySimplificationService(config);
+ StandaloneBoundaryImporter importer = new StandaloneBoundaryImporter(simplService, config);
+ importer.importBoundaries(Collections.singletonList(tempPbfFile.toString()), tempOutputDir.toString());
+ }
+
+ @AfterAll
+ static void tearDown() {
+ if (tempOutputDir != null && Files.exists(tempOutputDir)) {
+ deleteDirectory(tempOutputDir.toFile());
+ }
+ if (tempPbfFile != null && Files.exists(tempPbfFile)) {
+ tempPbfFile.toFile().delete();
+ }
+ }
+
+ @Test
+ void testOutputDatabasesExist() {
+ assertTrue(Files.exists(tempOutputDir.resolve("h3_to_osm")), "h3_to_osm directory should exist");
+ assertTrue(Files.exists(tempOutputDir.resolve("region_metadata")), "region_metadata directory should exist");
+ assertTrue(Files.exists(tempOutputDir.resolve("region_geometry")), "region_geometry directory should exist");
+ assertTrue(Files.exists(tempOutputDir.resolve("osm_names.tsv")), "osm_names.tsv should exist");
+ }
+
+ @Test
+ void testH3ToOsmHasEntries() throws RocksDBException {
+ Path dbPath = tempOutputDir.resolve("h3_to_osm");
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, dbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ assertTrue(it.isValid(), "h3_to_osm should have at least one entry");
+
+ byte[] key = it.key();
+ assertEquals(8, key.length, "H3 cell key should be 8 bytes");
+
+ byte[] val = it.value();
+ assertTrue(val.length >= 8, "H3 value should be at least 8 bytes");
+ assertEquals(0, val.length % 8, "H3 value should be a multiple of 8 bytes");
+
+ int count = 0;
+ it.seekToFirst();
+ while (it.isValid()) {
+ count++;
+ it.next();
+ }
+ System.out.println("Total h3_to_osm entries: " + count);
+ assertTrue(count > 100, "Should have more than 100 H3 cell entries, got: " + count);
+ }
+ }
+
+ @Test
+ void testRegionMetadataFormat() throws RocksDBException {
+ Path dbPath = tempOutputDir.resolve("region_metadata");
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, dbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ assertTrue(it.isValid(), "region_metadata should have entries");
+
+ byte[] key = it.key();
+ assertEquals(8, key.length, "Region metadata key should be 8 bytes (OSM ID)");
+
+ byte[] val = it.value();
+ assertEquals(12, val.length, "Value should be 12 bytes (8-byte cell count + 4-byte resolution)");
+
+ ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN);
+ long cellCount = bb.getLong();
+ int resolution = bb.getInt();
+
+ assertTrue(cellCount > 0, "Cell count should be positive, got: " + cellCount);
+ assertTrue(resolution >= 4 && resolution <= 9,
+ "Resolution should be between 4 and 9, got: " + resolution);
+
+ int count = 0;
+ it.seekToFirst();
+ while (it.isValid()) {
+ count++;
+ it.next();
+ }
+ System.out.println("Total region_metadata entries: " + count);
+ assertTrue(count >= 10, "Should have at least 10 boundaries, got: " + count);
+ }
+ }
+
+ @Test
+ void testAllRegionMetadataEntriesHaveValidResolution() throws RocksDBException {
+ Path dbPath = tempOutputDir.resolve("region_metadata");
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, dbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ int checked = 0;
+ while (it.isValid()) {
+ byte[] val = it.value();
+ assertEquals(12, val.length,
+ "Every region_metadata entry should be 12 bytes");
+
+ ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN);
+ long cellCount = bb.getLong();
+ int resolution = bb.getInt();
+
+ assertTrue(cellCount > 0,
+ "Cell count should be positive for entry " + checked);
+ assertTrue(resolution >= 4 && resolution <= 9,
+ "Resolution should be 4-9 for entry " + checked + ", got: " + resolution);
+
+ checked++;
+ it.next();
+ }
+ System.out.println("Verified " + checked + " region_metadata entries");
+ assertTrue(checked >= 10, "Should verify at least 10 entries, got: " + checked);
+ }
+ }
+
+ @Test
+ void testRegionGeometryHasEntries() throws RocksDBException {
+ Path dbPath = tempOutputDir.resolve("region_geometry");
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, dbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ assertTrue(it.isValid(), "region_geometry should have entries");
+
+ byte[] key = it.key();
+ assertEquals(8, key.length, "Region geometry key should be 8 bytes");
+
+ byte[] val = it.value();
+ assertTrue(val.length > 0, "WKB value should not be empty");
+
+ int count = 0;
+ it.seekToFirst();
+ while (it.isValid()) {
+ count++;
+ it.next();
+ }
+ System.out.println("Total region_geometry entries: " + count);
+ assertTrue(count >= 10, "Should have at least 10 geometries, got: " + count);
+ }
+ }
+
+ @Test
+ void testOsmNamesFileNotEmpty() throws IOException {
+ Path namesPath = tempOutputDir.resolve("osm_names.tsv");
+ assertTrue(Files.size(namesPath) > 0, "osm_names.tsv should not be empty");
+
+ String firstLine = Files.readAllLines(namesPath).getFirst();
+ assertTrue(firstLine.contains("\t"), "osm_names.tsv should be tab-separated");
+ System.out.println("osm_names.tsv first line: " + firstLine);
+ }
+
+ @Test
+ void testH3CellsMapToValidOsmIds() throws RocksDBException {
+ Set metadataOsmIds = new HashSet<>();
+ Path metaDbPath = tempOutputDir.resolve("region_metadata");
+
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, metaDbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ while (it.isValid()) {
+ long osmId = ByteBuffer.wrap(it.key()).order(ByteOrder.BIG_ENDIAN).getLong();
+ metadataOsmIds.add(osmId);
+ it.next();
+ }
+ }
+ assertFalse(metadataOsmIds.isEmpty(), "Should have metadata entries to cross-reference");
+
+ Path h3DbPath = tempOutputDir.resolve("h3_to_osm");
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, h3DbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ boolean foundMatch = false;
+ int checked = 0;
+ while (it.isValid() && checked < 100) {
+ byte[] val = it.value();
+ for (int i = 0; i < val.length; i += 8) {
+ long osmId = ByteBuffer.wrap(val, i, 8).order(ByteOrder.BIG_ENDIAN).getLong();
+ if (metadataOsmIds.contains(osmId)) {
+ foundMatch = true;
+ break;
+ }
+ }
+ if (foundMatch) break;
+ it.next();
+ checked++;
+ }
+ assertTrue(foundMatch,
+ "H3 cells should reference OSM IDs present in region_metadata");
+ }
+ }
+
+ @Test
+ void testRegionGeometryMatchesMetadata() throws RocksDBException {
+ Set metadataOsmIds = new HashSet<>();
+ Path metaDbPath = tempOutputDir.resolve("region_metadata");
+
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, metaDbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ while (it.isValid()) {
+ metadataOsmIds.add(
+ ByteBuffer.wrap(it.key()).order(ByteOrder.BIG_ENDIAN).getLong());
+ it.next();
+ }
+ }
+
+ Set geometryOsmIds = new HashSet<>();
+ Path geomDbPath = tempOutputDir.resolve("region_geometry");
+
+ try (Options opts = new Options().setCreateIfMissing(false);
+ RocksDB db = RocksDB.open(opts, geomDbPath.toString())) {
+
+ var it = db.newIterator();
+ it.seekToFirst();
+ while (it.isValid()) {
+ geometryOsmIds.add(
+ ByteBuffer.wrap(it.key()).order(ByteOrder.BIG_ENDIAN).getLong());
+ it.next();
+ }
+ }
+
+ assertEquals(metadataOsmIds, geometryOsmIds,
+ "Same OSM IDs should exist in region_metadata and region_geometry");
+ }
+
+ @Test
+ void testNoTemporaryFilesLeaked() {
+ Path tmpDir = tempOutputDir.resolve("tmp");
+ assertFalse(Files.exists(tmpDir),
+ "Temporary directory should be cleaned up after import");
+ }
+
+ private static void deleteDirectory(java.io.File dir) {
+ if (dir.isDirectory()) {
+ java.io.File[] children = dir.listFiles();
+ if (children != null) {
+ for (java.io.File child : children) {
+ deleteDirectory(child);
+ }
+ }
+ }
+ dir.delete();
+ }
+}
diff --git a/src/test/resources/schleswig-holstein-boundaries.osm.pbf b/src/test/resources/schleswig-holstein-boundaries.osm.pbf
new file mode 100644
index 0000000..12c9ab1
Binary files /dev/null and b/src/test/resources/schleswig-holstein-boundaries.osm.pbf differ