From 8a567782fc944603fb74dcbd367d705eea6adf6c Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sat, 11 Jul 2026 08:58:28 +0200 Subject: [PATCH 01/29] feat: add standalone H3-based boundary importer with RocksDB support - Integrated `StandaloneBoundaryImporter` service to enable standalone administrative boundary imports. - Added H3 (Uber H3) dependency for high-resolution geospatial indexing. - Updated import logic to process boundaries from PBF files and store outputs in RocksDB databases. - Enhanced geometry processing with support for buffering, simplification, and H3 polygon-to-cell mapping. --- pom.xml | 8 +- .../paikka/PaikkaApplication.java | 4 + .../importer/StandaloneBoundaryImporter.java | 454 ++++++++++++++++++ 3 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java diff --git a/pom.xml b/pom.xml index bb40e4e..c0b0dfd 100644 --- a/pom.xml +++ b/pom.xml @@ -45,7 +45,13 @@ s2-geometry 2.0.0 - + + + com.uber + h3 + 4.4.0 + + org.rocksdb diff --git a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java index 382c99d..07bb7e8 100644 --- a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java +++ b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java @@ -17,6 +17,7 @@ package com.dedicatedcode.paikka; import com.dedicatedcode.paikka.service.importer.ImportService; +import com.dedicatedcode.paikka.service.importer.StandaloneBoundaryImporter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -34,6 +35,9 @@ public class PaikkaApplication implements CommandLineRunner { @Autowired private ImportService importService; + @Autowired + private StandaloneBoundaryImporter standaloneBoundaryImporter; + static void main(String[] args) { for (String arg : args) { if ("-h".equals(arg) || "--help".equals(arg)) { diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java new file mode 100644 index 0000000..71637ed --- /dev/null +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -0,0 +1,454 @@ +/* + * This file is part of paikka. + * + * Paikka is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License + * as published by the Free Software Foundation, either version 3 or + * any later version. + * + * Paikka is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * You should have received a copy of the GNU Affero General Public License + * along with Paikka. If not, see . + */ + +package com.dedicatedcode.paikka.service.importer; + +import com.uber.h3core.H3Core; +import com.uber.h3core.util.LatLng; +import de.topobyte.osm4j.core.model.iface.*; +import de.topobyte.osm4j.pbf.seq.PbfIterator; +import org.locationtech.jts.geom.*; +import org.locationtech.jts.io.WKBWriter; +import org.rocksdb.*; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Standalone H3-based administrative boundary importer for Paikka. + *

+ * Reads a pre-filtered boundaries_only.pbf (Nodes -> Ways -> Relations ordered) + * and produces three RocksDB databases for offline mobile lookup: + * - h3_to_osm : H3_CELL_ID (uint64) -> List[OSM_ID] (raw byte array) + * - region_metadata : OSM_ID -> total cell count (int) + * - region_geometry : OSM_ID -> simplified WKB (bytes) + */ +@Service +public class StandaloneBoundaryImporter { + + private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); + private static final int H3_RESOLUTION = 9; + private static final double BUFFER_DISTANCE = 0.0001; // ~11m at equator, ensures border cells + + private final GeometrySimplificationService geometrySimplificationService; + private final H3Core h3; + + public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplificationService) throws Exception { + this.geometrySimplificationService = geometrySimplificationService; + this.h3 = H3Core.newInstance(); // Uber H3-Java 4.x + } + + // ============================ PUBLIC API ============================ + + public void importBoundaries(String pbfPath, String outputDir) throws Exception { + RocksDB.loadLibrary(); + Path out = Paths.get(outputDir); + Path tmp = out.resolve("tmp"); + Files.createDirectories(out); + Files.createDirectories(tmp); + + Path nodeCachePath = tmp.resolve("node_cache"); + Path wayCachePath = tmp.resolve("way_cache"); + Path h3ToOsmPath = out.resolve("h3_to_osm"); + Path regionMetaPath = out.resolve("region_metadata"); + Path regionGeomPath = out.resolve("region_geometry"); + + cleanup(nodeCachePath); + cleanup(wayCachePath); + cleanup(h3ToOsmPath); + cleanup(regionMetaPath); + cleanup(regionGeomPath); + + // Shared RocksDB options (inline with ImportService style) + BlockBasedTableConfig tableCfg = new BlockBasedTableConfig() + .setBlockSize(64 * 1024) + .setFilterPolicy(new BloomFilter(10, false)); + Options cacheOpts = new Options() + .setCreateIfMissing(true) + .setTableFormatConfig(tableCfg) + .setCompressionType(CompressionType.LZ4_COMPRESSION) + .setWriteBufferSize(512 * 1024 * 1024) + .setMaxWriteBufferNumber(3) + .setLevel0FileNumCompactionTrigger(4); + Options finalOpts = new Options() + .setCreateIfMissing(true) + .setTableFormatConfig(tableCfg) + .setCompressionType(CompressionType.ZSTD_COMPRESSION) + .setWriteBufferSize(256 * 1024 * 1024); + WriteOptions wo = new WriteOptions().setDisableWAL(true); + + try ( + RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString()); + RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString()); + RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString()); + RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString()); + RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()) + ) { + // ---------- SINGLE PASS ---------- + PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); + + // Phase 1 & 2: Stream nodes and ways (cached) + WriteBatch nodeBatch = new WriteBatch(); + WriteBatch wayBatch = new WriteBatch(); + AtomicLong phaseCounter = new AtomicLong(); + + while (iterator.hasNext()) { + EntityContainer c = iterator.next(); + if (c.getType() == EntityType.Node) { + // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair + OsmNode n = (OsmNode) c.getEntity(); + ByteBuffer bb = ByteBuffer.allocate(16) + .putDouble(n.getLatitude()) + .putDouble(n.getLongitude()); + nodeBatch.put(longToBytes(n.getId()), bb.array()); + if (phaseCounter.incrementAndGet() % 100_000 == 0) { + nodeCache.write(wo, nodeBatch); + nodeBatch.clear(); + } + } else if (c.getType() == EntityType.Way) { + // PHASE 2: Cache way node-id sequences (long[] as raw bytes) + OsmWay w = (OsmWay) c.getEntity(); + long[] ids = new long[w.getNumberOfNodes()]; + for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); + wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); + if (phaseCounter.incrementAndGet() % 50_000 == 0) { + wayCache.write(wo, wayBatch); + wayBatch.clear(); + } + } else if (c.getType() == EntityType.Relation) { + // PHASE 3: Relations (ways already fully cached above) + break; // Relations come after ways in ordered PBF; switch mode + } + } + nodeCache.write(wo, nodeBatch); + wayCache.write(wo, wayBatch); + nodeBatch.close(); + wayBatch.close(); + + // Re-open iterator for Phase 3 (or use two iterators; here we reuse file) + + // Phase 3: Process Relations (separate iterator pass is fine since PBF is local) + try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) { + PbfIterator relIter = new PbfIterator(is, false); + + List relations = new ArrayList<>(); + while (relIter.hasNext()) { + EntityContainer c = relIter.next(); + if (c.getType() == EntityType.Relation) { + OsmRelation r = (OsmRelation) c.getEntity(); + if (isAdministrativeBoundary(r)) { + relations.add(buildRelationStub(r)); + } + } + } + + // Process each relation: stitch geometry, H3 polyfill, write outputs + WriteBatch h3Batch = new WriteBatch(); + WriteBatch metaBatch = new WriteBatch(); + WriteBatch geomBatch = new WriteBatch(); + + for (RelationStub stub : relations) { + Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); + if (geom == null || geom.isEmpty() || !geom.isValid()) continue; + + // Buffer to include border-touching cells + Geometry buffered = geom.buffer(BUFFER_DISTANCE); + Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); + if (simplified == null || simplified.isEmpty()) simplified = buffered; + + // ---- H3 Polyfill ---- + List cells = polygonToCellsH3(simplified); + if (cells.isEmpty()) continue; + + // h3_to_osm : append OSM_ID to each cell (dedup) + for (long cell : cells) { + byte[] key = longToBytes(cell); + byte[] existing = h3ToOsm.get(key); + byte[] updated = appendOsmIdToArray(existing, stub.osmId); + h3Batch.put(key, updated); + } + + // region_metadata : OSM_ID -> cell count (int) + metaBatch.put(longToBytes(stub.osmId), intToBytes(cells.size())); + + // region_geometry : OSM_ID -> simplified WKB + byte[] wkb = new WKBWriter().write(simplified); + geomBatch.put(longToBytes(stub.osmId), wkb); + } + + h3ToOsm.write(wo, h3Batch); + regionMeta.write(wo, metaBatch); + regionGeom.write(wo, geomBatch); + h3Batch.close(); + metaBatch.close(); + geomBatch.close(); + wo.close(); + } + + // Compact finals + h3ToOsm.compactRange(); + regionMeta.compactRange(); + regionGeom.compactRange(); + } + + // Cleanup tmp + cleanup(tmp); + System.out.println("[StandaloneBoundaryImporter] Import complete. Temporary caches removed."); + } + + // ============================ GEOMETRY STITCHING ============================ + + /** + * Builds a JTS MultiPolygon from relation outer/inner way members. + * Rings are stitched by coordinate continuation (same logic as ImportService.buildConnectedRings). + */ + private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB wayCache) { + List> outerRings = stitchRings(stub.outerWays, nodeCache, wayCache); + List> innerRings = stitchRings(stub.innerWays, nodeCache, wayCache); + if (outerRings.isEmpty()) return null; + + List polygons = new ArrayList<>(); + for (List outer : outerRings) { + try { + LinearRing shell = GEOMETRY_FACTORY.createLinearRing(outer.toArray(new Coordinate[0])); + List holes = new ArrayList<>(); + for (List inner : innerRings) { + try { + holes.add(GEOMETRY_FACTORY.createLinearRing(inner.toArray(new Coordinate[0]))); + } catch (Exception ignored) { + } + } + Polygon p = GEOMETRY_FACTORY.createPolygon(shell, holes.toArray(new LinearRing[0])); + if (p.isValid()) polygons.add(p); + } catch (Exception ignored) { + } + } + if (polygons.isEmpty()) return null; + return polygons.size() == 1 ? polygons.get(0) : GEOMETRY_FACTORY.createMultiPolygon(polygons.toArray(new Polygon[0])); + } + + private List> stitchRings(List wayIds, RocksDB nodeCache, RocksDB wayCache) { + Map> wayCoords = new HashMap<>(); + for (long wid : wayIds) { + try { + byte[] seq = wayCache.get(longToBytes(wid)); + if (seq == null) continue; + long[] nodeIds = bytesToLongArray(seq); + List coords = resolveCoordinates(nodeIds, nodeCache); + if (coords != null && coords.size() >= 2) wayCoords.put(wid, coords); + } catch (RocksDBException e) { /* skip */ } + } + List> rings = new ArrayList<>(); + Set used = new HashSet<>(); + while (used.size() < wayCoords.size()) { + Long start = wayCoords.keySet().stream().filter(id -> !used.contains(id)).findFirst().orElse(null); + if (start == null) break; + List ring = new ArrayList<>(wayCoords.get(start)); + used.add(start); + boolean extended; + do { + extended = false; + Coordinate end = ring.get(ring.size() - 1); + for (Map.Entry> e : wayCoords.entrySet()) { + if (used.contains(e.getKey())) continue; + List w = e.getValue(); + if (end.equals2D(w.get(0))) { + ring.addAll(w.subList(1, w.size())); + used.add(e.getKey()); + extended = true; + break; + } else if (end.equals2D(w.get(w.size() - 1))) { + List rev = new ArrayList<>(w); + Collections.reverse(rev); + ring.addAll(rev.subList(1, rev.size())); + used.add(e.getKey()); + extended = true; + break; + } + } + } while (extended); + if (ring.size() >= 3 && !ring.get(0).equals2D(ring.get(ring.size() - 1))) + ring.add(new Coordinate(ring.get(0))); + if (ring.size() >= 4) rings.add(ring); + } + return rings; + } + + private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) { + try { + List keys = new ArrayList<>(nodeIds.length); + for (long id : nodeIds) keys.add(longToBytes(id)); + List vals = nodeCache.multiGetAsList(keys); + List coords = new ArrayList<>(nodeIds.length); + for (byte[] v : vals) { + if (v != null && v.length == 16) { + ByteBuffer bb = ByteBuffer.wrap(v); + double lat = bb.getDouble(0); + double lon = bb.getDouble(8); + coords.add(new Coordinate(lon, lat)); // JTS uses (x=lon, y=lat) + } else return null; + } + return coords; + } catch (RocksDBException e) { + return null; + } + } + + // ============================ H3 POLYFILL ============================ + + /** + * Converts a JTS Geometry to H3 cells at resolution 9. + * Uses h3.polygonToCells with LatLng vertices. Multipolygons are expanded. + */ + private List polygonToCellsH3(Geometry geom) { + List cells = new ArrayList<>(); + int num = geom.getNumGeometries(); + for (int i = 0; i < num; i++) { + Geometry part = geom.getGeometryN(i); + if (!(part instanceof Polygon poly)) continue; + List outer = toLatLng(poly.getExteriorRing().getCoordinates()); + List> holes = new ArrayList<>(); + for (int h = 0; h < poly.getNumInteriorRing(); h++) { + holes.add(toLatLng(poly.getInteriorRingN(h).getCoordinates())); + } + try { + List partCells = h3.polygonToCells(outer, holes, H3_RESOLUTION); + cells.addAll(partCells); + } catch (Exception e) { + // skip invalid loop + } + } + return cells; + } + + private List toLatLng(Coordinate[] coords) { + List list = new ArrayList<>(coords.length); + for (Coordinate c : coords) { + list.add(new LatLng(c.y, c.x)); // lat, lon + } + return list; + } + + // ============================ BYTE UTILS ============================ + + private byte[] longToBytes(long v) { + return ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(v).array(); + } + + private long bytesToLong(byte[] b) { + return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getLong(); + } + + private byte[] longArrayToBytes(long[] arr) { + ByteBuffer bb = ByteBuffer.allocate(8 * arr.length).order(ByteOrder.BIG_ENDIAN); + for (long v : arr) bb.putLong(v); + return bb.array(); + } + + private long[] bytesToLongArray(byte[] b) { + ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN); + long[] arr = new long[b.length / 8]; + for (int i = 0; i < arr.length; i++) arr[i] = bb.getLong(); + return arr; + } + + private byte[] intToBytes(int v) { + return ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(v).array(); + } + + /** + * Appends an OSM_ID to a raw byte array of longs, preventing duplicates. + * Format: sequence of 8-byte big-endian longs. + */ + private byte[] appendOsmIdToArray(byte[] existing, long osmId) { + if (existing == null || existing.length == 0) { + return longToBytes(osmId); + } + int count = existing.length / 8; + for (int i = 0; i < count; i++) { + long val = ByteBuffer.wrap(existing, i * 8, 8).order(ByteOrder.BIG_ENDIAN).getLong(); + if (val == osmId) return existing; // duplicate + } + ByteBuffer bb = ByteBuffer.allocate(existing.length + 8).order(ByteOrder.BIG_ENDIAN); + bb.put(existing); + bb.putLong(osmId); + return bb.array(); + } + + // ============================ OSM HELPERS ============================ + + private boolean isAdministrativeBoundary(OsmRelation r) { + boolean boundary = false, adminLevel = false; + for (int i = 0; i < r.getNumberOfTags(); i++) { + OsmTag t = r.getTag(i); + if ("boundary".equals(t.getKey()) && "administrative".equals(t.getValue())) boundary = true; + if ("admin_level".equals(t.getKey())) adminLevel = true; + if ("type".equals(t.getKey()) && "boundary".equals(t.getValue())) boundary = true; + } + return boundary && adminLevel; + } + + private RelationStub buildRelationStub(OsmRelation r) { + List outer = new ArrayList<>(); + List inner = new ArrayList<>(); + int level = 10; + for (int i = 0; i < r.getNumberOfMembers(); i++) { + OsmRelationMember m = r.getMember(i); + if (m.getType() == EntityType.Way) { + String role = m.getRole(); + if ("outer".equals(role) || role == null || role.isEmpty()) outer.add(m.getId()); + else if ("inner".equals(role)) inner.add(m.getId()); + } + } + for (int i = 0; i < r.getNumberOfTags(); i++) { + OsmTag t = r.getTag(i); + if ("admin_level".equals(t.getKey())) { + try { + level = Integer.parseInt(t.getValue()); + } catch (NumberFormatException ignored) { + } + } + } + return new RelationStub(r.getId(), level, outer, inner); + } + + private record RelationStub(long osmId, int adminLevel, List outerWays, List innerWays) { + } + + private void cleanup(Path p) { + if (Files.exists(p)) { + try { + Files.walk(p).sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.delete(path); + } catch (IOException e) { + System.err.println("warn: " + e.getMessage()); + } + }); + } catch (IOException e) { + System.err.println("Failed cleanup: " + p + " -> " + e.getMessage()); + } + } + } +} \ No newline at end of file From 882e5cc8eccd910537582f44d2bed847d7a2b9e8 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sat, 11 Jul 2026 09:02:59 +0200 Subject: [PATCH 02/29] feat: add --boundary-import mode to utilize StandaloneBoundaryImporter --- .../paikka/PaikkaApplication.java | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java index 07bb7e8..52fb029 100644 --- a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java +++ b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java @@ -50,14 +50,17 @@ static void main(String[] args) { // Check if this is import mode boolean isImportMode = false; + boolean isBoundaryImportMode = false; for (String arg : args) { if ("--import".equals(arg)) { isImportMode = true; - break; + } + if ("--boundary-import".equals(arg)) { + isBoundaryImportMode = true; } } - if (isImportMode) { + if (isImportMode || isBoundaryImportMode) { logger.info("Starting in import mode"); app.setWebApplicationType(org.springframework.boot.WebApplicationType.NONE); System.setProperty("paikka.import-mode", "true"); @@ -87,6 +90,7 @@ private static void printApiInfo() { @Override public void run(String... args) throws Exception { boolean isImportMode = false; + boolean isBoundaryImportMode = false; List pbfFiles = new ArrayList<>(); String dataDir = "./data"; Set usedArgIndices = new HashSet<>(); @@ -96,6 +100,8 @@ public void run(String... args) throws Exception { String arg = args[i]; if ("--import".equals(arg)) { isImportMode = true; + } else if ("--boundary-import".equals(arg)) { + isBoundaryImportMode = true; } else if ("--pbf-file".equals(arg)) { if (i + 1 >= args.length) { logger.error("Missing --pbf-file value"); System.exit(1); } String value = args[ i + 1]; @@ -116,7 +122,7 @@ public void run(String... args) throws Exception { if (usedArgIndices.contains(i)) continue; String arg = args[i]; if (arg.startsWith("--")) continue; // Skip unrecognized flags - if (isImportMode) pbfFiles.add(arg.trim()); + if (isImportMode || isBoundaryImportMode) pbfFiles.add(arg.trim()); } if (isImportMode) { @@ -132,6 +138,21 @@ public void run(String... args) throws Exception { logger.error("Import failed", e); System.exit(1); } + } else if (isBoundaryImportMode) { + if (pbfFiles.isEmpty()) { + logger.error("Boundary import mode requires at least one PBF file"); + printImportUsage(); + System.exit(1); + } + try { + for (String pbfFile : pbfFiles) { + standaloneBoundaryImporter.importBoundaries(pbfFile, dataDir); + } + System.exit(0); + } catch (Exception e) { + logger.error("Boundary import failed", e); + System.exit(1); + } } else { printApiInfo(); } @@ -159,23 +180,30 @@ private static void printHelp() { System.out.println(" Imports OpenStreetMap PBF files into the Paikka datastore."); System.out.println(" All specified PBF files are combined into a single final datastore."); + System.out.println("\n 3. Boundary Import Mode (requires --boundary-import flag):"); + System.out.println(" Imports administrative boundaries from OpenStreetMap PBF files into the Paikka datastore."); + System.out.println(" All specified PBF files are processed."); + System.out.println("\nImport Mode Options:"); - System.out.println(" --import Enable import mode (required for data import)"); + System.out.println(" --import Enable standard import mode (required for data import)"); + System.out.println(" --boundary-import Enable boundary import mode"); System.out.println(" --pbf-file Specify PBF file(s). Supports multiple formats:"); System.out.println(" • Comma-separated list: --pbf-file \"file1.pbf,file2.pbf\""); System.out.println(" • Repeated flags: --pbf-file file1.pbf --pbf-file file2.pbf"); System.out.println(" --data-dir Path to data directory (default: ./data)"); - System.out.println(" Positional arguments (after all flags) are treated as PBF files in import mode"); + System.out.println(" Positional arguments (after all flags) are treated as PBF files in import modes"); System.out.println("\nImport Examples:"); - System.out.println(" # Single PBF file"); + System.out.println(" # Single PBF file (Standard Import)"); System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm.pbf"); - System.out.println(" # Multiple PBFs (comma-separated)"); + System.out.println(" # Multiple PBFs (comma-separated) (Standard Import)"); System.out.println(" java -jar paikka.jar --import --pbf-file \"/data/osm1.pbf,/data/osm2.pbf\" --data-dir ./data"); - System.out.println(" # Multiple PBFs (repeated --pbf-file flags)"); + System.out.println(" # Multiple PBFs (repeated --pbf-file flags) (Standard Import)"); System.out.println(" java -jar paikka.jar --import --pbf-file /data/osm1.pbf --pbf-file /data/osm2.pbf"); - System.out.println(" # Multiple PBFs (trailing positional arguments)"); + System.out.println(" # Multiple PBFs (trailing positional arguments) (Standard Import)"); System.out.println(" java -jar paikka.jar --import /data/osm1.pbf /data/osm2.pbf"); + System.out.println(" # Boundary Import"); + System.out.println(" java -jar paikka.jar --boundary-import --pbf-file /data/boundaries.pbf --data-dir ./data"); } private static void printImportUsage() { From 7bc128fa70007d12a9f043e719e94d390cb145b5 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sat, 11 Jul 2026 09:10:35 +0200 Subject: [PATCH 03/29] added multi file suport for boundaries import --- .gitignore | 1 + .../paikka/PaikkaApplication.java | 4 +- .../importer/StandaloneBoundaryImporter.java | 185 +++++++++--------- 3 files changed, 96 insertions(+), 94 deletions(-) diff --git a/.gitignore b/.gitignore index 29a6c52..cbe522e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ build/ .aider* /data/ /.idea/ +/data-boundaries/ diff --git a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java index 52fb029..62bee27 100644 --- a/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java +++ b/src/main/java/com/dedicatedcode/paikka/PaikkaApplication.java @@ -145,9 +145,7 @@ public void run(String... args) throws Exception { System.exit(1); } try { - for (String pbfFile : pbfFiles) { - standaloneBoundaryImporter.importBoundaries(pbfFile, dataDir); - } + standaloneBoundaryImporter.importBoundaries(pbfFiles, dataDir); System.exit(0); } catch (Exception e) { logger.error("Boundary import failed", e); diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 71637ed..b03436e 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -61,7 +61,7 @@ public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplifi // ============================ PUBLIC API ============================ - public void importBoundaries(String pbfPath, String outputDir) throws Exception { + public void importBoundaries(List pbfPaths, String outputDir) throws Exception { RocksDB.loadLibrary(); Path out = Paths.get(outputDir); Path tmp = out.resolve("tmp"); @@ -105,105 +105,108 @@ public void importBoundaries(String pbfPath, String outputDir) throws Exception RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString()); RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()) ) { - // ---------- SINGLE PASS ---------- - PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); - - // Phase 1 & 2: Stream nodes and ways (cached) - WriteBatch nodeBatch = new WriteBatch(); - WriteBatch wayBatch = new WriteBatch(); - AtomicLong phaseCounter = new AtomicLong(); - - while (iterator.hasNext()) { - EntityContainer c = iterator.next(); - if (c.getType() == EntityType.Node) { - // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair - OsmNode n = (OsmNode) c.getEntity(); - ByteBuffer bb = ByteBuffer.allocate(16) - .putDouble(n.getLatitude()) - .putDouble(n.getLongitude()); - nodeBatch.put(longToBytes(n.getId()), bb.array()); - if (phaseCounter.incrementAndGet() % 100_000 == 0) { - nodeCache.write(wo, nodeBatch); - nodeBatch.clear(); - } - } else if (c.getType() == EntityType.Way) { - // PHASE 2: Cache way node-id sequences (long[] as raw bytes) - OsmWay w = (OsmWay) c.getEntity(); - long[] ids = new long[w.getNumberOfNodes()]; - for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); - wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); - if (phaseCounter.incrementAndGet() % 50_000 == 0) { - wayCache.write(wo, wayBatch); - wayBatch.clear(); + for (String pbfPath : pbfPaths) { + // ---------- SINGLE PASS ---------- + PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); + + // Phase 1 & 2: Stream nodes and ways (cached) + WriteBatch nodeBatch = new WriteBatch(); + WriteBatch wayBatch = new WriteBatch(); + AtomicLong phaseCounter = new AtomicLong(); + + while (iterator.hasNext()) { + EntityContainer c = iterator.next(); + if (c.getType() == EntityType.Node) { + // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair + OsmNode n = (OsmNode) c.getEntity(); + ByteBuffer bb = ByteBuffer.allocate(16) + .putDouble(n.getLatitude()) + .putDouble(n.getLongitude()); + nodeBatch.put(longToBytes(n.getId()), bb.array()); + if (phaseCounter.incrementAndGet() % 100_000 == 0) { + nodeCache.write(wo, nodeBatch); + nodeBatch.clear(); + } + } else if (c.getType() == EntityType.Way) { + // PHASE 2: Cache way node-id sequences (long[] as raw bytes) + OsmWay w = (OsmWay) c.getEntity(); + long[] ids = new long[w.getNumberOfNodes()]; + for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); + wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); + if (phaseCounter.incrementAndGet() % 50_000 == 0) { + wayCache.write(wo, wayBatch); + wayBatch.clear(); + } + } else if (c.getType() == EntityType.Relation) { + // PHASE 3: Relations (ways already fully cached above) + break; // Relations come after ways in ordered PBF; switch mode } - } else if (c.getType() == EntityType.Relation) { - // PHASE 3: Relations (ways already fully cached above) - break; // Relations come after ways in ordered PBF; switch mode } - } - nodeCache.write(wo, nodeBatch); - wayCache.write(wo, wayBatch); - nodeBatch.close(); - wayBatch.close(); - - // Re-open iterator for Phase 3 (or use two iterators; here we reuse file) - - // Phase 3: Process Relations (separate iterator pass is fine since PBF is local) - try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) { - PbfIterator relIter = new PbfIterator(is, false); - - List relations = new ArrayList<>(); - while (relIter.hasNext()) { - EntityContainer c = relIter.next(); - if (c.getType() == EntityType.Relation) { - OsmRelation r = (OsmRelation) c.getEntity(); - if (isAdministrativeBoundary(r)) { - relations.add(buildRelationStub(r)); + nodeCache.write(wo, nodeBatch); + wayCache.write(wo, wayBatch); + nodeBatch.close(); + wayBatch.close(); + + // Re-open iterator for Phase 3 (or use two iterators; here we reuse file) + + // Phase 3: Process Relations (separate iterator pass is fine since PBF is local) + try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) { + PbfIterator relIter = new PbfIterator(is, false); + + List relations = new ArrayList<>(); + while (relIter.hasNext()) { + EntityContainer c = relIter.next(); + if (c.getType() == EntityType.Relation) { + OsmRelation r = (OsmRelation) c.getEntity(); + if (isAdministrativeBoundary(r)) { + relations.add(buildRelationStub(r)); + } } } - } - // Process each relation: stitch geometry, H3 polyfill, write outputs - WriteBatch h3Batch = new WriteBatch(); - WriteBatch metaBatch = new WriteBatch(); - WriteBatch geomBatch = new WriteBatch(); - - for (RelationStub stub : relations) { - Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); - if (geom == null || geom.isEmpty() || !geom.isValid()) continue; - - // Buffer to include border-touching cells - Geometry buffered = geom.buffer(BUFFER_DISTANCE); - Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); - if (simplified == null || simplified.isEmpty()) simplified = buffered; - - // ---- H3 Polyfill ---- - List cells = polygonToCellsH3(simplified); - if (cells.isEmpty()) continue; - - // h3_to_osm : append OSM_ID to each cell (dedup) - for (long cell : cells) { - byte[] key = longToBytes(cell); - byte[] existing = h3ToOsm.get(key); - byte[] updated = appendOsmIdToArray(existing, stub.osmId); - h3Batch.put(key, updated); - } + // Process each relation: stitch geometry, H3 polyfill, write outputs + WriteBatch h3Batch = new WriteBatch(); + WriteBatch metaBatch = new WriteBatch(); + WriteBatch geomBatch = new WriteBatch(); + + for (RelationStub stub : relations) { + Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); + if (geom == null || geom.isEmpty() || !geom.isValid()) continue; + + // Buffer to include border-touching cells + Geometry buffered = geom.buffer(BUFFER_DISTANCE); + Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); + if (simplified == null || simplified.isEmpty()) simplified = buffered; + + // ---- H3 Polyfill ---- + List cells = polygonToCellsH3(simplified); + if (cells.isEmpty()) continue; + + // h3_to_osm : append OSM_ID to each cell (dedup) + for (long cell : cells) { + byte[] key = longToBytes(cell); + byte[] existing = h3ToOsm.get(key); + byte[] updated = appendOsmIdToArray(existing, stub.osmId); + h3Batch.put(key, updated); + } - // region_metadata : OSM_ID -> cell count (int) - metaBatch.put(longToBytes(stub.osmId), intToBytes(cells.size())); + // region_metadata : OSM_ID -> cell count (int) + metaBatch.put(longToBytes(stub.osmId), intToBytes(cells.size())); + + // region_geometry : OSM_ID -> simplified WKB + byte[] wkb = new WKBWriter().write(simplified); + geomBatch.put(longToBytes(stub.osmId), wkb); + } - // region_geometry : OSM_ID -> simplified WKB - byte[] wkb = new WKBWriter().write(simplified); - geomBatch.put(longToBytes(stub.osmId), wkb); + h3ToOsm.write(wo, h3Batch); + regionMeta.write(wo, metaBatch); + regionGeom.write(wo, geomBatch); + h3Batch.close(); + metaBatch.close(); + geomBatch.close(); + wo.close(); } - h3ToOsm.write(wo, h3Batch); - regionMeta.write(wo, metaBatch); - regionGeom.write(wo, geomBatch); - h3Batch.close(); - metaBatch.close(); - geomBatch.close(); - wo.close(); } // Compact finals From 6bcbf829dfc2c004368d94d4b3d53078f27fb622 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sat, 11 Jul 2026 09:21:51 +0200 Subject: [PATCH 04/29] feat: add BoundaryImportStatistics with progress tracking and error reporting --- .../importer/BoundaryImportStatistics.java | 354 ++++++++++++++++++ .../importer/StandaloneBoundaryImporter.java | 32 +- 2 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java new file mode 100644 index 0000000..602e46c --- /dev/null +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/BoundaryImportStatistics.java @@ -0,0 +1,354 @@ +/* + * This file is part of paikka. + * + * Paikka is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License + * as published by the Free Software Foundation, either version 3 or + * any later version. + * + * Paikka is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * You should have received a copy of the GNU Affero General Public License + * along with Paikka. If not, see . + */ + +package com.dedicatedcode.paikka.service.importer; + +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class BoundaryImportStatistics { + + public enum Stage { + CACHING_NODES_WAYS("Caching Nodes & Ways"), + PROCESSING_RELATIONS("Processing Relations & H3"), + OVERALL("Overall"); + + private final String shortName; + + Stage(String shortName) { + this.shortName = shortName; + } + + @Override + public String toString() { + return this.shortName; + } + } + + public enum Kind { + READ("Read/IO"), + DECODE("Decode"), + GEOMETRY("Geometry"), + STORE("Store/Write"), + OVERALL("Overall"); + + private final String shortName; + + Kind(String shortName) { + this.shortName = shortName; + } + + @Override + public String toString() { + return shortName; + } + } + + private static final double DEGRADED_WARN_RATE = 1e-4; // 0.01% = 1 in 10,000 + private static final int ERROR_SAMPLE_LIMIT = 50; + + private final AtomicLong errorsTotal = new AtomicLong(0); + private final ConcurrentHashMap errorBuckets = new ConcurrentHashMap<>(); + private final ConcurrentLinkedQueue errorSamples = new ConcurrentLinkedQueue<>(); + + private final AtomicLong nodesCached = new AtomicLong(0); + private final AtomicLong waysCached = new AtomicLong(0); + private final AtomicLong relationsFound = new AtomicLong(0); + private final AtomicLong relationsProcessed = new AtomicLong(0); + private final AtomicLong h3CellsGenerated = new AtomicLong(0); + + private volatile String currentPhase = "Initializing"; + private volatile boolean running = true; + private final long startTime = System.currentTimeMillis(); + private volatile long phaseStartTime = System.currentTimeMillis(); + private long totalTime; + + private final int TOTAL_STEPS = 2; + private int currentStep = 0; + + public long getNodesCached() { + return nodesCached.get(); + } + + public void incrementNodesCached() { + nodesCached.incrementAndGet(); + } + + public long getWaysCached() { + return waysCached.get(); + } + + public void incrementWaysCached() { + waysCached.incrementAndGet(); + } + + public long getRelationsFound() { + return relationsFound.get(); + } + + public void incrementRelationsFound() { + relationsFound.incrementAndGet(); + } + + public long getRelationsProcessed() { + return relationsProcessed.get(); + } + + public void incrementRelationsProcessed() { + relationsProcessed.incrementAndGet(); + } + + public long getH3CellsGenerated() { + return h3CellsGenerated.get(); + } + + public void addH3CellsGenerated(long count) { + h3CellsGenerated.addAndGet(count); + } + + public String getCurrentPhase() { + return currentPhase; + } + + public void setCurrentPhase(int step, String phase) { + this.currentPhase = phase; + this.phaseStartTime = System.currentTimeMillis(); + this.currentStep = step; + } + + public long getPhaseStartTime() { + return phaseStartTime; + } + + public boolean isRunning() { + return running; + } + + public void stop() { + this.running = false; + } + + public long getStartTime() { + return startTime; + } + + public long getTotalTime() { + return totalTime; + } + + public void setTotalTime(long t) { + this.totalTime = t; + } + + public void recordError(Stage stage, Kind kind, Long osmId, String operation, Exception e) { + errorsTotal.incrementAndGet(); + + String safePhase = stage.toString(); + String safeKind = kind.toString(); + String safeOp = operation != null ? operation : "-"; + String ex = (e != null) ? e.getClass().getSimpleName() : "Exception"; + String bucketKey = safePhase + "|" + safeKind + "|" + safeOp + "|" + ex; + + errorBuckets.computeIfAbsent(bucketKey, k -> new AtomicLong(0)).incrementAndGet(); + + if (errorSamples.size() < ERROR_SAMPLE_LIMIT) { + String msg = (e != null ? e.getMessage() : null); + errorSamples.add( + "phase=" + safePhase + + " kind=" + safeKind + + " id=" + (osmId != null ? osmId : "-") + + " op=" + safeOp + + " ex=" + (e != null ? e.getClass().getName() : "java.lang.Exception") + + (msg != null ? " msg=" + msg : "") + ); + } + } + + public long getErrorsTotal() { + return errorsTotal.get(); + } + + public String getMemoryStats() { + Runtime r = Runtime.getRuntime(); + long used = (r.totalMemory() - r.freeMemory()) / 1024 / 1024 / 1024; + long max = r.maxMemory() / 1024 / 1024 / 1024; + return String.format("%dGB/%dGB", used, max); + } + + public void startProgressReporter() { + boolean isTty = System.console() != null; + + Thread.ofPlatform().daemon().start(() -> { + while (isRunning()) { + long elapsed = System.currentTimeMillis() - getStartTime(); + long phaseElapsed = System.currentTimeMillis() - getPhaseStartTime(); + double phaseSeconds = phaseElapsed / 1000.0; + + String phase = getCurrentPhase(); + StringBuilder sb = new StringBuilder(); + + if (isTty) { + sb.append("\r\033[K"); + } + + sb.append(String.format("\033[1;90m[%d/%d]\033[0m ", currentStep, TOTAL_STEPS)); + + if (phase.contains("1.1")) { + long nodesPerSec = phaseSeconds > 0 ? (long) (getNodesCached() / phaseSeconds) : 0; + sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mCaching Nodes & Ways\033[0m", formatTime(elapsed))); + sb.append(String.format(" │ \033[32mNodes:\033[0m %s \033[33m(%s/s)\033[0m", + formatCompactNumber(getNodesCached()), formatCompactRate(nodesPerSec))); + sb.append(String.format(" │ \033[34mWays:\033[0m %s", formatCompactNumber(getWaysCached()))); + } else if (phase.contains("2.1")) { + long relsPerSec = phaseSeconds > 0 ? (long) (getRelationsProcessed() / phaseSeconds) : 0; + double percentage = getRelationsFound() > 0 ? (double) getRelationsProcessed() / getRelationsFound() * 100.0 : 0.0; + sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mProcessing Relations & H3\033[0m", formatTime(elapsed))); + sb.append(String.format(" │ \033[32mRelations:\033[0m %s/%s \033[33m(%s/s)\033[0m", + formatCompactNumber(getRelationsProcessed()), formatCompactNumber(getRelationsFound()), formatCompactRate(relsPerSec))); + sb.append(String.format(" │ \033[35mProgress:\033[0m %.2f%%", percentage)); + sb.append(String.format(" │ \033[36mH3 Cells:\033[0m %s", formatCompactNumber(getH3CellsGenerated()))); + } else { + sb.append(String.format("\033[1;36m[%s]\033[0m %s", formatTime(elapsed), phase)); + } + + sb.append(String.format(" │ \033[31mHeap:\033[0m %s", getMemoryStats())); + + if (isTty) { + System.out.print(sb); + System.out.flush(); + } else { + System.out.println(sb); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + break; + } + } + if (isTty) System.out.println(); + }); + } + + public void printFinalStatistics() { + System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("🎯 BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m"); + + long totalTime = Math.max(1, getTotalTime()); + double totalSeconds = totalTime / 1000.0; + + System.out.printf("\n\033[1;37m⏱️ Total Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime())); + + System.out.println("\033[1;37m📊 Processing Summary:\033[0m"); + System.out.println("┌────────────────────┬─────────────────┬─────────────────┐"); + System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │"); + System.out.println("├────────────────────┼─────────────────┼─────────────────┤"); + System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n", + formatCompactNumber(getNodesCached()), + formatCompactNumber((long) (getNodesCached() / totalSeconds))); + System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n", + formatCompactNumber(getWaysCached()), + formatCompactNumber((long) (getWaysCached() / totalSeconds))); + System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n", + formatCompactNumber(getRelationsFound()), + formatCompactNumber((long) (getRelationsFound() / totalSeconds))); + System.out.printf("│ \033[36mRelations Processed\033[0m│ %15s │ %13s/s │%n", + formatCompactNumber(getRelationsProcessed()), + formatCompactNumber((long) (getRelationsProcessed() / totalSeconds))); + System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n", + formatCompactNumber(getH3CellsGenerated()), + formatCompactNumber((long) (getH3CellsGenerated() / totalSeconds))); + System.out.println("└────────────────────┴─────────────────┴─────────────────┘"); + + System.out.println(); + } + + public void printOutcomeAndErrors() { + long err = getErrorsTotal(); + long denominator = Math.max(1L, getRelationsFound()); + double rate = (double) err / (double) denominator; + + String outcome = (rate >= DEGRADED_WARN_RATE) ? "DEGRADED" : "OK"; + System.out.println("\n\033[1;36mIMPORT OUTCOME: " + outcome + + " | errors=" + err + + " | relationsFound=" + denominator + + " | errorRate=" + String.format(Locale.ROOT, "%.6f%%", rate * 100.0) + + "\033[0m"); + + if (err == 0) { + return; + } + + System.err.println("\n=== Boundary import errors summary (best-effort) ==="); + System.err.println("totalErrors=" + err); + System.err.println("topBuckets=" + Math.min(10, errorBuckets.size()) + "/" + errorBuckets.size()); + + errorBuckets.entrySet().stream() + .sorted((a, b) -> Long.compare(b.getValue().get(), a.getValue().get())) + .limit(10) + .forEach(e -> System.err.println(" " + e.getValue().get() + "x " + e.getKey())); + + if (!errorSamples.isEmpty()) { + System.err.println("\nSamples (first " + errorSamples.size() + "):"); + for (String s : errorSamples) { + System.err.println(" " + s); + } + } + + System.err.println("=== End boundary import errors summary ===\n"); + } + + private String formatTime(long ms) { + long s = ms / 1000; + return String.format("%d:%02d:%02d", s / 3600, (s % 3600) / 60, s % 60); + } + + private String formatCompactNumber(long n) { + if (n < 1000) return String.valueOf(n); + if (n < 1_000_000) return String.format("%.2fk", n / 1000.0); + return String.format("%.3fM", n / 1_000_000.0); + } + + private String formatCompactRate(long n) { + if (n < 1000) return String.valueOf(n); + if (n < 1_000_000) return String.format("%.1fk", n / 1000.0); + return String.format("%.1fM", n / 1_000_000.0); + } + + private String centerText(String text) { + int pad = (80 - text.length()) / 2; + return " ".repeat(Math.max(0, pad)) + text; + } + + public void printPhaseHeader(String phase) { + System.out.println("\n\033[1;36m" + "─".repeat(80) + "\n" + phase + "\n" + "─".repeat(80) + "\033[0m"); + } + + public void printSuccess() { + System.out.println("\n\033[1;32m" + "=".repeat(80) + "\n" + centerText("BOUNDARY IMPORT COMPLETED SUCCESSFULLY") + "\n" + "=".repeat(80) + "\033[0m"); + } + + public void printError(String message) { + System.out.println("\n\033[1;31m" + "=".repeat(80) + "\n" + centerText(message) + "\n" + "=".repeat(80) + "\033[0m"); + } + + public void printPhaseSummary(String phaseName, long phaseStartTime) { + long phaseTime = System.currentTimeMillis() - phaseStartTime; + System.out.printf("\n\u001B[1;32m✓ %s COMPLETED\u001B[0m \u001B[2m(%s)\u001B[0m%n", phaseName, formatTime(phaseTime)); + } +} diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index b03436e..64a5afc 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -53,10 +53,12 @@ public class StandaloneBoundaryImporter { private final GeometrySimplificationService geometrySimplificationService; private final H3Core h3; + private final BoundaryImportStatistics stats; public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplificationService) throws Exception { this.geometrySimplificationService = geometrySimplificationService; this.h3 = H3Core.newInstance(); // Uber H3-Java 4.x + this.stats = new BoundaryImportStatistics(); } // ============================ PUBLIC API ============================ @@ -98,6 +100,8 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc .setWriteBufferSize(256 * 1024 * 1024); WriteOptions wo = new WriteOptions().setDisableWAL(true); + stats.startProgressReporter(); + try ( RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString()); RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString()); @@ -106,6 +110,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()) ) { for (String pbfPath : pbfPaths) { + stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways"); // ---------- SINGLE PASS ---------- PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); @@ -123,6 +128,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc .putDouble(n.getLatitude()) .putDouble(n.getLongitude()); nodeBatch.put(longToBytes(n.getId()), bb.array()); + stats.incrementNodesCached(); if (phaseCounter.incrementAndGet() % 100_000 == 0) { nodeCache.write(wo, nodeBatch); nodeBatch.clear(); @@ -133,6 +139,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc long[] ids = new long[w.getNumberOfNodes()]; for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); + stats.incrementWaysCached(); if (phaseCounter.incrementAndGet() % 50_000 == 0) { wayCache.write(wo, wayBatch); wayBatch.clear(); @@ -147,6 +154,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc nodeBatch.close(); wayBatch.close(); + stats.setCurrentPhase(2, "2.1: Processing Relations & H3"); // Re-open iterator for Phase 3 (or use two iterators; here we reuse file) // Phase 3: Process Relations (separate iterator pass is fine since PBF is local) @@ -160,6 +168,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc OsmRelation r = (OsmRelation) c.getEntity(); if (isAdministrativeBoundary(r)) { relations.add(buildRelationStub(r)); + stats.incrementRelationsFound(); } } } @@ -182,6 +191,9 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc List cells = polygonToCellsH3(simplified); if (cells.isEmpty()) continue; + stats.incrementRelationsProcessed(); + stats.addH3CellsGenerated(cells.size()); + // h3_to_osm : append OSM_ID to each cell (dedup) for (long cell : cells) { byte[] key = longToBytes(cell); @@ -215,6 +227,11 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc regionGeom.compactRange(); } + stats.stop(); + stats.setTotalTime(System.currentTimeMillis() - stats.getStartTime()); + stats.printFinalStatistics(); + stats.printOutcomeAndErrors(); + // Cleanup tmp cleanup(tmp); System.out.println("[StandaloneBoundaryImporter] Import complete. Temporary caches removed."); @@ -239,12 +256,14 @@ private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB for (List inner : innerRings) { try { holes.add(GEOMETRY_FACTORY.createLinearRing(inner.toArray(new Coordinate[0]))); - } catch (Exception ignored) { + } catch (Exception e) { + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId, "createLinearRing-inner", e); } } Polygon p = GEOMETRY_FACTORY.createPolygon(shell, holes.toArray(new LinearRing[0])); if (p.isValid()) polygons.add(p); - } catch (Exception ignored) { + } catch (Exception e) { + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId, "buildMultiPolygon", e); } } if (polygons.isEmpty()) return null; @@ -260,7 +279,9 @@ private List> stitchRings(List wayIds, RocksDB nodeCache, long[] nodeIds = bytesToLongArray(seq); List coords = resolveCoordinates(nodeIds, nodeCache); if (coords != null && coords.size() >= 2) wayCoords.put(wid, coords); - } catch (RocksDBException e) { /* skip */ } + } catch (RocksDBException e) { + stats.recordError(BoundaryImportStatistics.Stage.CACHING_NODES_WAYS, BoundaryImportStatistics.Kind.STORE, wid, "stitchRings", e); + } } List> rings = new ArrayList<>(); Set used = new HashSet<>(); @@ -314,6 +335,7 @@ private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) { } return coords; } catch (RocksDBException e) { + stats.recordError(BoundaryImportStatistics.Stage.CACHING_NODES_WAYS, BoundaryImportStatistics.Kind.STORE, null, "resolveCoordinates", e); return null; } } @@ -339,7 +361,7 @@ private List polygonToCellsH3(Geometry geom) { List partCells = h3.polygonToCells(outer, holes, H3_RESOLUTION); cells.addAll(partCells); } catch (Exception e) { - // skip invalid loop + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, null, "polygonToCellsH3", e); } } return cells; @@ -454,4 +476,4 @@ private void cleanup(Path p) { } } } -} \ No newline at end of file +} From 2b8645a828818a4414f9fb39a71feabbd09bebe4 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sat, 11 Jul 2026 09:47:36 +0200 Subject: [PATCH 05/29] feat: add multi-threading to StandaloneBoundaryImporter for H3 cell generation --- .../importer/StandaloneBoundaryImporter.java | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 64a5afc..9744c7a 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -16,6 +16,7 @@ package com.dedicatedcode.paikka.service.importer; +import com.dedicatedcode.paikka.config.PaikkaConfiguration; import com.uber.h3core.H3Core; import com.uber.h3core.util.LatLng; import de.topobyte.osm4j.core.model.iface.*; @@ -33,6 +34,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; /** @@ -52,11 +57,13 @@ public class StandaloneBoundaryImporter { private static final double BUFFER_DISTANCE = 0.0001; // ~11m at equator, ensures border cells private final GeometrySimplificationService geometrySimplificationService; + private final PaikkaConfiguration paikkaConfiguration; private final H3Core h3; private final BoundaryImportStatistics stats; - public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplificationService) throws Exception { + public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplificationService, PaikkaConfiguration paikkaConfiguration) throws Exception { this.geometrySimplificationService = geometrySimplificationService; + this.paikkaConfiguration = paikkaConfiguration; this.h3 = H3Core.newInstance(); // Uber H3-Java 4.x this.stats = new BoundaryImportStatistics(); } @@ -174,40 +181,56 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc } // Process each relation: stitch geometry, H3 polyfill, write outputs - WriteBatch h3Batch = new WriteBatch(); - WriteBatch metaBatch = new WriteBatch(); - WriteBatch geomBatch = new WriteBatch(); + int threads = paikkaConfiguration.getImportConfiguration().getThreads(); + ExecutorService executor = Executors.newFixedThreadPool(threads); + List> futures = new ArrayList<>(); for (RelationStub stub : relations) { - Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); - if (geom == null || geom.isEmpty() || !geom.isValid()) continue; + futures.add(executor.submit(() -> { + Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); + if (geom == null || geom.isEmpty() || !geom.isValid()) return null; - // Buffer to include border-touching cells - Geometry buffered = geom.buffer(BUFFER_DISTANCE); - Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); - if (simplified == null || simplified.isEmpty()) simplified = buffered; + // Buffer to include border-touching cells + Geometry buffered = geom.buffer(BUFFER_DISTANCE); + Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); + if (simplified == null || simplified.isEmpty()) simplified = buffered; - // ---- H3 Polyfill ---- - List cells = polygonToCellsH3(simplified); - if (cells.isEmpty()) continue; + // ---- H3 Polyfill ---- + List cells = polygonToCellsH3(simplified); + if (cells.isEmpty()) return null; + + return new ProcessedRelation(stub.osmId, cells, simplified); + })); + } + + executor.shutdown(); + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + + WriteBatch h3Batch = new WriteBatch(); + WriteBatch metaBatch = new WriteBatch(); + WriteBatch geomBatch = new WriteBatch(); + + for (Future future : futures) { + ProcessedRelation pr = future.get(); + if (pr == null) continue; stats.incrementRelationsProcessed(); - stats.addH3CellsGenerated(cells.size()); + stats.addH3CellsGenerated(pr.cells().size()); // h3_to_osm : append OSM_ID to each cell (dedup) - for (long cell : cells) { + for (long cell : pr.cells()) { byte[] key = longToBytes(cell); byte[] existing = h3ToOsm.get(key); - byte[] updated = appendOsmIdToArray(existing, stub.osmId); + byte[] updated = appendOsmIdToArray(existing, pr.osmId()); h3Batch.put(key, updated); } // region_metadata : OSM_ID -> cell count (int) - metaBatch.put(longToBytes(stub.osmId), intToBytes(cells.size())); + metaBatch.put(longToBytes(pr.osmId()), intToBytes(pr.cells().size())); // region_geometry : OSM_ID -> simplified WKB - byte[] wkb = new WKBWriter().write(simplified); - geomBatch.put(longToBytes(stub.osmId), wkb); + byte[] wkb = new WKBWriter().write(pr.simplified()); + geomBatch.put(longToBytes(pr.osmId()), wkb); } h3ToOsm.write(wo, h3Batch); @@ -461,6 +484,9 @@ private RelationStub buildRelationStub(OsmRelation r) { private record RelationStub(long osmId, int adminLevel, List outerWays, List innerWays) { } + private record ProcessedRelation(long osmId, List cells, Geometry simplified) { + } + private void cleanup(Path p) { if (Files.exists(p)) { try { From 66f301654472e8b9a0e0eecc791b490a64195cbc Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sat, 11 Jul 2026 09:54:58 +0200 Subject: [PATCH 06/29] refactor: implement parallel processing with temporary databases for planet-scale imports --- .../importer/StandaloneBoundaryImporter.java | 99 +++++++++++++------ 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 9744c7a..7a91923 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -34,6 +34,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -83,11 +84,18 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc Path regionMetaPath = out.resolve("region_metadata"); Path regionGeomPath = out.resolve("region_geometry"); + Path tmpH3ToOsmPath = tmp.resolve("tmp_h3_to_osm"); + Path tmpRegionMetaPath = tmp.resolve("tmp_region_metadata"); + Path tmpRegionGeomPath = tmp.resolve("tmp_region_geometry"); + cleanup(nodeCachePath); cleanup(wayCachePath); cleanup(h3ToOsmPath); cleanup(regionMetaPath); cleanup(regionGeomPath); + cleanup(tmpH3ToOsmPath); + cleanup(tmpRegionMetaPath); + cleanup(tmpRegionGeomPath); // Shared RocksDB options (inline with ImportService style) BlockBasedTableConfig tableCfg = new BlockBasedTableConfig() @@ -105,16 +113,19 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc .setTableFormatConfig(tableCfg) .setCompressionType(CompressionType.ZSTD_COMPRESSION) .setWriteBufferSize(256 * 1024 * 1024); - WriteOptions wo = new WriteOptions().setDisableWAL(true); stats.startProgressReporter(); try ( + WriteOptions wo = new WriteOptions().setDisableWAL(true); RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString()); RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString()); RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString()); RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString()); - RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()) + RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()); + RocksDB tmpH3ToOsm = RocksDB.open(cacheOpts, tmpH3ToOsmPath.toString()); + RocksDB tmpRegionMeta = RocksDB.open(cacheOpts, tmpRegionMetaPath.toString()); + RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString()) ) { for (String pbfPath : pbfPaths) { stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways"); @@ -183,10 +194,11 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // Process each relation: stitch geometry, H3 polyfill, write outputs int threads = paikkaConfiguration.getImportConfiguration().getThreads(); ExecutorService executor = Executors.newFixedThreadPool(threads); - List> futures = new ArrayList<>(); + ExecutorCompletionService ecs = new ExecutorCompletionService<>(executor); + int submitted = 0; for (RelationStub stub : relations) { - futures.add(executor.submit(() -> { + ecs.submit(() -> { Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); if (geom == null || geom.isEmpty() || !geom.isValid()) return null; @@ -200,50 +212,43 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc if (cells.isEmpty()) return null; return new ProcessedRelation(stub.osmId, cells, simplified); - })); + }); + submitted++; } - executor.shutdown(); - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - - WriteBatch h3Batch = new WriteBatch(); - WriteBatch metaBatch = new WriteBatch(); - WriteBatch geomBatch = new WriteBatch(); - - for (Future future : futures) { + for (int i = 0; i < submitted; i++) { + Future future = ecs.take(); ProcessedRelation pr = future.get(); if (pr == null) continue; stats.incrementRelationsProcessed(); stats.addH3CellsGenerated(pr.cells().size()); - // h3_to_osm : append OSM_ID to each cell (dedup) + // Write to temporary databases for (long cell : pr.cells()) { byte[] key = longToBytes(cell); - byte[] existing = h3ToOsm.get(key); + byte[] existing = tmpH3ToOsm.get(key); byte[] updated = appendOsmIdToArray(existing, pr.osmId()); - h3Batch.put(key, updated); + tmpH3ToOsm.put(wo, key, updated); } - // region_metadata : OSM_ID -> cell count (int) - metaBatch.put(longToBytes(pr.osmId()), intToBytes(pr.cells().size())); + tmpRegionMeta.put(wo, longToBytes(pr.osmId()), intToBytes(pr.cells().size())); - // region_geometry : OSM_ID -> simplified WKB byte[] wkb = new WKBWriter().write(pr.simplified()); - geomBatch.put(longToBytes(pr.osmId()), wkb); + tmpRegionGeom.put(wo, longToBytes(pr.osmId()), wkb); } - h3ToOsm.write(wo, h3Batch); - regionMeta.write(wo, metaBatch); - regionGeom.write(wo, geomBatch); - h3Batch.close(); - metaBatch.close(); - geomBatch.close(); - wo.close(); + executor.shutdown(); + executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } - } + stats.setCurrentPhase(3, "3.1: Compacting Final Databases"); + // Final Step: Copy from temporary DBs to final DBs in sorted order + copyDb(tmpRegionMeta, regionMeta); + copyDb(tmpRegionGeom, regionGeom); + copyH3Db(tmpH3ToOsm, h3ToOsm); + // Compact finals h3ToOsm.compactRange(); regionMeta.compactRange(); @@ -260,6 +265,44 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc System.out.println("[StandaloneBoundaryImporter] Import complete. Temporary caches removed."); } + private void copyDb(RocksDB source, RocksDB target) throws RocksDBException { + try (RocksIterator it = source.newIterator(); WriteOptions wo = new WriteOptions().setDisableWAL(true)) { + it.seekToFirst(); + while (it.isValid()) { + target.put(wo, it.key(), it.value()); + it.next(); + } + } + } + + private void copyH3Db(RocksDB source, RocksDB target) throws RocksDBException { + try (RocksIterator it = source.newIterator(); WriteOptions wo = new WriteOptions().setDisableWAL(true)) { + it.seekToFirst(); + while (it.isValid()) { + byte[] key = it.key(); + byte[] newVal = it.value(); + byte[] existing = target.get(key); + if (existing == null) { + target.put(wo, key, newVal); + } else { + byte[] merged = mergeOsmIdArrays(existing, newVal); + target.put(wo, key, merged); + } + it.next(); + } + } + } + + private byte[] mergeOsmIdArrays(byte[] existing, byte[] newVal) { + ByteBuffer bb = ByteBuffer.wrap(newVal).order(ByteOrder.BIG_ENDIAN); + byte[] current = existing; + while (bb.hasRemaining()) { + long osmId = bb.getLong(); + current = appendOsmIdToArray(current, osmId); + } + return current; + } + // ============================ GEOMETRY STITCHING ============================ /** From f9a4704d3553df0c0c0cd35329b0a720b7fc1305 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sat, 11 Jul 2026 10:24:12 +0200 Subject: [PATCH 07/29] refactor: implement producer-consumer pattern for memory-efficient relation processing --- .../importer/StandaloneBoundaryImporter.java | 162 +++++++++++------- 1 file changed, 99 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 7a91923..76cae1c 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -34,10 +34,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -97,7 +98,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc cleanup(tmpRegionMetaPath); cleanup(tmpRegionGeomPath); - // Shared RocksDB options (inline with ImportService style) + // Shared Rocksoptions (inline with ImportService style) BlockBasedTableConfig tableCfg = new BlockBasedTableConfig() .setBlockSize(64 * 1024) .setFilterPolicy(new BloomFilter(10, false)); @@ -179,67 +180,105 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) { PbfIterator relIter = new PbfIterator(is, false); - List relations = new ArrayList<>(); - while (relIter.hasNext()) { - EntityContainer c = relIter.next(); - if (c.getType() == EntityType.Relation) { - OsmRelation r = (OsmRelation) c.getEntity(); - if (isAdministrativeBoundary(r)) { - relations.add(buildRelationStub(r)); - stats.incrementRelationsFound(); - } - } - } - - // Process each relation: stitch geometry, H3 polyfill, write outputs int threads = paikkaConfiguration.getImportConfiguration().getThreads(); ExecutorService executor = Executors.newFixedThreadPool(threads); - ExecutorCompletionService ecs = new ExecutorCompletionService<>(executor); - - int submitted = 0; - for (RelationStub stub : relations) { - ecs.submit(() -> { - Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); - if (geom == null || geom.isEmpty() || !geom.isValid()) return null; - - // Buffer to include border-touching cells - Geometry buffered = geom.buffer(BUFFER_DISTANCE); - Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel); - if (simplified == null || simplified.isEmpty()) simplified = buffered; - - // ---- H3 Polyfill ---- - List cells = polygonToCellsH3(simplified); - if (cells.isEmpty()) return null; - - return new ProcessedRelation(stub.osmId, cells, simplified); - }); - submitted++; - } - - for (int i = 0; i < submitted; i++) { - Future future = ecs.take(); - ProcessedRelation pr = future.get(); - if (pr == null) continue; - - stats.incrementRelationsProcessed(); - stats.addH3CellsGenerated(pr.cells().size()); - - // Write to temporary databases - for (long cell : pr.cells()) { - byte[] key = longToBytes(cell); - byte[] existing = tmpH3ToOsm.get(key); - byte[] updated = appendOsmIdToArray(existing, pr.osmId()); - tmpH3ToOsm.put(wo, key, updated); + BlockingQueue> queue = new LinkedBlockingQueue<>(100); + List POISON_PILL = List.of(); + + // Producer thread + Thread producer = new Thread(() -> { + try { + List batch = new ArrayList<>(100); + while (relIter.hasNext()) { + EntityContainer c = relIter.next(); + if (c.getType() == EntityType.Relation) { + OsmRelation r = (OsmRelation) c.getEntity(); + if (isAdministrativeBoundary(r)) { + stats.incrementRelationsFound(); + batch.add(buildRelationStub(r)); + if (batch.size() >= 100) { + queue.put(batch); + batch = new ArrayList<>(100); + } + } + } + } + if (!batch.isEmpty()) { + queue.put(batch); + } + } catch (Exception e) { + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.READ, null, "producer-thread", e); + } finally { + for (int i = 0; i < threads; i++) { + try { + queue.put(POISON_PILL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } } - - tmpRegionMeta.put(wo, longToBytes(pr.osmId()), intToBytes(pr.cells().size())); - - byte[] wkb = new WKBWriter().write(pr.simplified()); - tmpRegionGeom.put(wo, longToBytes(pr.osmId()), wkb); + }); + + producer.start(); + + // Consumer threads + List> futures = new ArrayList<>(); + for (int i = 0; i < threads; i++) { + futures.add(executor.submit(() -> { + try { + while (true) { + List batch = queue.take(); + if (batch == POISON_PILL) break; + + for (RelationStub stub : batch) { + try { + Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); + if (geom == null || geom.isEmpty() || !geom.isValid()) continue; + + // Buffer to include border-touching cells + Geometry buffered = geom.buffer(BUFFER_DISTANCE); + Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel()); + if (simplified == null || simplified.isEmpty()) simplified = buffered; + + // ---- H3 Polyfill ---- + List cells = polygonToCellsH3(simplified); + if (cells.isEmpty()) continue; + + stats.incrementRelationsProcessed(); + stats.addH3CellsGenerated(cells.size()); + + // Write to temporary databases + for (long cell : cells) { + byte[] key = longToBytes(cell); + synchronized (tmpH3ToOsm) { + byte[] existing = tmpH3ToOsm.get(key); + byte[] updated = appendOsmIdToArray(existing, stub.osmId()); + tmpH3ToOsm.put(wo, key, updated); + } + } + + tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes(cells.size())); + + byte[] wkb = new WKBWriter().write(simplified); + tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb); + } catch (Exception e) { + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e); + } + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + })); } + // Wait for consumers to finish + for (Future f : futures) { + f.get(); + } executor.shutdown(); executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); + producer.join(); } } @@ -310,8 +349,8 @@ private byte[] mergeOsmIdArrays(byte[] existing, byte[] newVal) { * Rings are stitched by coordinate continuation (same logic as ImportService.buildConnectedRings). */ private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB wayCache) { - List> outerRings = stitchRings(stub.outerWays, nodeCache, wayCache); - List> innerRings = stitchRings(stub.innerWays, nodeCache, wayCache); + List> outerRings = stitchRings(stub.outerWays(), nodeCache, wayCache); + List> innerRings = stitchRings(stub.innerWays(), nodeCache, wayCache); if (outerRings.isEmpty()) return null; List polygons = new ArrayList<>(); @@ -323,13 +362,13 @@ private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB try { holes.add(GEOMETRY_FACTORY.createLinearRing(inner.toArray(new Coordinate[0]))); } catch (Exception e) { - stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId, "createLinearRing-inner", e); + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "createLinearRing-inner", e); } } Polygon p = GEOMETRY_FACTORY.createPolygon(shell, holes.toArray(new LinearRing[0])); if (p.isValid()) polygons.add(p); } catch (Exception e) { - stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId, "buildMultiPolygon", e); + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "buildMultiPolygon", e); } } if (polygons.isEmpty()) return null; @@ -527,9 +566,6 @@ private RelationStub buildRelationStub(OsmRelation r) { private record RelationStub(long osmId, int adminLevel, List outerWays, List innerWays) { } - private record ProcessedRelation(long osmId, List cells, Geometry simplified) { - } - private void cleanup(Path p) { if (Files.exists(p)) { try { From 938465e25a6b56c0a3ca9b5b81491a6cee4896ab Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sat, 11 Jul 2026 18:12:51 +0200 Subject: [PATCH 08/29] feat: make geometry simplification tolerances configurable - Added `SimplificationConfiguration` to `PaikkaConfiguration` to support configurable tolerances for geometry simplification. - Replaced hardcoded tolerance values in `GeometrySimplificationService` with values from configuration. - Updated tests to inject `PaikkaConfiguration` into `GeometrySimplificationService`. - Adjusted test properties and application configuration files to define default tolerance values. --- .../paikka/config/PaikkaConfiguration.java | 58 +++++++++++++ .../GeometrySimplificationService.java | 35 +++++--- .../importer/StandaloneBoundaryImporter.java | 84 ++++++++++--------- src/main/resources/application-dev.properties | 4 + src/main/resources/application.properties | 7 ++ .../GeometrySimplificationServiceTest.java | 8 +- .../paikka/service/ImportServiceTest.java | 2 +- .../resources/application-test.properties | 2 +- 8 files changed, 142 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/config/PaikkaConfiguration.java b/src/main/java/com/dedicatedcode/paikka/config/PaikkaConfiguration.java index 20e3f85..0df893f 100644 --- a/src/main/java/com/dedicatedcode/paikka/config/PaikkaConfiguration.java +++ b/src/main/java/com/dedicatedcode/paikka/config/PaikkaConfiguration.java @@ -32,6 +32,8 @@ public class PaikkaConfiguration { private ImportConfiguration importConfiguration; @Name("query") private QueryConfiguration queryConfiguration; + @Name("simplification") + private SimplificationConfiguration simplificationConfiguration; public ImportConfiguration getImportConfiguration() { return importConfiguration; @@ -65,6 +67,14 @@ public void setStatsDbPath(String statsDbPath) { this.statsDbPath = statsDbPath; } + public SimplificationConfiguration getSimplificationConfiguration() { + return simplificationConfiguration; + } + + public void setSimplificationConfiguration(SimplificationConfiguration simplificationConfiguration) { + this.simplificationConfiguration = simplificationConfiguration; + } + public static class ImportConfiguration { private int threads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); @@ -88,6 +98,54 @@ public void setChunkSize(int chunkSize) { } + public static class SimplificationConfiguration { + private double continentTolerance; + private double countryTolerance; + private double stateTolerance; + private double poiTolerance; + private double defaultTolerance; + + public double getContinentTolerance() { + return continentTolerance; + } + + public void setContinentTolerance(double continentTolerance) { + this.continentTolerance = continentTolerance; + } + + public double getCountryTolerance() { + return countryTolerance; + } + + public void setCountryTolerance(double countryTolerance) { + this.countryTolerance = countryTolerance; + } + + public double getStateTolerance() { + return stateTolerance; + } + + public void setStateTolerance(double stateTolerance) { + this.stateTolerance = stateTolerance; + } + + public double getPoiTolerance() { + return poiTolerance; + } + + public void setPoiTolerance(double poiTolerance) { + this.poiTolerance = poiTolerance; + } + + public double getDefaultTolerance() { + return defaultTolerance; + } + + public void setDefaultTolerance(double defaultTolerance) { + this.defaultTolerance = defaultTolerance; + } + } + public static class QueryConfiguration { /** diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java b/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java index 04907db..64b5efc 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java @@ -16,6 +16,7 @@ package com.dedicatedcode.paikka.service.importer; +import com.dedicatedcode.paikka.config.PaikkaConfiguration; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; import org.springframework.stereotype.Service; @@ -26,13 +27,20 @@ */ @Service public class GeometrySimplificationService { - - // Tolerance guidelines from implementation blueprint - private static final double COUNTRY_TOLERANCE = 0.00045; // 50 meters for country borders - private static final double STATE_TOLERANCE = 0.00009; // 10 meters for state/city - private static final double POI_TOLERANCE = 0.000018; // 2 meters for POI boundaries - private static final double DEFAULT_TOLERANCE = 0.000045; // 5 meters default - + + private final double continentTolerance; + private final double countryTolerance; + private final double stateTolerance; + private final double poiTolerance; + private final double defaultTolerance; + + public GeometrySimplificationService(PaikkaConfiguration paikkaConfiguration) { + this.continentTolerance = paikkaConfiguration.getSimplificationConfiguration().getContinentTolerance(); + this.countryTolerance = paikkaConfiguration.getSimplificationConfiguration().getCountryTolerance(); + this.stateTolerance = paikkaConfiguration.getSimplificationConfiguration().getStateTolerance(); + this.poiTolerance = paikkaConfiguration.getSimplificationConfiguration().getPoiTolerance(); + this.defaultTolerance = paikkaConfiguration.getSimplificationConfiguration().getDefaultTolerance(); + } /** * Simplify geometry using Douglas-Peucker algorithm with default tolerance. * @@ -40,7 +48,7 @@ public class GeometrySimplificationService { * @return Simplified geometry */ public Geometry simplify(Geometry geometry) { - return simplify(geometry, DEFAULT_TOLERANCE); + return simplify(geometry, defaultTolerance); } /** @@ -82,11 +90,12 @@ public Geometry simplifyByAdminLevel(Geometry geometry, int adminLevel) { if (geometry == null) { return null; } - + double tolerance = switch (adminLevel) { - case 2 -> COUNTRY_TOLERANCE; // Country - case 4, 6 -> STATE_TOLERANCE; // State/Region - default -> DEFAULT_TOLERANCE; + case 1 -> continentTolerance; // Continent / Supranational + case 2 -> countryTolerance; // Country + case 4, 6 -> stateTolerance; // State/Region + default -> defaultTolerance; }; return simplify(geometry, tolerance); @@ -99,7 +108,7 @@ public Geometry simplifyByAdminLevel(Geometry geometry, int adminLevel) { * @return Simplified geometry with POI-appropriate tolerance */ public Geometry simplifyPoiBoundary(Geometry geometry) { - return simplify(geometry, POI_TOLERANCE); + return simplify(geometry, poiTolerance); } /** diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 76cae1c..7ea610f 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -118,7 +118,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc stats.startProgressReporter(); try ( - WriteOptions wo = new WriteOptions().setDisableWAL(true); RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString()); RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString()); RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString()); @@ -130,48 +129,51 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc ) { for (String pbfPath : pbfPaths) { stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways"); - // ---------- SINGLE PASS ---------- - PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); - - // Phase 1 & 2: Stream nodes and ways (cached) - WriteBatch nodeBatch = new WriteBatch(); - WriteBatch wayBatch = new WriteBatch(); - AtomicLong phaseCounter = new AtomicLong(); - - while (iterator.hasNext()) { - EntityContainer c = iterator.next(); - if (c.getType() == EntityType.Node) { - // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair - OsmNode n = (OsmNode) c.getEntity(); - ByteBuffer bb = ByteBuffer.allocate(16) - .putDouble(n.getLatitude()) - .putDouble(n.getLongitude()); - nodeBatch.put(longToBytes(n.getId()), bb.array()); - stats.incrementNodesCached(); - if (phaseCounter.incrementAndGet() % 100_000 == 0) { - nodeCache.write(wo, nodeBatch); - nodeBatch.clear(); - } - } else if (c.getType() == EntityType.Way) { - // PHASE 2: Cache way node-id sequences (long[] as raw bytes) - OsmWay w = (OsmWay) c.getEntity(); - long[] ids = new long[w.getNumberOfNodes()]; - for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); - wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); - stats.incrementWaysCached(); - if (phaseCounter.incrementAndGet() % 50_000 == 0) { - wayCache.write(wo, wayBatch); - wayBatch.clear(); + try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) { + + // ---------- SINGLE PASS ---------- + PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false); + + // Phase 1 & 2: Stream nodes and ways (cached) + WriteBatch nodeBatch = new WriteBatch(); + WriteBatch wayBatch = new WriteBatch(); + AtomicLong phaseCounter = new AtomicLong(); + + while (iterator.hasNext()) { + EntityContainer c = iterator.next(); + if (c.getType() == EntityType.Node) { + // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair + OsmNode n = (OsmNode) c.getEntity(); + ByteBuffer bb = ByteBuffer.allocate(16) + .putDouble(n.getLatitude()) + .putDouble(n.getLongitude()); + nodeBatch.put(longToBytes(n.getId()), bb.array()); + stats.incrementNodesCached(); + if (phaseCounter.incrementAndGet() % 100_000 == 0) { + nodeCache.write(wo, nodeBatch); + nodeBatch.clear(); + } + } else if (c.getType() == EntityType.Way) { + // PHASE 2: Cache way node-id sequences (long[] as raw bytes) + OsmWay w = (OsmWay) c.getEntity(); + long[] ids = new long[w.getNumberOfNodes()]; + for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i); + wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids)); + stats.incrementWaysCached(); + if (phaseCounter.incrementAndGet() % 50_000 == 0) { + wayCache.write(wo, wayBatch); + wayBatch.clear(); + } + } else if (c.getType() == EntityType.Relation) { + // PHASE 3: Relations (ways already fully cached above) + break; // Relations come after ways in ordered PBF; switch mode } - } else if (c.getType() == EntityType.Relation) { - // PHASE 3: Relations (ways already fully cached above) - break; // Relations come after ways in ordered PBF; switch mode } + nodeCache.write(wo, nodeBatch); + wayCache.write(wo, wayBatch); + nodeBatch.close(); + wayBatch.close(); } - nodeCache.write(wo, nodeBatch); - wayCache.write(wo, wayBatch); - nodeBatch.close(); - wayBatch.close(); stats.setCurrentPhase(2, "2.1: Processing Relations & H3"); // Re-open iterator for Phase 3 (or use two iterators; here we reuse file) @@ -225,7 +227,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc List> futures = new ArrayList<>(); for (int i = 0; i < threads; i++) { futures.add(executor.submit(() -> { - try { + try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) { while (true) { List batch = queue.take(); if (batch == POISON_PILL) break; diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 315e3bf..6e8bd58 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -5,3 +5,7 @@ paikka.data-dir=./data spring.thymeleaf.cache=false paikka.admin.password=test + +paikka.import.threads=10 + + diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 49537ed..daf5669 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -23,6 +23,13 @@ paikka.query.max-results=500 paikka.query.default-results=10 paikka.query.base-url=http://localhost:8080 + +paikka.simplification.continent-tolerance=0.005 +paikka.simplification.country-tolerance=0.00045 +paikka.simplification.state-tolerance=0.00009 +paikka.simplification.poi-tolerance=0.000018 +paikka.simplification.default-tolerance=0.000045 + paikka.stats-db-path=./data/stats.db paikka.stats-db.flush=0/10 * * * * * diff --git a/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java index 89edffb..752b397 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java @@ -17,12 +17,14 @@ package com.dedicatedcode.paikka.service; import com.dedicatedcode.paikka.IntegrationTest; +import com.dedicatedcode.paikka.config.PaikkaConfiguration; import com.dedicatedcode.paikka.service.importer.GeometrySimplificationService; import org.junit.jupiter.api.Test; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LinearRing; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; @@ -30,8 +32,10 @@ @IntegrationTest class GeometrySimplificationServiceTest { - - private final GeometrySimplificationService service = new GeometrySimplificationService(); + + @Autowired + private GeometrySimplificationService service; + private final GeometryFactory geometryFactory = new GeometryFactory(); @Test diff --git a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java index 0cd4751..75bb808 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java @@ -65,7 +65,7 @@ void setUp() throws Exception { PaikkaConfiguration.ImportConfiguration importConfiguration = new PaikkaConfiguration.ImportConfiguration(); importConfiguration.setThreads(2); config.setImportConfiguration(importConfiguration); - GeometrySimplificationService geometrySimplificationService = new GeometrySimplificationService(); + GeometrySimplificationService geometrySimplificationService = new GeometrySimplificationService(config); S2Helper s2Helper = new S2Helper(); ImportService importService = new ImportService(s2Helper, geometrySimplificationService, config, "1.0.0"); diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index 7f048ee..5083d88 100644 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -1,5 +1,5 @@ paikka.data-dir=${java.io.tmpdir}/paikka-test-data -"paikka.stats-db-path=memory +paikka.stats-db-path=memory paikka.query.base-url=http://localhost:8080 paikka.admin.password=test paikka.stats-db.flush=- From 31d44b4e32bb75163c7fcc73077a121c9256960e Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 08:04:35 +0200 Subject: [PATCH 09/29] feat: enhance boundary importer with detailed geometry validation and logging - Introduced SLF4J logger to `StandaloneBoundaryImporter` for comprehensive logging. - Added detailed geometry validation with repair logic for invalid geometries. - Replaced `polygonToCellsH3` with optimized `processCellsH3Stream` for better scalability. - Improved handling of simplified geometries and batch processing for H3 cells. - Made simplification tolerances configurable in `application-dev.properties`. --- .../importer/StandaloneBoundaryImporter.java | 112 ++++++++++++------ src/main/resources/application-dev.properties | 13 +- 2 files changed, 91 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 7ea610f..ae3a50e 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -24,6 +24,8 @@ import org.locationtech.jts.geom.*; import org.locationtech.jts.io.WKBWriter; import org.rocksdb.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.IOException; @@ -34,12 +36,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; /** @@ -47,12 +44,13 @@ *

* Reads a pre-filtered boundaries_only.pbf (Nodes -> Ways -> Relations ordered) * and produces three RocksDB databases for offline mobile lookup: - * - h3_to_osm : H3_CELL_ID (uint64) -> List[OSM_ID] (raw byte array) - * - region_metadata : OSM_ID -> total cell count (int) - * - region_geometry : OSM_ID -> simplified WKB (bytes) + * - h3_to_osm: H3_CELL_ID (uint64) -> List[OSM_ID] (raw byte array) + * - region_metadata: OSM_ID -> total cell count (int) + * - region_geometry: OSM_ID -> simplified WKB (bytes) */ @Service public class StandaloneBoundaryImporter { + private static final Logger logger = LoggerFactory.getLogger(StandaloneBoundaryImporter.class); private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); private static final int H3_RESOLUTION = 9; @@ -233,36 +231,52 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc if (batch == POISON_PILL) break; for (RelationStub stub : batch) { + if (stub.adminLevel() <= 3) { + logger.info("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel()); + } try { Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); - if (geom == null || geom.isEmpty() || !geom.isValid()) continue; + if (geom == null || geom.isEmpty()) { + logger.warn("Relation OSM ID: {} [Admin Level: {}] Geometry is null or empty", stub.osmId(), stub.adminLevel()); + continue; + } + // Repair invalid geometries using buffer(0) + if (!geom.isValid()) { + logger.warn("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel()); + geom = geom.buffer(0); + if (geom == null || geom.isEmpty() || !geom.isValid()) { + logger.error("Relation OSM ID: {} [Admin Level: {}] Geometry repair failed", stub.osmId(), stub.adminLevel()); + continue; + } + } // Buffer to include border-touching cells Geometry buffered = geom.buffer(BUFFER_DISTANCE); Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel()); - if (simplified == null || simplified.isEmpty()) simplified = buffered; - + if (simplified == null || simplified.isEmpty()) { + logger.warn("Simplified Geometry is invalid for OSM ID: {}", stub.osmId()); + simplified = buffered; + } + if (stub.adminLevel() <= 3) { + logger.info("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId()); + } // ---- H3 Polyfill ---- - List cells = polygonToCellsH3(simplified); - if (cells.isEmpty()) continue; + AtomicLong cellCount = new AtomicLong(0); + long startTime = System.currentTimeMillis(); + processCellsH3Stream(simplified, stub.osmId(), wo, tmpH3ToOsm, cellCount); + if (stub.adminLevel() <= 3) logger.info("H3 Polyfill took {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + if (cellCount.get() == 0) continue; stats.incrementRelationsProcessed(); - stats.addH3CellsGenerated(cells.size()); - - // Write to temporary databases - for (long cell : cells) { - byte[] key = longToBytes(cell); - synchronized (tmpH3ToOsm) { - byte[] existing = tmpH3ToOsm.get(key); - byte[] updated = appendOsmIdToArray(existing, stub.osmId()); - tmpH3ToOsm.put(wo, key, updated); - } - } - - tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes(cells.size())); + stats.addH3CellsGenerated((int) cellCount.get()); + startTime = System.currentTimeMillis(); + tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes((int) cellCount.get())); + if (stub.adminLevel() <= 3) logger.info("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + startTime = System.currentTimeMillis(); byte[] wkb = new WKBWriter().write(simplified); tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb); + if (stub.adminLevel() <= 3) logger.info("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); } catch (Exception e) { stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e); } @@ -453,8 +467,7 @@ private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) { * Converts a JTS Geometry to H3 cells at resolution 9. * Uses h3.polygonToCells with LatLng vertices. Multipolygons are expanded. */ - private List polygonToCellsH3(Geometry geom) { - List cells = new ArrayList<>(); + private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) { int num = geom.getNumGeometries(); for (int i = 0; i < num; i++) { Geometry part = geom.getGeometryN(i); @@ -465,15 +478,48 @@ private List polygonToCellsH3(Geometry geom) { holes.add(toLatLng(poly.getInteriorRingN(h).getCoordinates())); } try { - List partCells = h3.polygonToCells(outer, holes, H3_RESOLUTION); - cells.addAll(partCells); + List batch = new ArrayList<>(10_000); + h3.polygonToCells(outer, holes, H3_RESOLUTION).forEach(cell -> { + batch.add(cell); + if (batch.size() >= 10_000) { + try { + processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); + } catch (RocksDBException e) { + throw new RuntimeException(e); + } + batch.clear(); + } + }); + if (!batch.isEmpty()) { + processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); + } } catch (Exception e) { - stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, null, "polygonToCellsH3", e); + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3Stream", e); } } - return cells; } + private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) throws RocksDBException { + List keys = new ArrayList<>(cells.size()); + for (long cell : cells) { + keys.add(longToBytes(cell)); + } + + // Synchronize to prevent race conditions when multiple threads update the same H3 cell + synchronized (tmpH3ToOsm) { + List existingValues = tmpH3ToOsm.multiGetAsList(keys); + try (WriteBatch writeBatch = new WriteBatch()) { + for (int i = 0; i < cells.size(); i++) { + cellCount.incrementAndGet(); + byte[] key = keys.get(i); + byte[] existing = existingValues.get(i); + byte[] updated = appendOsmIdToArray(existing, osmId); + writeBatch.put(key, updated); + } + tmpH3ToOsm.write(wo, writeBatch); + } + } + } private List toLatLng(Coordinate[] coords) { List list = new ArrayList<>(coords.length); for (Coordinate c : coords) { diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 6e8bd58..17de3d9 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -4,8 +4,19 @@ paikka.data-dir=./data spring.thymeleaf.cache=false +logging.level.com.dedicatedcode=DEBUG + paikka.admin.password=test paikka.import.threads=10 - +# Aggressive simplification tolerances (in degrees) +# ~5.5 km tolerance for continents (admin_level=1) +paikka.simplification.continent-tolerance=0.05 +# ~1.1 km tolerance for countries (admin_level=2) +paikka.simplification.country-tolerance=0.01 +# ~550 meters tolerance for states/regions (admin_level=4,6) +paikka.simplification.state-tolerance=0.005 +# Keep default and POI tolerances smaller to preserve local accuracy +paikka.simplification.default-tolerance=0.0001 +paikka.simplification.poi-tolerance=0.000018 From ee5fe5e29ac243040749c89c032e7fddd8fe1d5d Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 08:07:42 +0200 Subject: [PATCH 10/29] feat: implement variable H3 resolution by admin level for boundary import --- .../importer/StandaloneBoundaryImporter.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index ae3a50e..68e0ad9 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -53,7 +53,6 @@ public class StandaloneBoundaryImporter { private static final Logger logger = LoggerFactory.getLogger(StandaloneBoundaryImporter.class); private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); - private static final int H3_RESOLUTION = 9; private static final double BUFFER_DISTANCE = 0.0001; // ~11m at equator, ensures border cells private final GeometrySimplificationService geometrySimplificationService; @@ -260,11 +259,14 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc if (stub.adminLevel() <= 3) { logger.info("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId()); } + + int resolution = getResolutionForAdminLevel(stub.adminLevel()); + // ---- H3 Polyfill ---- AtomicLong cellCount = new AtomicLong(0); long startTime = System.currentTimeMillis(); - processCellsH3Stream(simplified, stub.osmId(), wo, tmpH3ToOsm, cellCount); - if (stub.adminLevel() <= 3) logger.info("H3 Polyfill took {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + processCellsH3Stream(simplified, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution); + if (stub.adminLevel() <= 3) logger.info("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId()); if (cellCount.get() == 0) continue; stats.incrementRelationsProcessed(); @@ -464,10 +466,21 @@ private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) { // ============================ H3 POLYFILL ============================ /** - * Converts a JTS Geometry to H3 cells at resolution 9. - * Uses h3.polygonToCells with LatLng vertices. Multipolygons are expanded. + * Determines the H3 resolution based on the administrative level. + * Lower admin levels (countries) use lower resolutions to save space. + * Higher admin levels (cities) use higher resolutions for accuracy. */ - private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) { + private int getResolutionForAdminLevel(int adminLevel) { + if (adminLevel <= 2) return 4; // Continents/Countries + if (adminLevel <= 6) return 6; // States/Regions + return 9; // Districts/Cities + } + + /** + * Converts a JTS Geometry to H3 cells at the specified resolution. + * Uses h3.polygonToCellsStream with LatLng vertices. Multipolygons are expanded. + */ + private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount, int resolution) { int num = geom.getNumGeometries(); for (int i = 0; i < num; i++) { Geometry part = geom.getGeometryN(i); @@ -479,7 +492,7 @@ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, Ro } try { List batch = new ArrayList<>(10_000); - h3.polygonToCells(outer, holes, H3_RESOLUTION).forEach(cell -> { + h3.polygonToCellsStream(outer, holes, resolution).forEach(cell -> { batch.add(cell); if (batch.size() >= 10_000) { try { From 3d0c369ec40897c1f2bfe4dc344c699910b690dd Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 08:40:11 +0200 Subject: [PATCH 11/29] refactor: replace polygonToCellsStream with polygonToCells method --- .../paikka/service/importer/StandaloneBoundaryImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 68e0ad9..6e08071 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -492,7 +492,7 @@ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, Ro } try { List batch = new ArrayList<>(10_000); - h3.polygonToCellsStream(outer, holes, resolution).forEach(cell -> { + h3.polygonToCells(outer, holes, resolution).forEach(cell -> { batch.add(cell); if (batch.size() >= 10_000) { try { From 186fc67c5609ea49e160d75b6c39f7c3d0e8f827 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 08:40:13 +0200 Subject: [PATCH 12/29] fix: count administrative boundary relations in first pass for accurate progress tracking --- .../service/importer/StandaloneBoundaryImporter.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 6e08071..e21f4ec 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -162,8 +162,11 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc wayBatch.clear(); } } else if (c.getType() == EntityType.Relation) { - // PHASE 3: Relations (ways already fully cached above) - break; // Relations come after ways in ordered PBF; switch mode + // PHASE 3: Count administrative boundaries for accurate progress tracking + OsmRelation r = (OsmRelation) c.getEntity(); + if (isAdministrativeBoundary(r)) { + stats.incrementRelationsFound(); + } } } nodeCache.write(wo, nodeBatch); @@ -193,7 +196,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc if (c.getType() == EntityType.Relation) { OsmRelation r = (OsmRelation) c.getEntity(); if (isAdministrativeBoundary(r)) { - stats.incrementRelationsFound(); batch.add(buildRelationStub(r)); if (batch.size() >= 100) { queue.put(batch); From eda7501d90b7752192a387893206a93acf8d519d Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 10:14:02 +0200 Subject: [PATCH 13/29] downgrade: reduce logging verbosity in StandaloneBoundaryImporter and update default logging level to ERROR in development config --- .../importer/StandaloneBoundaryImporter.java | 19 ++++++++++++------- src/main/resources/application-dev.properties | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index e21f4ec..9c603a0 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -233,7 +233,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc for (RelationStub stub : batch) { if (stub.adminLevel() <= 3) { - logger.info("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel()); + logger.debug("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel()); } try { Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); @@ -244,7 +244,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // Repair invalid geometries using buffer(0) if (!geom.isValid()) { - logger.warn("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel()); + logger.debug("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel()); geom = geom.buffer(0); if (geom == null || geom.isEmpty() || !geom.isValid()) { logger.error("Relation OSM ID: {} [Admin Level: {}] Geometry repair failed", stub.osmId(), stub.adminLevel()); @@ -259,7 +259,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc simplified = buffered; } if (stub.adminLevel() <= 3) { - logger.info("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId()); + logger.debug("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId()); } int resolution = getResolutionForAdminLevel(stub.adminLevel()); @@ -268,7 +268,9 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc AtomicLong cellCount = new AtomicLong(0); long startTime = System.currentTimeMillis(); processCellsH3Stream(simplified, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution); - if (stub.adminLevel() <= 3) logger.info("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId()); + if (stub.adminLevel() <= 3) { + logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId()); + } if (cellCount.get() == 0) continue; stats.incrementRelationsProcessed(); @@ -276,11 +278,15 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc startTime = System.currentTimeMillis(); tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes((int) cellCount.get())); - if (stub.adminLevel() <= 3) logger.info("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + if (stub.adminLevel() <= 3) { + logger.debug("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + } startTime = System.currentTimeMillis(); byte[] wkb = new WKBWriter().write(simplified); tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb); - if (stub.adminLevel() <= 3) logger.info("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + if (stub.adminLevel() <= 3) { + logger.debug("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId()); + } } catch (Exception e) { stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e); } @@ -321,7 +327,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // Cleanup tmp cleanup(tmp); - System.out.println("[StandaloneBoundaryImporter] Import complete. Temporary caches removed."); } private void copyDb(RocksDB source, RocksDB target) throws RocksDBException { diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 17de3d9..63b8fce 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -4,7 +4,7 @@ paikka.data-dir=./data spring.thymeleaf.cache=false -logging.level.com.dedicatedcode=DEBUG +logging.level.com.dedicatedcode=ERROR paikka.admin.password=test From db4699018c3da383087a2482fae8b9fb61b12ea1 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 10:31:22 +0200 Subject: [PATCH 14/29] feat: add BoundaryLookupService for H3-based administrative boundary queries --- .../paikka/service/BoundaryLookupService.java | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java diff --git a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java new file mode 100644 index 0000000..ab51021 --- /dev/null +++ b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java @@ -0,0 +1,126 @@ +package com.dedicatedcode.paikka.service; + +import com.dedicatedcode.paikka.config.PaikkaConfiguration; +import com.uber.h3core.H3Core; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import jakarta.annotation.PreDestroy; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@Service +public class BoundaryLookupService { + private static final Logger logger = LoggerFactory.getLogger(BoundaryLookupService.class); + + private final H3Core h3; + private final RocksDB h3ToOsmDb; + private final RocksDB regionMetadataDb; + + public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exception { + this.h3 = H3Core.newInstance(); + RocksDB.loadLibrary(); + + // Assuming the databases are stored in a 'boundaries' folder within the data directory. + // Adjust this path if your importer outputs to a different location. + Path dataDir = Paths.get(paikkaConfiguration.getDataDir()); + Path h3ToOsmPath = dataDir.resolve("h3_to_osm"); + Path regionMetaPath = dataDir.resolve("region_metadata"); + + Options options = new Options().setReadOnly(true); + + logger.info("Opening RocksDB databases for boundary lookup..."); + this.h3ToOsmDb = RocksDB.open(options, h3ToOsmPath.toString()); + this.regionMetadataDb = RocksDB.open(options, regionMetaPath.toString()); + logger.info("RocksDB databases opened successfully."); + } + + /** + * Looks up the administrative boundaries for a given coordinate. + * Returns a list of boundaries (OSM ID and total cell count) that contain this point. + */ + public List lookup(double lat, double lng) { + Set osmIds = new HashSet<>(); + + // The importer uses different resolutions based on admin level. + // We must query all three to get the full hierarchy (City, State, Country). + + // Resolution 9 (Districts/Cities - Admin Level >= 7) + long cellRes9 = h3.latLngToCell(lat, lng, 9); + osmIds.addAll(getOsmIdsForCell(cellRes9)); + + // Resolution 6 (States/Regions - Admin Level 3-6) + long cellRes6 = h3.latLngToCell(lat, lng, 6); + osmIds.addAll(getOsmIdsForCell(cellRes6)); + + // Resolution 4 (Countries/Continents - Admin Level <= 2) + long cellRes4 = h3.latLngToCell(lat, lng, 4); + osmIds.addAll(getOsmIdsForCell(cellRes4)); + + List results = new ArrayList<>(); + for (Long osmId : osmIds) { + int totalCells = getTotalCells(osmId); + results.add(new BoundaryInfo(osmId, totalCells)); + } + + return results; + } + + private Set getOsmIdsForCell(long cellId) { + Set ids = new HashSet<>(); + try { + byte[] val = h3ToOsmDb.get(longToBytes(cellId)); + if (val != null) { + ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN); + while (bb.hasRemaining()) { + ids.add(bb.getLong()); + } + } + } catch (RocksDBException e) { + logger.error("Failed to lookup H3 cell {} in h3_to_osm", cellId, e); + } + return ids; + } + + private int getTotalCells(long osmId) { + try { + byte[] val = regionMetadataDb.get(longToBytes(osmId)); + if (val != null && val.length == 4) { + return ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN).getInt(); + } + } catch (RocksDBException e) { + logger.error("Failed to lookup metadata for OSM ID {} in region_metadata", osmId, e); + } + return -1; // Indicates unknown total + } + + private byte[] longToBytes(long v) { + return ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(v).array(); + } + + @PreDestroy + public void cleanup() { + if (this.h3ToOsmDb != null) { + this.h3ToOsmDb.close(); + } + if (this.regionMetadataDb != null) { + this.regionMetadataDb.close(); + } + } + + /** + * Represents a boundary containing the queried point. + * Reitti can use the totalCells to calculate the percentage visited. + */ + public record BoundaryInfo(long osmId, int totalCells) {} +} From d043d856046389b9ea0805def0731f127081ac44 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 10:46:23 +0200 Subject: [PATCH 15/29] =?UTF-8?q?test:=20add=20integration=20test=20for=20?= =?UTF-8?q?BoundaryLookupService=20with=20L=C3=BCbeck=20coordinates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/BoundaryLookupServiceTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java diff --git a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java new file mode 100644 index 0000000..7595cf7 --- /dev/null +++ b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java @@ -0,0 +1,41 @@ +package com.dedicatedcode.paikka.service; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@IntegrationTest +@TestPropertySource(properties = "paikka.data-dir=/home/daniel/projects/paikka/data-boundaries") +class BoundaryLookupServiceTest { + + @Autowired + private BoundaryLookupService boundaryLookupService; + + @Test + void shouldLookupBoundariesForLuebeck() { + // Coordinates for Luebeck + double lat = 53.86422; + double lng = 10.69120; + + List results = boundaryLookupService.lookup(lat, lng); + + assertFalse(results.isEmpty(), "Should find at least one boundary for Luebeck coordinates"); + + List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); + + // Lübeck (Ebene 6) + assertTrue(osmIds.contains(62422L), "Should contain Lübeck (OSM ID 62422)"); + // Schleswig-Holstein + assertTrue(osmIds.contains(4388L), "Should contain Schleswig-Holstein (OSM ID 4388)"); + // Germany + assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); + + // Verify that we have some total cells calculated + assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); + } +} From e68842b9e8e7f60a5e433c0539f5a07e611ce9a3 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 18:08:15 +0200 Subject: [PATCH 16/29] feat: expand boundary lookup tests and improve database flexibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added unit tests for boundary lookup with additional locations (e.g., Pölzig). - Adjusted RocksDB options to remove read-only constraint for enhanced flexibility. - Updated assertions to verify total cell counts and boundary hierarchy integrity. --- .../paikka/service/BoundaryLookupService.java | 5 ++- .../service/BoundaryLookupServiceTest.java | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java index ab51021..01a28cd 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java @@ -37,7 +37,7 @@ public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exc Path h3ToOsmPath = dataDir.resolve("h3_to_osm"); Path regionMetaPath = dataDir.resolve("region_metadata"); - Options options = new Options().setReadOnly(true); + Options options = new Options(); logger.info("Opening RocksDB databases for boundary lookup..."); this.h3ToOsmDb = RocksDB.open(options, h3ToOsmPath.toString()); @@ -50,14 +50,13 @@ public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exc * Returns a list of boundaries (OSM ID and total cell count) that contain this point. */ public List lookup(double lat, double lng) { - Set osmIds = new HashSet<>(); // The importer uses different resolutions based on admin level. // We must query all three to get the full hierarchy (City, State, Country). // Resolution 9 (Districts/Cities - Admin Level >= 7) long cellRes9 = h3.latLngToCell(lat, lng, 9); - osmIds.addAll(getOsmIdsForCell(cellRes9)); + Set osmIds = new HashSet<>(getOsmIdsForCell(cellRes9)); // Resolution 6 (States/Regions - Admin Level 3-6) long cellRes6 = h3.latLngToCell(lat, lng, 6); diff --git a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java index 7595cf7..6a6af9d 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java @@ -1,5 +1,6 @@ package com.dedicatedcode.paikka.service; +import com.dedicatedcode.paikka.IntegrationTest; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; @@ -29,9 +30,37 @@ void shouldLookupBoundariesForLuebeck() { List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); // Lübeck (Ebene 6) - assertTrue(osmIds.contains(62422L), "Should contain Lübeck (OSM ID 62422)"); + assertTrue(osmIds.contains(367855L), "Should contain Innenstadt (OSM ID 367855)"); + assertTrue(osmIds.contains(27027L), "Should contain Lübeck (OSM ID 62422)"); // Schleswig-Holstein - assertTrue(osmIds.contains(4388L), "Should contain Schleswig-Holstein (OSM ID 4388)"); + assertTrue(osmIds.contains(51529L), "Should contain Schleswig-Holstein (OSM ID 51529)"); + // Germany + assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); + + // Verify that we have some total cells calculated + assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); + } + + @Test + void shouldLookupBoundariesForPoelzig() { + // Coordinates for Pölzig + double lat = 50.957171; + double lng = 12.208514; + + List results = boundaryLookupService.lookup(lat, lng); + + assertFalse(results.isEmpty(), "Should find at least one boundary for Pölzig coordinates"); + + List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); + + // Pölzig (Ebene 8) + assertTrue(osmIds.contains(2532078L), "Should contain (OSM ID 2532078)"); + // Am Brahmetal (Ebene 7) + assertTrue(osmIds.contains(2907045L), "Should contain (OSM ID 2907045)"); + // Greiz (Ebene 6) + assertTrue(osmIds.contains(62445L), "Should contain (OSM ID 62445)"); + // Thüringen (Ebene 4) + assertTrue(osmIds.contains(62366L), "Should contain (OSM ID 62366)"); // Germany assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); From c47bf1a4821ace14b346057799a7f906e098ccc0 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 18:35:11 +0200 Subject: [PATCH 17/29] feat: add osm_to_h3 database support for boundary cell lookup --- .../paikka/service/BoundaryLookupService.java | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java index 01a28cd..dd13796 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java @@ -26,22 +26,23 @@ public class BoundaryLookupService { private final H3Core h3; private final RocksDB h3ToOsmDb; private final RocksDB regionMetadataDb; + private final RocksDB osmToH3Db; public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exception { this.h3 = H3Core.newInstance(); RocksDB.loadLibrary(); - // Assuming the databases are stored in a 'boundaries' folder within the data directory. - // Adjust this path if your importer outputs to a different location. Path dataDir = Paths.get(paikkaConfiguration.getDataDir()); Path h3ToOsmPath = dataDir.resolve("h3_to_osm"); Path regionMetaPath = dataDir.resolve("region_metadata"); + Path osmToH3Path = dataDir.resolve("osm_to_h3"); Options options = new Options(); - + logger.info("Opening RocksDB databases for boundary lookup..."); this.h3ToOsmDb = RocksDB.open(options, h3ToOsmPath.toString()); this.regionMetadataDb = RocksDB.open(options, regionMetaPath.toString()); + this.osmToH3Db = RocksDB.open(options, osmToH3Path.toString()); logger.info("RocksDB databases opened successfully."); } @@ -50,19 +51,12 @@ public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exc * Returns a list of boundaries (OSM ID and total cell count) that contain this point. */ public List lookup(double lat, double lng) { - - // The importer uses different resolutions based on admin level. - // We must query all three to get the full hierarchy (City, State, Country). - - // Resolution 9 (Districts/Cities - Admin Level >= 7) long cellRes9 = h3.latLngToCell(lat, lng, 9); Set osmIds = new HashSet<>(getOsmIdsForCell(cellRes9)); - // Resolution 6 (States/Regions - Admin Level 3-6) long cellRes6 = h3.latLngToCell(lat, lng, 6); osmIds.addAll(getOsmIdsForCell(cellRes6)); - // Resolution 4 (Countries/Continents - Admin Level <= 2) long cellRes4 = h3.latLngToCell(lat, lng, 4); osmIds.addAll(getOsmIdsForCell(cellRes4)); @@ -75,6 +69,37 @@ public List lookup(double lat, double lng) { return results; } + /** + * Fetches the H3 cells for a specific lat,lon in all needed resolutions. + */ + public Set getCellsForPoint(double lat, double lng) { + Set cells = new HashSet<>(); + cells.add(h3.latLngToCell(lat, lng, 9)); + cells.add(h3.latLngToCell(lat, lng, 6)); + cells.add(h3.latLngToCell(lat, lng, 4)); + return cells; + } + + /** + * Fetches all H3 cells belonging to a specific boundary (OSM ID). + * This can be used to compare visited cells against the total cells of a boundary. + */ + public Set getCellsForBoundary(long osmId) { + Set cells = new HashSet<>(); + try { + byte[] val = osmToH3Db.get(longToBytes(osmId)); + if (val != null) { + ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN); + while (bb.hasRemaining()) { + cells.add(bb.getLong()); + } + } + } catch (RocksDBException e) { + logger.error("Failed to lookup cells for OSM ID {} in osm_to_h3", osmId, e); + } + return cells; + } + private Set getOsmIdsForCell(long cellId) { Set ids = new HashSet<>(); try { @@ -115,6 +140,9 @@ public void cleanup() { if (this.regionMetadataDb != null) { this.regionMetadataDb.close(); } + if (this.osmToH3Db != null) { + this.osmToH3Db.close(); + } } /** @@ -122,4 +150,4 @@ public void cleanup() { * Reitti can use the totalCells to calculate the percentage visited. */ public record BoundaryInfo(long osmId, int totalCells) {} -} +} \ No newline at end of file From 5628155644526d12e93eb3001113e55e677b2b78 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 18:35:12 +0200 Subject: [PATCH 18/29] refactor: align BoundaryLookupService resolutions with importer logic --- .../paikka/service/BoundaryLookupService.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java index dd13796..73e8153 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java @@ -49,16 +49,19 @@ public BoundaryLookupService(PaikkaConfiguration paikkaConfiguration) throws Exc /** * Looks up the administrative boundaries for a given coordinate. * Returns a list of boundaries (OSM ID and total cell count) that contain this point. + * Uses the same resolution logic as StandaloneBoundaryImporter. */ public List lookup(double lat, double lng) { - long cellRes9 = h3.latLngToCell(lat, lng, 9); - Set osmIds = new HashSet<>(getOsmIdsForCell(cellRes9)); - - long cellRes6 = h3.latLngToCell(lat, lng, 6); - osmIds.addAll(getOsmIdsForCell(cellRes6)); - - long cellRes4 = h3.latLngToCell(lat, lng, 4); - osmIds.addAll(getOsmIdsForCell(cellRes4)); + Set osmIds = new HashSet<>(); + + // Query all resolutions used by the importer (4, 6, 9) + // This covers all admin levels: 4 for countries/continents, 6 for states/regions, 9 for districts/cities + int[] resolutions = {4, 6, 9}; + + for (int resolution : resolutions) { + long cellId = h3.latLngToCell(lat, lng, resolution); + osmIds.addAll(getOsmIdsForCell(cellId)); + } List results = new ArrayList<>(); for (Long osmId : osmIds) { @@ -70,13 +73,19 @@ public List lookup(double lat, double lng) { } /** - * Fetches the H3 cells for a specific lat,lon in all needed resolutions. + * Fetches the H3 cells for a specific lat,lon in all resolutions used by the importer. + * Uses the same resolution logic as StandaloneBoundaryImporter. */ public Set getCellsForPoint(double lat, double lng) { Set cells = new HashSet<>(); - cells.add(h3.latLngToCell(lat, lng, 9)); - cells.add(h3.latLngToCell(lat, lng, 6)); - cells.add(h3.latLngToCell(lat, lng, 4)); + + // Use the same resolutions as the importer (4, 6, 9) + int[] resolutions = {4, 6, 9}; + + for (int resolution : resolutions) { + cells.add(h3.latLngToCell(lat, lng, resolution)); + } + return cells; } @@ -150,4 +159,4 @@ public void cleanup() { * Reitti can use the totalCells to calculate the percentage visited. */ public record BoundaryInfo(long osmId, int totalCells) {} -} \ No newline at end of file +} From 5e8ffc4180bc0012e7ed98336c521ab36e888256 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 18:46:32 +0200 Subject: [PATCH 19/29] feat: add getCellsWithBoundaries method for visit tracking --- .../paikka/service/BoundaryLookupService.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java index 73e8153..92ef728 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java +++ b/src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java @@ -154,9 +154,35 @@ public void cleanup() { } } + /** + * Returns H3 cells for a point along with their associated OSM boundary IDs. + * Useful for Reitti to track which boundaries are associated with each visited cell. + */ + public List getCellsWithBoundaries(double lat, double lng) { + List result = new ArrayList<>(); + + int[] resolutions = {4, 6, 9}; + + for (int resolution : resolutions) { + long cellId = h3.latLngToCell(lat, lng, resolution); + Set osmIds = getOsmIdsForCell(cellId); + if (!osmIds.isEmpty()) { + result.add(new CellWithBoundaries(cellId, resolution, osmIds)); + } + } + + return result; + } + /** * Represents a boundary containing the queried point. * Reitti can use the totalCells to calculate the percentage visited. */ public record BoundaryInfo(long osmId, int totalCells) {} + + /** + * Represents an H3 cell with its associated boundary OSM IDs. + * Useful for tracking which boundaries are affected when a cell is visited. + */ + public record CellWithBoundaries(long cellId, int resolution, Set osmIds) {} } From 4a10042465296754cc59bceb2c6c93e168991f47 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Sun, 12 Jul 2026 19:39:15 +0200 Subject: [PATCH 20/29] test: add test for getCellsForPoint method in BoundaryLookupService --- .../service/BoundaryLookupServiceTest.java | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java index 6a6af9d..e78ebdd 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java @@ -6,9 +6,9 @@ import org.springframework.test.context.TestPropertySource; import java.util.List; +import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @IntegrationTest @TestPropertySource(properties = "paikka.data-dir=/home/daniel/projects/paikka/data-boundaries") @@ -19,7 +19,6 @@ class BoundaryLookupServiceTest { @Test void shouldLookupBoundariesForLuebeck() { - // Coordinates for Luebeck double lat = 53.86422; double lng = 10.69120; @@ -29,21 +28,16 @@ void shouldLookupBoundariesForLuebeck() { List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); - // Lübeck (Ebene 6) assertTrue(osmIds.contains(367855L), "Should contain Innenstadt (OSM ID 367855)"); assertTrue(osmIds.contains(27027L), "Should contain Lübeck (OSM ID 62422)"); - // Schleswig-Holstein assertTrue(osmIds.contains(51529L), "Should contain Schleswig-Holstein (OSM ID 51529)"); - // Germany assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); - // Verify that we have some total cells calculated assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); } @Test void shouldLookupBoundariesForPoelzig() { - // Coordinates for Pölzig double lat = 50.957171; double lng = 12.208514; @@ -53,18 +47,22 @@ void shouldLookupBoundariesForPoelzig() { List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); - // Pölzig (Ebene 8) assertTrue(osmIds.contains(2532078L), "Should contain (OSM ID 2532078)"); - // Am Brahmetal (Ebene 7) assertTrue(osmIds.contains(2907045L), "Should contain (OSM ID 2907045)"); - // Greiz (Ebene 6) assertTrue(osmIds.contains(62445L), "Should contain (OSM ID 62445)"); - // Thüringen (Ebene 4) assertTrue(osmIds.contains(62366L), "Should contain (OSM ID 62366)"); - // Germany assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); - // Verify that we have some total cells calculated assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); } -} + + @Test + void shouldGetCellsForPoint() { + double lat = 53.86422; + double lng = 10.69120; + + Set cells = boundaryLookupService.getCellsForPoint(lat, lng); + + assertEquals(3, cells.size(), "Should return exactly 3 cells for the different resolutions"); + } +} \ No newline at end of file From 9e1ac298665a85c79e7015788fcf87f46db285e2 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 19:39:16 +0200 Subject: [PATCH 21/29] test: add tests for getCellsWithBoundaries method --- .../service/BoundaryLookupServiceTest.java | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java index e78ebdd..74a5e39 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java @@ -65,4 +65,52 @@ void shouldGetCellsForPoint() { assertEquals(3, cells.size(), "Should return exactly 3 cells for the different resolutions"); } -} \ No newline at end of file + + @Test + void shouldGetCellsWithBoundaries() { + double lat = 53.86422; + double lng = 10.69120; + + List cellsWithBoundaries = boundaryLookupService.getCellsWithBoundaries(lat, lng); + + assertFalse(cellsWithBoundaries.isEmpty(), "Should return cells with boundaries for Luebeck coordinates"); + + // Should have cells for different resolutions (4, 6, 9) + Set resolutions = cellsWithBoundaries.stream() + .map(BoundaryLookupService.CellWithBoundaries::resolution) + .collect(java.util.stream.Collectors.toSet()); + assertTrue(resolutions.contains(4), "Should include resolution 4 (countries/continents)"); + assertTrue(resolutions.contains(6), "Should include resolution 6 (states/regions)"); + assertTrue(resolutions.contains(9), "Should include resolution 9 (districts/cities)"); + + // Each cell should have associated OSM IDs + for (BoundaryLookupService.CellWithBoundaries cell : cellsWithBoundaries) { + assertFalse(cell.osmIds().isEmpty(), "Each cell should have associated OSM boundary IDs"); + assertTrue(cell.cellId() > 0, "Cell ID should be positive"); + } + + // Should contain expected OSM IDs across all cells + Set allOsmIds = cellsWithBoundaries.stream() + .flatMap(cell -> cell.osmIds().stream()) + .collect(java.util.stream.Collectors.toSet()); + + assertTrue(allOsmIds.contains(367855L), "Should contain Innenstadt (OSM ID 367855)"); + assertTrue(allOsmIds.contains(27027L), "Should contain Lübeck (OSM ID 27027)"); + assertTrue(allOsmIds.contains(51529L), "Should contain Schleswig-Holstein (OSM ID 51529)"); + assertTrue(allOsmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); + } + + @Test + void shouldGetCellsWithBoundariesForEmptyArea() { + // Use coordinates in the middle of the ocean where no boundaries should exist + double lat = 0.0; + double lng = 0.0; + + List cellsWithBoundaries = boundaryLookupService.getCellsWithBoundaries(lat, lng); + + // Should return empty list or cells with no OSM IDs + assertTrue(cellsWithBoundaries.isEmpty() || + cellsWithBoundaries.stream().allMatch(cell -> cell.osmIds().isEmpty()), + "Should return no cells with boundaries for ocean coordinates"); + } +} From d434e0991899c8570f3ec9b24404e5d8a38f439d Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Sun, 12 Jul 2026 19:48:03 +0200 Subject: [PATCH 22/29] perf: optimize boundary importer with thread-local batching and geometry simplification --- .../importer/StandaloneBoundaryImporter.java | 73 +++++++++++++------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 9c603a0..11975f9 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -226,6 +226,8 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc List> futures = new ArrayList<>(); for (int i = 0; i < threads; i++) { futures.add(executor.submit(() -> { + // Thread-local batch for H3 updates to reduce contention + Map> threadLocalH3Batch = new HashMap<>(); try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) { while (true) { List batch = queue.take(); @@ -251,13 +253,14 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc continue; } } - // Buffer to include border-touching cells - Geometry buffered = geom.buffer(BUFFER_DISTANCE); - Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(buffered, stub.adminLevel()); + // Simplify first to reduce H3 cell count, then buffer + Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(geom, stub.adminLevel()); if (simplified == null || simplified.isEmpty()) { - logger.warn("Simplified Geometry is invalid for OSM ID: {}", stub.osmId()); - simplified = buffered; + logger.warn("Simplified Geometry is invalid for OSM ID: {}, using original", stub.osmId()); + simplified = geom; } + // Buffer to include border-touching cells + Geometry buffered = simplified.buffer(BUFFER_DISTANCE); if (stub.adminLevel() <= 3) { logger.debug("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId()); } @@ -267,7 +270,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // ---- H3 Polyfill ---- AtomicLong cellCount = new AtomicLong(0); long startTime = System.currentTimeMillis(); - processCellsH3Stream(simplified, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution); + processCellsH3StreamThreadLocal(buffered, stub.osmId(), threadLocalH3Batch, cellCount, resolution); if (stub.adminLevel() <= 3) { logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId()); } @@ -291,6 +294,9 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e); } } + + // Flush thread-local H3 batch at end of batch processing + flushThreadLocalH3Batch(threadLocalH3Batch, wo, tmpH3ToOsm); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -486,8 +492,9 @@ private int getResolutionForAdminLevel(int adminLevel) { /** * Converts a JTS Geometry to H3 cells at the specified resolution. * Uses h3.polygonToCellsStream with LatLng vertices. Multipolygons are expanded. + * Thread-local version that accumulates updates in memory to reduce database contention. */ - private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount, int resolution) { + private void processCellsH3StreamThreadLocal(Geometry geom, long osmId, Map> threadLocalBatch, AtomicLong cellCount, int resolution) { int num = geom.getNumGeometries(); for (int i = 0; i < num; i++) { Geometry part = geom.getGeometryN(i); @@ -498,25 +505,49 @@ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, Ro holes.add(toLatLng(poly.getInteriorRingN(h).getCoordinates())); } try { - List batch = new ArrayList<>(10_000); h3.polygonToCells(outer, holes, resolution).forEach(cell -> { - batch.add(cell); - if (batch.size() >= 10_000) { - try { - processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); - } catch (RocksDBException e) { - throw new RuntimeException(e); - } - batch.clear(); - } + threadLocalBatch.computeIfAbsent(cell, k -> new HashSet<>()).add(osmId); + cellCount.incrementAndGet(); }); - if (!batch.isEmpty()) { - processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); - } } catch (Exception e) { - stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3Stream", e); + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3StreamThreadLocal", e); + } + } + } + + /** + * Flushes the thread-local H3 batch to RocksDB. + * This reduces contention by batching multiple updates per thread. + */ + private void flushThreadLocalH3Batch(Map> threadLocalBatch, WriteOptions wo, RocksDB tmpH3ToOsm) throws RocksDBException { + if (threadLocalBatch.isEmpty()) return; + + List keys = new ArrayList<>(); + for (Long cellId : threadLocalBatch.keySet()) { + keys.add(longToBytes(cellId)); + } + + // Synchronize only for the batch flush, not individual operations + synchronized (tmpH3ToOsm) { + List existingValues = tmpH3ToOsm.multiGetAsList(keys); + try (WriteBatch writeBatch = new WriteBatch()) { + int i = 0; + for (Map.Entry> entry : threadLocalBatch.entrySet()) { + byte[] key = keys.get(i); + byte[] existing = existingValues.get(i); + + byte[] updated = existing; + for (Long osmId : entry.getValue()) { + updated = appendOsmIdToArray(updated, osmId); + } + writeBatch.put(key, updated); + i++; + } + tmpH3ToOsm.write(wo, writeBatch); } } + + threadLocalBatch.clear(); } private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) throws RocksDBException { From a21a09d291559d698dd36c5fe96c57941080c6bd Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Mon, 13 Jul 2026 07:37:52 +0200 Subject: [PATCH 23/29] refactor: replace deprecated list methods and add RocksDBException handling --- .../importer/StandaloneBoundaryImporter.java | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 11975f9..5edd86d 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -300,6 +300,8 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc } } catch (InterruptedException e) { Thread.currentThread().interrupt(); + } catch (RocksDBException e) { + throw new RuntimeException(e); } })); } @@ -403,7 +405,7 @@ private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB } } if (polygons.isEmpty()) return null; - return polygons.size() == 1 ? polygons.get(0) : GEOMETRY_FACTORY.createMultiPolygon(polygons.toArray(new Polygon[0])); + return polygons.size() == 1 ? polygons.getFirst() : GEOMETRY_FACTORY.createMultiPolygon(polygons.toArray(new Polygon[0])); } private List> stitchRings(List wayIds, RocksDB nodeCache, RocksDB wayCache) { @@ -429,16 +431,16 @@ private List> stitchRings(List wayIds, RocksDB nodeCache, boolean extended; do { extended = false; - Coordinate end = ring.get(ring.size() - 1); + Coordinate end = ring.getLast(); for (Map.Entry> e : wayCoords.entrySet()) { if (used.contains(e.getKey())) continue; List w = e.getValue(); - if (end.equals2D(w.get(0))) { + if (end.equals2D(w.getFirst())) { ring.addAll(w.subList(1, w.size())); used.add(e.getKey()); extended = true; break; - } else if (end.equals2D(w.get(w.size() - 1))) { + } else if (end.equals2D(w.getLast())) { List rev = new ArrayList<>(w); Collections.reverse(rev); ring.addAll(rev.subList(1, rev.size())); @@ -448,8 +450,8 @@ private List> stitchRings(List wayIds, RocksDB nodeCache, } } } while (extended); - if (ring.size() >= 3 && !ring.get(0).equals2D(ring.get(ring.size() - 1))) - ring.add(new Coordinate(ring.get(0))); + if (ring.size() >= 3 && !ring.getFirst().equals2D(ring.getLast())) + ring.add(new Coordinate(ring.getFirst())); if (ring.size() >= 4) rings.add(ring); } return rings; @@ -534,9 +536,8 @@ private void flushThreadLocalH3Batch(Map> threadLocalBatch, Writ int i = 0; for (Map.Entry> entry : threadLocalBatch.entrySet()) { byte[] key = keys.get(i); - byte[] existing = existingValues.get(i); - - byte[] updated = existing; + + byte[] updated = existingValues.get(i); for (Long osmId : entry.getValue()) { updated = appendOsmIdToArray(updated, osmId); } @@ -585,10 +586,6 @@ private byte[] longToBytes(long v) { return ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(v).array(); } - private long bytesToLong(byte[] b) { - return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getLong(); - } - private byte[] longArrayToBytes(long[] arr) { ByteBuffer bb = ByteBuffer.allocate(8 * arr.length).order(ByteOrder.BIG_ENDIAN); for (long v : arr) bb.putLong(v); From 5fa646b5655ba18ee9ed4c137bbdd2827b4b9ba7 Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Mon, 13 Jul 2026 07:37:54 +0200 Subject: [PATCH 24/29] fix: revert thread-local batching and enhance compression to prevent OOM --- .../importer/StandaloneBoundaryImporter.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 5edd86d..9115ad0 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -110,7 +110,17 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc .setCreateIfMissing(true) .setTableFormatConfig(tableCfg) .setCompressionType(CompressionType.ZSTD_COMPRESSION) - .setWriteBufferSize(256 * 1024 * 1024); + .setWriteBufferSize(256 * 1024 * 1024) + .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION) + .setCompressionPerLevel(List.of( + CompressionType.NO_COMPRESSION, + CompressionType.NO_COMPRESSION, + CompressionType.LZ4_COMPRESSION, + CompressionType.LZ4_COMPRESSION, + CompressionType.ZSTD_COMPRESSION, + CompressionType.ZSTD_COMPRESSION, + CompressionType.ZSTD_COMPRESSION + )); stats.startProgressReporter(); @@ -226,8 +236,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc List> futures = new ArrayList<>(); for (int i = 0; i < threads; i++) { futures.add(executor.submit(() -> { - // Thread-local batch for H3 updates to reduce contention - Map> threadLocalH3Batch = new HashMap<>(); try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) { while (true) { List batch = queue.take(); @@ -270,7 +278,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // ---- H3 Polyfill ---- AtomicLong cellCount = new AtomicLong(0); long startTime = System.currentTimeMillis(); - processCellsH3StreamThreadLocal(buffered, stub.osmId(), threadLocalH3Batch, cellCount, resolution); + processCellsH3Stream(buffered, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution); if (stub.adminLevel() <= 3) { logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId()); } @@ -294,9 +302,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e); } } - - // Flush thread-local H3 batch at end of batch processing - flushThreadLocalH3Batch(threadLocalH3Batch, wo, tmpH3ToOsm); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); From 3aced51d78759f7419853c7701143fa2b11f31ed Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Mon, 13 Jul 2026 07:38:11 +0200 Subject: [PATCH 25/29] refactor: replace thread-local H3 processing with batched approach --- .../importer/StandaloneBoundaryImporter.java | 54 ++++++------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 9115ad0..5d8728b 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -499,9 +499,8 @@ private int getResolutionForAdminLevel(int adminLevel) { /** * Converts a JTS Geometry to H3 cells at the specified resolution. * Uses h3.polygonToCellsStream with LatLng vertices. Multipolygons are expanded. - * Thread-local version that accumulates updates in memory to reduce database contention. */ - private void processCellsH3StreamThreadLocal(Geometry geom, long osmId, Map> threadLocalBatch, AtomicLong cellCount, int resolution) { + private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount, int resolution) { int num = geom.getNumGeometries(); for (int i = 0; i < num; i++) { Geometry part = geom.getGeometryN(i); @@ -512,48 +511,25 @@ private void processCellsH3StreamThreadLocal(Geometry geom, long osmId, Map batch = new ArrayList<>(5_000); // Reduced batch size to prevent OOM h3.polygonToCells(outer, holes, resolution).forEach(cell -> { - threadLocalBatch.computeIfAbsent(cell, k -> new HashSet<>()).add(osmId); - cellCount.incrementAndGet(); - }); - } catch (Exception e) { - stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3StreamThreadLocal", e); - } - } - } - - /** - * Flushes the thread-local H3 batch to RocksDB. - * This reduces contention by batching multiple updates per thread. - */ - private void flushThreadLocalH3Batch(Map> threadLocalBatch, WriteOptions wo, RocksDB tmpH3ToOsm) throws RocksDBException { - if (threadLocalBatch.isEmpty()) return; - - List keys = new ArrayList<>(); - for (Long cellId : threadLocalBatch.keySet()) { - keys.add(longToBytes(cellId)); - } - - // Synchronize only for the batch flush, not individual operations - synchronized (tmpH3ToOsm) { - List existingValues = tmpH3ToOsm.multiGetAsList(keys); - try (WriteBatch writeBatch = new WriteBatch()) { - int i = 0; - for (Map.Entry> entry : threadLocalBatch.entrySet()) { - byte[] key = keys.get(i); - - byte[] updated = existingValues.get(i); - for (Long osmId : entry.getValue()) { - updated = appendOsmIdToArray(updated, osmId); + batch.add(cell); + if (batch.size() >= 5_000) { + try { + processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); + } catch (RocksDBException e) { + throw new RuntimeException(e); + } + batch.clear(); } - writeBatch.put(key, updated); - i++; + }); + if (!batch.isEmpty()) { + processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount); } - tmpH3ToOsm.write(wo, writeBatch); + } catch (Exception e) { + stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3Stream", e); } } - - threadLocalBatch.clear(); } private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) throws RocksDBException { From cdacd0f896e73bbafb50b131edd72037330d70ea Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Tue, 28 Jul 2026 16:38:03 +0200 Subject: [PATCH 26/29] finished h3 index generation --- .gitignore | 1 + scripts/create-h3-bundle.sh | 128 +++++++++++ scripts/filter_boundaries.sh | 75 +++++++ scripts/import-boundaries.sh | 200 ++++++++++++++++++ scripts/upload-h3-bundle.sh | 144 +++++++++++++ .../paikka/config/PaikkaConfiguration.java | 1 - .../paikka/service/BoundaryLookupService.java | 188 ---------------- .../service/importer/OsmNameStreamer.java | 106 ++++++++++ .../importer/StandaloneBoundaryImporter.java | 15 +- .../service/BoundaryLookupServiceTest.java | 116 ---------- .../paikka/service/ImportServiceTest.java | 7 + 11 files changed, 670 insertions(+), 311 deletions(-) create mode 100755 scripts/create-h3-bundle.sh create mode 100755 scripts/filter_boundaries.sh create mode 100755 scripts/import-boundaries.sh create mode 100755 scripts/upload-h3-bundle.sh delete mode 100644 src/main/java/com/dedicatedcode/paikka/service/BoundaryLookupService.java create mode 100644 src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java delete mode 100644 src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java diff --git a/.gitignore b/.gitignore index cbe522e..3d06e25 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ build/ /data/ /.idea/ /data-boundaries/ +/scripts/.env diff --git a/scripts/create-h3-bundle.sh b/scripts/create-h3-bundle.sh new file mode 100755 index 0000000..e098d7a --- /dev/null +++ b/scripts/create-h3-bundle.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# +# This file is part of paikka. +# +# Paikka is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 or +# any later version. +# +# Paikka is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# along with Paikka. If not, see . +# + +# Exit immediately if a command exits with a non-zero status +set -e + +# Default values +VERSION=$(date +%Y-%m-%d)-v1 +DOWNLOAD_BASE_URL="https://h3-osm.dedicatedcode.com" +OUTPUT_DIR="./dist" + +# Function to display usage instructions +usage() { + echo "Usage: $0 -d [OPTIONS]" + echo "" + echo "Required:" + echo " -d, --db-dir Path to the directory containing h3_to_osm, region_metadata, and region_geometry" + echo "" + echo "Options:" + echo " -v, --version Version string for the bundle (default: $VERSION)" + echo " -u, --url Base CDN URL where the zip will be hosted (default: $DOWNLOAD_BASE_URL)" + echo " -o, --output-dir Where to write the ZIP and manifest.json (default: $OUTPUT_DIR)" + echo " -h, --help Show this help message" + exit 1 +} + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + -d|--db-dir) DB_DIR="$2"; shift ;; + -v|--version) VERSION="$2"; shift ;; + -u|--url) DOWNLOAD_BASE_URL="$2"; shift ;; + -o|--output-dir) OUTPUT_DIR="$2"; shift ;; + -h|--help) usage ;; + *) echo "Unknown parameter passed: $1"; usage ;; + esac + shift +done + +# Validate required argument +if [ -z "$DB_DIR" ]; then + echo "Error: Database source directory (-d / --db-dir) is required." + usage +fi + +# Ensure absolute paths +DB_DIR_ABS=$(cd "$DB_DIR" && pwd) +mkdir -p "$OUTPUT_DIR" +OUTPUT_DIR_ABS=$(cd "$OUTPUT_DIR" && pwd) + +# Check that the three essential database directories actually exist +REQUIRED_DIRS=("h3_to_osm" "region_metadata" "region_geometry") +for dir in "${REQUIRED_DIRS[@]}"; do + if [ ! -d "$DB_DIR_ABS/$dir" ]; then + echo "Error: Required directory '$dir' not found inside $DB_DIR_ABS" + exit 1 + fi +done + +echo "==========================================" +echo "Preparing H3 RocksDB Bundle" +echo "Version: $VERSION" +echo "Source: $DB_DIR_ABS" +echo "Output: $OUTPUT_DIR_ABS" +echo "==========================================" + +ZIP_FILENAME="h3-rocksdb-${VERSION}.zip" +ZIP_PATH="$OUTPUT_DIR_ABS/$ZIP_FILENAME" + +# 1. Clean up any pre-existing zip at the target location to avoid mixing versions +rm -f "$ZIP_PATH" + +echo "Creating ZIP archive..." +# We run zip inside the source directory so that the subfolders are at the ROOT of the zip. +# -r: recursive +# -q: quiet +# -x "**/LOCK": IMPORTANT. Excludes RocksDB native file system locks which prevent startup on target systems. +( + cd "$DB_DIR_ABS" + zip -r -q "$ZIP_PATH" h3_to_osm region_metadata region_geometry osm_names.tsv -x "**/LOCK" +) + +echo "Calculating bundle metadata..." +# Check OS to use the correct parameters for 'stat' and 'sha256' tools (Mac vs Linux) +if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS + SIZE_BYTES=$(stat -f%z "$ZIP_PATH") + SHA256=$(shasum -a 256 "$ZIP_PATH" | awk '{print $1}') +else + # Linux / WSL + SIZE_BYTES=$(stat -c%s "$ZIP_PATH") + SHA256=$(sha256sum "$ZIP_PATH" | awk '{print $1}') +fi + +# 2. Build the exact Manifest structure the Spring Boot Lifecycle Manager expects +MANIFEST_PATH="$OUTPUT_DIR_ABS/manifest.json" +DOWNLOAD_URL="${DOWNLOAD_BASE_URL}/${ZIP_FILENAME}" + +cat < "$MANIFEST_PATH" +{ + "version": "${VERSION}", + "downloadUrl": "${DOWNLOAD_URL}", + "sha256": "${SHA256}", + "sizeBytes": ${SIZE_BYTES} +} +EOF + +echo "==========================================" +echo "Success! Package files generated:" +echo "Archive: $ZIP_PATH ($(numfmt --to=iec --suffix=B $SIZE_BYTES) / $SIZE_BYTES bytes)" +echo "Checksum: $SHA256" +echo "Manifest: $MANIFEST_PATH" +echo "==========================================" \ No newline at end of file diff --git a/scripts/filter_boundaries.sh b/scripts/filter_boundaries.sh new file mode 100755 index 0000000..318bc71 --- /dev/null +++ b/scripts/filter_boundaries.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Project Paikka - Lite PBF Filter +# Filters OSM PBF files to keep only POIs and Administrative Boundaries + +# Usage function +usage() { + echo "Usage: $0 " + echo "" + echo "Filters an OSM PBF file to keep only boundaries relevant for RAITTI:" + echo " - Points of Interest (amenity, shop, tourism, leisure, etc.)" + echo " - Administrative boundaries" + echo "" + echo "Arguments:" + echo " input_file Path to the input OSM PBF file" + echo " output_file Path for the filtered output PBF file" + echo "" + echo "Examples:" + echo " $0 planet-latest.osm.pbf planet-filtered.osm.pbf" + echo " $0 europe-latest.osm.pbf europe-paikka.osm.pbf" + echo "" + echo "Requirements:" + echo " - osmium-tool must be installed" + echo " - Sufficient disk space for output file" + exit 1 +} + +# Check if correct number of arguments provided +if [ $# -ne 2 ]; then + echo "Error: Incorrect number of arguments" + echo "" + usage +fi + +INPUT_FILE="$1" +OUTPUT_FILE="$2" + +# Check if input file exists +if [ ! -f "$INPUT_FILE" ]; then + echo "Error: Input file '$INPUT_FILE' does not exist" + exit 1 +fi + +# Check if osmium is available +if ! command -v osmium &> /dev/null; then + echo "Error: osmium-tool is not installed" + echo "Install with: sudo apt-get install osmium-tool (Ubuntu/Debian)" + echo "Or: brew install osmium-tool (macOS)" + exit 1 +fi + +echo "Starting OSM PBF filtering for PAIKKA..." +echo "Input file: $INPUT_FILE" +echo "Output file: $OUTPUT_FILE" +echo "" +osmium tags-filter "$INPUT_FILE" r/boundary=administrative -o "$OUTPUT_FILE" --overwrite + +if [ $? -eq 0 ]; then + echo "" + echo "✓ Filter complete: $OUTPUT_FILE created" + echo "✓ Input file size: $(du -h "$INPUT_FILE" | cut -f1)" + echo "✓ Output file size: $(du -h "$OUTPUT_FILE" | cut -f1)" + + # Calculate size reduction + INPUT_SIZE=$(stat -c%s "$INPUT_FILE" 2>/dev/null || stat -f%z "$INPUT_FILE" 2>/dev/null) + OUTPUT_SIZE=$(stat -c%s "$OUTPUT_FILE" 2>/dev/null || stat -f%z "$OUTPUT_FILE" 2>/dev/null) + + if [ -n "$INPUT_SIZE" ] && [ -n "$OUTPUT_SIZE" ] && [ "$INPUT_SIZE" -gt 0 ]; then + REDUCTION=$(( (INPUT_SIZE - OUTPUT_SIZE) * 100 / INPUT_SIZE )) + echo "✓ Size reduction: ${REDUCTION}%" + fi +else + echo "Error: Filtering failed" + exit 1 +fi diff --git a/scripts/import-boundaries.sh b/scripts/import-boundaries.sh new file mode 100755 index 0000000..47e664d --- /dev/null +++ b/scripts/import-boundaries.sh @@ -0,0 +1,200 @@ +#!/bin/bash + +# PAIKKA Import Script +# Runs PAIKKA in import mode with required JVM flags + +# Usage function +usage() { + echo "Usage: $0 [OPTIONS] " + echo "" + echo "Imports OSM PBF data into Reitti H3 format" + echo "" + echo "Required Arguments:" + echo " pbf_file Path to the OSM PBF file to import" + echo "" + echo "Options:" + echo " --jar-file PATH Path to the PAIKKA jar file (auto-detected if not provided)" + echo " --data-dir PATH Directory to store processed data (default: ./)" + echo " --memory SIZE JVM heap size (default: 16g)" + echo " --threads NUM Maximum number of import threads (default: half of CPU cores)" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 planet-latest.osm.pbf" + echo " $0 --jar-file /app/app.jar --data-dir /opt/paikka/data europe-latest.osm.pbf" + echo " $0 --memory 32g --threads 8 germany-latest.osm.pbf austria-latest.osm.pbf" + echo " $0 --data-dir ./data --memory 16g oceania-latest.osm.pbf" + echo "" + echo "Requirements:" + echo " - Java 25 or higher" + echo " - PAIKKA jar file in target/ directory or provided via --jar-file" + echo " - Sufficient RAM (recommended: 32GB+ for planet)" + exit 1 +} + +# Default values +JAR_FILE="" +DATA_DIR="./" +MEMORY="16g" +THREADS="" +PBF_FILES=() + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --jar-file) + JAR_FILE="$2" + shift 2 + ;; + --data-dir) + DATA_DIR="$2" + shift 2 + ;; + --memory) + MEMORY="$2" + shift 2 + ;; + --threads) + THREADS="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -*) + echo "Error: Unknown option: $1" + echo "" + usage + ;; + *) + PBF_FILES+=("$1") + shift + ;; + esac +done + +# Check if any PBF file argument is provided +if [ ${#PBF_FILES} -eq 0 ]; then + echo "Error: At least one PBF file argument required" + echo "" + usage +fi + +# Verify each PBF file exists +for f in ${PBF_FILES[@]}; do + if [ ! -f "$f" ]; then + echo "Error: PBF file '$f' does not exist" + exit 1 + fi +done + +# Find PAIKKA jar file if not provided +if [ -z "$JAR_FILE" ]; then + JAR_FILE=$(find target -name "paikka-*.jar" -not -name "*-sources.jar" | head -1) + + if [ -z "$JAR_FILE" ]; then + echo "Error: PAIKKA jar file not found in target/ directory" + echo "Please run 'mvn clean package' first or provide jar file path via --jar-file" + exit 1 + fi +fi + +# Verify jar file exists +if [ ! -f "$JAR_FILE" ]; then + echo "Error: JAR file '$JAR_FILE' does not exist" + exit 1 +fi + +echo "Starting PAIKKA import..." +echo "PBF files: ${PBF_FILES[*]}" +echo "Data dir: $DATA_DIR" +echo "Memory: $MEMORY" +echo "JAR file: $JAR_FILE" +if [ -n "$THREADS" ]; then + echo "Threads: $THREADS" +fi +echo "" + +# Check available system memory +AVAILABLE_MEM_KB=$(grep MemAvailable /proc/meminfo | awk '{print $2}') +AVAILABLE_MEM_GB=$((AVAILABLE_MEM_KB / 1024 / 1024)) + +echo "System memory: ${AVAILABLE_MEM_GB}GB available" +echo "Requested heap: $MEMORY" + +# Build JVM arguments with memory management optimizations +JVM_ARGS="-Xmx$MEMORY -Xms$MEMORY" +JVM_ARGS="$JVM_ARGS -XX:+UseG1GC" +JVM_ARGS="$JVM_ARGS -XX:MaxGCPauseMillis=200" +JVM_ARGS="$JVM_ARGS -XX:+UnlockExperimentalVMOptions" +JVM_ARGS="$JVM_ARGS -XX:+UseTransparentHugePages" +JVM_ARGS="$JVM_ARGS --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-exports=java.base/sun.nio.ch=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-opens=jdk.compiler/com.sun.tools.javac=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-opens=java.base/java.lang=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-opens=java.base/java.io=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --add-opens=java.base/java.util=ALL-UNNAMED" +JVM_ARGS="$JVM_ARGS --enable-native-access=ALL-UNNAMED" + +# Add thread configuration if specified +if [ -n "$THREADS" ]; then + JVM_ARGS="$JVM_ARGS -Dpaikka.import.threads=$THREADS" +fi + +# Run PAIKKA import with required JVM flags +java $JVM_ARGS \ + -jar "$JAR_FILE" \ + --boundary-import \ + --data-dir "$DATA_DIR" \ + "${PBF_FILES[@]}" + +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo "" + echo "✓ Import completed successfully" +elif [ $EXIT_CODE -eq 134 ]; then + echo "" + echo "✗ Import failed: Process was killed (likely out of memory)" + echo "💡 Try reducing heap size or adding more RAM" + echo " Current heap: $MEMORY, Available: ${AVAILABLE_MEM_GB}GB" + exit 1 +else + echo "" + echo "✗ Import failed with exit code: $EXIT_CODE" + exit 1 +fi + +if [ $EXIT_CODE -eq 0 ]; then + echo "" + echo "✓ Import completed successfully" + echo "✓ Data directory: $DATA_DIR" + + # Clean up temporary files to save disk space + echo "" + echo "🧹 Cleaning up temporary files..." + + # Remove node_cache + TEMP_DIR="$DATA_DIR/node_cache" + if [ -d "$TEMP_DIR" ]; then + echo " Removing temporary directory: $TEMP_DIR" + rm -rf "$TEMP_DIR" + echo " ✓ Node cache cleaned up" + fi + + # Remove grid_index + GRID_DIR="$DATA_DIR/grid_index" + if [ -d "$GRID_DIR" ]; then + echo " Removing temporary directory: $GRID_DIR" + rm -rf "$GRID_DIR" + echo " ✓ Grid index cleaned up" + fi + +else + echo "" + echo "✗ Import failed" + exit 1 +fi diff --git a/scripts/upload-h3-bundle.sh b/scripts/upload-h3-bundle.sh new file mode 100755 index 0000000..878fbe4 --- /dev/null +++ b/scripts/upload-h3-bundle.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +set -e + +# --- Configuration & Defaults --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="$SCRIPT_DIR/.env" +DIST_DIR="$SCRIPT_DIR/dist" # Default fallback if no directory parameter is provided + +# Show help/usage instructions +usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -d, --dist-dir Path to folder containing the ZIP file and manifest.json (Default: $DIST_DIR)" + echo " -h, --help Show this help message" + exit 1 +} + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case $1 in + -d|--dist-dir) DIST_DIR="$2"; shift ;; + -h|--help) usage ;; + *) echo "Unknown parameter: $1"; usage ;; + esac + shift +done + +# Load credentials from .env file +if [ -f "$ENV_FILE" ]; then + source "$ENV_FILE" +else + echo "Error: Configuration file .env was not found at: $ENV_FILE" + exit 1 +fi + +# AWS CLI check +if ! command -v aws &> /dev/null; then + echo "Error: The AWS CLI is not installed on this server." + exit 1 +fi + +# Validate specified dist directory path and convert to absolute path +if [ ! -d "$DIST_DIR" ]; then + echo "Error: The specified dist directory does not exist: $DIST_DIR" + exit 1 +fi +DIST_DIR_ABS=$(cd "$DIST_DIR" && pwd) + +# Setup endpoint URL for Cloudflare R2 +R2_ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" + +# Set environment variables for AWS CLI +export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" +export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY" +export AWS_DEFAULT_REGION="auto" + +# Resolve local files +MANIFEST_FILE="$DIST_DIR_ABS/manifest.json" + +if [ ! -f "$MANIFEST_FILE" ]; then + echo "Error: manifest.json was not found in folder $DIST_DIR_ABS." + echo "Please ensure the bundle script was executed successfully there." + exit 1 +fi + +# Extract download URL and filename from the local manifest.json +ZIP_URL=$(grep -o '"downloadUrl": *"[^"]*"' "$MANIFEST_FILE" | grep -o '"[^"]*"$' | tr -d '"') +ZIP_FILENAME=$(basename "$ZIP_URL") +ZIP_FILE="$DIST_DIR_ABS/$ZIP_FILENAME" + +if [ ! -f "$ZIP_FILE" ]; then + echo "Error: The bundle ZIP file was not found at: $ZIP_FILE" + exit 1 +fi + +# Normalize remote paths +REMOTE_PREFIX="" +if [ -n "$R2_PATH" ]; then + REMOTE_PREFIX="${R2_PATH%/}/" +fi + +REMOTE_ZIP_KEY="${REMOTE_PREFIX}${ZIP_FILENAME}" +REMOTE_MANIFEST_KEY="${REMOTE_PREFIX}manifest.json" + +echo "==========================================" +echo "Uploading H3 RocksDB Bundle to R2" +echo "Source Dir: $DIST_DIR_ABS" +echo "Bundle: $ZIP_FILENAME" +echo "Bucket: $R2_BUCKET" +echo "Prefix: ${REMOTE_PREFIX:-[root]}" +echo "==========================================" + +# 1. Upload the heavy ZIP file first +echo "Uploading $ZIP_FILENAME..." +aws s3 cp "$ZIP_FILE" "s3://$R2_BUCKET/$REMOTE_ZIP_KEY" \ + --endpoint-url "$R2_ENDPOINT" \ + --cache-control "public, max-age=31536000, immutable" + +# 2. Upload manifest last for atomic update execution +echo "Uploading manifest.json..." +aws s3 cp "$MANIFEST_FILE" "s3://$R2_BUCKET/$REMOTE_MANIFEST_KEY" \ + --endpoint-url "$R2_ENDPOINT" \ + --cache-control "no-cache, no-store, must-revalidate" \ + --content-type "application/json" + +echo "Files successfully uploaded." +echo "------------------------------------------" +echo "Applying retention policy (Keeping only the 2 newest ZIP files)..." + +# 3. List all ZIPs in the bucket, sorted chronologically (oldest first) +ZIPS_IN_BUCKET=$(aws s3api list-objects-v2 \ + --endpoint-url "$R2_ENDPOINT" \ + --bucket "$R2_BUCKET" \ + --prefix "$REMOTE_PREFIX" \ + --query "Contents[?ends_with(Key, '.zip')] | sort_by(@, &LastModified)[].Key" \ + --output text) + +# Convert output into a Bash array +read -r -a ZIP_ARRAY <<< "$ZIPS_IN_BUCKET" +TOTAL_ZIPS=${#ZIP_ARRAY[@]} + +echo "$TOTAL_ZIPS ZIP file(s) found in bucket." + +# If more than 2 versions are present, clean up the oldest +if [ "$TOTAL_ZIPS" -gt 2 ]; then + DELETE_COUNT=$((TOTAL_ZIPS - 2)) + echo "Retaining the 2 newest versions. Deleting $DELETE_COUNT older version(s)..." + + for ((i=0; i lookup(double lat, double lng) { - Set osmIds = new HashSet<>(); - - // Query all resolutions used by the importer (4, 6, 9) - // This covers all admin levels: 4 for countries/continents, 6 for states/regions, 9 for districts/cities - int[] resolutions = {4, 6, 9}; - - for (int resolution : resolutions) { - long cellId = h3.latLngToCell(lat, lng, resolution); - osmIds.addAll(getOsmIdsForCell(cellId)); - } - - List results = new ArrayList<>(); - for (Long osmId : osmIds) { - int totalCells = getTotalCells(osmId); - results.add(new BoundaryInfo(osmId, totalCells)); - } - - return results; - } - - /** - * Fetches the H3 cells for a specific lat,lon in all resolutions used by the importer. - * Uses the same resolution logic as StandaloneBoundaryImporter. - */ - public Set getCellsForPoint(double lat, double lng) { - Set cells = new HashSet<>(); - - // Use the same resolutions as the importer (4, 6, 9) - int[] resolutions = {4, 6, 9}; - - for (int resolution : resolutions) { - cells.add(h3.latLngToCell(lat, lng, resolution)); - } - - return cells; - } - - /** - * Fetches all H3 cells belonging to a specific boundary (OSM ID). - * This can be used to compare visited cells against the total cells of a boundary. - */ - public Set getCellsForBoundary(long osmId) { - Set cells = new HashSet<>(); - try { - byte[] val = osmToH3Db.get(longToBytes(osmId)); - if (val != null) { - ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN); - while (bb.hasRemaining()) { - cells.add(bb.getLong()); - } - } - } catch (RocksDBException e) { - logger.error("Failed to lookup cells for OSM ID {} in osm_to_h3", osmId, e); - } - return cells; - } - - private Set getOsmIdsForCell(long cellId) { - Set ids = new HashSet<>(); - try { - byte[] val = h3ToOsmDb.get(longToBytes(cellId)); - if (val != null) { - ByteBuffer bb = ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN); - while (bb.hasRemaining()) { - ids.add(bb.getLong()); - } - } - } catch (RocksDBException e) { - logger.error("Failed to lookup H3 cell {} in h3_to_osm", cellId, e); - } - return ids; - } - - private int getTotalCells(long osmId) { - try { - byte[] val = regionMetadataDb.get(longToBytes(osmId)); - if (val != null && val.length == 4) { - return ByteBuffer.wrap(val).order(ByteOrder.BIG_ENDIAN).getInt(); - } - } catch (RocksDBException e) { - logger.error("Failed to lookup metadata for OSM ID {} in region_metadata", osmId, e); - } - return -1; // Indicates unknown total - } - - private byte[] longToBytes(long v) { - return ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(v).array(); - } - - @PreDestroy - public void cleanup() { - if (this.h3ToOsmDb != null) { - this.h3ToOsmDb.close(); - } - if (this.regionMetadataDb != null) { - this.regionMetadataDb.close(); - } - if (this.osmToH3Db != null) { - this.osmToH3Db.close(); - } - } - - /** - * Returns H3 cells for a point along with their associated OSM boundary IDs. - * Useful for Reitti to track which boundaries are associated with each visited cell. - */ - public List getCellsWithBoundaries(double lat, double lng) { - List result = new ArrayList<>(); - - int[] resolutions = {4, 6, 9}; - - for (int resolution : resolutions) { - long cellId = h3.latLngToCell(lat, lng, resolution); - Set osmIds = getOsmIdsForCell(cellId); - if (!osmIds.isEmpty()) { - result.add(new CellWithBoundaries(cellId, resolution, osmIds)); - } - } - - return result; - } - - /** - * Represents a boundary containing the queried point. - * Reitti can use the totalCells to calculate the percentage visited. - */ - public record BoundaryInfo(long osmId, int totalCells) {} - - /** - * Represents an H3 cell with its associated boundary OSM IDs. - * Useful for tracking which boundaries are affected when a cell is visited. - */ - public record CellWithBoundaries(long cellId, int resolution, Set osmIds) {} -} diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java b/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java new file mode 100644 index 0000000..75261da --- /dev/null +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java @@ -0,0 +1,106 @@ +package com.dedicatedcode.paikka.service.importer; +import de.topobyte.osm4j.core.model.iface.OsmEntity; +import de.topobyte.osm4j.core.model.iface.OsmTag; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; + +public class OsmNameStreamer implements AutoCloseable { + private final BufferedWriter writer; + + public OsmNameStreamer(String outputPath) throws IOException { + this.writer = new BufferedWriter(new FileWriter(outputPath)); + } + + public void processEntity(OsmEntity entity, String type) throws IOException { + long id = entity.getId(); + int numTags = entity.getNumberOfTags(); + + StringBuilder jsonBuilder = new StringBuilder(); + jsonBuilder.append("{"); + boolean hasNames = false; + + for (int i = 0; i < numTags; i++) { + OsmTag tag = entity.getTag(i); + String key = tag.getKey(); + + if ("name".equals(key) || (key != null && key.startsWith("name:"))) { + String value = tag.getValue(); + if (value != null && !value.isBlank()) { + if (hasNames) { + jsonBuilder.append(","); + } + hasNames = true; + + // Build standard JSON key-value pairs + jsonBuilder.append("\"").append(escapeJson(key)).append("\":") + .append("\"").append(escapeJson(value)).append("\""); + } + } + } + jsonBuilder.append("}"); + + if (hasNames) { + String jsonString = jsonBuilder.toString(); + + // Escape the finished JSON string specifically for PG Text-Mode COPY rules + String postgresSafeJson = escapeForPostgresCopy(jsonString); + + // Writes exactly 3 columns matching your schema: osm_id, osm_type, all_names + this.writer.write(id + "\t" + type + "\t" + postgresSafeJson + "\n"); + } + } + + /** + * Step 1: Encodes values to safely fit inside a JSON string property + */ + private String escapeJson(String value) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < value.length(); i++) { + char ch = value.charAt(i); + switch (ch) { + case '"': sb.append("\\\""); break; + case '\\': sb.append("\\\\"); break; + case '\b': sb.append("\\b"); break; + case '\f': sb.append("\\f"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + case '\t': sb.append("\\t"); break; + default: + if (ch < ' ') { + String ss = Integer.toHexString(ch); + sb.append("\\u"); + sb.repeat("0", 4 - ss.length()); + sb.append(ss.toUpperCase()); + } else { + sb.append(ch); + } + } + } + return sb.toString(); + } + + /** + * Step 2: Escapes control characters so Postgres COPY doesn't misinterpret them + */ + private String escapeForPostgresCopy(String text) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < text.length(); i++) { + char ch = text.charAt(i); + switch (ch) { + case '\\': sb.append("\\\\"); break; // Crucial for nested JSON backslashes + case '\t': sb.append("\\t"); break; + case '\n': sb.append("\\n"); break; + case '\r': sb.append("\\r"); break; + default: sb.append(ch); + } + } + return sb.toString(); + } + + @Override + public void close() throws IOException { + writer.flush(); + writer.close(); + } +} \ No newline at end of file diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java index 5d8728b..2c2c04f 100644 --- a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java +++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java @@ -81,6 +81,7 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc Path h3ToOsmPath = out.resolve("h3_to_osm"); Path regionMetaPath = out.resolve("region_metadata"); Path regionGeomPath = out.resolve("region_geometry"); + Path nameSql = out.resolve("osm_names.tsv"); Path tmpH3ToOsmPath = tmp.resolve("tmp_h3_to_osm"); Path tmpRegionMetaPath = tmp.resolve("tmp_region_metadata"); @@ -132,7 +133,8 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString()); RocksDB tmpH3ToOsm = RocksDB.open(cacheOpts, tmpH3ToOsmPath.toString()); RocksDB tmpRegionMeta = RocksDB.open(cacheOpts, tmpRegionMetaPath.toString()); - RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString()) + RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString()); + OsmNameStreamer nameStreamer = new OsmNameStreamer(nameSql.toString()) ) { for (String pbfPath : pbfPaths) { stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways"); @@ -205,6 +207,11 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc EntityContainer c = relIter.next(); if (c.getType() == EntityType.Relation) { OsmRelation r = (OsmRelation) c.getEntity(); + try { + nameStreamer.processEntity(r, "R"); + } catch (IOException e) { + logger.warn("Failed to stream name for relation ID: {}", r.getId(), e); + } if (isAdministrativeBoundary(r)) { batch.add(buildRelationStub(r)); if (batch.size() >= 100) { @@ -248,7 +255,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc try { Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache); if (geom == null || geom.isEmpty()) { - logger.warn("Relation OSM ID: {} [Admin Level: {}] Geometry is null or empty", stub.osmId(), stub.adminLevel()); continue; } @@ -264,7 +270,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc // Simplify first to reduce H3 cell count, then buffer Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(geom, stub.adminLevel()); if (simplified == null || simplified.isEmpty()) { - logger.warn("Simplified Geometry is invalid for OSM ID: {}, using original", stub.osmId()); simplified = geom; } // Buffer to include border-touching cells @@ -305,8 +310,6 @@ public void importBoundaries(List pbfPaths, String outputDir) throws Exc } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - } catch (RocksDBException e) { - throw new RuntimeException(e); } })); } @@ -492,7 +495,7 @@ private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) { */ private int getResolutionForAdminLevel(int adminLevel) { if (adminLevel <= 2) return 4; // Continents/Countries - if (adminLevel <= 6) return 6; // States/Regions + if (adminLevel <= 5) return 6; // States/Regions return 9; // Districts/Cities } diff --git a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java deleted file mode 100644 index 74a5e39..0000000 --- a/src/test/java/com/dedicatedcode/paikka/service/BoundaryLookupServiceTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.dedicatedcode.paikka.service; - -import com.dedicatedcode.paikka.IntegrationTest; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.TestPropertySource; - -import java.util.List; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; - -@IntegrationTest -@TestPropertySource(properties = "paikka.data-dir=/home/daniel/projects/paikka/data-boundaries") -class BoundaryLookupServiceTest { - - @Autowired - private BoundaryLookupService boundaryLookupService; - - @Test - void shouldLookupBoundariesForLuebeck() { - double lat = 53.86422; - double lng = 10.69120; - - List results = boundaryLookupService.lookup(lat, lng); - - assertFalse(results.isEmpty(), "Should find at least one boundary for Luebeck coordinates"); - - List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); - - assertTrue(osmIds.contains(367855L), "Should contain Innenstadt (OSM ID 367855)"); - assertTrue(osmIds.contains(27027L), "Should contain Lübeck (OSM ID 62422)"); - assertTrue(osmIds.contains(51529L), "Should contain Schleswig-Holstein (OSM ID 51529)"); - assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); - - assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); - } - - @Test - void shouldLookupBoundariesForPoelzig() { - double lat = 50.957171; - double lng = 12.208514; - - List results = boundaryLookupService.lookup(lat, lng); - - assertFalse(results.isEmpty(), "Should find at least one boundary for Pölzig coordinates"); - - List osmIds = results.stream().map(BoundaryLookupService.BoundaryInfo::osmId).toList(); - - assertTrue(osmIds.contains(2532078L), "Should contain (OSM ID 2532078)"); - assertTrue(osmIds.contains(2907045L), "Should contain (OSM ID 2907045)"); - assertTrue(osmIds.contains(62445L), "Should contain (OSM ID 62445)"); - assertTrue(osmIds.contains(62366L), "Should contain (OSM ID 62366)"); - assertTrue(osmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); - - assertTrue(results.stream().anyMatch(b -> b.totalCells() > 0), "At least one boundary should have a positive total cell count"); - } - - @Test - void shouldGetCellsForPoint() { - double lat = 53.86422; - double lng = 10.69120; - - Set cells = boundaryLookupService.getCellsForPoint(lat, lng); - - assertEquals(3, cells.size(), "Should return exactly 3 cells for the different resolutions"); - } - - @Test - void shouldGetCellsWithBoundaries() { - double lat = 53.86422; - double lng = 10.69120; - - List cellsWithBoundaries = boundaryLookupService.getCellsWithBoundaries(lat, lng); - - assertFalse(cellsWithBoundaries.isEmpty(), "Should return cells with boundaries for Luebeck coordinates"); - - // Should have cells for different resolutions (4, 6, 9) - Set resolutions = cellsWithBoundaries.stream() - .map(BoundaryLookupService.CellWithBoundaries::resolution) - .collect(java.util.stream.Collectors.toSet()); - assertTrue(resolutions.contains(4), "Should include resolution 4 (countries/continents)"); - assertTrue(resolutions.contains(6), "Should include resolution 6 (states/regions)"); - assertTrue(resolutions.contains(9), "Should include resolution 9 (districts/cities)"); - - // Each cell should have associated OSM IDs - for (BoundaryLookupService.CellWithBoundaries cell : cellsWithBoundaries) { - assertFalse(cell.osmIds().isEmpty(), "Each cell should have associated OSM boundary IDs"); - assertTrue(cell.cellId() > 0, "Cell ID should be positive"); - } - - // Should contain expected OSM IDs across all cells - Set allOsmIds = cellsWithBoundaries.stream() - .flatMap(cell -> cell.osmIds().stream()) - .collect(java.util.stream.Collectors.toSet()); - - assertTrue(allOsmIds.contains(367855L), "Should contain Innenstadt (OSM ID 367855)"); - assertTrue(allOsmIds.contains(27027L), "Should contain Lübeck (OSM ID 27027)"); - assertTrue(allOsmIds.contains(51529L), "Should contain Schleswig-Holstein (OSM ID 51529)"); - assertTrue(allOsmIds.contains(51477L), "Should contain Germany (OSM ID 51477)"); - } - - @Test - void shouldGetCellsWithBoundariesForEmptyArea() { - // Use coordinates in the middle of the ocean where no boundaries should exist - double lat = 0.0; - double lng = 0.0; - - List cellsWithBoundaries = boundaryLookupService.getCellsWithBoundaries(lat, lng); - - // Should return empty list or cells with no OSM IDs - assertTrue(cellsWithBoundaries.isEmpty() || - cellsWithBoundaries.stream().allMatch(cell -> cell.osmIds().isEmpty()), - "Should return no cells with boundaries for ocean coordinates"); - } -} diff --git a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java index 75bb808..44c535d 100644 --- a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java +++ b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java @@ -65,6 +65,13 @@ void setUp() throws Exception { PaikkaConfiguration.ImportConfiguration importConfiguration = new PaikkaConfiguration.ImportConfiguration(); importConfiguration.setThreads(2); config.setImportConfiguration(importConfiguration); + PaikkaConfiguration.SimplificationConfiguration simplificationConfiguration = new PaikkaConfiguration.SimplificationConfiguration(); + simplificationConfiguration.setContinentTolerance(0.005); + simplificationConfiguration.setCountryTolerance(0.00045); + simplificationConfiguration.setStateTolerance(0.00009); + simplificationConfiguration.setPoiTolerance(0.000018); + simplificationConfiguration.setDefaultTolerance(0.000045); + config.setSimplificationConfiguration(simplificationConfiguration); GeometrySimplificationService geometrySimplificationService = new GeometrySimplificationService(config); S2Helper s2Helper = new S2Helper(); From 382fafbaca6886f1629005dd5d909409ee623e3a Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Tue, 28 Jul 2026 17:36:28 +0200 Subject: [PATCH 27/29] feat: add H3 bundle pipeline script for streamlined data processing and upload - Introduced `scripts/update-h3.sh` to automate H3 bundle generation. - Supports configurable options for memory, threads, and input paths. - Includes filtering, data import, bundling, and optional upload to R2 storage. - Enhanced error handling and directory preparation for seamless execution. --- scripts/update-h3.sh | 381 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 scripts/update-h3.sh diff --git a/scripts/update-h3.sh b/scripts/update-h3.sh new file mode 100644 index 0000000..2085010 --- /dev/null +++ b/scripts/update-h3.sh @@ -0,0 +1,381 @@ +#!/bin/bash + +# +# This file is part of paikka. +# +# Paikka is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 or +# any later version. +# +# Paikka is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# You should have received a copy of the GNU Affero General Public License +# along with Paikka. If not, see . +# + +# ============================================================================== +# PAIKKA H3 Bundle Pipeline +# ============================================================================== +# Single command to download, filter, import, zip, and upload H3 bundles. +# +# Usage: +# ./scripts/build-h3-bundle.sh [OPTIONS] +# +# Options: +# --env-file PATH Path to .env file (default: ./scripts/.env) +# --data-dir PATH Directory for import data (default: ./data) +# --jar-file PATH Path to PAIKKA jar (auto-detected if not provided) +# --memory SIZE JVM heap size (default: 16g) +# --threads NUM Import threads (default: 10) +# --pbf-file PATH Use local PBF file instead of downloading +# --version STR Bundle version (default: YYYY-MM-DD-v1) +# --output-dir PATH Bundle output directory (default: ./dist) +# --no-upload Skip R2 upload (local bundle only) +# -h, --help Show this help message +# ============================================================================== + +set -e +set -o pipefail + +# ============================================================================== +# SCRIPT CONFIGURATION AND GLOBAL DEFAULTS +# ============================================================================== + +# --- General Settings --- +PLANET_URL="https://planet.osm.org/pbf/planet-latest.osm.pbf" +LOCAL_WORK_DIR="$(pwd)" +PBF_INPUT_FILE="planet-latest.osm.pbf" +PBF_FILTERED_FILE="planet-filtered.pbf" +DOCKER_IMAGE="dedicatedcode/paikka:latest" + +# --- Local Paths --- +DOWNLOAD_DIR="${DOWNLOAD_DIR:-$LOCAL_WORK_DIR}" +DATA_DIR="${DATA_DIR:-$LOCAL_WORK_DIR}" + +# --- Import Settings --- +IMPORT_MEMORY="${IMPORT_MEMORY:-16g}" +IMPORT_THREADS="${IMPORT_THREADS:-10}" +JAR_FILE="${JAR_FILE:-}" + +# --- Bundle Settings --- +VERSION="${VERSION:-$(date +%Y-%m-%d)-v1}" +DOWNLOAD_BASE_URL="${DOWNLOAD_BASE_URL:-https://h3-osm.dedicatedcode.com}" +BUNDLE_OUTPUT_DIR="${BUNDLE_OUTPUT_DIR:-$LOCAL_WORK_DIR/dist}" + +# --- R2 Upload Settings (from .env) --- +R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-}" +R2_ACCESS_KEY_ID="${R2_ACCESS_KEY_ID:-}" +R2_SECRET_ACCESS_KEY="${R2_SECRET_ACCESS_KEY:-}" +R2_BUCKET="${R2_BUCKET:-}" +R2_PATH="${R2_PATH:-}" + +# --- Script Flags --- +PBF_INPUT_PATH="" +NO_UPLOAD=false + +# ============================================================================== +# HELPER FUNCTIONS +# ============================================================================== + +log() { + echo -e "\n[$(date +'%Y-%m-%d %H:%M:%S')] --- $1 ---" +} + +# ============================================================================== +# CORE LOGIC FUNCTIONS +# ============================================================================== + +### +# Parses command-line arguments and loads environment configuration. +### +parse_args_and_configure() { + log "Step 0: Parsing arguments and setting configuration" + + # Load .env file if it exists (environment variables can override these) + ENV_FILE="./scripts/.env" + if [ -f "$ENV_FILE" ]; then + echo "Loading configuration from $ENV_FILE" + set -a + source "$ENV_FILE" + set +a + fi + + # Parse command-line arguments (highest precedence) + while [[ $# -gt 0 ]]; do + case $1 in + --env-file) + ENV_FILE="$2" + if [ -f "$ENV_FILE" ]; then + echo "Loading configuration from $ENV_FILE" + set -a + source "$ENV_FILE" + set +a + else + echo "Error: .env file not found: $ENV_FILE" + exit 1 + fi + shift 2 + ;; + --data-dir) + DATA_DIR="$2" + shift 2 + ;; + --jar-file) + JAR_FILE="$2" + shift 2 + ;; + --memory) + IMPORT_MEMORY="$2" + shift 2 + ;; + --threads) + IMPORT_THREADS="$2" + shift 2 + ;; + --pbf-file) + PBF_INPUT_PATH="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --output-dir) + BUNDLE_OUTPUT_DIR="$2" + shift 2 + ;; + --no-upload) + NO_UPLOAD=true + shift + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --env-file PATH Path to .env file (default: ./scripts/.env)" + echo " --data-dir PATH Directory for import data (default: ./data)" + echo " --jar-file PATH Path to PAIKKA jar (auto-detected if not provided)" + echo " --memory SIZE JVM heap size (default: 16g)" + echo " --threads NUM Import threads (default: 10)" + echo " --pbf-file PATH Use local PBF file instead of downloading" + echo " --version STR Bundle version (default: YYYY-MM-DD-v1)" + echo " --output-dir PATH Bundle output directory (default: ./dist)" + echo " --no-upload Skip R2 upload (local bundle only)" + echo " -h, --help Show this help message" + exit 1 + ;; + *) + echo "Error: Unknown option: $1" + echo "" + echo "Usage: $0 [OPTIONS]" + exit 1 + ;; + esac + done + + # Re-apply environment variable defaults (env vars take precedence over .env file) + DOWNLOAD_DIR="${DOWNLOAD_DIR:-$LOCAL_WORK_DIR}" + DATA_DIR="${DATA_DIR:-$LOCAL_WORK_DIR}" + IMPORT_MEMORY="${IMPORT_MEMORY:-16g}" + IMPORT_THREADS="${IMPORT_THREADS:-10}" + VERSION="${VERSION:-$(date +%Y-%m-%d)-v1}" + BUNDLE_OUTPUT_DIR="${BUNDLE_OUTPUT_DIR:-$LOCAL_WORK_DIR/dist}" + + # Validate PBF input if provided + if [ -n "$PBF_INPUT_PATH" ] && [ ! -f "$PBF_INPUT_PATH" ]; then + echo "Error: PBF file not found: $PBF_INPUT_PATH" + exit 1 + fi + + # Auto-detect JAR file if not provided + if [ -z "$JAR_FILE" ]; then + JAR_FILE=$(find target -name "paikka-*.jar" -not -name "*-sources.jar" 2>/dev/null | head -1) + fi + + # Validate JAR file + if [ -n "$JAR_FILE" ] && [ ! -f "$JAR_FILE" ]; then + echo "Error: JAR file not found: $JAR_FILE" + exit 1 + fi + + # Display configuration + echo "==========================================" + echo "H3 Bundle Pipeline Configuration" + echo "==========================================" + echo " Data directory: $DATA_DIR" + echo " Import memory: $IMPORT_MEMORY" + echo " Import threads: $IMPORT_THREADS" + echo " JAR file: ${JAR_FILE:-auto-detect}" + echo " Bundle version: $VERSION" + echo " Bundle output: $BUNDLE_OUTPUT_DIR" + echo " Skip upload: $NO_UPLOAD" + if [ -n "$PBF_INPUT_PATH" ]; then + echo " PBF input: $PBF_INPUT_PATH" + else + echo " PBF input: Download from $PLANET_URL" + fi + echo "==========================================" + + # Validate R2 upload settings (only if upload is enabled) + if [ "$NO_UPLOAD" = false ]; then + local missing_vars=() + [ -z "$R2_ACCOUNT_ID" ] && missing_vars+=("R2_ACCOUNT_ID") + [ -z "$R2_ACCESS_KEY_ID" ] && missing_vars+=("R2_ACCESS_KEY_ID") + [ -z "$R2_SECRET_ACCESS_KEY" ] && missing_vars+=("R2_SECRET_ACCESS_KEY") + [ -z "$R2_BUCKET" ] && missing_vars+=("R2_BUCKET") + + if [ ${#missing_vars} -gt 0 ]; then + echo "" + echo "Error: Missing required R2 configuration:" + for var in ${missing_vars}; do + echo " - $var" + done + echo "" + echo "Provide via --env-file, .env file, or environment variables." + exit 1 + fi + fi +} + +### +# LOCAL: Creates the necessary working directories. +### +local_prepare_directories() { + log "Step 1: Preparing directories" + mkdir -p "$DOWNLOAD_DIR" + mkdir -p "$DATA_DIR" + mkdir -p "$BUNDLE_OUTPUT_DIR" +} + +### +# LOCAL: Downloads the latest OSM planet file. +### +local_download_planet_file() { + if [ -n "$PBF_INPUT_PATH" ]; then + log "Step 2a: Using provided PBF file – skipping download" + return 0 + fi + + log "Step 2a: Downloading latest OSM planet file" + cd "$DOWNLOAD_DIR" + wget -N "$PLANET_URL" +} + +### +# LOCAL: Pulls the latest version of the Paikka Docker image. +### +local_pull_docker_image() { + log "Step 2b: Pulling latest Docker image: $DOCKER_IMAGE" + sudo docker pull "$DOCKER_IMAGE" +} + +### +# LOCAL: Filters the PBF file using the Paikka container. +### +local_filter_pbf() { + log "Step 3: Filtering PBF file" + + if [ -n "$PBF_INPUT_PATH" ]; then + INPUT_DIR="$(dirname "$PBF_INPUT_PATH")" + INPUT_FILE="$(basename "$PBF_INPUT_PATH")" + sudo docker run --rm \ + -v "$INPUT_DIR":/input \ + -v "$DOWNLOAD_DIR":/data \ + "$DOCKER_IMAGE" prepare-boundaries "/input/$INPUT_FILE" "/data/$PBF_FILTERED_FILE" + else + sudo docker run --rm \ + -v "$DOWNLOAD_DIR":/data \ + "$DOCKER_IMAGE" prepare-boundaries "/data/$PBF_INPUT_FILE" "/data/$PBF_FILTERED_FILE" + fi +} + +### +# LOCAL: Runs the Java H3 import. +### +local_import_h3() { + log "Step 4: Running H3 import" + + local PBF_TO_IMPORT="$DOWNLOAD_DIR/$PBF_FILTERED_FILE" + + cd "$LOCAL_WORK_DIR" + ./scripts/import-boundaries.sh \ + --jar-file "$JAR_FILE" \ + --data-dir "$DATA_DIR" \ + --memory "$IMPORT_MEMORY" \ + --threads "$IMPORT_THREADS" \ + "$PBF_TO_IMPORT" +} + +### +# LOCAL: Removes intermediate PBF files. +### +local_cleanup_pbf() { + log "Step 5: Cleaning up intermediate PBF files" + cd "$DOWNLOAD_DIR" + rm -f "$PBF_FILTERED_FILE" + if [ -z "$PBF_INPUT_PATH" ]; then + rm -f "$PBF_INPUT_FILE" + fi + echo "Cleaned up filtered PBF file" +} + +### +# LOCAL: Creates the H3 RocksDB bundle ZIP and manifest. +### +local_create_bundle() { + log "Step 6: Creating H3 bundle" + + ./scripts/create-h3-bundle.sh \ + --db-dir "$DATA_DIR" \ + --version "$VERSION" \ + --url "$DOWNLOAD_BASE_URL" \ + --output-dir "$BUNDLE_OUTPUT_DIR" +} + +### +# LOCAL: Uploads the bundle to Cloudflare R2. +### +local_upload_bundle() { + if [ "$NO_UPLOAD" = true ]; then + log "Step 7: Skipping R2 upload (--no-upload)" + return 0 + fi + + log "Step 7: Uploading bundle to R2" + + ./scripts/upload-h3-bundle.sh \ + --dist-dir "$BUNDLE_OUTPUT_DIR" +} + +# ============================================================================== +# MAIN ORCHESTRATION FUNCTION +# ============================================================================== + +main() { + parse_args_and_configure "$@" + local_prepare_directories + local_download_planet_file + local_pull_docker_image + local_filter_pbf + local_import_h3 + local_cleanup_pbf + local_create_bundle + local_upload_bundle + + log "H3 bundle pipeline completed successfully" + echo "==========================================" + echo " Bundle: $BUNDLE_OUTPUT_DIR/h3-rocksdb-${VERSION}.zip" + echo " Manifest: $BUNDLE_OUTPUT_DIR/manifest.json" + echo "==========================================" +} + +# ============================================================================== +# SCRIPT ENTRYPOINT +# ============================================================================== + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi \ No newline at end of file From 329c3ddc156251fedaa8817dd90d66af3186d59c Mon Sep 17 00:00:00 2001 From: "Daniel Graf (aider-ce)" Date: Tue, 28 Jul 2026 17:39:40 +0200 Subject: [PATCH 28/29] feat: add H3 bundle pipeline script with Docker integration --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 8a15915..00d837e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,11 @@ COPY scripts/* $APP_HOME/ RUN ln -s $APP_HOME/filter_osm.sh /usr/bin/prepare RUN ln -s $APP_HOME/import.sh /usr/bin/import +RUN ln -s $APP_HOME/import-boundaries.sh /usr/bin/import-boundaries RUN chmod +x /usr/bin/prepare RUN chmod +x /usr/bin/import +RUN chmod +x /usr/bin/import-boundaries # Create a script to start the application with configurable UID/GID RUN cat <<'EOF' > /entrypoint.sh @@ -62,6 +64,10 @@ elif [ "$1" = "import" ]; then echo "Running script: $1" shift exec runuser -u paikka -- import --jar-file "$APP_HOME/app.jar" "$@" +elif [ "$1" = "import-boundaries" ]; then + echo "Running script: $1" + shift + exec runuser -u paikka -- import-boundaries --jar-file "$APP_HOME/app.jar" "$@" fi # Default: Execute the Java application From 16e47f9dea4d5117f1e90c64ea50271a65e5d377 Mon Sep 17 00:00:00 2001 From: Daniel Graf Date: Tue, 28 Jul 2026 18:56:30 +0200 Subject: [PATCH 29/29] feat: add support for boundary preparation and streamline cleanup - Added `prepare-boundaries` command with symlink and execution setup in Dockerfile. - Updated `update-h3.sh` to rename filtered PBF output and disable Docker image pull. - Fixed `filter_boundaries.sh` to correct typo in description. - Simplified `import-boundaries.sh` by removing unnecessary cleanup steps. --- .gitignore | 2 +- Dockerfile | 4 +++- scripts/filter_boundaries.sh | 2 +- scripts/import-boundaries.sh | 20 -------------------- scripts/update-h3.sh | 4 ++-- 5 files changed, 7 insertions(+), 25 deletions(-) mode change 100644 => 100755 scripts/update-h3.sh diff --git a/.gitignore b/.gitignore index 3d06e25..a824681 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,4 @@ build/ /data/ /.idea/ /data-boundaries/ -/scripts/.env +/.env diff --git a/Dockerfile b/Dockerfile index 00d837e..75239ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,10 +29,12 @@ COPY --chown=paikka:paikka target/*.jar $APP_HOME/app.jar COPY scripts/* $APP_HOME/ RUN ln -s $APP_HOME/filter_osm.sh /usr/bin/prepare +RUN ln -s $APP_HOME/filter_boundaries.sh /usr/bin/prepare-boundaries RUN ln -s $APP_HOME/import.sh /usr/bin/import RUN ln -s $APP_HOME/import-boundaries.sh /usr/bin/import-boundaries RUN chmod +x /usr/bin/prepare +RUN chmod +x /usr/bin/prepare-boundaries RUN chmod +x /usr/bin/import RUN chmod +x /usr/bin/import-boundaries @@ -56,7 +58,7 @@ chown -R paikka:paikka $STATS_DIR cd $DATA_DIR # Check if the first argument is a known script -if [ "$1" = "prepare" ]; then +if [ "$1" = "prepare" ] || [ "$1" = "prepare-boundaries" ]; then echo "Running script: $1" shift exec runuser -u paikka -- prepare "$@" diff --git a/scripts/filter_boundaries.sh b/scripts/filter_boundaries.sh index 318bc71..a098f9b 100755 --- a/scripts/filter_boundaries.sh +++ b/scripts/filter_boundaries.sh @@ -7,7 +7,7 @@ usage() { echo "Usage: $0 " echo "" - echo "Filters an OSM PBF file to keep only boundaries relevant for RAITTI:" + echo "Filters an OSM PBF file to keep only boundaries relevant for REITTI:" echo " - Points of Interest (amenity, shop, tourism, leisure, etc.)" echo " - Administrative boundaries" echo "" diff --git a/scripts/import-boundaries.sh b/scripts/import-boundaries.sh index 47e664d..3a60b1a 100755 --- a/scripts/import-boundaries.sh +++ b/scripts/import-boundaries.sh @@ -172,27 +172,7 @@ if [ $EXIT_CODE -eq 0 ]; then echo "" echo "✓ Import completed successfully" echo "✓ Data directory: $DATA_DIR" - - # Clean up temporary files to save disk space - echo "" - echo "🧹 Cleaning up temporary files..." - - # Remove node_cache - TEMP_DIR="$DATA_DIR/node_cache" - if [ -d "$TEMP_DIR" ]; then - echo " Removing temporary directory: $TEMP_DIR" - rm -rf "$TEMP_DIR" - echo " ✓ Node cache cleaned up" - fi - # Remove grid_index - GRID_DIR="$DATA_DIR/grid_index" - if [ -d "$GRID_DIR" ]; then - echo " Removing temporary directory: $GRID_DIR" - rm -rf "$GRID_DIR" - echo " ✓ Grid index cleaned up" - fi - else echo "" echo "✗ Import failed" diff --git a/scripts/update-h3.sh b/scripts/update-h3.sh old mode 100644 new mode 100755 index 2085010..5abdbca --- a/scripts/update-h3.sh +++ b/scripts/update-h3.sh @@ -48,7 +48,7 @@ set -o pipefail PLANET_URL="https://planet.osm.org/pbf/planet-latest.osm.pbf" LOCAL_WORK_DIR="$(pwd)" PBF_INPUT_FILE="planet-latest.osm.pbf" -PBF_FILTERED_FILE="planet-filtered.pbf" +PBF_FILTERED_FILE="planet-boundaries-filtered.pbf" DOCKER_IMAGE="dedicatedcode/paikka:latest" # --- Local Paths --- @@ -358,7 +358,7 @@ main() { parse_args_and_configure "$@" local_prepare_directories local_download_planet_file - local_pull_docker_image +# local_pull_docker_image local_filter_pbf local_import_h3 local_cleanup_pbf