errorSamples = new ConcurrentLinkedQueue<>();
+
+ private final AtomicLong nodesCached = new AtomicLong(0);
+ private final AtomicLong waysCached = new AtomicLong(0);
+ private final AtomicLong relationsFound = new AtomicLong(0);
+ private final AtomicLong relationsProcessed = new AtomicLong(0);
+ private final AtomicLong h3CellsGenerated = new AtomicLong(0);
+
+ private volatile String currentPhase = "Initializing";
+ private volatile boolean running = true;
+ private final long startTime = System.currentTimeMillis();
+ private volatile long phaseStartTime = System.currentTimeMillis();
+ private long totalTime;
+
+ private final int TOTAL_STEPS = 2;
+ private int currentStep = 0;
+
+ public long getNodesCached() {
+ return nodesCached.get();
+ }
+
+ public void incrementNodesCached() {
+ nodesCached.incrementAndGet();
+ }
+
+ public long getWaysCached() {
+ return waysCached.get();
+ }
+
+ public void incrementWaysCached() {
+ waysCached.incrementAndGet();
+ }
+
+ public long getRelationsFound() {
+ return relationsFound.get();
+ }
+
+ public void incrementRelationsFound() {
+ relationsFound.incrementAndGet();
+ }
+
+ public long getRelationsProcessed() {
+ return relationsProcessed.get();
+ }
+
+ public void incrementRelationsProcessed() {
+ relationsProcessed.incrementAndGet();
+ }
+
+ public long getH3CellsGenerated() {
+ return h3CellsGenerated.get();
+ }
+
+ public void addH3CellsGenerated(long count) {
+ h3CellsGenerated.addAndGet(count);
+ }
+
+ public String getCurrentPhase() {
+ return currentPhase;
+ }
+
+ public void setCurrentPhase(int step, String phase) {
+ this.currentPhase = phase;
+ this.phaseStartTime = System.currentTimeMillis();
+ this.currentStep = step;
+ }
+
+ public long getPhaseStartTime() {
+ return phaseStartTime;
+ }
+
+ public boolean isRunning() {
+ return running;
+ }
+
+ public void stop() {
+ this.running = false;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public long getTotalTime() {
+ return totalTime;
+ }
+
+ public void setTotalTime(long t) {
+ this.totalTime = t;
+ }
+
+ public void recordError(Stage stage, Kind kind, Long osmId, String operation, Exception e) {
+ errorsTotal.incrementAndGet();
+
+ String safePhase = stage.toString();
+ String safeKind = kind.toString();
+ String safeOp = operation != null ? operation : "-";
+ String ex = (e != null) ? e.getClass().getSimpleName() : "Exception";
+ String bucketKey = safePhase + "|" + safeKind + "|" + safeOp + "|" + ex;
+
+ errorBuckets.computeIfAbsent(bucketKey, k -> new AtomicLong(0)).incrementAndGet();
+
+ if (errorSamples.size() < ERROR_SAMPLE_LIMIT) {
+ String msg = (e != null ? e.getMessage() : null);
+ errorSamples.add(
+ "phase=" + safePhase
+ + " kind=" + safeKind
+ + " id=" + (osmId != null ? osmId : "-")
+ + " op=" + safeOp
+ + " ex=" + (e != null ? e.getClass().getName() : "java.lang.Exception")
+ + (msg != null ? " msg=" + msg : "")
+ );
+ }
+ }
+
+ public long getErrorsTotal() {
+ return errorsTotal.get();
+ }
+
+ public String getMemoryStats() {
+ Runtime r = Runtime.getRuntime();
+ long used = (r.totalMemory() - r.freeMemory()) / 1024 / 1024 / 1024;
+ long max = r.maxMemory() / 1024 / 1024 / 1024;
+ return String.format("%dGB/%dGB", used, max);
+ }
+
+ public void startProgressReporter() {
+ boolean isTty = System.console() != null;
+
+ Thread.ofPlatform().daemon().start(() -> {
+ while (isRunning()) {
+ long elapsed = System.currentTimeMillis() - getStartTime();
+ long phaseElapsed = System.currentTimeMillis() - getPhaseStartTime();
+ double phaseSeconds = phaseElapsed / 1000.0;
+
+ String phase = getCurrentPhase();
+ StringBuilder sb = new StringBuilder();
+
+ if (isTty) {
+ sb.append("\r\033[K");
+ }
+
+ sb.append(String.format("\033[1;90m[%d/%d]\033[0m ", currentStep, TOTAL_STEPS));
+
+ if (phase.contains("1.1")) {
+ long nodesPerSec = phaseSeconds > 0 ? (long) (getNodesCached() / phaseSeconds) : 0;
+ sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mCaching Nodes & Ways\033[0m", formatTime(elapsed)));
+ sb.append(String.format(" │ \033[32mNodes:\033[0m %s \033[33m(%s/s)\033[0m",
+ formatCompactNumber(getNodesCached()), formatCompactRate(nodesPerSec)));
+ sb.append(String.format(" │ \033[34mWays:\033[0m %s", formatCompactNumber(getWaysCached())));
+ } else if (phase.contains("2.1")) {
+ long relsPerSec = phaseSeconds > 0 ? (long) (getRelationsProcessed() / phaseSeconds) : 0;
+ double percentage = getRelationsFound() > 0 ? (double) getRelationsProcessed() / getRelationsFound() * 100.0 : 0.0;
+ sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mProcessing Relations & H3\033[0m", formatTime(elapsed)));
+ sb.append(String.format(" │ \033[32mRelations:\033[0m %s/%s \033[33m(%s/s)\033[0m",
+ formatCompactNumber(getRelationsProcessed()), formatCompactNumber(getRelationsFound()), formatCompactRate(relsPerSec)));
+ sb.append(String.format(" │ \033[35mProgress:\033[0m %.2f%%", percentage));
+ sb.append(String.format(" │ \033[36mH3 Cells:\033[0m %s", formatCompactNumber(getH3CellsGenerated())));
+ } else {
+ sb.append(String.format("\033[1;36m[%s]\033[0m %s", formatTime(elapsed), phase));
+ }
+
+ sb.append(String.format(" │ \033[31mHeap:\033[0m %s", getMemoryStats()));
+
+ if (isTty) {
+ System.out.print(sb);
+ System.out.flush();
+ } else {
+ System.out.println(sb);
+ }
+
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ if (isTty) System.out.println();
+ });
+ }
+
+ public void printFinalStatistics() {
+ System.out.println("\n\033[1;36m" + "═".repeat(80) + "\n" + centerText("🎯 BOUNDARY IMPORT STATISTICS") + "\n" + "═".repeat(80) + "\033[0m");
+
+ long totalTime = Math.max(1, getTotalTime());
+ double totalSeconds = totalTime / 1000.0;
+
+ System.out.printf("\n\033[1;37m⏱️ Total Import Time:\033[0m \033[1;33m%s\033[0m%n%n", formatTime(getTotalTime()));
+
+ System.out.println("\033[1;37m📊 Processing Summary:\033[0m");
+ System.out.println("┌────────────────────┬─────────────────┬─────────────────┐");
+ System.out.println("│ \033[1mEntity Type\033[0m │ \033[1mTotal Count\033[0m │ \033[1mAvg Speed\033[0m │");
+ System.out.println("├────────────────────┼─────────────────┼─────────────────┤");
+ System.out.printf("│ \033[32mNodes Cached\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber(getNodesCached()),
+ formatCompactNumber((long) (getNodesCached() / totalSeconds)));
+ System.out.printf("│ \033[34mWays Cached\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber(getWaysCached()),
+ formatCompactNumber((long) (getWaysCached() / totalSeconds)));
+ System.out.printf("│ \033[35mRelations Found\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber(getRelationsFound()),
+ formatCompactNumber((long) (getRelationsFound() / totalSeconds)));
+ System.out.printf("│ \033[36mRelations Processed\033[0m│ %15s │ %13s/s │%n",
+ formatCompactNumber(getRelationsProcessed()),
+ formatCompactNumber((long) (getRelationsProcessed() / totalSeconds)));
+ System.out.printf("│ \033[33mH3 Cells Generated\033[0m │ %15s │ %13s/s │%n",
+ formatCompactNumber(getH3CellsGenerated()),
+ formatCompactNumber((long) (getH3CellsGenerated() / totalSeconds)));
+ System.out.println("└────────────────────┴─────────────────┴─────────────────┘");
+
+ System.out.println();
+ }
+
+ public void printOutcomeAndErrors() {
+ long err = getErrorsTotal();
+ long denominator = Math.max(1L, getRelationsFound());
+ double rate = (double) err / (double) denominator;
+
+ String outcome = (rate >= DEGRADED_WARN_RATE) ? "DEGRADED" : "OK";
+ System.out.println("\n\033[1;36mIMPORT OUTCOME: " + outcome
+ + " | errors=" + err
+ + " | relationsFound=" + denominator
+ + " | errorRate=" + String.format(Locale.ROOT, "%.6f%%", rate * 100.0)
+ + "\033[0m");
+
+ if (err == 0) {
+ return;
+ }
+
+ System.err.println("\n=== Boundary import errors summary (best-effort) ===");
+ System.err.println("totalErrors=" + err);
+ System.err.println("topBuckets=" + Math.min(10, errorBuckets.size()) + "/" + errorBuckets.size());
+
+ errorBuckets.entrySet().stream()
+ .sorted((a, b) -> Long.compare(b.getValue().get(), a.getValue().get()))
+ .limit(10)
+ .forEach(e -> System.err.println(" " + e.getValue().get() + "x " + e.getKey()));
+
+ if (!errorSamples.isEmpty()) {
+ System.err.println("\nSamples (first " + errorSamples.size() + "):");
+ for (String s : errorSamples) {
+ System.err.println(" " + s);
+ }
+ }
+
+ System.err.println("=== End boundary import errors summary ===\n");
+ }
+
+ private String formatTime(long ms) {
+ long s = ms / 1000;
+ return String.format("%d:%02d:%02d", s / 3600, (s % 3600) / 60, s % 60);
+ }
+
+ private String formatCompactNumber(long n) {
+ if (n < 1000) return String.valueOf(n);
+ if (n < 1_000_000) return String.format("%.2fk", n / 1000.0);
+ return String.format("%.3fM", n / 1_000_000.0);
+ }
+
+ private String formatCompactRate(long n) {
+ if (n < 1000) return String.valueOf(n);
+ if (n < 1_000_000) return String.format("%.1fk", n / 1000.0);
+ return String.format("%.1fM", n / 1_000_000.0);
+ }
+
+ private String centerText(String text) {
+ int pad = (80 - text.length()) / 2;
+ return " ".repeat(Math.max(0, pad)) + text;
+ }
+
+ public void printPhaseHeader(String phase) {
+ System.out.println("\n\033[1;36m" + "─".repeat(80) + "\n" + phase + "\n" + "─".repeat(80) + "\033[0m");
+ }
+
+ public void printSuccess() {
+ System.out.println("\n\033[1;32m" + "=".repeat(80) + "\n" + centerText("BOUNDARY IMPORT COMPLETED SUCCESSFULLY") + "\n" + "=".repeat(80) + "\033[0m");
+ }
+
+ public void printError(String message) {
+ System.out.println("\n\033[1;31m" + "=".repeat(80) + "\n" + centerText(message) + "\n" + "=".repeat(80) + "\033[0m");
+ }
+
+ public void printPhaseSummary(String phaseName, long phaseStartTime) {
+ long phaseTime = System.currentTimeMillis() - phaseStartTime;
+ System.out.printf("\n\u001B[1;32m✓ %s COMPLETED\u001B[0m \u001B[2m(%s)\u001B[0m%n", phaseName, formatTime(phaseTime));
+ }
+}
diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java b/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java
index 04907db..64b5efc 100644
--- a/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java
+++ b/src/main/java/com/dedicatedcode/paikka/service/importer/GeometrySimplificationService.java
@@ -16,6 +16,7 @@
package com.dedicatedcode.paikka.service.importer;
+import com.dedicatedcode.paikka.config.PaikkaConfiguration;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.simplify.DouglasPeuckerSimplifier;
import org.springframework.stereotype.Service;
@@ -26,13 +27,20 @@
*/
@Service
public class GeometrySimplificationService {
-
- // Tolerance guidelines from implementation blueprint
- private static final double COUNTRY_TOLERANCE = 0.00045; // 50 meters for country borders
- private static final double STATE_TOLERANCE = 0.00009; // 10 meters for state/city
- private static final double POI_TOLERANCE = 0.000018; // 2 meters for POI boundaries
- private static final double DEFAULT_TOLERANCE = 0.000045; // 5 meters default
-
+
+ private final double continentTolerance;
+ private final double countryTolerance;
+ private final double stateTolerance;
+ private final double poiTolerance;
+ private final double defaultTolerance;
+
+ public GeometrySimplificationService(PaikkaConfiguration paikkaConfiguration) {
+ this.continentTolerance = paikkaConfiguration.getSimplificationConfiguration().getContinentTolerance();
+ this.countryTolerance = paikkaConfiguration.getSimplificationConfiguration().getCountryTolerance();
+ this.stateTolerance = paikkaConfiguration.getSimplificationConfiguration().getStateTolerance();
+ this.poiTolerance = paikkaConfiguration.getSimplificationConfiguration().getPoiTolerance();
+ this.defaultTolerance = paikkaConfiguration.getSimplificationConfiguration().getDefaultTolerance();
+ }
/**
* Simplify geometry using Douglas-Peucker algorithm with default tolerance.
*
@@ -40,7 +48,7 @@ public class GeometrySimplificationService {
* @return Simplified geometry
*/
public Geometry simplify(Geometry geometry) {
- return simplify(geometry, DEFAULT_TOLERANCE);
+ return simplify(geometry, defaultTolerance);
}
/**
@@ -82,11 +90,12 @@ public Geometry simplifyByAdminLevel(Geometry geometry, int adminLevel) {
if (geometry == null) {
return null;
}
-
+
double tolerance = switch (adminLevel) {
- case 2 -> COUNTRY_TOLERANCE; // Country
- case 4, 6 -> STATE_TOLERANCE; // State/Region
- default -> DEFAULT_TOLERANCE;
+ case 1 -> continentTolerance; // Continent / Supranational
+ case 2 -> countryTolerance; // Country
+ case 4, 6 -> stateTolerance; // State/Region
+ default -> defaultTolerance;
};
return simplify(geometry, tolerance);
@@ -99,7 +108,7 @@ public Geometry simplifyByAdminLevel(Geometry geometry, int adminLevel) {
* @return Simplified geometry with POI-appropriate tolerance
*/
public Geometry simplifyPoiBoundary(Geometry geometry) {
- return simplify(geometry, POI_TOLERANCE);
+ return simplify(geometry, poiTolerance);
}
/**
diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java b/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java
new file mode 100644
index 0000000..75261da
--- /dev/null
+++ b/src/main/java/com/dedicatedcode/paikka/service/importer/OsmNameStreamer.java
@@ -0,0 +1,106 @@
+package com.dedicatedcode.paikka.service.importer;
+import de.topobyte.osm4j.core.model.iface.OsmEntity;
+import de.topobyte.osm4j.core.model.iface.OsmTag;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class OsmNameStreamer implements AutoCloseable {
+ private final BufferedWriter writer;
+
+ public OsmNameStreamer(String outputPath) throws IOException {
+ this.writer = new BufferedWriter(new FileWriter(outputPath));
+ }
+
+ public void processEntity(OsmEntity entity, String type) throws IOException {
+ long id = entity.getId();
+ int numTags = entity.getNumberOfTags();
+
+ StringBuilder jsonBuilder = new StringBuilder();
+ jsonBuilder.append("{");
+ boolean hasNames = false;
+
+ for (int i = 0; i < numTags; i++) {
+ OsmTag tag = entity.getTag(i);
+ String key = tag.getKey();
+
+ if ("name".equals(key) || (key != null && key.startsWith("name:"))) {
+ String value = tag.getValue();
+ if (value != null && !value.isBlank()) {
+ if (hasNames) {
+ jsonBuilder.append(",");
+ }
+ hasNames = true;
+
+ // Build standard JSON key-value pairs
+ jsonBuilder.append("\"").append(escapeJson(key)).append("\":")
+ .append("\"").append(escapeJson(value)).append("\"");
+ }
+ }
+ }
+ jsonBuilder.append("}");
+
+ if (hasNames) {
+ String jsonString = jsonBuilder.toString();
+
+ // Escape the finished JSON string specifically for PG Text-Mode COPY rules
+ String postgresSafeJson = escapeForPostgresCopy(jsonString);
+
+ // Writes exactly 3 columns matching your schema: osm_id, osm_type, all_names
+ this.writer.write(id + "\t" + type + "\t" + postgresSafeJson + "\n");
+ }
+ }
+
+ /**
+ * Step 1: Encodes values to safely fit inside a JSON string property
+ */
+ private String escapeJson(String value) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < value.length(); i++) {
+ char ch = value.charAt(i);
+ switch (ch) {
+ case '"': sb.append("\\\""); break;
+ case '\\': sb.append("\\\\"); break;
+ case '\b': sb.append("\\b"); break;
+ case '\f': sb.append("\\f"); break;
+ case '\n': sb.append("\\n"); break;
+ case '\r': sb.append("\\r"); break;
+ case '\t': sb.append("\\t"); break;
+ default:
+ if (ch < ' ') {
+ String ss = Integer.toHexString(ch);
+ sb.append("\\u");
+ sb.repeat("0", 4 - ss.length());
+ sb.append(ss.toUpperCase());
+ } else {
+ sb.append(ch);
+ }
+ }
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Step 2: Escapes control characters so Postgres COPY doesn't misinterpret them
+ */
+ private String escapeForPostgresCopy(String text) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < text.length(); i++) {
+ char ch = text.charAt(i);
+ switch (ch) {
+ case '\\': sb.append("\\\\"); break; // Crucial for nested JSON backslashes
+ case '\t': sb.append("\\t"); break;
+ case '\n': sb.append("\\n"); break;
+ case '\r': sb.append("\\r"); break;
+ default: sb.append(ch);
+ }
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.flush();
+ writer.close();
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java
new file mode 100644
index 0000000..2c2c04f
--- /dev/null
+++ b/src/main/java/com/dedicatedcode/paikka/service/importer/StandaloneBoundaryImporter.java
@@ -0,0 +1,664 @@
+/*
+ * This file is part of paikka.
+ *
+ * Paikka is free software: you can redistribute it and/or
+ * modify it under the terms of the GNU Affero General Public License
+ * as published by the Free Software Foundation, either version 3 or
+ * any later version.
+ *
+ * Paikka is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied
+ * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Affero General Public License for more details.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Paikka. If not, see .
+ */
+
+package com.dedicatedcode.paikka.service.importer;
+
+import com.dedicatedcode.paikka.config.PaikkaConfiguration;
+import com.uber.h3core.H3Core;
+import com.uber.h3core.util.LatLng;
+import de.topobyte.osm4j.core.model.iface.*;
+import de.topobyte.osm4j.pbf.seq.PbfIterator;
+import org.locationtech.jts.geom.*;
+import org.locationtech.jts.io.WKBWriter;
+import org.rocksdb.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Standalone H3-based administrative boundary importer for Paikka.
+ *
+ * Reads a pre-filtered boundaries_only.pbf (Nodes -> Ways -> Relations ordered)
+ * and produces three RocksDB databases for offline mobile lookup:
+ * - h3_to_osm: H3_CELL_ID (uint64) -> List[OSM_ID] (raw byte array)
+ * - region_metadata: OSM_ID -> total cell count (int)
+ * - region_geometry: OSM_ID -> simplified WKB (bytes)
+ */
+@Service
+public class StandaloneBoundaryImporter {
+ private static final Logger logger = LoggerFactory.getLogger(StandaloneBoundaryImporter.class);
+
+ private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory();
+ private static final double BUFFER_DISTANCE = 0.0001; // ~11m at equator, ensures border cells
+
+ private final GeometrySimplificationService geometrySimplificationService;
+ private final PaikkaConfiguration paikkaConfiguration;
+ private final H3Core h3;
+ private final BoundaryImportStatistics stats;
+
+ public StandaloneBoundaryImporter(GeometrySimplificationService geometrySimplificationService, PaikkaConfiguration paikkaConfiguration) throws Exception {
+ this.geometrySimplificationService = geometrySimplificationService;
+ this.paikkaConfiguration = paikkaConfiguration;
+ this.h3 = H3Core.newInstance(); // Uber H3-Java 4.x
+ this.stats = new BoundaryImportStatistics();
+ }
+
+ // ============================ PUBLIC API ============================
+
+ public void importBoundaries(List pbfPaths, String outputDir) throws Exception {
+ RocksDB.loadLibrary();
+ Path out = Paths.get(outputDir);
+ Path tmp = out.resolve("tmp");
+ Files.createDirectories(out);
+ Files.createDirectories(tmp);
+
+ Path nodeCachePath = tmp.resolve("node_cache");
+ Path wayCachePath = tmp.resolve("way_cache");
+ Path h3ToOsmPath = out.resolve("h3_to_osm");
+ Path regionMetaPath = out.resolve("region_metadata");
+ Path regionGeomPath = out.resolve("region_geometry");
+ Path nameSql = out.resolve("osm_names.tsv");
+
+ Path tmpH3ToOsmPath = tmp.resolve("tmp_h3_to_osm");
+ Path tmpRegionMetaPath = tmp.resolve("tmp_region_metadata");
+ Path tmpRegionGeomPath = tmp.resolve("tmp_region_geometry");
+
+ cleanup(nodeCachePath);
+ cleanup(wayCachePath);
+ cleanup(h3ToOsmPath);
+ cleanup(regionMetaPath);
+ cleanup(regionGeomPath);
+ cleanup(tmpH3ToOsmPath);
+ cleanup(tmpRegionMetaPath);
+ cleanup(tmpRegionGeomPath);
+
+ // Shared Rocksoptions (inline with ImportService style)
+ BlockBasedTableConfig tableCfg = new BlockBasedTableConfig()
+ .setBlockSize(64 * 1024)
+ .setFilterPolicy(new BloomFilter(10, false));
+ Options cacheOpts = new Options()
+ .setCreateIfMissing(true)
+ .setTableFormatConfig(tableCfg)
+ .setCompressionType(CompressionType.LZ4_COMPRESSION)
+ .setWriteBufferSize(512 * 1024 * 1024)
+ .setMaxWriteBufferNumber(3)
+ .setLevel0FileNumCompactionTrigger(4);
+ Options finalOpts = new Options()
+ .setCreateIfMissing(true)
+ .setTableFormatConfig(tableCfg)
+ .setCompressionType(CompressionType.ZSTD_COMPRESSION)
+ .setWriteBufferSize(256 * 1024 * 1024)
+ .setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION)
+ .setCompressionPerLevel(List.of(
+ CompressionType.NO_COMPRESSION,
+ CompressionType.NO_COMPRESSION,
+ CompressionType.LZ4_COMPRESSION,
+ CompressionType.LZ4_COMPRESSION,
+ CompressionType.ZSTD_COMPRESSION,
+ CompressionType.ZSTD_COMPRESSION,
+ CompressionType.ZSTD_COMPRESSION
+ ));
+
+ stats.startProgressReporter();
+
+ try (
+ RocksDB nodeCache = RocksDB.open(cacheOpts, nodeCachePath.toString());
+ RocksDB wayCache = RocksDB.open(cacheOpts, wayCachePath.toString());
+ RocksDB h3ToOsm = RocksDB.open(finalOpts, h3ToOsmPath.toString());
+ RocksDB regionMeta = RocksDB.open(finalOpts, regionMetaPath.toString());
+ RocksDB regionGeom = RocksDB.open(finalOpts, regionGeomPath.toString());
+ RocksDB tmpH3ToOsm = RocksDB.open(cacheOpts, tmpH3ToOsmPath.toString());
+ RocksDB tmpRegionMeta = RocksDB.open(cacheOpts, tmpRegionMetaPath.toString());
+ RocksDB tmpRegionGeom = RocksDB.open(cacheOpts, tmpRegionGeomPath.toString());
+ OsmNameStreamer nameStreamer = new OsmNameStreamer(nameSql.toString())
+ ) {
+ for (String pbfPath : pbfPaths) {
+ stats.setCurrentPhase(1, "1.1: Caching Nodes & Ways");
+ try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+
+ // ---------- SINGLE PASS ----------
+ PbfIterator iterator = new PbfIterator(Files.newInputStream(Paths.get(pbfPath)), false);
+
+ // Phase 1 & 2: Stream nodes and ways (cached)
+ WriteBatch nodeBatch = new WriteBatch();
+ WriteBatch wayBatch = new WriteBatch();
+ AtomicLong phaseCounter = new AtomicLong();
+
+ while (iterator.hasNext()) {
+ EntityContainer c = iterator.next();
+ if (c.getType() == EntityType.Node) {
+ // PHASE 1: Cache node coordinates (lat, lon) as 16-byte double pair
+ OsmNode n = (OsmNode) c.getEntity();
+ ByteBuffer bb = ByteBuffer.allocate(16)
+ .putDouble(n.getLatitude())
+ .putDouble(n.getLongitude());
+ nodeBatch.put(longToBytes(n.getId()), bb.array());
+ stats.incrementNodesCached();
+ if (phaseCounter.incrementAndGet() % 100_000 == 0) {
+ nodeCache.write(wo, nodeBatch);
+ nodeBatch.clear();
+ }
+ } else if (c.getType() == EntityType.Way) {
+ // PHASE 2: Cache way node-id sequences (long[] as raw bytes)
+ OsmWay w = (OsmWay) c.getEntity();
+ long[] ids = new long[w.getNumberOfNodes()];
+ for (int i = 0; i < w.getNumberOfNodes(); i++) ids[i] = w.getNodeId(i);
+ wayBatch.put(longToBytes(w.getId()), longArrayToBytes(ids));
+ stats.incrementWaysCached();
+ if (phaseCounter.incrementAndGet() % 50_000 == 0) {
+ wayCache.write(wo, wayBatch);
+ wayBatch.clear();
+ }
+ } else if (c.getType() == EntityType.Relation) {
+ // PHASE 3: Count administrative boundaries for accurate progress tracking
+ OsmRelation r = (OsmRelation) c.getEntity();
+ if (isAdministrativeBoundary(r)) {
+ stats.incrementRelationsFound();
+ }
+ }
+ }
+ nodeCache.write(wo, nodeBatch);
+ wayCache.write(wo, wayBatch);
+ nodeBatch.close();
+ wayBatch.close();
+ }
+
+ stats.setCurrentPhase(2, "2.1: Processing Relations & H3");
+ // Re-open iterator for Phase 3 (or use two iterators; here we reuse file)
+
+ // Phase 3: Process Relations (separate iterator pass is fine since PBF is local)
+ try (InputStream is = Files.newInputStream(Paths.get(pbfPath))) {
+ PbfIterator relIter = new PbfIterator(is, false);
+
+ int threads = paikkaConfiguration.getImportConfiguration().getThreads();
+ ExecutorService executor = Executors.newFixedThreadPool(threads);
+ BlockingQueue> queue = new LinkedBlockingQueue<>(100);
+ List POISON_PILL = List.of();
+
+ // Producer thread
+ Thread producer = new Thread(() -> {
+ try {
+ List batch = new ArrayList<>(100);
+ while (relIter.hasNext()) {
+ EntityContainer c = relIter.next();
+ if (c.getType() == EntityType.Relation) {
+ OsmRelation r = (OsmRelation) c.getEntity();
+ try {
+ nameStreamer.processEntity(r, "R");
+ } catch (IOException e) {
+ logger.warn("Failed to stream name for relation ID: {}", r.getId(), e);
+ }
+ if (isAdministrativeBoundary(r)) {
+ batch.add(buildRelationStub(r));
+ if (batch.size() >= 100) {
+ queue.put(batch);
+ batch = new ArrayList<>(100);
+ }
+ }
+ }
+ }
+ if (!batch.isEmpty()) {
+ queue.put(batch);
+ }
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.READ, null, "producer-thread", e);
+ } finally {
+ for (int i = 0; i < threads; i++) {
+ try {
+ queue.put(POISON_PILL);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ });
+
+ producer.start();
+
+ // Consumer threads
+ List> futures = new ArrayList<>();
+ for (int i = 0; i < threads; i++) {
+ futures.add(executor.submit(() -> {
+ try (WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+ while (true) {
+ List batch = queue.take();
+ if (batch == POISON_PILL) break;
+
+ for (RelationStub stub : batch) {
+ if (stub.adminLevel() <= 3) {
+ logger.debug("Processing relation OSM ID: {} [Admin Level: {}]", stub.osmId(), stub.adminLevel());
+ }
+ try {
+ Geometry geom = buildMultiPolygon(stub, nodeCache, wayCache);
+ if (geom == null || geom.isEmpty()) {
+ continue;
+ }
+
+ // Repair invalid geometries using buffer(0)
+ if (!geom.isValid()) {
+ logger.debug("Relation OSM ID: {} [Admin Level: {}] Geometry is invalid, attempting repair", stub.osmId(), stub.adminLevel());
+ geom = geom.buffer(0);
+ if (geom == null || geom.isEmpty() || !geom.isValid()) {
+ logger.error("Relation OSM ID: {} [Admin Level: {}] Geometry repair failed", stub.osmId(), stub.adminLevel());
+ continue;
+ }
+ }
+ // Simplify first to reduce H3 cell count, then buffer
+ Geometry simplified = geometrySimplificationService.simplifyByAdminLevel(geom, stub.adminLevel());
+ if (simplified == null || simplified.isEmpty()) {
+ simplified = geom;
+ }
+ // Buffer to include border-touching cells
+ Geometry buffered = simplified.buffer(BUFFER_DISTANCE);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("Simplified Geometry: {} points for OSM ID: {}", simplified.getNumPoints(), stub.osmId());
+ }
+
+ int resolution = getResolutionForAdminLevel(stub.adminLevel());
+
+ // ---- H3 Polyfill ----
+ AtomicLong cellCount = new AtomicLong(0);
+ long startTime = System.currentTimeMillis();
+ processCellsH3Stream(buffered, stub.osmId(), wo, tmpH3ToOsm, cellCount, resolution);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("H3 Polyfill (Res {}) took {}ms for OSM ID: {}", resolution, System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ if (cellCount.get() == 0) continue;
+
+ stats.incrementRelationsProcessed();
+ stats.addH3CellsGenerated((int) cellCount.get());
+
+ startTime = System.currentTimeMillis();
+ tmpRegionMeta.put(wo, longToBytes(stub.osmId()), intToBytes((int) cellCount.get()));
+ if (stub.adminLevel() <= 3) {
+ logger.debug("H3 Cells written to tmpRegionMeta in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ startTime = System.currentTimeMillis();
+ byte[] wkb = new WKBWriter().write(simplified);
+ tmpRegionGeom.put(wo, longToBytes(stub.osmId()), wkb);
+ if (stub.adminLevel() <= 3) {
+ logger.debug("WKB written to tmpRegionGeom in {}ms for OSM ID: {}", System.currentTimeMillis() - startTime, stub.osmId());
+ }
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "process-relation", e);
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }));
+ }
+
+ // Wait for consumers to finish
+ for (Future f : futures) {
+ f.get();
+ }
+ executor.shutdown();
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
+ producer.join();
+ }
+ }
+
+ stats.setCurrentPhase(3, "3.1: Compacting Final Databases");
+ // Final Step: Copy from temporary DBs to final DBs in sorted order
+ copyDb(tmpRegionMeta, regionMeta);
+ copyDb(tmpRegionGeom, regionGeom);
+ copyH3Db(tmpH3ToOsm, h3ToOsm);
+
+ // Compact finals
+ h3ToOsm.compactRange();
+ regionMeta.compactRange();
+ regionGeom.compactRange();
+ }
+
+ stats.stop();
+ stats.setTotalTime(System.currentTimeMillis() - stats.getStartTime());
+ stats.printFinalStatistics();
+ stats.printOutcomeAndErrors();
+
+ // Cleanup tmp
+ cleanup(tmp);
+ }
+
+ private void copyDb(RocksDB source, RocksDB target) throws RocksDBException {
+ try (RocksIterator it = source.newIterator(); WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+ it.seekToFirst();
+ while (it.isValid()) {
+ target.put(wo, it.key(), it.value());
+ it.next();
+ }
+ }
+ }
+
+ private void copyH3Db(RocksDB source, RocksDB target) throws RocksDBException {
+ try (RocksIterator it = source.newIterator(); WriteOptions wo = new WriteOptions().setDisableWAL(true)) {
+ it.seekToFirst();
+ while (it.isValid()) {
+ byte[] key = it.key();
+ byte[] newVal = it.value();
+ byte[] existing = target.get(key);
+ if (existing == null) {
+ target.put(wo, key, newVal);
+ } else {
+ byte[] merged = mergeOsmIdArrays(existing, newVal);
+ target.put(wo, key, merged);
+ }
+ it.next();
+ }
+ }
+ }
+
+ private byte[] mergeOsmIdArrays(byte[] existing, byte[] newVal) {
+ ByteBuffer bb = ByteBuffer.wrap(newVal).order(ByteOrder.BIG_ENDIAN);
+ byte[] current = existing;
+ while (bb.hasRemaining()) {
+ long osmId = bb.getLong();
+ current = appendOsmIdToArray(current, osmId);
+ }
+ return current;
+ }
+
+ // ============================ GEOMETRY STITCHING ============================
+
+ /**
+ * Builds a JTS MultiPolygon from relation outer/inner way members.
+ * Rings are stitched by coordinate continuation (same logic as ImportService.buildConnectedRings).
+ */
+ private Geometry buildMultiPolygon(RelationStub stub, RocksDB nodeCache, RocksDB wayCache) {
+ List> outerRings = stitchRings(stub.outerWays(), nodeCache, wayCache);
+ List> innerRings = stitchRings(stub.innerWays(), nodeCache, wayCache);
+ if (outerRings.isEmpty()) return null;
+
+ List polygons = new ArrayList<>();
+ for (List outer : outerRings) {
+ try {
+ LinearRing shell = GEOMETRY_FACTORY.createLinearRing(outer.toArray(new Coordinate[0]));
+ List holes = new ArrayList<>();
+ for (List inner : innerRings) {
+ try {
+ holes.add(GEOMETRY_FACTORY.createLinearRing(inner.toArray(new Coordinate[0])));
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "createLinearRing-inner", e);
+ }
+ }
+ Polygon p = GEOMETRY_FACTORY.createPolygon(shell, holes.toArray(new LinearRing[0]));
+ if (p.isValid()) polygons.add(p);
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, stub.osmId(), "buildMultiPolygon", e);
+ }
+ }
+ if (polygons.isEmpty()) return null;
+ return polygons.size() == 1 ? polygons.getFirst() : GEOMETRY_FACTORY.createMultiPolygon(polygons.toArray(new Polygon[0]));
+ }
+
+ private List> stitchRings(List wayIds, RocksDB nodeCache, RocksDB wayCache) {
+ Map> wayCoords = new HashMap<>();
+ for (long wid : wayIds) {
+ try {
+ byte[] seq = wayCache.get(longToBytes(wid));
+ if (seq == null) continue;
+ long[] nodeIds = bytesToLongArray(seq);
+ List coords = resolveCoordinates(nodeIds, nodeCache);
+ if (coords != null && coords.size() >= 2) wayCoords.put(wid, coords);
+ } catch (RocksDBException e) {
+ stats.recordError(BoundaryImportStatistics.Stage.CACHING_NODES_WAYS, BoundaryImportStatistics.Kind.STORE, wid, "stitchRings", e);
+ }
+ }
+ List> rings = new ArrayList<>();
+ Set used = new HashSet<>();
+ while (used.size() < wayCoords.size()) {
+ Long start = wayCoords.keySet().stream().filter(id -> !used.contains(id)).findFirst().orElse(null);
+ if (start == null) break;
+ List ring = new ArrayList<>(wayCoords.get(start));
+ used.add(start);
+ boolean extended;
+ do {
+ extended = false;
+ Coordinate end = ring.getLast();
+ for (Map.Entry> e : wayCoords.entrySet()) {
+ if (used.contains(e.getKey())) continue;
+ List w = e.getValue();
+ if (end.equals2D(w.getFirst())) {
+ ring.addAll(w.subList(1, w.size()));
+ used.add(e.getKey());
+ extended = true;
+ break;
+ } else if (end.equals2D(w.getLast())) {
+ List rev = new ArrayList<>(w);
+ Collections.reverse(rev);
+ ring.addAll(rev.subList(1, rev.size()));
+ used.add(e.getKey());
+ extended = true;
+ break;
+ }
+ }
+ } while (extended);
+ if (ring.size() >= 3 && !ring.getFirst().equals2D(ring.getLast()))
+ ring.add(new Coordinate(ring.getFirst()));
+ if (ring.size() >= 4) rings.add(ring);
+ }
+ return rings;
+ }
+
+ private List resolveCoordinates(long[] nodeIds, RocksDB nodeCache) {
+ try {
+ List keys = new ArrayList<>(nodeIds.length);
+ for (long id : nodeIds) keys.add(longToBytes(id));
+ List vals = nodeCache.multiGetAsList(keys);
+ List coords = new ArrayList<>(nodeIds.length);
+ for (byte[] v : vals) {
+ if (v != null && v.length == 16) {
+ ByteBuffer bb = ByteBuffer.wrap(v);
+ double lat = bb.getDouble(0);
+ double lon = bb.getDouble(8);
+ coords.add(new Coordinate(lon, lat)); // JTS uses (x=lon, y=lat)
+ } else return null;
+ }
+ return coords;
+ } catch (RocksDBException e) {
+ stats.recordError(BoundaryImportStatistics.Stage.CACHING_NODES_WAYS, BoundaryImportStatistics.Kind.STORE, null, "resolveCoordinates", e);
+ return null;
+ }
+ }
+
+ // ============================ H3 POLYFILL ============================
+
+ /**
+ * Determines the H3 resolution based on the administrative level.
+ * Lower admin levels (countries) use lower resolutions to save space.
+ * Higher admin levels (cities) use higher resolutions for accuracy.
+ */
+ private int getResolutionForAdminLevel(int adminLevel) {
+ if (adminLevel <= 2) return 4; // Continents/Countries
+ if (adminLevel <= 5) return 6; // States/Regions
+ return 9; // Districts/Cities
+ }
+
+ /**
+ * Converts a JTS Geometry to H3 cells at the specified resolution.
+ * Uses h3.polygonToCellsStream with LatLng vertices. Multipolygons are expanded.
+ */
+ private void processCellsH3Stream(Geometry geom, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount, int resolution) {
+ int num = geom.getNumGeometries();
+ for (int i = 0; i < num; i++) {
+ Geometry part = geom.getGeometryN(i);
+ if (!(part instanceof Polygon poly)) continue;
+ List outer = toLatLng(poly.getExteriorRing().getCoordinates());
+ List> holes = new ArrayList<>();
+ for (int h = 0; h < poly.getNumInteriorRing(); h++) {
+ holes.add(toLatLng(poly.getInteriorRingN(h).getCoordinates()));
+ }
+ try {
+ List batch = new ArrayList<>(5_000); // Reduced batch size to prevent OOM
+ h3.polygonToCells(outer, holes, resolution).forEach(cell -> {
+ batch.add(cell);
+ if (batch.size() >= 5_000) {
+ try {
+ processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount);
+ } catch (RocksDBException e) {
+ throw new RuntimeException(e);
+ }
+ batch.clear();
+ }
+ });
+ if (!batch.isEmpty()) {
+ processH3Batch(batch, osmId, wo, tmpH3ToOsm, cellCount);
+ }
+ } catch (Exception e) {
+ stats.recordError(BoundaryImportStatistics.Stage.PROCESSING_RELATIONS, BoundaryImportStatistics.Kind.GEOMETRY, osmId, "processCellsH3Stream", e);
+ }
+ }
+ }
+
+ private void processH3Batch(List cells, long osmId, WriteOptions wo, RocksDB tmpH3ToOsm, AtomicLong cellCount) throws RocksDBException {
+ List keys = new ArrayList<>(cells.size());
+ for (long cell : cells) {
+ keys.add(longToBytes(cell));
+ }
+
+ // Synchronize to prevent race conditions when multiple threads update the same H3 cell
+ synchronized (tmpH3ToOsm) {
+ List existingValues = tmpH3ToOsm.multiGetAsList(keys);
+ try (WriteBatch writeBatch = new WriteBatch()) {
+ for (int i = 0; i < cells.size(); i++) {
+ cellCount.incrementAndGet();
+ byte[] key = keys.get(i);
+ byte[] existing = existingValues.get(i);
+ byte[] updated = appendOsmIdToArray(existing, osmId);
+ writeBatch.put(key, updated);
+ }
+ tmpH3ToOsm.write(wo, writeBatch);
+ }
+ }
+ }
+ private List toLatLng(Coordinate[] coords) {
+ List list = new ArrayList<>(coords.length);
+ for (Coordinate c : coords) {
+ list.add(new LatLng(c.y, c.x)); // lat, lon
+ }
+ return list;
+ }
+
+ // ============================ BYTE UTILS ============================
+
+ private byte[] longToBytes(long v) {
+ return ByteBuffer.allocate(8).order(ByteOrder.BIG_ENDIAN).putLong(v).array();
+ }
+
+ private byte[] longArrayToBytes(long[] arr) {
+ ByteBuffer bb = ByteBuffer.allocate(8 * arr.length).order(ByteOrder.BIG_ENDIAN);
+ for (long v : arr) bb.putLong(v);
+ return bb.array();
+ }
+
+ private long[] bytesToLongArray(byte[] b) {
+ ByteBuffer bb = ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN);
+ long[] arr = new long[b.length / 8];
+ for (int i = 0; i < arr.length; i++) arr[i] = bb.getLong();
+ return arr;
+ }
+
+ private byte[] intToBytes(int v) {
+ return ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(v).array();
+ }
+
+ /**
+ * Appends an OSM_ID to a raw byte array of longs, preventing duplicates.
+ * Format: sequence of 8-byte big-endian longs.
+ */
+ private byte[] appendOsmIdToArray(byte[] existing, long osmId) {
+ if (existing == null || existing.length == 0) {
+ return longToBytes(osmId);
+ }
+ int count = existing.length / 8;
+ for (int i = 0; i < count; i++) {
+ long val = ByteBuffer.wrap(existing, i * 8, 8).order(ByteOrder.BIG_ENDIAN).getLong();
+ if (val == osmId) return existing; // duplicate
+ }
+ ByteBuffer bb = ByteBuffer.allocate(existing.length + 8).order(ByteOrder.BIG_ENDIAN);
+ bb.put(existing);
+ bb.putLong(osmId);
+ return bb.array();
+ }
+
+ // ============================ OSM HELPERS ============================
+
+ private boolean isAdministrativeBoundary(OsmRelation r) {
+ boolean boundary = false, adminLevel = false;
+ for (int i = 0; i < r.getNumberOfTags(); i++) {
+ OsmTag t = r.getTag(i);
+ if ("boundary".equals(t.getKey()) && "administrative".equals(t.getValue())) boundary = true;
+ if ("admin_level".equals(t.getKey())) adminLevel = true;
+ if ("type".equals(t.getKey()) && "boundary".equals(t.getValue())) boundary = true;
+ }
+ return boundary && adminLevel;
+ }
+
+ private RelationStub buildRelationStub(OsmRelation r) {
+ List outer = new ArrayList<>();
+ List inner = new ArrayList<>();
+ int level = 10;
+ for (int i = 0; i < r.getNumberOfMembers(); i++) {
+ OsmRelationMember m = r.getMember(i);
+ if (m.getType() == EntityType.Way) {
+ String role = m.getRole();
+ if ("outer".equals(role) || role == null || role.isEmpty()) outer.add(m.getId());
+ else if ("inner".equals(role)) inner.add(m.getId());
+ }
+ }
+ for (int i = 0; i < r.getNumberOfTags(); i++) {
+ OsmTag t = r.getTag(i);
+ if ("admin_level".equals(t.getKey())) {
+ try {
+ level = Integer.parseInt(t.getValue());
+ } catch (NumberFormatException ignored) {
+ }
+ }
+ }
+ return new RelationStub(r.getId(), level, outer, inner);
+ }
+
+ private record RelationStub(long osmId, int adminLevel, List outerWays, List innerWays) {
+ }
+
+ private void cleanup(Path p) {
+ if (Files.exists(p)) {
+ try {
+ Files.walk(p).sorted(Comparator.reverseOrder()).forEach(path -> {
+ try {
+ Files.delete(path);
+ } catch (IOException e) {
+ System.err.println("warn: " + e.getMessage());
+ }
+ });
+ } catch (IOException e) {
+ System.err.println("Failed cleanup: " + p + " -> " + e.getMessage());
+ }
+ }
+ }
+}
diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties
index 315e3bf..63b8fce 100644
--- a/src/main/resources/application-dev.properties
+++ b/src/main/resources/application-dev.properties
@@ -4,4 +4,19 @@ paikka.data-dir=./data
spring.thymeleaf.cache=false
+logging.level.com.dedicatedcode=ERROR
+
paikka.admin.password=test
+
+paikka.import.threads=10
+
+# Aggressive simplification tolerances (in degrees)
+# ~5.5 km tolerance for continents (admin_level=1)
+paikka.simplification.continent-tolerance=0.05
+# ~1.1 km tolerance for countries (admin_level=2)
+paikka.simplification.country-tolerance=0.01
+# ~550 meters tolerance for states/regions (admin_level=4,6)
+paikka.simplification.state-tolerance=0.005
+# Keep default and POI tolerances smaller to preserve local accuracy
+paikka.simplification.default-tolerance=0.0001
+paikka.simplification.poi-tolerance=0.000018
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index 49537ed..daf5669 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -23,6 +23,13 @@ paikka.query.max-results=500
paikka.query.default-results=10
paikka.query.base-url=http://localhost:8080
+
+paikka.simplification.continent-tolerance=0.005
+paikka.simplification.country-tolerance=0.00045
+paikka.simplification.state-tolerance=0.00009
+paikka.simplification.poi-tolerance=0.000018
+paikka.simplification.default-tolerance=0.000045
+
paikka.stats-db-path=./data/stats.db
paikka.stats-db.flush=0/10 * * * * *
diff --git a/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java
index 89edffb..752b397 100644
--- a/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java
+++ b/src/test/java/com/dedicatedcode/paikka/service/GeometrySimplificationServiceTest.java
@@ -17,12 +17,14 @@
package com.dedicatedcode.paikka.service;
import com.dedicatedcode.paikka.IntegrationTest;
+import com.dedicatedcode.paikka.config.PaikkaConfiguration;
import com.dedicatedcode.paikka.service.importer.GeometrySimplificationService;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.LinearRing;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
@@ -30,8 +32,10 @@
@IntegrationTest
class GeometrySimplificationServiceTest {
-
- private final GeometrySimplificationService service = new GeometrySimplificationService();
+
+ @Autowired
+ private GeometrySimplificationService service;
+
private final GeometryFactory geometryFactory = new GeometryFactory();
@Test
diff --git a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java
index 0cd4751..44c535d 100644
--- a/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java
+++ b/src/test/java/com/dedicatedcode/paikka/service/ImportServiceTest.java
@@ -65,7 +65,14 @@ void setUp() throws Exception {
PaikkaConfiguration.ImportConfiguration importConfiguration = new PaikkaConfiguration.ImportConfiguration();
importConfiguration.setThreads(2);
config.setImportConfiguration(importConfiguration);
- GeometrySimplificationService geometrySimplificationService = new GeometrySimplificationService();
+ PaikkaConfiguration.SimplificationConfiguration simplificationConfiguration = new PaikkaConfiguration.SimplificationConfiguration();
+ simplificationConfiguration.setContinentTolerance(0.005);
+ simplificationConfiguration.setCountryTolerance(0.00045);
+ simplificationConfiguration.setStateTolerance(0.00009);
+ simplificationConfiguration.setPoiTolerance(0.000018);
+ simplificationConfiguration.setDefaultTolerance(0.000045);
+ config.setSimplificationConfiguration(simplificationConfiguration);
+ GeometrySimplificationService geometrySimplificationService = new GeometrySimplificationService(config);
S2Helper s2Helper = new S2Helper();
ImportService importService = new ImportService(s2Helper, geometrySimplificationService, config, "1.0.0");
diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties
index 7f048ee..5083d88 100644
--- a/src/test/resources/application-test.properties
+++ b/src/test/resources/application-test.properties
@@ -1,5 +1,5 @@
paikka.data-dir=${java.io.tmpdir}/paikka-test-data
-"paikka.stats-db-path=memory
+paikka.stats-db-path=memory
paikka.query.base-url=http://localhost:8080
paikka.admin.password=test
paikka.stats-db.flush=-