From b95b9cbe668c45317cf7fa1f16c4ddfad44e44d5 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Tue, 11 Aug 2026 11:44:04 -0400 Subject: [PATCH 1/2] raster: read a palette numbered from 0 --- src/raster/bsb.zig | 57 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/src/raster/bsb.zig b/src/raster/bsb.zig index 946fbf6c..daeadfab 100644 --- a/src/raster/bsb.zig +++ b/src/raster/bsb.zig @@ -87,8 +87,15 @@ pub const Header = struct { /// sheet already on WGS84. dtm_lat: f64 = 0, dtm_lon: f64 = 0, - /// `RGB/n` — the palette, index 1..n (index 0 is unused by the format). + /// `RGB/n` — the palette, in the file's own numbering. Address it as + /// `palette[pixel - palette_base]`, or through `rgbAt`. palette: []Rgb = &.{}, + /// The number the file gives its FIRST palette entry. BSB numbers from 1 + /// and treats pixel value 0 as no data. Sheets in the wild number from 0, + /// and then 0 is a real colour. A parser that assumes one of the two reads + /// every colour off by one, and on a 0-based sheet indexes before the + /// start of the array. + palette_base: u8 = 1, /// `PLY/n,lat,lon` — the border polygon. Cuts the paper collar off, so /// quilted sheets butt at their neat lines instead of overprinting each /// other's margins. Stands in for M_COVR in the archive metadata. @@ -143,9 +150,10 @@ pub const Chart = struct { pub fn rgbAt(c: Chart, x: u32, y: u32) Rgb { if (x >= c.header.width or y >= c.header.height) return .{ .r = 0, .g = 0, .b = 0 }; const idx = c.pixels[@as(usize, y) * c.header.width + x]; - // The format numbers the palette from 1. - if (idx == 0 or idx > c.header.palette.len) return .{ .r = 0, .g = 0, .b = 0 }; - return c.header.palette[idx - 1]; + if (idx < c.header.palette_base) return .{ .r = 0, .g = 0, .b = 0 }; + const at = idx - c.header.palette_base; + if (at >= c.header.palette.len) return .{ .r = 0, .g = 0, .b = 0 }; + return c.header.palette[at]; } }; @@ -298,6 +306,8 @@ pub fn parseHeader(a: Allocator, bytes: []const u8) Error!Header { var h: Header = .{}; var palette = std.ArrayList(Rgb).empty; + // The lowest RGB index the file used, which decides the base. + var palette_min: u32 = std.math.maxInt(u32); var border = std.ArrayList([2]f64).empty; var refs = std.ArrayList(Header.Ref).empty; @@ -357,9 +367,12 @@ pub fn parseHeader(a: Allocator, bytes: []const u8) Error!Header { const r = parseU32(n.next() orelse "") orelse continue; const g = parseU32(n.next() orelse "") orelse continue; const b = parseU32(n.next() orelse "") orelse continue; - // The format numbers from 1 and is not obliged to be in order. - while (palette.items.len < idx) try palette.append(a, .{ .r = 0, .g = 0, .b = 0 }); - palette.items[idx - 1] = .{ .r = @intCast(r & 255), .g = @intCast(g & 255), .b = @intCast(b & 255) }; + // Stored at its own number and shifted at the end, because the + // base is not known until every RGB line has been read. Entries + // are not obliged to be in order either. + while (palette.items.len <= idx) try palette.append(a, .{ .r = 0, .g = 0, .b = 0 }); + palette.items[idx] = .{ .r = @intCast(r & 255), .g = @intCast(g & 255), .b = @intCast(b & 255) }; + palette_min = @min(palette_min, idx); } else if (std.mem.eql(u8, key, "PLY")) { var n = std.mem.splitScalar(u8, rest, ','); _ = n.next(); @@ -398,6 +411,10 @@ pub fn parseHeader(a: Allocator, bytes: []const u8) Error!Header { if (h.depth == 0) return Error.BadHeader; h.data_off = off; + // Nothing wrote slot 0 on a 1-based sheet, so dropping it restores the + // numbering the rest of the engine reads. + h.palette_base = if (palette_min == 0) 0 else 1; + if (h.palette_base == 1 and palette.items.len > 0) _ = palette.orderedRemove(0); h.palette = palette.items; h.border = border.items; h.refs = refs.items; @@ -534,6 +551,32 @@ test "KNP fields" { try testing.expectEqualStrings("", fieldOf(rest, "SP").?); } +test "a palette numbered from 0 is read at its own base" { + // BSB numbers its palette from 1. Sheets in the wild number from 0, and + // then pixel value 0 is a real colour. Reading such a sheet as 1-based + // indexes one before the start of the array, which on a release build is a + // write to whatever is there. + // parseHeader allocates into a caller arena, as decode does. + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + const zero = + "BSB/NA=T,RA=2,2\r\nRGB/0,10,20,30\r\nRGB/1,40,50,60\r\n\x1a\x00\x01"; + const h0 = try parseHeader(a, zero); + try testing.expectEqual(@as(u8, 0), h0.palette_base); + try testing.expectEqual(@as(usize, 2), h0.palette.len); + try testing.expectEqual(@as(u8, 10), h0.palette[0].r); + try testing.expectEqual(@as(u8, 40), h0.palette[1].r); + + const one = + "BSB/NA=T,RA=2,2\r\nRGB/1,10,20,30\r\nRGB/2,40,50,60\r\n\x1a\x00\x01"; + const h1 = try parseHeader(a, one); + try testing.expectEqual(@as(u8, 1), h1.palette_base); + try testing.expectEqual(@as(usize, 2), h1.palette.len); + try testing.expectEqual(@as(u8, 10), h1.palette[0].r); + try testing.expectEqual(@as(u8, 40), h1.palette[1].r); +} + test "polynomial parse and evaluate" { const p = parsePoly("3,-1.7614611556e+002,5.7055880833e-003,-1.3260547082e-015"); try testing.expectEqual(@as(u8, 3), p.order); From a58a693f00aeb8a90cd3211aefb3cf0cb4381353 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Tue, 11 Aug 2026 11:44:04 -0400 Subject: [PATCH 2/2] abi: bake charts and raster charts from a caller-owned list --- include/tile57.h | 56 ++++++++++- src/capi.zig | 208 +++++++++++++++++++++++++++++++++++++++++ src/raster/bakebsb.zig | 4 +- 3 files changed, 266 insertions(+), 2 deletions(-) diff --git a/include/tile57.h b/include/tile57.h index 9bf550da..43f8a1a9 100644 --- a/include/tile57.h +++ b/include/tile57.h @@ -244,6 +244,52 @@ tile57_status tile57_bake_tree(const char *in_dir, const char *out_dir, uint32_t tile57_bake_progress progress, void *progress_ctx, uint32_t *out_baked, tile57_error *err); +/* Names the chart that just finished, by its INDEX into the caller's in_paths. + * Charts bake concurrently, so the count in tile57_bake_progress cannot say + * which chart a step belongs to. Called from worker threads, out of order, so + * it must be thread-safe. Takes the same context as the progress callback. + * NULL to skip. */ +typedef void (*tile57_bake_label)(void *ctx, uint32_t index); + +/* Bake in_paths[i] to out_paths[i], in parallel, freeing each archive right + * after its write (peak memory is the worker count, not n). Both arrays are + * caller-allocated and length `n`; parent directories of the outputs must + * exist. Each output gets an .sha content-hash sidecar. + * + * This is tile57_bake_tree with the CALLER choosing the list and its ORDER. + * Two things follow from that. `label` can name a finished chart by index, so + * no string crosses the ABI. And a host that wants the coarse charts first + * sorts its own list that way, which is what makes a partly finished bake + * usable: an overview and a coastal chart cover the whole passage, where the + * same number of harbor cells covers one river. + * + * `progress` may CANCEL by returning false, at the next chart boundary, and a + * cancelled bake is TILE57_OK with *out_baked counting what finished. Unlike + * tile57_bake_tree this does NOT skip work already done: the caller chose the + * list, so the caller decides what is already baked. */ +tile57_status tile57_bake_files(const char *const *in_paths, const char *const *out_paths, + size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + +/* tile57_bake_files for RASTER charts: BSB/KAP in, one PMTiles archive of PNG + * tiles out per chart. Same contract — caller-owned lists, `label` names a + * finished chart by index, `progress` may cancel, *out_baked counts what was + * written. + * + * A chart that cannot be baked is SKIPPED, not fatal: a BSB set often carries + * a sheet with no georeference or no border, and one of those must not cost + * the mariner the rest of the set. Compare *out_baked with `n` to find out. + * + * What this writes are raster charts: tile57_chart_get_info reports is_raster + * on each, and they compose through tile57_compose_rasters, never through the + * vector compositor. `workers` is a memory bound and one large ocean sheet + * decodes to roughly 180 megapixels, so keep the count small. */ +tile57_status tile57_bake_rasters(const char *const *in_paths, const char *const *out_paths, + size_t n, uint32_t workers, + tile57_bake_progress progress, tile57_bake_label label, + void *progress_ctx, uint32_t *out_baked, tile57_error *err); + /* Read a PMTiles archive's metadata JSON blob (decompressed) into *out / * *out_len (free with tile57_free); NULL/0 when the archive carries none. A * per-chart bake embeds the chart's M_COVR coverage + cscl + date/name under a @@ -317,7 +363,14 @@ typedef enum { * tile57_tile_type); a host passes it to tile57_style_template so the renderer * decodes the tiles correctly. native_scale is the compilation scale (1:N) the * bake embedded in the archive metadata; 0 = unknown (a composed/foreign - * archive — derive the scale from the zoom band instead). */ + * archive — derive the scale from the zoom band instead). + * + * is_raster separates a raster archive (a baked RNC, or imagery) from a + * vector chart. Both open, and both carry coverage and a scale, so nothing + * else here tells them apart. tile_type cannot carry it: TILE57_TILE_TYPE_MLT + * is 2, and 2 is PNG in the PMTiles header. A host that composes vector charts + * must skip an archive with is_raster set, or the tiles decode as a format + * they are not. */ typedef struct { uint8_t min_zoom, max_zoom; uint32_t bands; /* bitmask of navigational bands present */ @@ -325,6 +378,7 @@ typedef struct { bool has_anchor; double anchor_lat, anchor_lon, anchor_zoom; uint8_t tile_type; /* tile57_tile_type */ int32_t native_scale; + bool is_raster; /* tiles are images, not vector tiles */ } tile57_info; void tile57_chart_get_info(tile57_chart *chart, tile57_info *out); diff --git a/src/capi.zig b/src/capi.zig index 9c028ac0..0342425c 100644 --- a/src/capi.zig +++ b/src/capi.zig @@ -385,6 +385,205 @@ export fn tile57_bake_tree( return OK; } +/// Bake the cells at `in_paths` to the archives at `out_paths`, in parallel. +/// The CALLER owns the list, so `label` names each finished chart by its index +/// into it and no string crosses the ABI. See tile57.h. +export fn tile57_bake_files( + in_paths: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const ins = in_paths orelse return failWith(err, .badarg, "in_paths must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + for (0..n) |i| { + in_list[i] = std.mem.span(ins[i]); + out_list[i] = std.mem.span(outs[i]); + } + const baked = chart.bakeChartsToFiles(sharedIo(), in_list, out_list, null, workers, progress, progress_ctx, label); + if (out_baked) |p| p.* = @intCast(baked); + return OK; +} + +/// One raster bake in flight, shared by its workers. +const RasterJob = struct { + next: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), + done: std.atomic.Value(u32) = std.atomic.Value(u32).init(0), + cancel: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + in: []const []const u8, + out: []const []const u8, + ok: []bool, + progress: chart.BakeProgress, + label: chart.BakeLabel, + ctx: ?*anyopaque, +}; + +/// One raster chart: read, warp, write. It decodes whole (an ocean sheet is +/// about 180 Mpx) plus the pyramid it warps into, so this takes libc's +/// allocator and frees everything before the next one starts. +fn bakeOneRaster(io: std.Io, job: *RasterJob, i: usize) void { + const a = std.heap.c_allocator; + const stem = std.fs.path.stem(std.fs.path.basename(job.in[i])); + const kap = std.Io.Dir.cwd().readFileAlloc(io, job.in[i], a, .unlimited) catch return; + defer a.free(kap); + const baked = raster.bakebsb.bakeBytes(a, kap, stem) catch return; + defer a.free(baked.bytes); + std.Io.Dir.cwd().writeFile(io, .{ .sub_path = job.out[i], .data = baked.bytes }) catch return; + job.ok[i] = true; +} + +fn rasterWorker(job: *RasterJob) void { + const a = std.heap.c_allocator; + // Its own std.Io: a worker must not share the caller's. + const threaded = a.create(std.Io.Threaded) catch return; + threaded.* = .init(a, .{}); + defer { + threaded.deinit(); + a.destroy(threaded); + } + const io = threaded.io(); + while (true) { + if (job.cancel.load(.monotonic)) return; // a peer's progress callback said stop + const i = job.next.fetchAdd(1, .monotonic); + if (i >= job.in.len) return; + bakeOneRaster(io, job, i); + if (job.label) |lb| lb(job.ctx, @intCast(i)); + const d = job.done.fetchAdd(1, .monotonic) + 1; + if (job.progress) |cb| { + if (!cb(job.ctx, d, @intCast(job.in.len))) { + job.cancel.store(true, .monotonic); + return; + } + } + } +} + +/// Bake the BSB/KAP charts at `in_paths` to the archives at `out_paths`. This +/// is tile57_bake_files for raster charts. See tile57.h. +export fn tile57_bake_rasters( + in_paths: ?[*]const [*:0]const u8, + out_paths: ?[*]const [*:0]const u8, + n: usize, + workers: u32, + progress: chart.BakeProgress, + label: chart.BakeLabel, + progress_ctx: ?*anyopaque, + out_baked: ?*u32, + err: ?*CError, +) callconv(.c) c_int { + if (out_baked) |p| p.* = 0; + if (n == 0) return OK; + const ins = in_paths orelse return failWith(err, .badarg, "in_paths must not be null"); + const outs = out_paths orelse return failWith(err, .badarg, "out_paths must not be null"); + + const in_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(in_list); + const out_list = gpa.alloc([]const u8, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(out_list); + const ok = gpa.alloc(bool, n) catch return failWith(err, .nomem, "out of memory"); + defer gpa.free(ok); + @memset(ok, false); + for (0..n) |i| { + in_list[i] = std.mem.span(ins[i]); + out_list[i] = std.mem.span(outs[i]); + } + + var job = RasterJob{ + .in = in_list, + .out = out_list, + .ok = ok, + .progress = progress, + .label = label, + .ctx = progress_ctx, + }; + // A worker holds one whole chart, so `workers` is a MEMORY bound here, as + // it is for cells. + // + // EVERY worker is a thread of ours, and the calling thread only waits. + // Warping a sheet needs a deep stack: an ocean sheet decodes to about 180 + // megapixels. The cell bake can borrow the caller's thread safely, but a + // host that calls this from a pool thread with a small stack (libdispatch + // gives 512 KB) crashes inside the warp. Asking for the stack here is the + // only way to know it is there. + const stack = 16 * 1024 * 1024; + var threads: [8]std.Thread = undefined; + const want = @min(@max(workers, 1), @min(threads.len, n)); + var spawned: usize = 0; + while (spawned < want) : (spawned += 1) { + threads[spawned] = std.Thread.spawn(.{ .stack_size = stack }, rasterWorker, .{&job}) catch break; + } + if (spawned == 0) rasterWorker(&job); // nothing would run otherwise + for (threads[0..spawned]) |t| t.join(); + + var baked: u32 = 0; + for (ok) |o| { + if (o) baked += 1; + } + if (out_baked) |p| p.* = baked; + + // The ownership partition, beside the archives, as the ENC bake writes one. + // Without it every open rebuilds the quilt in memory, and a host with a + // thousand sheets pays for that on the boat instead of here, once. + if (baked > 0) writeRasterPartition(out_list, ok); + return OK; +} + +/// Build the quilt over the archives just written and leave it beside them. +/// Best effort: a library with no partition still draws, only slower. +fn writeRasterPartition(out_list: []const []const u8, ok: []const bool) void { + const io = sharedIo(); + var charts = std.ArrayList(*raster.RasterChart).empty; + defer { + for (charts.items) |rc| { + rc.close(); + gpa.destroy(rc); + } + charts.deinit(gpa); + } + var archives = std.ArrayList(compose.ChartArchive).empty; + defer archives.deinit(gpa); + + var dir: []const u8 = ""; + for (out_list, ok) |path, good| { + if (!good) continue; + const pz = gpa.dupeZ(u8, path) catch continue; + defer gpa.free(pz); + const opened = raster.RasterChart.open(io, gpa, pz, null) catch continue; + const rc = gpa.create(raster.RasterChart) catch continue; + rc.* = opened; + charts.append(gpa, rc) catch continue; + // One directory per chart, so the library is the directory above. + if (dir.len == 0) { + const own = std.fs.path.dirname(path) orelse continue; + dir = std.fs.path.dirname(own) orelse own; + } + const rd = rc.pmtilesReader() orelse continue; + const cov = rc.decodedCoverage() orelse continue; + archives.append(gpa, .{ .reader = rd, .cov = cov }) catch continue; + } + if (archives.items.len == 0 or dir.len == 0) return; + + const src = (compose.ComposeSource.openRasters(gpa, archives.items, null) catch return) orelse return; + defer src.deinit(); + const bytes = src.serializePartition(gpa) catch return; + defer gpa.free(bytes); + const path = std.fs.path.join(gpa, &.{ dir, "partition.tpart" }) catch return; + defer gpa.free(path); + std.Io.Dir.cwd().writeFile(io, .{ .sub_path = path, .data = bytes }) catch {}; +} + /// The metadata JSON blob of a PMTiles archive (decompressed) — e.g. the embedded /// per-cell "coverage" a single-cell bake carries — into *out / *out_len (free with /// tile57_free); NULL/0 when the archive carries none. See tile57.h. @@ -464,6 +663,7 @@ const CInfo = extern struct { anchor_zoom: f64, tile_type: u8, // the archive's stored encoding (TILE57_TILE_TYPE_*) native_scale: i32, // embedded compilation scale (1:N); 0 = derive from zoom band + is_raster: bool, // the archive stores pictures, not vector tiles }; // tile57_tile_type values (keep in sync with tile57.h). @@ -485,6 +685,14 @@ export fn tile57_chart_get_info(src: ?*Chart, out: ?*CInfo) callconv(.c) void { .mlt => TILE_TYPE_MLT, else => TILE_TYPE_MVT, }; + // A raster archive opens as a chart and carries coverage and a scale, so + // nothing above tells it apart from a vector chart. Its tiles are images. + // tile_type cannot carry this: TILE57_TILE_TYPE_MLT is 2, and 2 is PNG in + // the PMTiles header. + o.is_raster = switch (s.tileType()) { + .png, .jpeg, .webp, .avif => true, + else => false, + }; if (s.bounds()) |b| { o.has_bounds = true; o.west = b[0]; diff --git a/src/raster/bakebsb.zig b/src/raster/bakebsb.zig index 6470c768..329bcd79 100644 --- a/src/raster/bakebsb.zig +++ b/src/raster/bakebsb.zig @@ -396,6 +396,7 @@ fn warpTile(c: *Ctx, z: u8, x: u32, y: u32) Error!?[]u8 { var any = false; const src = c.chart.pixels; const pal = c.chart.header.palette; + const base = c.chart.header.palette_base; const stride: usize = c.chart.header.width; var j: usize = 0; @@ -423,7 +424,8 @@ fn warpTile(c: *Ctx, z: u8, x: u32, y: u32) Error!?[]u8 { const idx = src[@as(usize, @intFromFloat(sy)) * stride + @as(usize, @intFromFloat(sx))]; // The format numbers its palette from 1; 0 and anything past // the end is a malformed run, which shows as black. - const rgb = if (idx >= 1 and idx <= pal.len) pal[idx - 1] else bsb.Rgb{ .r = 0, .g = 0, .b = 0 }; + const at = if (idx >= base) idx - base else 255; + const rgb = if (at < pal.len) pal[at] else bsb.Rgb{ .r = 0, .g = 0, .b = 0 }; rs += rgb.r; gs += rgb.g; bs += rgb.b;