0.1.8 --- okb chunk digests verified on several contexts at once; openkal 0.19.0

.github/workflows/release-binaries.yml

@@ -14,7 +14,7 @@ on:
         description: "The release tag the binaries are built from and attached to (e.g. 0.1.3)"
         required: true
       platform:
-        description: "The openkal-platform whose hosted arrangement builds the tools: the one whose version group holds the implementations the tag pins (0.1.5: 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2)"
+        description: "The openkal-platform whose hosted arrangement builds the tools: the one whose version group holds the implementations the tag pins (0.1.5: 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2; 0.1.8: 0.20.0)"
         required: true
         default: "0.19.0"
       mcpp_ref:
@@ -31,7 +31,7 @@ env:
   # The platform whose hosted arrangement the programs are built over, named by
   # the dispatch beside the tag: a tag's tools pin exact implementations, and the
   # platform is the one whose version group holds them (0.1.4: 0.18.0; 0.1.5:
-  # 0.19.0, openkal-linux 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2).
+  # 0.19.0, openkal-linux 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2; 0.1.8: 0.20.0).
   PLATFORM_VERSION: ${{ github.event.inputs.platform }}
   XLINGS_VERSION: v2026.10.26.1
   XLINGS_NON_INTERACTIVE: '1'

README.md

@@ -68,6 +68,8 @@ its header and calls it, on openkal and on every other system alike ([docs/bind.
   the binding `openkal-bind` generates (including a callback); the boundary's refusals hold.
 - Once an image is loaded, the runner maps neither it nor the okb it came from, and holds its
   segments once (Linux, from the process's mappings).
+- An okb states its images' chunk digests, which okrun verifies on several contexts and
+  `okld bundle check` beside the whole digest; a changed byte is refused by either.
 - libokld's SHA-256, CRC-32, ZIP reader and writer (with ZIP64) and manifest reader hold
   against their vectors and refusals.
 

bind/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-bind"
-version     = "0.1.7"
+version     = "0.1.8"
 # `mcpp::xpkg_request', `mcpp::decision' (protocol 15).
 mcpp        = ">=2026.10.1.3"
 description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun."

docs/bind.md

@@ -24,10 +24,10 @@ sources      = []
 include_dirs = ["include"]
 
 [target.'cfg(not(os = "openkal"))'.dependencies]
-openkal-run = "0.1.7"
+openkal-run = "0.1.8"
 
 [build-dependencies]
-openkal-bind = "0.1.7"
+openkal-bind = "0.1.8"
 ```
 
 ```cpp
@@ -103,3 +103,4 @@ llvm toolchain's clang converts the call.
 | 0.1.5 | No change of its own; released with the archive, whose runner maps an image or okb file rather than reading it whole, and binds `openkal.map` for images (openkal 0.18). |
 | 0.1.6 | No change of its own; released with the archive, whose runner releases the file it ran once the image is loaded, and whose loader digests on the processor's SHA-256 instructions and writes each segment once. |
 | 0.1.7 | No change of its own; released with the archive, whose loader also digests on ARMv8's SHA-256 instructions on Linux where the processor states them. |
+| 0.1.8 | No change of its own; released with the archive, built against openkal 0.19.0 (the runner binds `kal_task_cores` and `kal_task_set_priority`), whose okb carries chunk digests the runner verifies on several contexts at once. |

docs/okb.md

@@ -74,6 +74,7 @@ A JSON object (RFC 8259) in UTF-8. Names are ASCII.
 | `entry` | The image entered, by its name within an architecture's directory. An architecture without its own `entry` takes this one. |
 | `architectures` | One object per architecture the okb holds, keyed by the architecture's name. `dir` is the directory of its images (default: the architecture's name); `entry` is the image it enters. At least one architecture is required. |
 | `files` | One object per member other than `okb.json` and `okb.sig`, keyed by the member's name. `sha256` is the SHA-256 of its content (64 hexadecimal digits); `size` is its length in bytes. |
+| `files.<member>.chunks` | Optional (okrun 0.1.8). `size` is a chunk's length in bytes, and `sha256` the SHA-256 of each `size` bytes of the member in order, the last as long as what remains; there are exactly as many as the member's size needs. `okld bundle` writes them for every image, in chunks of 1 MiB. A reader that verifies them need not digest the whole, and can verify them at once on several processors, where one SHA-256 of the whole cannot be shared out; a reader that does not know them verifies `sha256`, as before. |
 
 Unknown keys are ignored, so that a later minor addition does not need a new `okb` number.
 
@@ -97,7 +98,7 @@ A reader that runs an okb:
 3. selects the architecture of the machine it runs on, and refuses an okb that holds none for
    it, naming the architectures it holds;
 4. admits each member it uses: stored (C1), an image's bytes aligned (C2), and its SHA-256 the
-   one `files` states. A member that does not hold is refused; a reader does not decompress,
+   one `files` states --- or, where `files` states its chunks, each chunk's (§3). A member that does not hold is refused; a reader does not decompress,
    realign or copy a member to make it hold;
 5. resolves the entry and each image an image needs (DT_NEEDED) in this order:
    1. the architecture's directory, then the archive's root;
@@ -163,3 +164,4 @@ exports of the other images of its directory. It prints one line per violation.
 |---|---|---|
 | 1 | 0.1.2 | This document. |
 | 1 | 0.1.3 | `okld bundle --package`: an okb from a package mcpp staged. |
+| 1 | 0.1.8 | `files.<member>.chunks`: an image's chunk digests, written by `okld bundle`, verified by okrun on several contexts at once and by `okld bundle check` beside the whole digest. Still okb 1: a reader that does not know the key ignores it. |

examples/host/mcpp.toml

@@ -4,7 +4,7 @@
 # system of each architecture with the images built once.
 [package]
 name    = "okrun-host"
-version = "0.1.7"
+version = "0.1.8"
 
 [dependencies]
 openkal-run = { path = "../../run" }

ld/include/okld.h

@@ -181,6 +181,27 @@ int okld_okb_is_image(const char* name, okld_size name_len);
  * filled. Nothing is copied. */
 int okld_okb_admit(const void* archive, okld_u64 size, const struct okld_zip_member* member,
                    const void* json, okld_size json_size, struct okld_why* why);
+/* THE CHUNK DIGESTS okb.json may state for a member (`files.<member>.chunks',
+ * docs/okb.md, version 0.1.8): the SHA-256 of each `size' bytes of it in
+ * order, the last as long as what remains. 1 when they are stated, each given
+ * to `each' (which returns non-zero to stop), and `*chunk_size' set; 0 when
+ * not; negative when the manifest cannot be read. */
+int okld_okb_chunks(const void* json, okld_size json_size, const char* member, okld_u64* chunk_size,
+                    int (*each)(okld_u64 index, const unsigned char digest[32], void* ctx), void* ctx);
+/* Jobs run by the caller's means, all of them, before it returns: job(i, ctx)
+ * for each i below `count'. A loader has no threads of its own; a runner
+ * gives it its own (okrun: openkal tasks). */
+typedef void (*okld_parallel)(void (*job)(okld_u64 index, void* ctx), okld_u64 count, void* ctx, void* host);
+/* okld_okb_admit, but where the manifest states the member's chunk digests
+ * they are what is verified, as `parallel' runs them (null: one after
+ * another), instead of the SHA-256 of the whole: a member's admission is as
+ * fast as the processors together, where SHA-256 of one stream is not. With
+ * no chunks stated it is okld_okb_admit. */
+int okld_okb_admit_chunked(const void* archive, okld_u64 size, const struct okld_zip_member* member,
+                           const void* json, okld_size json_size, okld_parallel parallel, void* host,
+                           struct okld_why* why);
+/* The chunk size `okld bundle' writes. */
+#define OKLD_OKB_CHUNK 1048576u
 /* The architecture's name for an ELF machine (62, 183), or null. */
 const char* okld_okb_arch_of(okld_u32 machine);
 

ld/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-ld"
-version     = "0.1.7"
+version     = "0.1.8"
 description = "libokld: the ok format (openkal image format) --- its definition, a reader of bytes, a checker, a signer, and a loader. Freestanding: no C library, no exceptions."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]

ld/src/okb.cpp

@@ -30,6 +30,13 @@ struct reader {
     int (*on_file)(const char* name, okld_size len, void* ctx) = nullptr;   // okld_okb_each_file
     void* on_file_ctx = nullptr;
     int stopped = 0;
+    // okld_okb_chunks's member: its chunk size, and each chunk digest in order.
+    const char* chunk_member = nullptr;
+    u64* chunk_size = nullptr;
+    int (*on_chunk)(u64 index, const u8 digest[32], void* ctx) = nullptr;
+    void* on_chunk_ctx = nullptr;
+    u64 chunk_index = 0;
+    bool chunks_found = false;
 
     void fail(const char* s) { if (!failed) { failed = true; sentence(why, sizeof why) << s; } }
     void ws() { while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')) ++p; }
@@ -118,6 +125,16 @@ struct reader {
                 else put(a->entry, sizeof a->entry, s, n);
             }
         }
+        if (chunk_member) {
+            const char* chunk_digest[] = { "files", chunk_member, "chunks", "sha256", nullptr };
+            if (at_path(5, chunk_digest)) {
+                u8 d[32];
+                if (!hex_digest(s, n, d)) return;
+                chunks_found = true;
+                if (!stopped && on_chunk) stopped = on_chunk(chunk_index, d, on_chunk_ctx);
+                ++chunk_index;
+            }
+        }
         if (member) {
             const char* file_digest[] = { "files", member, "sha256" };
             if (at_path(3, file_digest)) {
@@ -132,6 +149,16 @@ struct reader {
         }
     }
 
+    bool hex_digest(const char* s, size n, u8 out[32]) {
+        if (n != 64) { fail("a sha256 that is not 64 hexadecimal digits"); return false; }
+        for (int i = 0; i < 32; ++i) {
+            const int hi = hex(s[2 * i]), lo = hex(s[2 * i + 1]);
+            if (hi < 0 || lo < 0) { fail("a sha256 that is not hexadecimal"); return false; }
+            out[i] = static_cast<u8>(hi * 16 + lo);
+        }
+        return true;
+    }
+
     okld_okb_arch* arch_named(const char* s, size n) {
         for (okld_u32 i = 0; i < m->arch_count; ++i)
             if (length(m->archs[i].name) == n) {
@@ -203,6 +230,10 @@ struct reader {
         if ((*p >= '0' && *p <= '9') || *p == '-') {
             u64 v = 0;
             if (number(v) && m && path_is("okb")) m->okb = static_cast<okld_u32>(v);
+            if (!failed && chunk_member && chunk_size) {
+                const char* size_path[] = { "files", chunk_member, "chunks", "size" };
+                if (at_path(4, size_path)) *chunk_size = v;
+            }
             return;
         }
         auto word = [&](const char* w) {
@@ -283,17 +314,26 @@ extern "C" int okld_okb_is_image(const char* name, okld_size n) {
     return ends_with(name, n, ".oke") || ends_with(name, n, ".okso") ? 1 : 0;
 }
 
-extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const okld_zip_member* m,
-                              const void* json, okld_size json_n, okld_why* why) {
-    char name[256];
+namespace {
+// A member's name, and what admission asks of its place: stored, within the
+// archive, an image's bytes at a multiple of 64 KiB.
+bool admit_layout(okld_u64 archive_size, const okld_zip_member* m, char (&name)[256], okld_why* why) {
     const size n = m->name_len < sizeof name - 1 ? m->name_len : sizeof name - 1;
     copy(name, m->name, n); name[n] = 0;
-    if (m->method != 0 || m->size != m->original) { say(why, name, ": not stored; an okb member is stored as it is (docs/okb.md)"); return -1; }
-    if (m->data > archive_size || m->size > archive_size - m->data) { say(why, name, ": its bytes extend beyond the archive"); return -1; }
+    if (m->method != 0 || m->size != m->original) { say(why, name, ": not stored; an okb member is stored as it is (docs/okb.md)"); return false; }
+    if (m->data > archive_size || m->size > archive_size - m->data) { say(why, name, ": its bytes extend beyond the archive"); return false; }
     if (okld_okb_is_image(name, n) && m->data % OKLD_OKB_ALIGNMENT != 0) {
         say(why, name, ": its bytes do not begin at a multiple of 64 KiB (docs/okb.md)");
-        return -1;
+        return false;
     }
+    return true;
+}
+}  // namespace
+
+extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const okld_zip_member* m,
+                              const void* json, okld_size json_n, okld_why* why) {
+    char name[256];
+    if (!admit_layout(archive_size, m, name, why)) return -1;
     const auto* bytes = static_cast<const u8*>(archive) + m->data;
     if (json) {
         u8 want[32], have[32];
@@ -312,3 +352,84 @@ extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const
 extern "C" const char* okld_okb_arch_of(okld_u32 machine) {
     return machine == 62 ? "x86_64" : machine == 183 ? "aarch64" : nullptr;
 }
+
+extern "C" int okld_okb_chunks(const void* json, okld_size n, const char* member, okld_u64* chunk_size,
+                               int (*each)(okld_u64 index, const unsigned char digest[32], void* ctx), void* ctx) {
+    u64 size = 0;
+    reader r{ static_cast<const char*>(json), static_cast<const char*>(json) + n };
+    r.chunk_member = member;
+    r.chunk_size = &size;
+    r.on_chunk = each;
+    r.on_chunk_ctx = ctx;
+    r.document();
+    if (r.failed) return -1;
+    if (!r.chunks_found) return 0;
+    if (chunk_size) *chunk_size = size;
+    return 1;
+}
+
+namespace {
+
+// THE CHUNKS ARE SPLIT AMONG THE JOBS BY INDEX, and each job reads the
+// manifest once for its own: the library holds no list of digests (it
+// allocates nothing), and a manifest is a few kilobytes against megabytes
+// digested. A job records the first chunk that did not hold, plus one.
+constexpr u64 kJobs = 8;
+
+struct chunk_check {
+    const u8* bytes; u64 size; u64 chunk;
+    const void* json; okld_size json_n; const char* name;
+    u64 jobs;
+    u64 bad[kJobs];
+};
+struct chunk_job { chunk_check* c; u64 job; };
+
+void check_chunks(okld_u64 job, void* ctx) {
+    auto& c = *static_cast<chunk_check*>(ctx);
+    chunk_job j{ &c, job };
+    okld_okb_chunks(c.json, c.json_n, c.name, nullptr, [](okld_u64 i, const unsigned char want[32], void* x) -> int {
+        auto& j = *static_cast<chunk_job*>(x);
+        auto& c = *j.c;
+        if (i % c.jobs != j.job) return 0;
+        const u64 at = i * c.chunk;
+        if (at >= c.size) { if (!c.bad[j.job]) c.bad[j.job] = i + 1; return 1; }
+        const u64 n = c.size - at < c.chunk ? c.size - at : c.chunk;
+        unsigned char have[32];
+        okld_sha256(c.bytes + at, n, have);
+        for (int k = 0; k < 32; ++k)
+            if (have[k] != want[k]) { if (!c.bad[j.job]) c.bad[j.job] = i + 1; return 1; }
+        return 0;
+    }, &j);
+}
+
+}  // namespace
+
+extern "C" int okld_okb_admit_chunked(const void* archive, okld_u64 archive_size, const okld_zip_member* m,
+                                      const void* json, okld_size json_n, okld_parallel parallel, void* host,
+                                      okld_why* why) {
+    if (!json) return okld_okb_admit(archive, archive_size, m, json, json_n, why);
+    char name[256];
+    if (!admit_layout(archive_size, m, name, why)) return -1;
+    u64 chunk = 0, count = 0;
+    const int stated = okld_okb_chunks(json, json_n, name, &chunk, [](okld_u64, const unsigned char*, void* x) -> int {
+        ++*static_cast<u64*>(x);
+        return 0;
+    }, &count);
+    if (stated < 0) { say(why, "okb.json cannot be read"); return -1; }
+    if (stated == 0) return okld_okb_admit(archive, archive_size, m, json, json_n, why);
+    // THE CHUNKS COVER THE MEMBER EXACTLY: as many as its size needs, none past it.
+    if (chunk == 0 || count != (m->size + chunk - 1) / chunk || (m->size == 0 && count != 0)) {
+        say(why, name, ": okb.json states chunks that do not cover it");
+        return -1;
+    }
+    chunk_check c{};
+    c.bytes = static_cast<const u8*>(archive) + m->data;
+    c.size = m->size; c.chunk = chunk;
+    c.json = json; c.json_n = json_n; c.name = name;
+    c.jobs = count < kJobs ? (count ? count : 1) : kJobs;
+    if (parallel) parallel(&check_chunks, c.jobs, &c, host);
+    else for (u64 j = 0; j < c.jobs; ++j) check_chunks(j, &c);
+    for (u64 j = 0; j < c.jobs; ++j)
+        if (c.bad[j]) { say(why, name, ": its content is not the one okb.json states (sha256 of a chunk)"); return -1; }
+    return 0;
+}

ld/src/okld.cppm

@@ -50,11 +50,15 @@ export using ::okld_okb_digest;
 export using ::okld_okb_each_file;
 export using ::okld_okb_is_image;
 export using ::okld_okb_admit;
+export using ::okld_okb_chunks;
+export using ::okld_okb_admit_chunked;
+export using ::okld_parallel;
 export using ::okld_okb_arch_of;
 
 export namespace okld {
 inline constexpr okld_u32 format_major = OKLD_FORMAT_MAJOR;
 inline constexpr okld_u32 format_minor = OKLD_FORMAT_MINOR;
 inline constexpr okld_u32 okb_version = OKLD_OKB_VERSION;
 inline constexpr okld_u32 okb_alignment = OKLD_OKB_ALIGNMENT;
+inline constexpr okld_u32 okb_chunk = OKLD_OKB_CHUNK;
 }