From 4ed2b2101d379a593e3b20508b7d10e0d9678dda Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sun, 1 Nov 2026 04:09:55 +0900 Subject: [PATCH] 0.1.8 --- okb chunk digests verified on several contexts at once; openkal 0.19.0 (kal_task_cores, kal_task_set_priority bound for images) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An okb's image member may state its chunk digests (okb.json `files..chunks`: the SHA-256 of each 1 MiB, docs/okb.md §3). `okld bundle` writes them for every image; okrun verifies them through okld_okb_admit_chunked on up to eight contexts (kal_task), where one SHA-256 of the whole member cannot be shared out; `okld bundle check` verifies them beside the whole digest, so an okb whose two disagree is refused. A reader that does not know the key verifies the whole digest, as before: still okb 1. Measured on a 68 MB image with SHA extensions: 0.10-0.12 s to start with 0.1.7, 0.04-0.07 s with 0.1.8; without them the whole digest took 0.3 s and more. libokld has no contexts of its own and allocates nothing: the caller runs the jobs (okld_parallel), and each job reads the manifest once for its own chunks. libokld's tests cover the chunks held one after another and as a caller runs them, a changed byte in the last chunk, chunks that do not cover the member, and the whole digest where none are stated. The runner is built against openkal 0.19.0 (openkal-linux 0.20.0, openkal-macos 0.17.0, openkal-windows 0.15.0), so images bind kal_task_cores and kal_task_set_priority. ld, run, tools and bind are 0.1.8. Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- .github/workflows/release-binaries.yml | 4 +- README.md | 2 + bind/mcpp.toml | 2 +- docs/bind.md | 5 +- docs/okb.md | 4 +- examples/host/mcpp.toml | 2 +- ld/include/okld.h | 21 ++++ ld/mcpp.toml | 2 +- ld/src/okb.cpp | 133 +++++++++++++++++++++++-- ld/src/okld.cppm | 4 + ld/tests/okb_test.cpp | 54 ++++++++++ run/mcpp.toml | 10 +- run/src/run.cpp | 26 ++++- tools/mcpp.toml | 2 +- tools/src/okld.cpp | 22 +++- 15 files changed, 270 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 9959710..f221cee 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -14,7 +14,7 @@ on: description: "The release tag the binaries are built from and attached to (e.g. 0.1.3)" required: true platform: - description: "The openkal-platform whose hosted arrangement builds the tools: the one whose version group holds the implementations the tag pins (0.1.5: 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2)" + description: "The openkal-platform whose hosted arrangement builds the tools: the one whose version group holds the implementations the tag pins (0.1.5: 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2; 0.1.8: 0.20.0)" required: true default: "0.19.0" mcpp_ref: @@ -31,7 +31,7 @@ env: # The platform whose hosted arrangement the programs are built over, named by # the dispatch beside the tag: a tag's tools pin exact implementations, and the # platform is the one whose version group holds them (0.1.4: 0.18.0; 0.1.5: - # 0.19.0, openkal-linux 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2). + # 0.19.0, openkal-linux 0.19.0; 0.1.6: 0.19.1; 0.1.7: 0.19.2; 0.1.8: 0.20.0). PLATFORM_VERSION: ${{ github.event.inputs.platform }} XLINGS_VERSION: v2026.10.26.1 XLINGS_NON_INTERACTIVE: '1' diff --git a/README.md b/README.md index c82eead..f41f92f 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ its header and calls it, on openkal and on every other system alike ([docs/bind. the binding `openkal-bind` generates (including a callback); the boundary's refusals hold. - Once an image is loaded, the runner maps neither it nor the okb it came from, and holds its segments once (Linux, from the process's mappings). +- An okb states its images' chunk digests, which okrun verifies on several contexts and + `okld bundle check` beside the whole digest; a changed byte is refused by either. - libokld's SHA-256, CRC-32, ZIP reader and writer (with ZIP64) and manifest reader hold against their vectors and refusals. diff --git a/bind/mcpp.toml b/bind/mcpp.toml index 4104af6..bb633cd 100644 --- a/bind/mcpp.toml +++ b/bind/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-bind" -version = "0.1.7" +version = "0.1.8" # `mcpp::xpkg_request', `mcpp::decision' (protocol 15). mcpp = ">=2026.10.1.3" description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun." diff --git a/docs/bind.md b/docs/bind.md index 979dac5..5c79fb8 100644 --- a/docs/bind.md +++ b/docs/bind.md @@ -24,10 +24,10 @@ sources = [] include_dirs = ["include"] [target.'cfg(not(os = "openkal"))'.dependencies] -openkal-run = "0.1.7" +openkal-run = "0.1.8" [build-dependencies] -openkal-bind = "0.1.7" +openkal-bind = "0.1.8" ``` ```cpp @@ -103,3 +103,4 @@ llvm toolchain's clang converts the call. | 0.1.5 | No change of its own; released with the archive, whose runner maps an image or okb file rather than reading it whole, and binds `openkal.map` for images (openkal 0.18). | | 0.1.6 | No change of its own; released with the archive, whose runner releases the file it ran once the image is loaded, and whose loader digests on the processor's SHA-256 instructions and writes each segment once. | | 0.1.7 | No change of its own; released with the archive, whose loader also digests on ARMv8's SHA-256 instructions on Linux where the processor states them. | +| 0.1.8 | No change of its own; released with the archive, built against openkal 0.19.0 (the runner binds `kal_task_cores` and `kal_task_set_priority`), whose okb carries chunk digests the runner verifies on several contexts at once. | diff --git a/docs/okb.md b/docs/okb.md index d97521f..d2f7d01 100644 --- a/docs/okb.md +++ b/docs/okb.md @@ -74,6 +74,7 @@ A JSON object (RFC 8259) in UTF-8. Names are ASCII. | `entry` | The image entered, by its name within an architecture's directory. An architecture without its own `entry` takes this one. | | `architectures` | One object per architecture the okb holds, keyed by the architecture's name. `dir` is the directory of its images (default: the architecture's name); `entry` is the image it enters. At least one architecture is required. | | `files` | One object per member other than `okb.json` and `okb.sig`, keyed by the member's name. `sha256` is the SHA-256 of its content (64 hexadecimal digits); `size` is its length in bytes. | +| `files..chunks` | Optional (okrun 0.1.8). `size` is a chunk's length in bytes, and `sha256` the SHA-256 of each `size` bytes of the member in order, the last as long as what remains; there are exactly as many as the member's size needs. `okld bundle` writes them for every image, in chunks of 1 MiB. A reader that verifies them need not digest the whole, and can verify them at once on several processors, where one SHA-256 of the whole cannot be shared out; a reader that does not know them verifies `sha256`, as before. | Unknown keys are ignored, so that a later minor addition does not need a new `okb` number. @@ -97,7 +98,7 @@ A reader that runs an okb: 3. selects the architecture of the machine it runs on, and refuses an okb that holds none for it, naming the architectures it holds; 4. admits each member it uses: stored (C1), an image's bytes aligned (C2), and its SHA-256 the - one `files` states. A member that does not hold is refused; a reader does not decompress, + one `files` states --- or, where `files` states its chunks, each chunk's (§3). A member that does not hold is refused; a reader does not decompress, realign or copy a member to make it hold; 5. resolves the entry and each image an image needs (DT_NEEDED) in this order: 1. the architecture's directory, then the archive's root; @@ -163,3 +164,4 @@ exports of the other images of its directory. It prints one line per violation. |---|---|---| | 1 | 0.1.2 | This document. | | 1 | 0.1.3 | `okld bundle --package`: an okb from a package mcpp staged. | +| 1 | 0.1.8 | `files..chunks`: an image's chunk digests, written by `okld bundle`, verified by okrun on several contexts at once and by `okld bundle check` beside the whole digest. Still okb 1: a reader that does not know the key ignores it. | diff --git a/examples/host/mcpp.toml b/examples/host/mcpp.toml index 8d51c85..1224335 100644 --- a/examples/host/mcpp.toml +++ b/examples/host/mcpp.toml @@ -4,7 +4,7 @@ # system of each architecture with the images built once. [package] name = "okrun-host" -version = "0.1.7" +version = "0.1.8" [dependencies] openkal-run = { path = "../../run" } diff --git a/ld/include/okld.h b/ld/include/okld.h index 5ec00c9..ddece3b 100644 --- a/ld/include/okld.h +++ b/ld/include/okld.h @@ -181,6 +181,27 @@ int okld_okb_is_image(const char* name, okld_size name_len); * filled. Nothing is copied. */ int okld_okb_admit(const void* archive, okld_u64 size, const struct okld_zip_member* member, const void* json, okld_size json_size, struct okld_why* why); +/* THE CHUNK DIGESTS okb.json may state for a member (`files..chunks', + * docs/okb.md, version 0.1.8): the SHA-256 of each `size' bytes of it in + * order, the last as long as what remains. 1 when they are stated, each given + * to `each' (which returns non-zero to stop), and `*chunk_size' set; 0 when + * not; negative when the manifest cannot be read. */ +int okld_okb_chunks(const void* json, okld_size json_size, const char* member, okld_u64* chunk_size, + int (*each)(okld_u64 index, const unsigned char digest[32], void* ctx), void* ctx); +/* Jobs run by the caller's means, all of them, before it returns: job(i, ctx) + * for each i below `count'. A loader has no threads of its own; a runner + * gives it its own (okrun: openkal tasks). */ +typedef void (*okld_parallel)(void (*job)(okld_u64 index, void* ctx), okld_u64 count, void* ctx, void* host); +/* okld_okb_admit, but where the manifest states the member's chunk digests + * they are what is verified, as `parallel' runs them (null: one after + * another), instead of the SHA-256 of the whole: a member's admission is as + * fast as the processors together, where SHA-256 of one stream is not. With + * no chunks stated it is okld_okb_admit. */ +int okld_okb_admit_chunked(const void* archive, okld_u64 size, const struct okld_zip_member* member, + const void* json, okld_size json_size, okld_parallel parallel, void* host, + struct okld_why* why); +/* The chunk size `okld bundle' writes. */ +#define OKLD_OKB_CHUNK 1048576u /* The architecture's name for an ELF machine (62, 183), or null. */ const char* okld_okb_arch_of(okld_u32 machine); diff --git a/ld/mcpp.toml b/ld/mcpp.toml index abddcde..8fd87ca 100644 --- a/ld/mcpp.toml +++ b/ld/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-ld" -version = "0.1.7" +version = "0.1.8" description = "libokld: the ok format (openkal image format) --- its definition, a reader of bytes, a checker, a signer, and a loader. Freestanding: no C library, no exceptions." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/ld/src/okb.cpp b/ld/src/okb.cpp index d652fd9..bf5bee6 100644 --- a/ld/src/okb.cpp +++ b/ld/src/okb.cpp @@ -30,6 +30,13 @@ struct reader { int (*on_file)(const char* name, okld_size len, void* ctx) = nullptr; // okld_okb_each_file void* on_file_ctx = nullptr; int stopped = 0; + // okld_okb_chunks's member: its chunk size, and each chunk digest in order. + const char* chunk_member = nullptr; + u64* chunk_size = nullptr; + int (*on_chunk)(u64 index, const u8 digest[32], void* ctx) = nullptr; + void* on_chunk_ctx = nullptr; + u64 chunk_index = 0; + bool chunks_found = false; void fail(const char* s) { if (!failed) { failed = true; sentence(why, sizeof why) << s; } } void ws() { while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')) ++p; } @@ -118,6 +125,16 @@ struct reader { else put(a->entry, sizeof a->entry, s, n); } } + if (chunk_member) { + const char* chunk_digest[] = { "files", chunk_member, "chunks", "sha256", nullptr }; + if (at_path(5, chunk_digest)) { + u8 d[32]; + if (!hex_digest(s, n, d)) return; + chunks_found = true; + if (!stopped && on_chunk) stopped = on_chunk(chunk_index, d, on_chunk_ctx); + ++chunk_index; + } + } if (member) { const char* file_digest[] = { "files", member, "sha256" }; if (at_path(3, file_digest)) { @@ -132,6 +149,16 @@ struct reader { } } + bool hex_digest(const char* s, size n, u8 out[32]) { + if (n != 64) { fail("a sha256 that is not 64 hexadecimal digits"); return false; } + for (int i = 0; i < 32; ++i) { + const int hi = hex(s[2 * i]), lo = hex(s[2 * i + 1]); + if (hi < 0 || lo < 0) { fail("a sha256 that is not hexadecimal"); return false; } + out[i] = static_cast(hi * 16 + lo); + } + return true; + } + okld_okb_arch* arch_named(const char* s, size n) { for (okld_u32 i = 0; i < m->arch_count; ++i) if (length(m->archs[i].name) == n) { @@ -203,6 +230,10 @@ struct reader { if ((*p >= '0' && *p <= '9') || *p == '-') { u64 v = 0; if (number(v) && m && path_is("okb")) m->okb = static_cast(v); + if (!failed && chunk_member && chunk_size) { + const char* size_path[] = { "files", chunk_member, "chunks", "size" }; + if (at_path(4, size_path)) *chunk_size = v; + } return; } auto word = [&](const char* w) { @@ -283,17 +314,26 @@ extern "C" int okld_okb_is_image(const char* name, okld_size n) { return ends_with(name, n, ".oke") || ends_with(name, n, ".okso") ? 1 : 0; } -extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const okld_zip_member* m, - const void* json, okld_size json_n, okld_why* why) { - char name[256]; +namespace { +// A member's name, and what admission asks of its place: stored, within the +// archive, an image's bytes at a multiple of 64 KiB. +bool admit_layout(okld_u64 archive_size, const okld_zip_member* m, char (&name)[256], okld_why* why) { const size n = m->name_len < sizeof name - 1 ? m->name_len : sizeof name - 1; copy(name, m->name, n); name[n] = 0; - if (m->method != 0 || m->size != m->original) { say(why, name, ": not stored; an okb member is stored as it is (docs/okb.md)"); return -1; } - if (m->data > archive_size || m->size > archive_size - m->data) { say(why, name, ": its bytes extend beyond the archive"); return -1; } + if (m->method != 0 || m->size != m->original) { say(why, name, ": not stored; an okb member is stored as it is (docs/okb.md)"); return false; } + if (m->data > archive_size || m->size > archive_size - m->data) { say(why, name, ": its bytes extend beyond the archive"); return false; } if (okld_okb_is_image(name, n) && m->data % OKLD_OKB_ALIGNMENT != 0) { say(why, name, ": its bytes do not begin at a multiple of 64 KiB (docs/okb.md)"); - return -1; + return false; } + return true; +} +} // namespace + +extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const okld_zip_member* m, + const void* json, okld_size json_n, okld_why* why) { + char name[256]; + if (!admit_layout(archive_size, m, name, why)) return -1; const auto* bytes = static_cast(archive) + m->data; if (json) { u8 want[32], have[32]; @@ -312,3 +352,84 @@ extern "C" int okld_okb_admit(const void* archive, okld_u64 archive_size, const extern "C" const char* okld_okb_arch_of(okld_u32 machine) { return machine == 62 ? "x86_64" : machine == 183 ? "aarch64" : nullptr; } + +extern "C" int okld_okb_chunks(const void* json, okld_size n, const char* member, okld_u64* chunk_size, + int (*each)(okld_u64 index, const unsigned char digest[32], void* ctx), void* ctx) { + u64 size = 0; + reader r{ static_cast(json), static_cast(json) + n }; + r.chunk_member = member; + r.chunk_size = &size; + r.on_chunk = each; + r.on_chunk_ctx = ctx; + r.document(); + if (r.failed) return -1; + if (!r.chunks_found) return 0; + if (chunk_size) *chunk_size = size; + return 1; +} + +namespace { + +// THE CHUNKS ARE SPLIT AMONG THE JOBS BY INDEX, and each job reads the +// manifest once for its own: the library holds no list of digests (it +// allocates nothing), and a manifest is a few kilobytes against megabytes +// digested. A job records the first chunk that did not hold, plus one. +constexpr u64 kJobs = 8; + +struct chunk_check { + const u8* bytes; u64 size; u64 chunk; + const void* json; okld_size json_n; const char* name; + u64 jobs; + u64 bad[kJobs]; +}; +struct chunk_job { chunk_check* c; u64 job; }; + +void check_chunks(okld_u64 job, void* ctx) { + auto& c = *static_cast(ctx); + chunk_job j{ &c, job }; + okld_okb_chunks(c.json, c.json_n, c.name, nullptr, [](okld_u64 i, const unsigned char want[32], void* x) -> int { + auto& j = *static_cast(x); + auto& c = *j.c; + if (i % c.jobs != j.job) return 0; + const u64 at = i * c.chunk; + if (at >= c.size) { if (!c.bad[j.job]) c.bad[j.job] = i + 1; return 1; } + const u64 n = c.size - at < c.chunk ? c.size - at : c.chunk; + unsigned char have[32]; + okld_sha256(c.bytes + at, n, have); + for (int k = 0; k < 32; ++k) + if (have[k] != want[k]) { if (!c.bad[j.job]) c.bad[j.job] = i + 1; return 1; } + return 0; + }, &j); +} + +} // namespace + +extern "C" int okld_okb_admit_chunked(const void* archive, okld_u64 archive_size, const okld_zip_member* m, + const void* json, okld_size json_n, okld_parallel parallel, void* host, + okld_why* why) { + if (!json) return okld_okb_admit(archive, archive_size, m, json, json_n, why); + char name[256]; + if (!admit_layout(archive_size, m, name, why)) return -1; + u64 chunk = 0, count = 0; + const int stated = okld_okb_chunks(json, json_n, name, &chunk, [](okld_u64, const unsigned char*, void* x) -> int { + ++*static_cast(x); + return 0; + }, &count); + if (stated < 0) { say(why, "okb.json cannot be read"); return -1; } + if (stated == 0) return okld_okb_admit(archive, archive_size, m, json, json_n, why); + // THE CHUNKS COVER THE MEMBER EXACTLY: as many as its size needs, none past it. + if (chunk == 0 || count != (m->size + chunk - 1) / chunk || (m->size == 0 && count != 0)) { + say(why, name, ": okb.json states chunks that do not cover it"); + return -1; + } + chunk_check c{}; + c.bytes = static_cast(archive) + m->data; + c.size = m->size; c.chunk = chunk; + c.json = json; c.json_n = json_n; c.name = name; + c.jobs = count < kJobs ? (count ? count : 1) : kJobs; + if (parallel) parallel(&check_chunks, c.jobs, &c, host); + else for (u64 j = 0; j < c.jobs; ++j) check_chunks(j, &c); + for (u64 j = 0; j < c.jobs; ++j) + if (c.bad[j]) { say(why, name, ": its content is not the one okb.json states (sha256 of a chunk)"); return -1; } + return 0; +} diff --git a/ld/src/okld.cppm b/ld/src/okld.cppm index 8904ee8..4fe8afb 100644 --- a/ld/src/okld.cppm +++ b/ld/src/okld.cppm @@ -50,6 +50,9 @@ export using ::okld_okb_digest; export using ::okld_okb_each_file; export using ::okld_okb_is_image; export using ::okld_okb_admit; +export using ::okld_okb_chunks; +export using ::okld_okb_admit_chunked; +export using ::okld_parallel; export using ::okld_okb_arch_of; export namespace okld { @@ -57,4 +60,5 @@ inline constexpr okld_u32 format_major = OKLD_FORMAT_MAJOR; inline constexpr okld_u32 format_minor = OKLD_FORMAT_MINOR; inline constexpr okld_u32 okb_version = OKLD_OKB_VERSION; inline constexpr okld_u32 okb_alignment = OKLD_OKB_ALIGNMENT; +inline constexpr okld_u32 okb_chunk = OKLD_OKB_CHUNK; } diff --git a/ld/tests/okb_test.cpp b/ld/tests/okb_test.cpp index 90ece4a..eb7b171 100644 --- a/ld/tests/okb_test.cpp +++ b/ld/tests/okb_test.cpp @@ -161,5 +161,59 @@ int main() { check(okld_okb_read_manifest(later, std::strlen(later), &man, &why) < 0 && std::strstr(why.text, "later version"), "a later okb version is refused"); check(okld_okb_read_manifest("{\"okb\": 1, \"entry\": ", 20, &man, &why) < 0, "a truncated manifest is refused"); + + // CHUNK DIGESTS (0.1.8): an image's chunks are verified instead of its + // whole digest, one after another or as the caller runs them. + { + std::string big(2 * OKLD_OKB_CHUNK + OKLD_OKB_CHUNK / 2, '\0'); + for (std::size_t i = 0; i < big.size(); ++i) big[i] = static_cast((i * 131) % 251); + unsigned char whole[32]; + okld_sha256(big.data(), big.size(), whole); + std::string list; + for (std::size_t at = 0; at < big.size(); at += OKLD_OKB_CHUNK) { + unsigned char c[32]; + okld_sha256(big.data() + at, std::min(OKLD_OKB_CHUNK, big.size() - at), c); + list += std::string(at ? ", " : "") + "\"" + hex(c, 32) + "\""; + } + const auto chunked = [&](const std::string& chunks) { + return "{\"okb\": 1, \"entry\": \"big.oke\", \"architectures\": {\"x86_64\": {}},\n" + " \"files\": {\"x86_64/big.oke\": {\"sha256\": \"" + hex(whole, 32) + "\", \"size\": " + + std::to_string(big.size()) + chunks + "}}}"; + }; + const std::string with = chunked(", \"chunks\": {\"size\": " + std::to_string(OKLD_OKB_CHUNK) + ", \"sha256\": [" + list + "]}"); + okld_u64 chunk = 0; + int counted = 0; + check(okld_okb_chunks(with.data(), with.size(), "x86_64/big.oke", &chunk, [](okld_u64, const unsigned char*, void* c) -> int { + ++*static_cast(c); + return 0; + }, &counted) == 1 && chunk == OKLD_OKB_CHUNK && counted == 3, "the manifest states three chunks of 1 MiB"); + const auto pack = [&](const std::string& json, const std::string& image) { + return write({ { "okb.json", json.data(), json.size(), 0 }, { "x86_64/big.oke", image.data(), image.size(), 1 } }); + }; + const auto admit = [&](const std::vector& a, const std::string& json, okld_parallel run, okld_why* w) { + okld_zip_member m{}; + if (okld_zip_find(a.data(), a.size(), "x86_64/big.oke", &m, w) != 0) return -2; + return okld_okb_admit_chunked(a.data(), a.size(), &m, json.data(), json.size(), run, nullptr, w); + }; + const auto good = pack(with, big); + check(admit(good, with, nullptr, &why) == 0, "its chunks hold, one after another"); + int jobs_run = 0; + static int* seen = nullptr; + seen = &jobs_run; + check(admit(good, with, [](void (*job)(okld_u64, void*), okld_u64 n, void* c, void*) { + for (okld_u64 i = n; i-- > 0;) { job(i, c); ++*seen; } + }, &why) == 0 && jobs_run == 3, "and as the caller runs them, in any order"); + std::string changed = big; + changed[2 * OKLD_OKB_CHUNK + 7] ^= 1; + check(admit(pack(with, changed), with, nullptr, &why) < 0 && std::strstr(why.text, "chunk"), + "a changed byte in the last chunk is refused, naming a chunk"); + const std::string short_list = chunked(", \"chunks\": {\"size\": " + std::to_string(OKLD_OKB_CHUNK) + ", \"sha256\": [" + + list.substr(0, list.rfind(',')) + "]}"); + check(admit(pack(short_list, big), short_list, nullptr, &why) < 0 && std::strstr(why.text, "cover"), + "chunks that do not cover the member are refused"); + const std::string without = chunked(""); + check(admit(pack(without, big), without, nullptr, &why) == 0, "without chunks, the whole digest is verified"); + check(admit(pack(without, changed), without, nullptr, &why) < 0, "and refuses a changed byte"); + } return failures; } diff --git a/run/mcpp.toml b/run/mcpp.toml index f0f098d..4aff245 100644 --- a/run/mcpp.toml +++ b/run/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-run" -version = "0.1.7" +version = "0.1.8" description = "libokrun: the runner's core --- an ok image's names bound to this process's one openkal implementation, in the image's calling convention, and its entry entered." license = "Apache-2.0" authors = ["mcpplibs"] @@ -11,17 +11,17 @@ repo = "https://github.com/mcpplibs/openkal-image" kind = "lib" [dependencies] -openkal = "0.18.0" +openkal = "0.19.0" openkal-ld = { path = "../ld" } [target.linux.dependencies] -openkal-linux = "0.19.0" +openkal-linux = "0.20.0" [target.macos.dependencies] -openkal-macos = "0.16.0" +openkal-macos = "0.17.0" [target.windows.dependencies] -openkal-windows = "0.14.0" +openkal-windows = "0.15.0" # The interfaces each system's implementation provides decide one unit: where # openkal.space is provided its names are bound, and elsewhere an image that diff --git a/run/src/run.cpp b/run/src/run.cpp index 049c171..5b94634 100644 --- a/run/src/run.cpp +++ b/run/src/run.cpp @@ -266,12 +266,36 @@ bool join(char* out, unsigned long cap, const char* a, const char* b, const char return true; } +// THE CHUNKS OF A MEMBER ARE DIGESTED ON SEVERAL CONTEXTS AT ONCE (okb.json's +// `chunks', 0.1.8): libokld has no contexts of its own and asks for these. The +// calling context runs the first job; one that cannot be started runs here too. +struct chunk_task { void (*job)(okld_u64, void*); okld_u64 index; void* ctx; }; +void run_chunk_task(void* p) { + auto* t = static_cast(p); + t->job(t->index, t->ctx); +} +void in_parallel(void (*job)(okld_u64, void*), okld_u64 count, void* ctx, void*) { + constexpr okld_u64 kMost = 16; + chunk_task tasks[kMost]; + kal_task started[kMost]; + bool running[kMost] = {}; + for (okld_u64 i = 1; i < count && i < kMost; ++i) { + tasks[i] = { job, i, ctx }; + running[i] = kal_task_start(&run_chunk_task, &tasks[i], &started[i]) == kal_ok; + if (!running[i]) job(i, ctx); + } + for (okld_u64 i = kMost; i < count; ++i) job(i, ctx); + if (count) job(0, ctx); + for (okld_u64 i = 1; i < count && i < kMost; ++i) + if (running[i]) kal_task_join(started[i]); +} + // A member of the archive by name, admitted (stored, aligned, its stated content). const void* archive_member(search& s, const char* member, unsigned long long& size) { okld_zip_member m{}; okld_why why{}; if (okld_zip_find(s.archive, s.archive_size, member, &m, &why) != 0) return nullptr; - if (okld_okb_admit(s.archive, s.archive_size, &m, s.json, s.json_size, &why) != 0) { + if (okld_okb_admit_chunked(s.archive, s.archive_size, &m, s.json, s.json_size, &in_parallel, nullptr, &why) != 0) { copy_text(s.refusal, sizeof s.refusal, why.text); return nullptr; } diff --git a/tools/mcpp.toml b/tools/mcpp.toml index 6be5a33..7008926 100644 --- a/tools/mcpp.toml +++ b/tools/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-tools" -version = "0.1.7" +version = "0.1.8" description = "okrun, which runs an ok image on this system, and okld, the format's tool on a build machine (check, note, sign, inspect)." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/tools/src/okld.cpp b/tools/src/okld.cpp index 09a0e5d..42f77c9 100644 --- a/tools/src/okld.cpp +++ b/tools/src/okld.cpp @@ -327,8 +327,20 @@ int bundle(const std::vector& args) { for (const auto& [k, m] : members) { unsigned char d[32]; okld_sha256(m.bytes.data(), m.bytes.size(), d); - json += std::format(" {}: {{ \"sha256\": \"{}\", \"size\": {} }}{}\n", json_string(k), hex(d, 32), - m.bytes.size(), ++i == members.size() ? "" : ","); + // AN IMAGE'S CHUNKS TOO (docs/okb.md, 0.1.8): a runner verifies them + // in parallel, where one SHA-256 of the whole cannot be shared out. + std::string chunks; + if (okld_okb_is_image(k.data(), k.size())) { + chunks = std::format(",\n \"chunks\": {{ \"size\": {}, \"sha256\": [", okld::okb_chunk); + for (std::size_t at = 0; at < m.bytes.size(); at += okld::okb_chunk) { + unsigned char c[32]; + okld_sha256(m.bytes.data() + at, std::min(okld::okb_chunk, m.bytes.size() - at), c); + chunks += std::format("{}\"{}\"", at ? ", " : " ", hex(c, 32)); + } + chunks += " ] }"; + } + json += std::format(" {}: {{ \"sha256\": \"{}\", \"size\": {}{} }}{}\n", json_string(k), hex(d, 32), + m.bytes.size(), chunks, ++i == members.size() ? "" : ","); } } json += " }\n}\n"; @@ -440,6 +452,12 @@ int bundle_check(const std::vector& args) { const auto n = member_name(m); if (n == "okb.json" || n == "okb.sig" || n.ends_with("/")) continue; if (okld_okb_admit(a.data(), a.size(), &m, text, json->size, &why) != 0) { fail(why.text); refused.insert(&m); } + // The chunks a runner verifies instead, where they are stated: both + // digests hold, or the okb is refused (docs/okb.md, 0.1.8). + else if (okld_okb_admit_chunked(a.data(), a.size(), &m, text, json->size, nullptr, nullptr, &why) != 0) { + fail(why.text); + refused.insert(&m); + } if (!okld_okb_is_image(n.data(), n.size())) continue; const auto slash = n.find('/'); by_dir[slash == std::string::npos ? std::string() : n.substr(0, slash)].push_back(&m);