diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40ee0f9..e1289f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,14 @@ jobs: out/x86_64/needed.oke out/x86_64/liboktwice.okso out/aarch64/needed.oke out/aarch64/liboktwice.okso "$OKLD" bundle out/signed.okb --sign "$RUNNER_TEMP/seed" --entry needed.oke \ out/x86_64/needed.oke out/x86_64/liboktwice.okso out/aarch64/needed.oke out/aarch64/liboktwice.okso - for okb in hello needed signed; do "$OKLD" bundle check "out/$okb.okb"; done + # From a universal package mcpp staged (mcpp-package.json, SPEC-014 §4.5). + python3 examples/package/stage.py staged out + "$OKLD" bundle out/package.okb --package staged/tree + cp -R staged/tree staged/changed + printf 'x' >> staged/changed/lib/x86_64/needed.oke + if "$OKLD" bundle out/changed.okb --package staged/changed 2> changed.txt; then echo "::error::a package whose image is not its stated content was bundled"; exit 1; fi + grep -q "mcpp-package.json states" changed.txt + for okb in hello needed signed package; do "$OKLD" bundle check "out/$okb.okb"; done "$OKLD" bundle check out/signed.okb --require-signature --trust "$(cat out/key.hex)" if "$OKLD" bundle check out/hello.okb --require-signature 2> unsigned.txt; then echo "::error::an unsigned okb passed --require-signature"; exit 1; fi grep -q "not signed" unsigned.txt @@ -200,6 +207,7 @@ jobs: grep -q "argc as kal_env_arg_count = 3" <<<"$out" || { echo "::error::the okb's entry and arguments"; exit 1; } n=$("$OKRUN" out/needed.okb | tr -d '\r') grep -q "twice(21) = 42, next_value() = 41" <<<"$n" || { echo "::error::an image the okb's entry needs, from the okb"; exit 1; } + "$OKRUN" out/package.okb | tr -d '\r' | grep -q "twice(21) = 42" || { echo "::error::the okb of a staged package"; exit 1; } "$OKRUN" --require-signature --trust "$key" out/signed.okb > /dev/null || { echo "::error::a signed okb was refused"; exit 1; } if "$OKRUN" --require-signature out/needed.okb 2> e4.txt; then echo "::error::an unsigned okb ran under --require-signature"; exit 1; fi grep -q "okb.json is not signed" e4.txt @@ -225,6 +233,13 @@ jobs: grep -q "library: twice(21) = 42, next_value() = 41, 42" lib.txt || { echo "::error::okrun_open_library"; exit 1; } "$host" okb out/needed.okb | tr -d '\r' | grep -q "twice(21) = 42" || { echo "::error::okrun_run_okb_bytes"; exit 1; } "$host" images "out/$a/needed.oke" "out/$a/liboktwice.okso" | tr -d '\r' | grep -q "twice(21) = 42" || { echo "::error::okrun_run_images"; exit 1; } + # The archive a program that carries its images holds: no okb.json, + # the images at the root, zipalign's field and bytes between members. + python3 examples/package/stage.py staged out + "$host" archive staged/carried.zip needed.oke | tr -d '\r' | grep -q "twice(21) = 42" || { echo "::error::an archive without okb.json, from memory"; exit 1; } + python3 -c "import sys; d=bytearray(open(sys.argv[1],'rb').read()); d[65536+200]^=1; open(sys.argv[2],'wb').write(d)" staged/carried.zip staged/bad.zip + if "$host" archive staged/bad.zip needed.oke 2> crc.txt; then echo "::error::a changed member ran"; exit 1; fi + grep -q "CRC-32" crc.txt echo " ok an okb and a group of images run from memory; a library's exports are called from here" - name: The binding openkal-bind generates run: | diff --git a/README.md b/README.md index 1528918..e765f8b 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ images are stored and aligned to 64 KiB ([docs/okb.md](docs/okb.md)): ```sh okld bundle app.okb --version 0.1.0 x86_64/app.oke aarch64/app.oke # okb.json, the images by architecture +okld bundle app.okb --package target/dist/app-0.1.0-openkal # or from a package mcpp staged okld bundle check app.okb # stored, aligned, digests, signatures okrun app.okb arguments... # this machine's architecture's entry ``` diff --git a/bind/mcpp.toml b/bind/mcpp.toml index 25422e7..c9217f7 100644 --- a/bind/mcpp.toml +++ b/bind/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-bind" -version = "0.1.2" +version = "0.1.3" # `mcpp::xpkg_request', `mcpp::decision' (protocol 15). mcpp = ">=2026.10.1.3" description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun." diff --git a/docs/bind.md b/docs/bind.md index 90b706f..9238a99 100644 --- a/docs/bind.md +++ b/docs/bind.md @@ -24,10 +24,10 @@ sources = [] include_dirs = ["include"] [target.'cfg(not(os = "openkal"))'.dependencies] -openkal-run = "0.1.2" +openkal-run = "0.1.3" [build-dependencies] -openkal-bind = "0.1.2" +openkal-bind = "0.1.3" ``` ```cpp @@ -98,3 +98,4 @@ llvm toolchain's clang converts the call. | openkal-bind | Statement | |---|---| | 0.1.2 | This document. Released with openkal-run 0.1.2, whose library interface it calls. | +| 0.1.3 | No change of its own; released with the archive. | diff --git a/docs/okb.md b/docs/okb.md index 2d4a3d7..d97521f 100644 --- a/docs/okb.md +++ b/docs/okb.md @@ -120,6 +120,7 @@ has no such interface in version 0.17. ``` okld bundle [--name ] [--version ] [--entry ] [--sign ] ... +okld bundle --package [--sign ] okld bundle check [--require-signature] [--trust ]... ``` @@ -128,6 +129,19 @@ architecture's directory, and writes `okb.json` with the digests; `--sign` signs and the manifest with one key. The archive is deterministic: no time is recorded, and the same images give the same bytes. The entry defaults to the first `.oke` named. +`--package` takes the images from a universal package that mcpp has staged (mcpp SPEC-014 §4.2, +§4.5), reading `mcpp-package.json` at the root of the tree: + +- the images are the files of role `image`, at `lib//`; +- each image's architecture, from its ELF header, must be the one its directory names; +- its content must be the `sha256` stated for it; +- each architecture's entry is the file `launch.entry` names for it; +- the okb's name and version are the package's. + +Files of every other role (launchers, loaders, resources, metadata) are not members of a +version 1 okb. This is the action of the `okb` package format that openkal-platform's plugin +provides. + `okld bundle check` verifies C1 to C3, the manifest, every digest, every signature (and the trust asked for), that each architecture's entry is present, that each image is of its directory's architecture, and that each image follows the convention (SPEC 12.2) with the @@ -148,3 +162,4 @@ exports of the other images of its directory. It prints one line per violation. | okb | okrun | Change | |---|---|---| | 1 | 0.1.2 | This document. | +| 1 | 0.1.3 | `okld bundle --package`: an okb from a package mcpp staged. | diff --git a/examples/host/mcpp.toml b/examples/host/mcpp.toml index 57779f0..89989e7 100644 --- a/examples/host/mcpp.toml +++ b/examples/host/mcpp.toml @@ -4,7 +4,7 @@ # system of each architecture with the images built once. [package] name = "okrun-host" -version = "0.1.2" +version = "0.1.3" [dependencies] openkal-run = { path = "../../run" } diff --git a/examples/host/src/main.cpp b/examples/host/src/main.cpp index abf96a8..51856dc 100644 --- a/examples/host/src/main.cpp +++ b/examples/host/src/main.cpp @@ -1,5 +1,8 @@ // okrun-host library /liboktwice.okso the library's exports, called from here // okrun-host okb [arguments...] an okb run from the bytes of memory +// okrun-host archive [arguments...] +// an archive of images without okb.json, as a +// program that carries its images holds one // okrun-host images // a group of images run from memory #include @@ -32,7 +35,7 @@ std::vector read(const char* path) { } // namespace int main(int argc, char** argv) { - if (argc < 3) { std::fprintf(stderr, "usage: okrun-host library | okb | images \n"); return 2; } + if (argc < 3) { std::fprintf(stderr, "usage: okrun-host library | okb | archive | images \n"); return 2; } okrun_options options{}; char why[512]; if (std::strcmp(argv[1], "library") == 0) { @@ -56,6 +59,13 @@ int main(int argc, char** argv) { std::fprintf(stderr, "okrun-host: %s\n", why); return 127; } + if (std::strcmp(argv[1], "archive") == 0 && argc >= 4) { + const auto bytes = read(argv[2]); + okrun_run_okb_bytes(bytes.data(), bytes.size(), argv[3], argc - 3, const_cast(argv + 3), + &options, why, sizeof why); + std::fprintf(stderr, "okrun-host: %s\n", why); + return 127; + } if (std::strcmp(argv[1], "images") == 0 && argc >= 4) { const auto app = read(argv[2]), lib = read(argv[3]); const okrun_image images[] = { diff --git a/examples/package/stage.py b/examples/package/stage.py new file mode 100644 index 0000000..6551527 --- /dev/null +++ b/examples/package/stage.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""stage.py -- a universal package of images as mcpp +stages it (mcpp SPEC-014 §4.2, §4.5), for `okld bundle --package', and an +archive of the same images without okb.json, as a program that carries them +holds it (zipalign's field 0xD935, and bytes between members). + + /tree/ lib//needed.oke, liboktwice.okso, a launcher, + a resource, mcpp-package.json + /carried.zip needed.oke and liboktwice.okso of this machine's + architecture at the root, stored, 64 KiB-aligned +""" +import hashlib, json, os, platform, shutil, struct, sys, zlib + +out, images = sys.argv[1], sys.argv[2] +tree = os.path.join(out, "tree") +files = [] + +def place(rel, data, role, leg=None): + path = os.path.join(tree, *rel.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + files.append({"path": rel, "role": role, "leg": leg, "sha256": hashlib.sha256(data).hexdigest()}) + +for arch in ("x86_64", "aarch64"): + for name in ("needed.oke", "liboktwice.okso"): + with open(os.path.join(images, arch, name), "rb") as f: + place(f"lib/{arch}/{name}", f.read(), "image", f"{arch}-openkal") +place("bin/needed", b"#!/bin/sh\n", "launcher") +place("share/needed/data.txt", b"a resource\n", "resource") +doc = { + "schema": 1, "name": "needed", "version": "0.1.0", "platform": "openkal-platform@0.17.2", + "linkage": "declared", "mode": "vendored", "legs": ["x86_64-openkal", "aarch64-openkal"], "systems": [], + "launch": {"runner": ["okrun", "{}"], + "entry": {"x86_64": "lib/x86_64/needed.oke", "aarch64": "lib/aarch64/needed.oke"}}, + "files": files, +} +with open(os.path.join(tree, "mcpp-package.json"), "w") as f: + json.dump(doc, f, indent=2) + +# The carried archive: this machine's images at the root. +arch = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "aarch64", "aarch64": "aarch64"}[platform.machine().lower()] +members = [] +for name in ("needed.oke", "liboktwice.okso"): + with open(os.path.join(images, arch, name), "rb") as f: + members.append((name, f.read())) +blob, central = bytearray(), bytearray() +for i, (name, data) in enumerate(members): + n = name.encode() + if i == 1: + blob += b"\0" * 3 # bytes between members, which no entry names + pad = (-(len(blob) + 30 + len(n))) % 65536 + if 0 < pad < 6: + blob += b"\0" * pad # too few for a field: bytes before the header + pad = 0 + extra = b"" + if pad: + # zipalign's field: id 0xD935, the alignment (2 bytes), zeros. + extra = struct.pack("carried; diff --git a/tools/mcpp.toml b/tools/mcpp.toml index 88a7960..843cd2e 100644 --- a/tools/mcpp.toml +++ b/tools/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-tools" -version = "0.1.2" +version = "0.1.3" description = "okrun, which runs an ok image on this system, and okld, the format's tool on a build machine (check, note, sign, inspect)." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/tools/src/okld.cpp b/tools/src/okld.cpp index e48ba7f..09a0e5d 100644 --- a/tools/src/okld.cpp +++ b/tools/src/okld.cpp @@ -90,6 +90,104 @@ std::string json_string(std::string_view v) { return out + "\""; } +// --- JSON, as mcpp writes `mcpp-package.json_value' (mcpp SPEC-014 §4.5) ------------ + +struct json_value { + enum kind_t { null, boolean, number, string, array, object } kind = null; + bool b = false; + double n = 0; + std::string s; + std::vector items; + std::map fields; + const json_value* get(std::string_view k) const { + auto it = fields.find(std::string(k)); + return it == fields.end() ? nullptr : &it->second; + } + std::string str(std::string_view k) const { + const json_value* v = get(k); + return v && v->kind == string ? v->s : std::string(); + } +}; + +struct json_reader { + std::string_view t; + std::size_t at = 0; + void ws() { while (at < t.size() && std::isspace(static_cast(t[at]))) ++at; } + bool text(std::string& out) { + if (at >= t.size() || t[at] != '"') return false; + ++at; + while (at < t.size() && t[at] != '"') { + char c = t[at++]; + if (c == '\\' && at < t.size()) { + const char e = t[at++]; + switch (e) { + case 'n': c = '\n'; break; case 't': c = '\t'; break; case 'r': c = '\r'; break; + case 'b': c = '\b'; break; case 'f': c = '\f'; break; + case 'u': { + if (at + 4 > t.size()) return false; + const unsigned v = static_cast(std::stoul(std::string(t.substr(at, 4)), nullptr, 16)); + at += 4; + if (v < 0x80) { out += static_cast(v); continue; } + if (v < 0x800) { out += static_cast(0xc0 | (v >> 6)); out += static_cast(0x80 | (v & 0x3f)); continue; } + out += static_cast(0xe0 | (v >> 12)); out += static_cast(0x80 | ((v >> 6) & 0x3f)); + out += static_cast(0x80 | (v & 0x3f)); + continue; + } + default: c = e; + } + } + out += c; + } + if (at >= t.size()) return false; + ++at; + return true; + } + bool value(json_value& v, int depth = 0) { + ws(); + if (at >= t.size() || depth > 64) return false; + const char c = t[at]; + if (c == '{') { + v.kind = json_value::object; ++at; ws(); + if (at < t.size() && t[at] == '}') { ++at; return true; } + for (;;) { + ws(); + std::string k; + if (!text(k)) return false; + ws(); + if (at >= t.size() || t[at] != ':') return false; + ++at; + if (!value(v.fields[k], depth + 1)) return false; + ws(); + if (at < t.size() && t[at] == ',') { ++at; continue; } + if (at < t.size() && t[at] == '}') { ++at; return true; } + return false; + } + } + if (c == '[') { + v.kind = json_value::array; ++at; ws(); + if (at < t.size() && t[at] == ']') { ++at; return true; } + for (;;) { + v.items.emplace_back(); + if (!value(v.items.back(), depth + 1)) return false; + ws(); + if (at < t.size() && t[at] == ',') { ++at; continue; } + if (at < t.size() && t[at] == ']') { ++at; return true; } + return false; + } + } + if (c == '"') { v.kind = json_value::string; return text(v.s); } + if (t.substr(at, 4) == "true") { v.kind = json_value::boolean; v.b = true; at += 4; return true; } + if (t.substr(at, 5) == "false") { v.kind = json_value::boolean; at += 5; return true; } + if (t.substr(at, 4) == "null") { at += 4; return true; } + const std::size_t from = at; + while (at < t.size() && (std::isdigit(static_cast(t[at])) || std::string_view("+-.eE").contains(t[at]))) ++at; + if (at == from) return false; + v.kind = json_value::number; + std::from_chars(t.data() + from, t.data() + at, v.n); + return true; + } +}; + std::string base_name(const std::string& path) { const auto at = path.find_last_of("/\\"); return at == std::string::npos ? path : path.substr(at + 1); @@ -97,10 +195,16 @@ std::string base_name(const std::string& path) { int bundle(const std::vector& args) { // okld bundle [--name ] [--version ] [--entry ] [--sign ] ... - if (args.size() < 3) { std::println(std::cerr, "usage: okld bundle [--name ] [--version ] [--entry ] [--sign ] ..."); return 2; } + // okld bundle --package [--sign ] + const char* usage = "usage: okld bundle [--name ] [--version ] [--entry ] [--sign ] ...\n" + " okld bundle --package [--sign ]"; + if (args.size() < 3) { std::println(std::cerr, "{}", usage); return 2; } const std::string out = args[1]; - std::string name, version, entry, seed_file; + std::string name, version, entry, seed_file, package; std::vector paths; + std::map expected_arch; // path -> the architecture its place names + std::map stated_digest; // path -> the sha256 mcpp-package.json states + std::map arch_entry; // architecture -> its entry's file name for (std::size_t i = 2; i < args.size(); ++i) { const auto& a = args[i]; auto value = [&](std::string& into) { @@ -112,9 +216,51 @@ int bundle(const std::vector& args) { else if (a == "--version") { if (!value(version)) return 2; } else if (a == "--entry") { if (!value(entry)) return 2; } else if (a == "--sign") { if (!value(seed_file)) return 2; } + else if (a == "--package") { if (!value(package)) return 2; } else if (a.starts_with("--")) { std::println(std::cerr, "okld: unknown option {}", a); return 2; } else paths.push_back(a); } + if (!package.empty()) { + // A PACKAGE MCPP STAGED (mcpp SPEC-014 §4.5): its images are the files + // of role `image', at `lib//'; its entry per architecture + // is `launch.entry'; its name and version are the package's. Every + // other role (launcher, loader, resource, metadata) is not a member of + // an okb of this version. + if (!paths.empty() || !entry.empty()) { std::println(std::cerr, "okld: --package names the images and the entry; give neither beside it"); return 2; } + const std::string at = (std::filesystem::path(package) / "mcpp-package.json").string(); + auto text = read_file(at); + if (!text) { std::println(std::cerr, "okld: {} cannot be read; --package names a tree mcpp staged", at); return 1; } + json_value doc; + json_reader r{ std::string_view(reinterpret_cast(text->data()), text->size()) }; + if (!r.value(doc) || doc.kind != json_value::object) { std::println(std::cerr, "okld: {} is not a JSON object", at); return 1; } + const json_value* schema = doc.get("schema"); + if (!schema || schema->kind != json_value::number || schema->n != 1) { std::println(std::cerr, "okld: {}: schema {} is not the one this okld reads (1)", at, schema && schema->kind == json_value::number ? std::format("{}", schema->n) : std::string("(none)")); return 1; } + if (name.empty()) name = doc.str("name"); + if (version.empty()) version = doc.str("version"); + const json_value* launch = doc.get("launch"); + const json_value* entries = launch ? launch->get("entry") : nullptr; + if (!entries || entries->kind != json_value::object || entries->fields.empty()) { std::println(std::cerr, "okld: {}: no launch.entry; the package is not a universal package of images", at); return 1; } + for (const auto& [arch, e] : entries->fields) { + if (e.kind != json_value::string || !e.s.starts_with("lib/" + arch + "/")) { std::println(std::cerr, "okld: {}: launch.entry.{} is not lib/{}/", at, arch, arch); return 1; } + arch_entry[arch] = base_name(e.s); + } + const json_value* files = doc.get("files"); + if (!files || files->kind != json_value::array) { std::println(std::cerr, "okld: {}: no files", at); return 1; } + for (const auto& f : files->items) { + if (f.str("role") != "image") continue; + const std::string rel = f.str("path"); + const auto slash = rel.find('/', 4); + if (!rel.starts_with("lib/") || slash == std::string::npos || rel.find('/', slash + 1) != std::string::npos) { + std::println(std::cerr, "okld: {}: the image {} is not at lib//", at, rel); + return 1; + } + const std::string path = (std::filesystem::path(package) / rel).string(); + paths.push_back(path); + expected_arch[path] = rel.substr(4, slash - 4); + if (const auto d = f.str("sha256"); !d.empty()) stated_digest[path] = d; + } + if (paths.empty()) { std::println(std::cerr, "okld: {}: no file of role `image'", at); return 1; } + } if (paths.empty()) { std::println(std::cerr, "okld: an okb holds at least one image"); return 2; } std::optional> seed; if (!seed_file.empty()) { @@ -135,6 +281,15 @@ int bundle(const std::vector& args) { if (!info.has_requirements) { std::println(std::cerr, "okld: {}: no requirements note; `okld note' it first", path); return 1; } const char* arch = okld_okb_arch_of(info.machine); if (!arch) { std::println(std::cerr, "okld: {}: an architecture okb has no directory for", path); return 1; } + if (auto e = expected_arch.find(path); e != expected_arch.end() && e->second != arch) { + std::println(std::cerr, "okld: {}: an image of {}, in the directory of {}", path, arch, e->second); + return 1; + } + if (auto d = stated_digest.find(path); d != stated_digest.end()) { + unsigned char have[32]; + okld_sha256(bytes->data(), bytes->size(), have); + if (hex(have, 32) != d->second) { std::println(std::cerr, "okld: {}: its content is not the one mcpp-package.json states (sha256)", path); return 1; } + } const auto file = base_name(path); if (!file.ends_with(".oke") && !file.ends_with(".okso")) { std::println(std::cerr, "okld: {}: an image is named .oke or .okso", path); return 1; } if (seed && okld_sign(bytes->data(), bytes->size(), seed->data(), &why) != 0) { std::println(std::cerr, "okld: {}: {}", path, why.text); return 1; } @@ -144,11 +299,16 @@ int bundle(const std::vector& args) { } std::set archs; for (const auto& [k, m] : members) archs.insert(m.arch); + if (entry.empty() && !arch_entry.empty()) entry = arch_entry.begin()->second; if (entry.empty()) for (const auto& [k, m] : members) if (m.file.ends_with(".oke")) { entry = m.file; break; } if (entry.empty()) { std::println(std::cerr, "okld: no .oke among the images, and no --entry"); return 1; } - for (const auto& a : archs) - if (!members.contains(a + "/" + entry)) { std::println(std::cerr, "okld: the architecture {} has no {}", a, entry); return 1; } + for (const auto& a : archs) { + if (!arch_entry.contains(a)) arch_entry[a] = entry; + if (!members.contains(a + "/" + arch_entry[a])) { std::println(std::cerr, "okld: the architecture {} has no {}", a, arch_entry[a]); return 1; } + } + for (const auto& [a, e] : arch_entry) + if (!archs.contains(a)) { std::println(std::cerr, "okld: an entry is stated for {}, which has no image", a); return 1; } if (name.empty()) name = entry.substr(0, entry.rfind('.')); // okb.json: keys in a fixed order, files by member name. @@ -159,7 +319,7 @@ int bundle(const std::vector& args) { std::size_t i = 0; for (const auto& a : archs) json += std::format(" {}: {{ \"dir\": {}, \"entry\": {} }}{}\n", json_string(a), json_string(a), - json_string(entry), ++i == archs.size() ? "" : ","); + json_string(arch_entry[a]), ++i == archs.size() ? "" : ","); } json += " },\n \"files\": {\n"; {