From 1f7dd76f43887e0430d915c17d4c7c1e983310ac Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Mon, 2 Nov 2026 10:01:14 +0900 Subject: [PATCH] 0.3.0 --- openkal 0.21: an image's instructions and read-only data mapped from its file; a relocation refused outside the writable segments okld_host gains `map`: the loader asks the host to make each segment no relocation writes a part of the file the image was read from (kal_exec_map_part) and copies it where the host declines --- bytes a program carries, an implementation without KAL_EXEC_PROP_FILE. okrun keeps the file it mapped open while it holds the bytes and answers for the file run, a library's file and a file read for an image another needs; an image stored aligned in an okb is mapped as well. A language server's worker made 1 200 page faults starting where it made 16 000, and its cold build of 176 modules was ready in 1.05 times the native time instead of 1.63. Each relocation (RELA, JMPREL, RELR) must write a writable segment, or the image is refused: a segment that is not writable may be the file's own pages. Instructions are made coherent for the copied executable segments only. openkal 0.21.0, openkal-linux 0.22.0, openkal-macos 0.19.0, openkal-windows 0.17.0. Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --- README.md | 9 ++++- bind/mcpp.toml | 2 +- docs/bind.md | 5 +-- ld/include/okld.h | 10 ++++++ ld/mcpp.toml | 2 +- ld/src/load.cpp | 89 +++++++++++++++++++++++++++++++++++++++++------ run/mcpp.toml | 10 +++--- run/src/run.cpp | 58 +++++++++++++++++++++++------- tools/mcpp.toml | 2 +- 9 files changed, 152 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 1b670b3..e6709de 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,14 @@ okrun app.okb arguments... # this mach ``` The images an image needs are found in the okb, then beside it, in `lib//` and `lib/`, and -in `OKRUN_PATH`. A program that carries its images gives their archive to +in `OKRUN_PATH`. + +**An image's instructions and read-only data are its file's pages** (0.3.0, openkal 0.21): where +the implementation reports `KAL_EXEC_PROP_FILE`, the loader makes each segment no relocation +writes a part of the file (`kal_exec_map_part`) rather than a copy, for an image file and for an +image stored aligned in an okb; the writable segments are copied. A program then starts as a +program of its system does: a language server's worker made 1 200 page faults where it made +16 000. A relocation that would write a segment that is not writable is refused (SPEC 12.2). A program that carries its images gives their archive to `okrun_run_okb_bytes`, which runs it from memory and writes nothing to disk. ## A measurement of an image diff --git a/bind/mcpp.toml b/bind/mcpp.toml index 88a6e3f..0bdb2a1 100644 --- a/bind/mcpp.toml +++ b/bind/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-bind" -version = "0.2.0" +version = "0.3.0" # `mcpp::xpkg_request', `mcpp::decision' (protocol 15). mcpp = ">=2026.10.1.3" description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun." diff --git a/docs/bind.md b/docs/bind.md index d1e6e61..a08f3d5 100644 --- a/docs/bind.md +++ b/docs/bind.md @@ -24,10 +24,10 @@ sources = [] include_dirs = ["include"] [target.'cfg(not(os = "openkal"))'.dependencies] -openkal-run = "0.2.0" +openkal-run = "0.3.0" [build-dependencies] -openkal-bind = "0.2.0" +openkal-bind = "0.3.0" ``` ```cpp @@ -105,4 +105,5 @@ llvm toolchain's clang converts the call. | 0.1.7 | No change of its own; released with the archive, whose loader also digests on ARMv8's SHA-256 instructions on Linux where the processor states them. | | 0.1.8 | No change of its own; released with the archive, built against openkal 0.19.0 (the runner binds `kal_task_cores` and `kal_task_set_priority`), whose okb carries chunk digests the runner verifies on several contexts at once. | | 0.2.0 | No change of its own; released with the archive, whose runner measures an image (okprof) and remembers an okb's admission. | +| 0.3.0 | No change of its own; released with the archive, whose loader maps an image's instructions and read-only data from its file where the implementation can (openkal 0.21, `kal_exec_map_part`). | | 0.1.9 | No change of its own; released with the archive, whose runner on macOS is built over openkal-macos 0.17.1 (`kal_task_current` without a system call). | diff --git a/ld/include/okld.h b/ld/include/okld.h index 192216f..2cedc00 100644 --- a/ld/include/okld.h +++ b/ld/include/okld.h @@ -239,6 +239,16 @@ struct okld_host { /* Ordinary memory for the loader's own records (kal_alloc, kal_free). */ void* (*alloc)(okld_size bytes, void* ctx); void (*free)(void* p, okld_size bytes, void* ctx); + /* Makes part of the region the bytes of the file the image was read from, + * rather than a copy of them (kal_exec_map_part, openkal 0.21): `size' + * bytes from `bytes' + `from', at `offset' in the region, executed where + * `execute' is non-zero. `from' is a multiple of 65536 (KAL_MAP_ALIGNMENT) + * and `offset' and `size' of the granularity. Zero when it did; anything + * else, and the loader copies the bytes, as it does where this is null --- + * for bytes that are not a file of their own, or an implementation that + * does not report KAL_EXEC_PROP_FILE. Version 0.3. */ + int (*map)(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from, + int execute, void* ctx); /* Non-zero: each image is announced to a debugger through the GDB JIT * interface (`__jit_debug_register_code'), with its addresses where it was * placed, so that gdb and lldb show its symbols and its source. */ diff --git a/ld/mcpp.toml b/ld/mcpp.toml index a5cf4fe..a5ff48f 100644 --- a/ld/mcpp.toml +++ b/ld/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-ld" -version = "0.2.0" +version = "0.3.0" description = "libokld: the ok format (openkal image format) --- its definition, a reader of bytes, a checker, a signer, and a loader. Freestanding: no C library, no exceptions." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/ld/src/load.cpp b/ld/src/load.cpp index a19d51c..e68bba5 100644 --- a/ld/src/load.cpp +++ b/ld/src/load.cpp @@ -84,6 +84,21 @@ void sync_instructions(unsigned char* from, unsigned char* to) { elf::u64 round_up(elf::u64 v, elf::u64 g) { return (v + g - 1) & ~(g - 1); } elf::u64 round_down(elf::u64 v, elf::u64 g) { return v & ~(g - 1); } +// The alignment of an offset into a file that is mapped: openkal's +// KAL_MAP_ALIGNMENT, to which every loadable segment is aligned (SPEC 12.2). +constexpr elf::u64 file_alignment = 65536; + +// The parts of the span relocations may write: the writable segments. +struct writable { + elf::u64 from[8] = {}, to[8] = {}; + unsigned count = 0; + bool holds(elf::u64 at, elf::u64 bytes) const { + for (unsigned k = 0; k < count; ++k) + if (at >= from[k] && at <= to[k] && bytes <= to[k] - at) return true; + return false; + } +}; + void say(okld_why* why, const char* a, const char* b = nullptr, const char* c = nullptr) { if (!why) return; sentence s(why->text, sizeof why->text); @@ -183,7 +198,11 @@ void withdraw(okld_module* m) { m->host->free(e, sizeof(jit_code_entry), m->host->ctx); } -int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why) { +// EACH RELOCATION WRITES A WRITABLE SEGMENT, or the image is refused: a +// segment that is not writable may be the file's own pages (okld_host::map), +// and a write the format does not allow (SPEC 12.2: no text relocation) is +// refused rather than made wherever its offset points. +int relocate(okld_module* m, const view& v, const okld_host* host, const writable& w, okld_why* why) { const elf::u64 b = reinterpret_cast(m->base); const bool x86 = v.eh->machine == elf::em_x86_64; auto symbol_address = [&](elf::u32 index, bool& ok) -> elf::u64 { @@ -205,6 +224,11 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why if (at == ~0ull || bytes == 0) return true; const auto* r = reinterpret_cast(m->base + at); for (elf::u64 k = 0; k < bytes / sizeof(elf::rela); ++k) { + if (elf::r_type(r[k].info) != (x86 ? elf::r_x86_64_none : elf::r_aarch64_none) + && !w.holds(r[k].offset, 8)) { + say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)"); + return false; + } auto* where = reinterpret_cast(m->base + r[k].offset); const elf::u32 t = elf::r_type(r[k].info); bool ok = true; @@ -237,17 +261,27 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why if (const elf::u64 at = v.dynamic(elf::dt_relr, ~0ull); at != ~0ull) { const elf::u64 bytes = v.dynamic(elf::dt_relrsz); const auto* e = reinterpret_cast(m->base + at); - elf::u64* where = nullptr; - for (elf::u64 k = 0; k < bytes / 8; ++k) { + elf::u64 next = 0; // the offset the first bit of a bitmap entry names + bool ok = true; + for (elf::u64 k = 0; ok && k < bytes / 8; ++k) { if ((e[k] & 1) == 0) { - where = reinterpret_cast(m->base + e[k]); - *where++ += b; + ok = w.holds(e[k], 8); + if (ok) *reinterpret_cast(m->base + e[k]) += b; + next = e[k] + 8; } else { elf::u64 bits = e[k] >> 1; - for (int i = 0; bits; ++i, bits >>= 1) if (bits & 1) where[i] += b; - where += 63; + for (elf::u64 i = 0; ok && bits; ++i, bits >>= 1) { + if (!(bits & 1)) continue; + ok = w.holds(next + 8 * i, 8); + if (ok) *reinterpret_cast(m->base + next + 8 * i) += b; + } + next += 63 * 8; } } + if (!ok) { + say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)"); + return -1; + } } return 0; } @@ -313,11 +347,41 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const // Each segment is its file bytes and then zeros to its size in memory. What // lies between segments is no segment's and is not written: a page the image // never touches is not made resident. + // + // A SEGMENT NOTHING WRITES IS THE FILE'S OWN PAGES where the host can make + // it so (0.3, openkal 0.21): instructions and read-only data are the file's + // bytes unchanged, and a copy of them made each program that started write + // and hold the whole of them --- 54 MB and 14 000 page faults before the + // first instruction of a language server, where a program of the + // environment pays for the pages it touches. Mapped from the alignment + // below the segment to the granule above its end, which no other segment + // shares; otherwise, and where the host declines, copied. + bool mapped[32] = {}; + writable w; for (unsigned k = 0; k < v.eh->phnum; ++k) { const auto& p = v.ph[k]; if (p.type != elf::pt_load) continue; if (p.offset + p.filesz > n) { say(why, "a segment extends beyond the file"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } if (p.filesz > p.memsz) { say(why, "a segment holds more of the file than its size in memory"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } + if (p.flags & elf::pf_w) { + if (w.count == 8) { say(why, "the image has more than eight writable segments"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } + w.from[w.count] = p.vaddr; w.to[w.count] = p.vaddr + p.memsz; ++w.count; + } else if (host->map && k < 32 && p.filesz == p.memsz && p.filesz != 0 + && p.vaddr % file_alignment == p.offset % file_alignment) { + const elf::u64 lo = round_down(p.vaddr, file_alignment), hi = round_up(p.vaddr + p.filesz, g); + const elf::u64 from = p.offset - (p.vaddr - lo); + bool alone = from + (hi - lo) <= n; + for (unsigned j = 0; alone && j < v.eh->phnum; ++j) { + const auto& q = v.ph[j]; + if (j == k || q.type != elf::pt_load) continue; + const elf::u64 qlo = round_down(q.vaddr, g), qhi = round_up(q.vaddr + q.memsz, g); + if (qlo < hi && lo < qhi) alone = false; + } + if (alone && host->map(m->base, lo, hi - lo, bytes, from, (p.flags & elf::pf_x) != 0, host->ctx) == 0) { + mapped[k] = true; + continue; + } + } copy(m->base + p.vaddr, static_cast(bytes) + p.offset, p.filesz); fill(m->base + p.vaddr + p.filesz, 0, p.memsz - p.filesz); } @@ -345,20 +409,23 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const m->init_array = v.dynamic(elf::dt_init_array); m->init_arraysz = v.dynamic(elf::dt_init_arraysz); m->fini_array = v.dynamic(elf::dt_fini_array); m->fini_arraysz = v.dynamic(elf::dt_fini_arraysz); - if (relocate(m, v, host, why) != 0) { discard(m); return -1; } + if (relocate(m, v, host, w, why) != 0) { discard(m); return -1; } - // Publish what executes; the rest stays writable. + // Publish what executes and was copied; the rest stays writable. A segment + // that is the file's pages is executable already, and its bytes reached + // memory through the environment rather than through this loader's data + // path, so its instructions need no maintenance here either. for (unsigned k = 0; k < v.eh->phnum; ++k) { const auto& p = v.ph[k]; - if (p.type != elf::pt_load || !(p.flags & elf::pf_x)) continue; + if (p.type != elf::pt_load || !(p.flags & elf::pf_x) || (k < 32 && mapped[k])) continue; const elf::u64 from = round_down(p.vaddr, g), to = round_up(p.vaddr + p.memsz, g); if (host->publish(m->base, from, to - from, host->ctx) != 0) { say(why, "the environment refused to make the image's instructions executable"); discard(m); return -1; } + sync_instructions(m->base + p.vaddr, m->base + p.vaddr + p.memsz); } - sync_instructions(m->base, m->base + m->span); lock(); m->next = g_modules; diff --git a/run/mcpp.toml b/run/mcpp.toml index 411745e..cd79f6c 100644 --- a/run/mcpp.toml +++ b/run/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-run" -version = "0.2.0" +version = "0.3.0" description = "libokrun: the runner's core --- an ok image's names bound to this process's one openkal implementation, in the image's calling convention, and its entry entered." license = "Apache-2.0" authors = ["mcpplibs"] @@ -11,17 +11,17 @@ repo = "https://github.com/mcpplibs/openkal-image" kind = "lib" [dependencies] -openkal = "0.20.0" +openkal = "0.21.0" openkal-ld = { path = "../ld" } [target.linux.dependencies] -openkal-linux = "0.21.0" +openkal-linux = "0.22.0" [target.macos.dependencies] -openkal-macos = "0.18.0" +openkal-macos = "0.19.0" [target.windows.dependencies] -openkal-windows = "0.16.0" +openkal-windows = "0.17.0" # The interfaces each system's implementation provides decide one unit: where # openkal.space is provided its names are bound, and elsewhere an image that diff --git a/run/src/run.cpp b/run/src/run.cpp index 851608f..6b0e61f 100644 --- a/run/src/run.cpp +++ b/run/src/run.cpp @@ -89,8 +89,15 @@ bool locate(const char* path, kal_dir& dir, const char*& rest) { } // A file's bytes: a region of the file (openkal.map), or a buffer it was read -// into where the file cannot be mapped. `cap' is zero for a region. -struct file_bytes { unsigned char* data = nullptr; unsigned long long size = 0, cap = 0; }; +// into where the file cannot be mapped. `cap' is zero for a region. A region's +// file stays open while its bytes are held, so that the loader can make part of +// an image the file's own pages (okld_host::map, 0.3). +struct file_bytes { + unsigned char* data = nullptr; + unsigned long long size = 0, cap = 0; + kal_file file{}; + bool open = false; +}; void release_bytes(unsigned char* data, unsigned long long size, unsigned long long cap) { if (!data) return; @@ -100,6 +107,7 @@ void release_bytes(unsigned char* data, unsigned long long size, unsigned long l void release_file(file_bytes& f) { release_bytes(f.data, f.size, f.cap); + if (f.open) kal_fs_close_file(f.file); f = {}; } @@ -118,6 +126,8 @@ bool map_all(kal_file f, file_bytes& out) { out.data = static_cast(p); out.size = info.size; out.cap = 0; + out.file = f; + out.open = true; return true; } @@ -128,7 +138,7 @@ bool read_all(const char* path, file_bytes& out, char* why, unsigned long long c unsigned long n = length(rest); kal_file f{}; if (kal_fs_open(dir, rest, n, KAL_OPEN_READ, &f) != kal_ok) { text(why, cap) << path << " cannot be opened"; return false; } - if (map_all(f, out)) { kal_fs_close_file(f); return true; } + if (map_all(f, out)) return true; const kal_stream s = kal_fs_stream(f); for (;;) { if (out.size == out.cap) { @@ -572,9 +582,35 @@ void release_needed(const void* bytes, okld_size, void* ctx) { } } +// Part of an image is the file's own pages where the bytes are a file this +// search holds (the file run, a library's, one read for an image another +// needs) at a multiple of the alignment, and the implementation reports +// KAL_EXEC_PROP_FILE (openkal 0.21). Bytes a program carries, a member of an +// archive and an implementation without the position are copied. +bool part_of(const file_bytes& f, const void* bytes, okld_size from, kal_u64& at) { + const auto* b = static_cast(bytes); + if (!f.open || !f.data || b < f.data || b >= f.data + f.size) return false; + at = static_cast(b - f.data) + from; + return at % KAL_MAP_ALIGNMENT == 0; +} + +int map_part(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from, int execute, + void* ctx) { + if (!(kal_exec_props() & KAL_EXEC_PROP_FILE)) return -1; + auto& s = *static_cast(ctx); + const kal_uintptr how = execute ? KAL_EXEC_MAP_EXECUTE : 0; + kal_u64 at = 0; + if (part_of(s.ran, bytes, from, at)) + return kal_exec_map_part(region, offset, size, s.ran.file, at, how) == kal_ok ? 0 : -1; + for (const auto& f : s.needed) + if (part_of(f, bytes, from, at)) + return kal_exec_map_part(region, offset, size, f.file, at, how) == kal_ok ? 0 : -1; + return -1; +} + void prepare_host(search& s) { okld_host& host = s.host; - host.reserve = &reserve; host.publish = &publish; host.release = &release; + host.reserve = &reserve; host.publish = &publish; host.release = &release; host.map = &map_part; host.granularity = &granularity; host.resolve = &resolve; host.alloc = &allocate; host.free = &deallocate; host.find_needed = &find_needed; host.release_bytes = &release_needed; @@ -793,9 +829,7 @@ extern "C" int okrun_run_images(const okrun_image* images, unsigned long count, struct okrun_library { okld_module* module = nullptr; - search s; - unsigned char* bytes = nullptr; // a file's, mapped or read here - unsigned long long size = 0, cap = 0; + search s; // `s.ran': the file it was read from, until it is loaded }; namespace { @@ -812,8 +846,7 @@ okrun_library* open_loaded(okrun_library* lib, const void* bytes, unsigned long } okld_initialise(lib->module, &call_image, nullptr); // The file a library was read from is released once it is loaded. - release_bytes(lib->bytes, lib->size, lib->cap); - lib->bytes = nullptr; lib->size = 0; lib->cap = 0; + release_ran(lib->s); return lib; } @@ -828,7 +861,7 @@ okrun_library* new_library(const okrun_options* options) { void free_library(okrun_library* lib) { if (!lib) return; - release_bytes(lib->bytes, lib->size, lib->cap); + release_ran(lib->s); kal_free(lib, sizeof(okrun_library), 16); } @@ -857,9 +890,8 @@ extern "C" okrun_library* okrun_open_library(const char* path, const okrun_optio } } directory_of(found, lib->s.directory, sizeof lib->s.directory); - lib->bytes = f.data; - lib->size = f.size; - lib->cap = f.cap; + lib->s.ran = f; + copy_text(lib->s.ran_path, sizeof lib->s.ran_path, found); if (!open_loaded(lib, f.data, f.size, found, why, cap)) { free_library(lib); return nullptr; } return lib; } diff --git a/tools/mcpp.toml b/tools/mcpp.toml index a50cf62..c3a67ab 100644 --- a/tools/mcpp.toml +++ b/tools/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-tools" -version = "0.2.0" +version = "0.3.0" description = "okrun, which runs an ok image on this system, and okld, the format's tool on a build machine (check, note, sign, inspect)." license = "Apache-2.0" authors = ["mcpplibs"]