diff --git a/README.md b/README.md index 1b670b3..e6709de 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,14 @@ okrun app.okb arguments... # this mach ``` The images an image needs are found in the okb, then beside it, in `lib//` and `lib/`, and -in `OKRUN_PATH`. A program that carries its images gives their archive to +in `OKRUN_PATH`. + +**An image's instructions and read-only data are its file's pages** (0.3.0, openkal 0.21): where +the implementation reports `KAL_EXEC_PROP_FILE`, the loader makes each segment no relocation +writes a part of the file (`kal_exec_map_part`) rather than a copy, for an image file and for an +image stored aligned in an okb; the writable segments are copied. A program then starts as a +program of its system does: a language server's worker made 1 200 page faults where it made +16 000. A relocation that would write a segment that is not writable is refused (SPEC 12.2). A program that carries its images gives their archive to `okrun_run_okb_bytes`, which runs it from memory and writes nothing to disk. ## A measurement of an image diff --git a/bind/mcpp.toml b/bind/mcpp.toml index 88a6e3f..0bdb2a1 100644 --- a/bind/mcpp.toml +++ b/bind/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-bind" -version = "0.2.0" +version = "0.3.0" # `mcpp::xpkg_request', `mcpp::decision' (protocol 15). mcpp = ">=2026.10.1.3" description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun." diff --git a/docs/bind.md b/docs/bind.md index d1e6e61..a08f3d5 100644 --- a/docs/bind.md +++ b/docs/bind.md @@ -24,10 +24,10 @@ sources = [] include_dirs = ["include"] [target.'cfg(not(os = "openkal"))'.dependencies] -openkal-run = "0.2.0" +openkal-run = "0.3.0" [build-dependencies] -openkal-bind = "0.2.0" +openkal-bind = "0.3.0" ``` ```cpp @@ -105,4 +105,5 @@ llvm toolchain's clang converts the call. | 0.1.7 | No change of its own; released with the archive, whose loader also digests on ARMv8's SHA-256 instructions on Linux where the processor states them. | | 0.1.8 | No change of its own; released with the archive, built against openkal 0.19.0 (the runner binds `kal_task_cores` and `kal_task_set_priority`), whose okb carries chunk digests the runner verifies on several contexts at once. | | 0.2.0 | No change of its own; released with the archive, whose runner measures an image (okprof) and remembers an okb's admission. | +| 0.3.0 | No change of its own; released with the archive, whose loader maps an image's instructions and read-only data from its file where the implementation can (openkal 0.21, `kal_exec_map_part`). | | 0.1.9 | No change of its own; released with the archive, whose runner on macOS is built over openkal-macos 0.17.1 (`kal_task_current` without a system call). | diff --git a/ld/include/okld.h b/ld/include/okld.h index 192216f..2cedc00 100644 --- a/ld/include/okld.h +++ b/ld/include/okld.h @@ -239,6 +239,16 @@ struct okld_host { /* Ordinary memory for the loader's own records (kal_alloc, kal_free). */ void* (*alloc)(okld_size bytes, void* ctx); void (*free)(void* p, okld_size bytes, void* ctx); + /* Makes part of the region the bytes of the file the image was read from, + * rather than a copy of them (kal_exec_map_part, openkal 0.21): `size' + * bytes from `bytes' + `from', at `offset' in the region, executed where + * `execute' is non-zero. `from' is a multiple of 65536 (KAL_MAP_ALIGNMENT) + * and `offset' and `size' of the granularity. Zero when it did; anything + * else, and the loader copies the bytes, as it does where this is null --- + * for bytes that are not a file of their own, or an implementation that + * does not report KAL_EXEC_PROP_FILE. Version 0.3. */ + int (*map)(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from, + int execute, void* ctx); /* Non-zero: each image is announced to a debugger through the GDB JIT * interface (`__jit_debug_register_code'), with its addresses where it was * placed, so that gdb and lldb show its symbols and its source. */ diff --git a/ld/mcpp.toml b/ld/mcpp.toml index a5cf4fe..a5ff48f 100644 --- a/ld/mcpp.toml +++ b/ld/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-ld" -version = "0.2.0" +version = "0.3.0" description = "libokld: the ok format (openkal image format) --- its definition, a reader of bytes, a checker, a signer, and a loader. Freestanding: no C library, no exceptions." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/ld/src/load.cpp b/ld/src/load.cpp index a19d51c..e68bba5 100644 --- a/ld/src/load.cpp +++ b/ld/src/load.cpp @@ -84,6 +84,21 @@ void sync_instructions(unsigned char* from, unsigned char* to) { elf::u64 round_up(elf::u64 v, elf::u64 g) { return (v + g - 1) & ~(g - 1); } elf::u64 round_down(elf::u64 v, elf::u64 g) { return v & ~(g - 1); } +// The alignment of an offset into a file that is mapped: openkal's +// KAL_MAP_ALIGNMENT, to which every loadable segment is aligned (SPEC 12.2). +constexpr elf::u64 file_alignment = 65536; + +// The parts of the span relocations may write: the writable segments. +struct writable { + elf::u64 from[8] = {}, to[8] = {}; + unsigned count = 0; + bool holds(elf::u64 at, elf::u64 bytes) const { + for (unsigned k = 0; k < count; ++k) + if (at >= from[k] && at <= to[k] && bytes <= to[k] - at) return true; + return false; + } +}; + void say(okld_why* why, const char* a, const char* b = nullptr, const char* c = nullptr) { if (!why) return; sentence s(why->text, sizeof why->text); @@ -183,7 +198,11 @@ void withdraw(okld_module* m) { m->host->free(e, sizeof(jit_code_entry), m->host->ctx); } -int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why) { +// EACH RELOCATION WRITES A WRITABLE SEGMENT, or the image is refused: a +// segment that is not writable may be the file's own pages (okld_host::map), +// and a write the format does not allow (SPEC 12.2: no text relocation) is +// refused rather than made wherever its offset points. +int relocate(okld_module* m, const view& v, const okld_host* host, const writable& w, okld_why* why) { const elf::u64 b = reinterpret_cast(m->base); const bool x86 = v.eh->machine == elf::em_x86_64; auto symbol_address = [&](elf::u32 index, bool& ok) -> elf::u64 { @@ -205,6 +224,11 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why if (at == ~0ull || bytes == 0) return true; const auto* r = reinterpret_cast(m->base + at); for (elf::u64 k = 0; k < bytes / sizeof(elf::rela); ++k) { + if (elf::r_type(r[k].info) != (x86 ? elf::r_x86_64_none : elf::r_aarch64_none) + && !w.holds(r[k].offset, 8)) { + say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)"); + return false; + } auto* where = reinterpret_cast(m->base + r[k].offset); const elf::u32 t = elf::r_type(r[k].info); bool ok = true; @@ -237,17 +261,27 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why if (const elf::u64 at = v.dynamic(elf::dt_relr, ~0ull); at != ~0ull) { const elf::u64 bytes = v.dynamic(elf::dt_relrsz); const auto* e = reinterpret_cast(m->base + at); - elf::u64* where = nullptr; - for (elf::u64 k = 0; k < bytes / 8; ++k) { + elf::u64 next = 0; // the offset the first bit of a bitmap entry names + bool ok = true; + for (elf::u64 k = 0; ok && k < bytes / 8; ++k) { if ((e[k] & 1) == 0) { - where = reinterpret_cast(m->base + e[k]); - *where++ += b; + ok = w.holds(e[k], 8); + if (ok) *reinterpret_cast(m->base + e[k]) += b; + next = e[k] + 8; } else { elf::u64 bits = e[k] >> 1; - for (int i = 0; bits; ++i, bits >>= 1) if (bits & 1) where[i] += b; - where += 63; + for (elf::u64 i = 0; ok && bits; ++i, bits >>= 1) { + if (!(bits & 1)) continue; + ok = w.holds(next + 8 * i, 8); + if (ok) *reinterpret_cast(m->base + next + 8 * i) += b; + } + next += 63 * 8; } } + if (!ok) { + say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)"); + return -1; + } } return 0; } @@ -313,11 +347,41 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const // Each segment is its file bytes and then zeros to its size in memory. What // lies between segments is no segment's and is not written: a page the image // never touches is not made resident. + // + // A SEGMENT NOTHING WRITES IS THE FILE'S OWN PAGES where the host can make + // it so (0.3, openkal 0.21): instructions and read-only data are the file's + // bytes unchanged, and a copy of them made each program that started write + // and hold the whole of them --- 54 MB and 14 000 page faults before the + // first instruction of a language server, where a program of the + // environment pays for the pages it touches. Mapped from the alignment + // below the segment to the granule above its end, which no other segment + // shares; otherwise, and where the host declines, copied. + bool mapped[32] = {}; + writable w; for (unsigned k = 0; k < v.eh->phnum; ++k) { const auto& p = v.ph[k]; if (p.type != elf::pt_load) continue; if (p.offset + p.filesz > n) { say(why, "a segment extends beyond the file"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } if (p.filesz > p.memsz) { say(why, "a segment holds more of the file than its size in memory"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } + if (p.flags & elf::pf_w) { + if (w.count == 8) { say(why, "the image has more than eight writable segments"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; } + w.from[w.count] = p.vaddr; w.to[w.count] = p.vaddr + p.memsz; ++w.count; + } else if (host->map && k < 32 && p.filesz == p.memsz && p.filesz != 0 + && p.vaddr % file_alignment == p.offset % file_alignment) { + const elf::u64 lo = round_down(p.vaddr, file_alignment), hi = round_up(p.vaddr + p.filesz, g); + const elf::u64 from = p.offset - (p.vaddr - lo); + bool alone = from + (hi - lo) <= n; + for (unsigned j = 0; alone && j < v.eh->phnum; ++j) { + const auto& q = v.ph[j]; + if (j == k || q.type != elf::pt_load) continue; + const elf::u64 qlo = round_down(q.vaddr, g), qhi = round_up(q.vaddr + q.memsz, g); + if (qlo < hi && lo < qhi) alone = false; + } + if (alone && host->map(m->base, lo, hi - lo, bytes, from, (p.flags & elf::pf_x) != 0, host->ctx) == 0) { + mapped[k] = true; + continue; + } + } copy(m->base + p.vaddr, static_cast(bytes) + p.offset, p.filesz); fill(m->base + p.vaddr + p.filesz, 0, p.memsz - p.filesz); } @@ -345,20 +409,23 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const m->init_array = v.dynamic(elf::dt_init_array); m->init_arraysz = v.dynamic(elf::dt_init_arraysz); m->fini_array = v.dynamic(elf::dt_fini_array); m->fini_arraysz = v.dynamic(elf::dt_fini_arraysz); - if (relocate(m, v, host, why) != 0) { discard(m); return -1; } + if (relocate(m, v, host, w, why) != 0) { discard(m); return -1; } - // Publish what executes; the rest stays writable. + // Publish what executes and was copied; the rest stays writable. A segment + // that is the file's pages is executable already, and its bytes reached + // memory through the environment rather than through this loader's data + // path, so its instructions need no maintenance here either. for (unsigned k = 0; k < v.eh->phnum; ++k) { const auto& p = v.ph[k]; - if (p.type != elf::pt_load || !(p.flags & elf::pf_x)) continue; + if (p.type != elf::pt_load || !(p.flags & elf::pf_x) || (k < 32 && mapped[k])) continue; const elf::u64 from = round_down(p.vaddr, g), to = round_up(p.vaddr + p.memsz, g); if (host->publish(m->base, from, to - from, host->ctx) != 0) { say(why, "the environment refused to make the image's instructions executable"); discard(m); return -1; } + sync_instructions(m->base + p.vaddr, m->base + p.vaddr + p.memsz); } - sync_instructions(m->base, m->base + m->span); lock(); m->next = g_modules; diff --git a/run/mcpp.toml b/run/mcpp.toml index 411745e..cd79f6c 100644 --- a/run/mcpp.toml +++ b/run/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-run" -version = "0.2.0" +version = "0.3.0" description = "libokrun: the runner's core --- an ok image's names bound to this process's one openkal implementation, in the image's calling convention, and its entry entered." license = "Apache-2.0" authors = ["mcpplibs"] @@ -11,17 +11,17 @@ repo = "https://github.com/mcpplibs/openkal-image" kind = "lib" [dependencies] -openkal = "0.20.0" +openkal = "0.21.0" openkal-ld = { path = "../ld" } [target.linux.dependencies] -openkal-linux = "0.21.0" +openkal-linux = "0.22.0" [target.macos.dependencies] -openkal-macos = "0.18.0" +openkal-macos = "0.19.0" [target.windows.dependencies] -openkal-windows = "0.16.0" +openkal-windows = "0.17.0" # The interfaces each system's implementation provides decide one unit: where # openkal.space is provided its names are bound, and elsewhere an image that diff --git a/run/src/run.cpp b/run/src/run.cpp index 851608f..6b0e61f 100644 --- a/run/src/run.cpp +++ b/run/src/run.cpp @@ -89,8 +89,15 @@ bool locate(const char* path, kal_dir& dir, const char*& rest) { } // A file's bytes: a region of the file (openkal.map), or a buffer it was read -// into where the file cannot be mapped. `cap' is zero for a region. -struct file_bytes { unsigned char* data = nullptr; unsigned long long size = 0, cap = 0; }; +// into where the file cannot be mapped. `cap' is zero for a region. A region's +// file stays open while its bytes are held, so that the loader can make part of +// an image the file's own pages (okld_host::map, 0.3). +struct file_bytes { + unsigned char* data = nullptr; + unsigned long long size = 0, cap = 0; + kal_file file{}; + bool open = false; +}; void release_bytes(unsigned char* data, unsigned long long size, unsigned long long cap) { if (!data) return; @@ -100,6 +107,7 @@ void release_bytes(unsigned char* data, unsigned long long size, unsigned long l void release_file(file_bytes& f) { release_bytes(f.data, f.size, f.cap); + if (f.open) kal_fs_close_file(f.file); f = {}; } @@ -118,6 +126,8 @@ bool map_all(kal_file f, file_bytes& out) { out.data = static_cast(p); out.size = info.size; out.cap = 0; + out.file = f; + out.open = true; return true; } @@ -128,7 +138,7 @@ bool read_all(const char* path, file_bytes& out, char* why, unsigned long long c unsigned long n = length(rest); kal_file f{}; if (kal_fs_open(dir, rest, n, KAL_OPEN_READ, &f) != kal_ok) { text(why, cap) << path << " cannot be opened"; return false; } - if (map_all(f, out)) { kal_fs_close_file(f); return true; } + if (map_all(f, out)) return true; const kal_stream s = kal_fs_stream(f); for (;;) { if (out.size == out.cap) { @@ -572,9 +582,35 @@ void release_needed(const void* bytes, okld_size, void* ctx) { } } +// Part of an image is the file's own pages where the bytes are a file this +// search holds (the file run, a library's, one read for an image another +// needs) at a multiple of the alignment, and the implementation reports +// KAL_EXEC_PROP_FILE (openkal 0.21). Bytes a program carries, a member of an +// archive and an implementation without the position are copied. +bool part_of(const file_bytes& f, const void* bytes, okld_size from, kal_u64& at) { + const auto* b = static_cast(bytes); + if (!f.open || !f.data || b < f.data || b >= f.data + f.size) return false; + at = static_cast(b - f.data) + from; + return at % KAL_MAP_ALIGNMENT == 0; +} + +int map_part(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from, int execute, + void* ctx) { + if (!(kal_exec_props() & KAL_EXEC_PROP_FILE)) return -1; + auto& s = *static_cast(ctx); + const kal_uintptr how = execute ? KAL_EXEC_MAP_EXECUTE : 0; + kal_u64 at = 0; + if (part_of(s.ran, bytes, from, at)) + return kal_exec_map_part(region, offset, size, s.ran.file, at, how) == kal_ok ? 0 : -1; + for (const auto& f : s.needed) + if (part_of(f, bytes, from, at)) + return kal_exec_map_part(region, offset, size, f.file, at, how) == kal_ok ? 0 : -1; + return -1; +} + void prepare_host(search& s) { okld_host& host = s.host; - host.reserve = &reserve; host.publish = &publish; host.release = &release; + host.reserve = &reserve; host.publish = &publish; host.release = &release; host.map = &map_part; host.granularity = &granularity; host.resolve = &resolve; host.alloc = &allocate; host.free = &deallocate; host.find_needed = &find_needed; host.release_bytes = &release_needed; @@ -793,9 +829,7 @@ extern "C" int okrun_run_images(const okrun_image* images, unsigned long count, struct okrun_library { okld_module* module = nullptr; - search s; - unsigned char* bytes = nullptr; // a file's, mapped or read here - unsigned long long size = 0, cap = 0; + search s; // `s.ran': the file it was read from, until it is loaded }; namespace { @@ -812,8 +846,7 @@ okrun_library* open_loaded(okrun_library* lib, const void* bytes, unsigned long } okld_initialise(lib->module, &call_image, nullptr); // The file a library was read from is released once it is loaded. - release_bytes(lib->bytes, lib->size, lib->cap); - lib->bytes = nullptr; lib->size = 0; lib->cap = 0; + release_ran(lib->s); return lib; } @@ -828,7 +861,7 @@ okrun_library* new_library(const okrun_options* options) { void free_library(okrun_library* lib) { if (!lib) return; - release_bytes(lib->bytes, lib->size, lib->cap); + release_ran(lib->s); kal_free(lib, sizeof(okrun_library), 16); } @@ -857,9 +890,8 @@ extern "C" okrun_library* okrun_open_library(const char* path, const okrun_optio } } directory_of(found, lib->s.directory, sizeof lib->s.directory); - lib->bytes = f.data; - lib->size = f.size; - lib->cap = f.cap; + lib->s.ran = f; + copy_text(lib->s.ran_path, sizeof lib->s.ran_path, found); if (!open_loaded(lib, f.data, f.size, found, why, cap)) { free_library(lib); return nullptr; } return lib; } diff --git a/tools/mcpp.toml b/tools/mcpp.toml index a50cf62..c3a67ab 100644 --- a/tools/mcpp.toml +++ b/tools/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "openkal-tools" -version = "0.2.0" +version = "0.3.0" description = "okrun, which runs an ok image on this system, and okld, the format's tool on a build machine (check, note, sign, inspect)." license = "Apache-2.0" authors = ["mcpplibs"]