0.3.0 --- openkal 0.21: an image's instructions and read-only data mapped from its file; a relocation refused outside the writable segments

README.md

@@ -41,7 +41,14 @@ okrun app.okb arguments...                                           # this mach
 ```
 
 The images an image needs are found in the okb, then beside it, in `lib/<arch>/` and `lib/`, and
-in `OKRUN_PATH`. A program that carries its images gives their archive to
+in `OKRUN_PATH`.
+
+**An image's instructions and read-only data are its file's pages** (0.3.0, openkal 0.21): where
+the implementation reports `KAL_EXEC_PROP_FILE`, the loader makes each segment no relocation
+writes a part of the file (`kal_exec_map_part`) rather than a copy, for an image file and for an
+image stored aligned in an okb; the writable segments are copied. A program then starts as a
+program of its system does: a language server's worker made 1 200 page faults where it made
+16 000. A relocation that would write a segment that is not writable is refused (SPEC 12.2). A program that carries its images gives their archive to
 `okrun_run_okb_bytes`, which runs it from memory and writes nothing to disk.
 
 ## A measurement of an image

bind/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-bind"
-version     = "0.2.0"
+version     = "0.3.0"
 # `mcpp::xpkg_request', `mcpp::decision' (protocol 15).
 mcpp        = ">=2026.10.1.3"
 description = "A build plugin: a library published as an ok image (.okso) and a C header is called from a program of any system, through a binding generated from the header and libokrun."

docs/bind.md

@@ -24,10 +24,10 @@ sources      = []
 include_dirs = ["include"]
 
 [target.'cfg(not(os = "openkal"))'.dependencies]
-openkal-run = "0.2.0"
+openkal-run = "0.3.0"
 
 [build-dependencies]
-openkal-bind = "0.2.0"
+openkal-bind = "0.3.0"
 ```
 
 ```cpp
@@ -105,4 +105,5 @@ llvm toolchain's clang converts the call.
 | 0.1.7 | No change of its own; released with the archive, whose loader also digests on ARMv8's SHA-256 instructions on Linux where the processor states them. |
 | 0.1.8 | No change of its own; released with the archive, built against openkal 0.19.0 (the runner binds `kal_task_cores` and `kal_task_set_priority`), whose okb carries chunk digests the runner verifies on several contexts at once. |
 | 0.2.0 | No change of its own; released with the archive, whose runner measures an image (okprof) and remembers an okb's admission. |
+| 0.3.0 | No change of its own; released with the archive, whose loader maps an image's instructions and read-only data from its file where the implementation can (openkal 0.21, `kal_exec_map_part`). |
 | 0.1.9 | No change of its own; released with the archive, whose runner on macOS is built over openkal-macos 0.17.1 (`kal_task_current` without a system call). |

ld/include/okld.h

@@ -239,6 +239,16 @@ struct okld_host {
     /* Ordinary memory for the loader's own records (kal_alloc, kal_free). */
     void*     (*alloc)(okld_size bytes, void* ctx);
     void      (*free)(void* p, okld_size bytes, void* ctx);
+    /* Makes part of the region the bytes of the file the image was read from,
+     * rather than a copy of them (kal_exec_map_part, openkal 0.21): `size'
+     * bytes from `bytes' + `from', at `offset' in the region, executed where
+     * `execute' is non-zero. `from' is a multiple of 65536 (KAL_MAP_ALIGNMENT)
+     * and `offset' and `size' of the granularity. Zero when it did; anything
+     * else, and the loader copies the bytes, as it does where this is null ---
+     * for bytes that are not a file of their own, or an implementation that
+     * does not report KAL_EXEC_PROP_FILE. Version 0.3. */
+    int       (*map)(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from,
+                     int execute, void* ctx);
     /* Non-zero: each image is announced to a debugger through the GDB JIT
      * interface (`__jit_debug_register_code'), with its addresses where it was
      * placed, so that gdb and lldb show its symbols and its source. */

ld/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-ld"
-version     = "0.2.0"
+version     = "0.3.0"
 description = "libokld: the ok format (openkal image format) --- its definition, a reader of bytes, a checker, a signer, and a loader. Freestanding: no C library, no exceptions."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]

ld/src/load.cpp

@@ -84,6 +84,21 @@ void sync_instructions(unsigned char* from, unsigned char* to) {
 elf::u64 round_up(elf::u64 v, elf::u64 g) { return (v + g - 1) & ~(g - 1); }
 elf::u64 round_down(elf::u64 v, elf::u64 g) { return v & ~(g - 1); }
 
+// The alignment of an offset into a file that is mapped: openkal's
+// KAL_MAP_ALIGNMENT, to which every loadable segment is aligned (SPEC 12.2).
+constexpr elf::u64 file_alignment = 65536;
+
+// The parts of the span relocations may write: the writable segments.
+struct writable {
+    elf::u64 from[8] = {}, to[8] = {};
+    unsigned count = 0;
+    bool holds(elf::u64 at, elf::u64 bytes) const {
+        for (unsigned k = 0; k < count; ++k)
+            if (at >= from[k] && at <= to[k] && bytes <= to[k] - at) return true;
+        return false;
+    }
+};
+
 void say(okld_why* why, const char* a, const char* b = nullptr, const char* c = nullptr) {
     if (!why) return;
     sentence s(why->text, sizeof why->text);
@@ -183,7 +198,11 @@ void withdraw(okld_module* m) {
     m->host->free(e, sizeof(jit_code_entry), m->host->ctx);
 }
 
-int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why) {
+// EACH RELOCATION WRITES A WRITABLE SEGMENT, or the image is refused: a
+// segment that is not writable may be the file's own pages (okld_host::map),
+// and a write the format does not allow (SPEC 12.2: no text relocation) is
+// refused rather than made wherever its offset points.
+int relocate(okld_module* m, const view& v, const okld_host* host, const writable& w, okld_why* why) {
     const elf::u64 b = reinterpret_cast<elf::u64>(m->base);
     const bool x86 = v.eh->machine == elf::em_x86_64;
     auto symbol_address = [&](elf::u32 index, bool& ok) -> elf::u64 {
@@ -205,6 +224,11 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why
         if (at == ~0ull || bytes == 0) return true;
         const auto* r = reinterpret_cast<const elf::rela*>(m->base + at);
         for (elf::u64 k = 0; k < bytes / sizeof(elf::rela); ++k) {
+            if (elf::r_type(r[k].info) != (x86 ? elf::r_x86_64_none : elf::r_aarch64_none)
+                && !w.holds(r[k].offset, 8)) {
+                say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)");
+                return false;
+            }
             auto* where = reinterpret_cast<elf::u64*>(m->base + r[k].offset);
             const elf::u32 t = elf::r_type(r[k].info);
             bool ok = true;
@@ -237,17 +261,27 @@ int relocate(okld_module* m, const view& v, const okld_host* host, okld_why* why
     if (const elf::u64 at = v.dynamic(elf::dt_relr, ~0ull); at != ~0ull) {
         const elf::u64 bytes = v.dynamic(elf::dt_relrsz);
         const auto* e = reinterpret_cast<const elf::u64*>(m->base + at);
-        elf::u64* where = nullptr;
-        for (elf::u64 k = 0; k < bytes / 8; ++k) {
+        elf::u64 next = 0;   // the offset the first bit of a bitmap entry names
+        bool ok = true;
+        for (elf::u64 k = 0; ok && k < bytes / 8; ++k) {
             if ((e[k] & 1) == 0) {
-                where = reinterpret_cast<elf::u64*>(m->base + e[k]);
-                *where++ += b;
+                ok = w.holds(e[k], 8);
+                if (ok) *reinterpret_cast<elf::u64*>(m->base + e[k]) += b;
+                next = e[k] + 8;
             } else {
                 elf::u64 bits = e[k] >> 1;
-                for (int i = 0; bits; ++i, bits >>= 1) if (bits & 1) where[i] += b;
-                where += 63;
+                for (elf::u64 i = 0; ok && bits; ++i, bits >>= 1) {
+                    if (!(bits & 1)) continue;
+                    ok = w.holds(next + 8 * i, 8);
+                    if (ok) *reinterpret_cast<elf::u64*>(m->base + next + 8 * i) += b;
+                }
+                next += 63 * 8;
             }
         }
+        if (!ok) {
+            say(why, "a relocation writes outside the image's writable segments (SPEC 12.2)");
+            return -1;
+        }
     }
     return 0;
 }
@@ -313,11 +347,41 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const
     // Each segment is its file bytes and then zeros to its size in memory. What
     // lies between segments is no segment's and is not written: a page the image
     // never touches is not made resident.
+    //
+    // A SEGMENT NOTHING WRITES IS THE FILE'S OWN PAGES where the host can make
+    // it so (0.3, openkal 0.21): instructions and read-only data are the file's
+    // bytes unchanged, and a copy of them made each program that started write
+    // and hold the whole of them --- 54 MB and 14 000 page faults before the
+    // first instruction of a language server, where a program of the
+    // environment pays for the pages it touches. Mapped from the alignment
+    // below the segment to the granule above its end, which no other segment
+    // shares; otherwise, and where the host declines, copied.
+    bool mapped[32] = {};
+    writable w;
     for (unsigned k = 0; k < v.eh->phnum; ++k) {
         const auto& p = v.ph[k];
         if (p.type != elf::pt_load) continue;
         if (p.offset + p.filesz > n) { say(why, "a segment extends beyond the file"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; }
         if (p.filesz > p.memsz) { say(why, "a segment holds more of the file than its size in memory"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; }
+        if (p.flags & elf::pf_w) {
+            if (w.count == 8) { say(why, "the image has more than eight writable segments"); host->release(m->base, m->span, host->ctx); host->free(m, sizeof *m, host->ctx); return -1; }
+            w.from[w.count] = p.vaddr; w.to[w.count] = p.vaddr + p.memsz; ++w.count;
+        } else if (host->map && k < 32 && p.filesz == p.memsz && p.filesz != 0
+                   && p.vaddr % file_alignment == p.offset % file_alignment) {
+            const elf::u64 lo = round_down(p.vaddr, file_alignment), hi = round_up(p.vaddr + p.filesz, g);
+            const elf::u64 from = p.offset - (p.vaddr - lo);
+            bool alone = from + (hi - lo) <= n;
+            for (unsigned j = 0; alone && j < v.eh->phnum; ++j) {
+                const auto& q = v.ph[j];
+                if (j == k || q.type != elf::pt_load) continue;
+                const elf::u64 qlo = round_down(q.vaddr, g), qhi = round_up(q.vaddr + q.memsz, g);
+                if (qlo < hi && lo < qhi) alone = false;
+            }
+            if (alone && host->map(m->base, lo, hi - lo, bytes, from, (p.flags & elf::pf_x) != 0, host->ctx) == 0) {
+                mapped[k] = true;
+                continue;
+            }
+        }
         copy(m->base + p.vaddr, static_cast<const unsigned char*>(bytes) + p.offset, p.filesz);
         fill(m->base + p.vaddr + p.filesz, 0, p.memsz - p.filesz);
     }
@@ -345,20 +409,23 @@ extern "C" int okld_load(const void* bytes, okld_size n, const char* name, const
     m->init_array = v.dynamic(elf::dt_init_array); m->init_arraysz = v.dynamic(elf::dt_init_arraysz);
     m->fini_array = v.dynamic(elf::dt_fini_array); m->fini_arraysz = v.dynamic(elf::dt_fini_arraysz);
 
-    if (relocate(m, v, host, why) != 0) { discard(m); return -1; }
+    if (relocate(m, v, host, w, why) != 0) { discard(m); return -1; }
 
-    // Publish what executes; the rest stays writable.
+    // Publish what executes and was copied; the rest stays writable. A segment
+    // that is the file's pages is executable already, and its bytes reached
+    // memory through the environment rather than through this loader's data
+    // path, so its instructions need no maintenance here either.
     for (unsigned k = 0; k < v.eh->phnum; ++k) {
         const auto& p = v.ph[k];
-        if (p.type != elf::pt_load || !(p.flags & elf::pf_x)) continue;
+        if (p.type != elf::pt_load || !(p.flags & elf::pf_x) || (k < 32 && mapped[k])) continue;
         const elf::u64 from = round_down(p.vaddr, g), to = round_up(p.vaddr + p.memsz, g);
         if (host->publish(m->base, from, to - from, host->ctx) != 0) {
             say(why, "the environment refused to make the image's instructions executable");
             discard(m);
             return -1;
         }
+        sync_instructions(m->base + p.vaddr, m->base + p.vaddr + p.memsz);
     }
-    sync_instructions(m->base, m->base + m->span);
 
     lock();
     m->next = g_modules;

run/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-run"
-version     = "0.2.0"
+version     = "0.3.0"
 description = "libokrun: the runner's core --- an ok image's names bound to this process's one openkal implementation, in the image's calling convention, and its entry entered."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]
@@ -11,17 +11,17 @@ repo        = "https://github.com/mcpplibs/openkal-image"
 kind = "lib"
 
 [dependencies]
-openkal = "0.20.0"
+openkal = "0.21.0"
 openkal-ld = { path = "../ld" }
 
 [target.linux.dependencies]
-openkal-linux = "0.21.0"
+openkal-linux = "0.22.0"
 
 [target.macos.dependencies]
-openkal-macos = "0.18.0"
+openkal-macos = "0.19.0"
 
 [target.windows.dependencies]
-openkal-windows = "0.16.0"
+openkal-windows = "0.17.0"
 
 # The interfaces each system's implementation provides decide one unit: where
 # openkal.space is provided its names are bound, and elsewhere an image that

run/src/run.cpp

@@ -89,8 +89,15 @@ bool locate(const char* path, kal_dir& dir, const char*& rest) {
 }
 
 // A file's bytes: a region of the file (openkal.map), or a buffer it was read
-// into where the file cannot be mapped. `cap' is zero for a region.
-struct file_bytes { unsigned char* data = nullptr; unsigned long long size = 0, cap = 0; };
+// into where the file cannot be mapped. `cap' is zero for a region. A region's
+// file stays open while its bytes are held, so that the loader can make part of
+// an image the file's own pages (okld_host::map, 0.3).
+struct file_bytes {
+    unsigned char* data = nullptr;
+    unsigned long long size = 0, cap = 0;
+    kal_file file{};
+    bool open = false;
+};
 
 void release_bytes(unsigned char* data, unsigned long long size, unsigned long long cap) {
     if (!data) return;
@@ -100,6 +107,7 @@ void release_bytes(unsigned char* data, unsigned long long size, unsigned long l
 
 void release_file(file_bytes& f) {
     release_bytes(f.data, f.size, f.cap);
+    if (f.open) kal_fs_close_file(f.file);
     f = {};
 }
 
@@ -118,6 +126,8 @@ bool map_all(kal_file f, file_bytes& out) {
     out.data = static_cast<unsigned char*>(p);
     out.size = info.size;
     out.cap = 0;
+    out.file = f;
+    out.open = true;
     return true;
 }
 
@@ -128,7 +138,7 @@ bool read_all(const char* path, file_bytes& out, char* why, unsigned long long c
     unsigned long n = length(rest);
     kal_file f{};
     if (kal_fs_open(dir, rest, n, KAL_OPEN_READ, &f) != kal_ok) { text(why, cap) << path << " cannot be opened"; return false; }
-    if (map_all(f, out)) { kal_fs_close_file(f); return true; }
+    if (map_all(f, out)) return true;
     const kal_stream s = kal_fs_stream(f);
     for (;;) {
         if (out.size == out.cap) {
@@ -572,9 +582,35 @@ void release_needed(const void* bytes, okld_size, void* ctx) {
     }
 }
 
+// Part of an image is the file's own pages where the bytes are a file this
+// search holds (the file run, a library's, one read for an image another
+// needs) at a multiple of the alignment, and the implementation reports
+// KAL_EXEC_PROP_FILE (openkal 0.21). Bytes a program carries, a member of an
+// archive and an implementation without the position are copied.
+bool part_of(const file_bytes& f, const void* bytes, okld_size from, kal_u64& at) {
+    const auto* b = static_cast<const unsigned char*>(bytes);
+    if (!f.open || !f.data || b < f.data || b >= f.data + f.size) return false;
+    at = static_cast<kal_u64>(b - f.data) + from;
+    return at % KAL_MAP_ALIGNMENT == 0;
+}
+
+int map_part(void* region, okld_size offset, okld_size size, const void* bytes, okld_size from, int execute,
+             void* ctx) {
+    if (!(kal_exec_props() & KAL_EXEC_PROP_FILE)) return -1;
+    auto& s = *static_cast<search*>(ctx);
+    const kal_uintptr how = execute ? KAL_EXEC_MAP_EXECUTE : 0;
+    kal_u64 at = 0;
+    if (part_of(s.ran, bytes, from, at))
+        return kal_exec_map_part(region, offset, size, s.ran.file, at, how) == kal_ok ? 0 : -1;
+    for (const auto& f : s.needed)
+        if (part_of(f, bytes, from, at))
+            return kal_exec_map_part(region, offset, size, f.file, at, how) == kal_ok ? 0 : -1;
+    return -1;
+}
+
 void prepare_host(search& s) {
     okld_host& host = s.host;
-    host.reserve = &reserve; host.publish = &publish; host.release = &release;
+    host.reserve = &reserve; host.publish = &publish; host.release = &release; host.map = &map_part;
     host.granularity = &granularity; host.resolve = &resolve;
     host.alloc = &allocate; host.free = &deallocate;
     host.find_needed = &find_needed; host.release_bytes = &release_needed;
@@ -793,9 +829,7 @@ extern "C" int okrun_run_images(const okrun_image* images, unsigned long count,
 
 struct okrun_library {
     okld_module*  module = nullptr;
-    search        s;
-    unsigned char* bytes = nullptr;   // a file's, mapped or read here
-    unsigned long long size = 0, cap = 0;
+    search        s;                  // `s.ran': the file it was read from, until it is loaded
 };
 
 namespace {
@@ -812,8 +846,7 @@ okrun_library* open_loaded(okrun_library* lib, const void* bytes, unsigned long
     }
     okld_initialise(lib->module, &call_image, nullptr);
     // The file a library was read from is released once it is loaded.
-    release_bytes(lib->bytes, lib->size, lib->cap);
-    lib->bytes = nullptr; lib->size = 0; lib->cap = 0;
+    release_ran(lib->s);
     return lib;
 }
 
@@ -828,7 +861,7 @@ okrun_library* new_library(const okrun_options* options) {
 
 void free_library(okrun_library* lib) {
     if (!lib) return;
-    release_bytes(lib->bytes, lib->size, lib->cap);
+    release_ran(lib->s);
     kal_free(lib, sizeof(okrun_library), 16);
 }
 
@@ -857,9 +890,8 @@ extern "C" okrun_library* okrun_open_library(const char* path, const okrun_optio
         }
     }
     directory_of(found, lib->s.directory, sizeof lib->s.directory);
-    lib->bytes = f.data;
-    lib->size = f.size;
-    lib->cap = f.cap;
+    lib->s.ran = f;
+    copy_text(lib->s.ran_path, sizeof lib->s.ran_path, found);
     if (!open_loaded(lib, f.data, f.size, found, why, cap)) { free_library(lib); return nullptr; }
     return lib;
 }

tools/mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "openkal-tools"
-version     = "0.2.0"
+version     = "0.3.0"
 description = "okrun, which runs an ok image on this system, and okld, the format's tool on a build machine (check, note, sign, inspect)."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]