HLE extensions: declared keys, key look-ups, raw reads and file filters

CMakeLists.txt

@@ -359,6 +359,7 @@ if(PSPRECOMP_BUILD_TESTS)
         "${PORTABLEKIT_ROOT}/tests/hle_extension_tests.cpp"
         "${PORTABLEKIT_ROOT}/tests/hle_extension_test_host.cpp"
         "${PORTABLEKIT_ROOT}/host/hle/hle_extensions.cpp"
+        "${PORTABLEKIT_ROOT}/host/extension_keys.cpp"
         "${PORTABLEKIT_ROOT}/examples/hle_extension/example_extension.cpp")
     target_include_directories(portablekit_hle_extension_tests PRIVATE "${PORTABLEKIT_ROOT}/host")
     target_link_libraries(portablekit_hle_extension_tests PRIVATE psprecomp_core)
@@ -407,3 +408,24 @@ if(PSPRECOMP_BUILD_TESTS)
     endif()
     set_target_properties(portablekit_host_check PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 endif()
+
+# A read filter on an open file, through the real IoFileMgrForUser handlers:
+# the system layer the check above builds, with a test's main.
+if(PSPRECOMP_BUILD_TESTS)
+    set(_io_test_sources ${_check_sources})
+    list(REMOVE_ITEM _io_test_sources "${PORTABLEKIT_ROOT}/host/main.cpp")
+    add_executable(portablekit_io_filter_tests "${PORTABLEKIT_ROOT}/tests/io_filter_tests.cpp" ${_io_test_sources})
+    target_compile_features(portablekit_io_filter_tests PRIVATE cxx_std_20)
+    target_compile_definitions(portablekit_io_filter_tests PRIVATE
+        PORTABLEKIT_NIDS_CSV="${PORTABLEKIT_ROOT}/configs/nids.csv")
+    target_include_directories(portablekit_io_filter_tests PRIVATE
+        "${PORTABLEKIT_ROOT}/host" "${PORTABLEKIT_ROOT}/third_party"
+        "${CMAKE_CURRENT_BINARY_DIR}/host_check_nids")
+    target_link_libraries(portablekit_io_filter_tests PRIVATE psprecomp_core Threads::Threads ${CMAKE_DL_LIBS})
+    _portablekit_link_hle_extensions(portablekit_io_filter_tests)
+    if(WIN32)
+        target_link_libraries(portablekit_io_filter_tests PRIVATE ws2_32 iphlpapi)
+    endif()
+    set_target_properties(portablekit_io_filter_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+    add_test(NAME portablekit_io_filter_tests COMMAND portablekit_io_filter_tests)
+endif()

apps/portablekit/src/app_main.cpp

@@ -424,13 +424,16 @@ int command_keys(const Arguments &args) {
         if (g_json) {
             std::vector<Json> problems;
             for (const std::string &problem : report.problems) problems.push_back(Json().field("problem", problem));
+            std::vector<Json> warnings;
+            for (const std::string &warning : report.warnings) warnings.push_back(Json().field("warning", warning));
             std::cout << Json().field("ok", ok).field("path", path_text(keys_file_path()))
                              .field("executables", report.can_decrypt_executables)
                              .field("saves", report.can_encrypt_saves)
                              .field("tags", static_cast<std::uint64_t>(report.tag_count))
-                             .array("problems", problems).str() << "\n";
+                             .array("problems", problems).array("warnings", warnings).str() << "\n";
         } else {
             for (const std::string &problem : report.problems) std::cerr << "  " << problem << "\n";
+            for (const std::string &warning : report.warnings) std::cerr << "  warning: " << warning << "\n";
             if (ok) std::cout << "Keys saved to " << path_text(keys_file_path()) << "\n";
         }
         return ok ? 0 : kKeysRejected;
@@ -440,18 +443,34 @@ int command_keys(const Arguments &args) {
     if (g_json) {
         std::vector<Json> problems;
         for (const std::string &problem : report.problems) problems.push_back(Json().field("problem", problem));
+        std::vector<Json> warnings;
+        for (const std::string &warning : report.warnings) warnings.push_back(Json().field("warning", warning));
+        std::vector<Json> declared;
+        for (const KeysReport::DeclaredStatus &key : report.declared)
+            declared.push_back(Json().field("name", key.name).field("module", key.module).field("present", key.present));
         std::cout << Json().field("ok", true).field("path", path_text(report.path)).field("found", report.file_found)
                          .field("executables", report.can_decrypt_executables)
                          .field("saves", report.can_encrypt_saves)
                          .field("tags", static_cast<std::uint64_t>(report.tag_count))
-                         .array("problems", problems).str() << "\n";
+                         .array("problems", problems).array("warnings", warnings)
+                         .array("extension_keys", declared).str() << "\n";
         return 0;
     }
     std::cout << "Keys file: " << path_text(report.path) << (report.file_found ? "" : " (none)") << "\n"
               << "  decrypt executables: " << (report.can_decrypt_executables ? "yes" : "no") << "\n"
               << "  encrypt saves:       " << (report.can_encrypt_saves ? "yes" : "no (saves are kept unencrypted)") << "\n"
               << "  tag keys:            " << report.tag_count << "\n";
     for (const std::string &problem : report.problems) std::cout << "  problem: " << problem << "\n";
+    for (const std::string &warning : report.warnings) std::cout << "  warning: " << warning << "\n";
+    // The keys HLE extension modules read, by module.
+    std::string module;
+    for (const KeysReport::DeclaredStatus &key : report.declared) {
+        if (key.module != module) {
+            module = key.module;
+            std::cout << "  keys " << module << " reads:\n";
+        }
+        std::cout << "    " << key.name << ": " << (key.present ? "present" : "missing") << "\n";
+    }
     return 0;
 }
 

apps/portablekit/src/keys_file.cpp

@@ -3,10 +3,13 @@
 #include "app_home.hpp"
 #include "text_format.hpp"
 
+#include "extension_keys.hpp"
 #include "psprecomp/sha256.hpp"
 
 #include <cstdlib>
 #include <mutex>
+#include <set>
+#include <string_view>
 
 namespace portablekit::app {
 namespace {
@@ -51,14 +54,55 @@ Key16 to_key(const std::vector<std::uint8_t> &bytes) {
     return key;
 }
 
+// What the reader says of a name it does not use, and what that becomes once
+// HLE extension modules had their say: nothing for a name one declares, a
+// warning for the rest (another program's keys may share the file).
+constexpr std::string_view kNotUsed = " is not a key this program uses.";
+
+void sort_out_unused_names(KeysReport &report, const std::set<std::string> &declared_found) {
+    std::vector<std::string> kept;
+    for (std::string &problem : report.problems) {
+        if (!problem.ends_with(kNotUsed)) {
+            kept.push_back(std::move(problem));
+            continue;
+        }
+        const std::string written = problem.substr(0, problem.size() - kNotUsed.size());
+        if (declared_found.contains(lower(written))) continue;
+        report.warnings.push_back(written + " is not a key this program or its extension modules use; it is ignored.");
+    }
+    report.problems = std::move(kept);
+}
+
+// The keys HLE extension modules declare, and whether the file gave them.
+void list_declared(KeysReport &report) {
+    for (const DeclaredKey &key : declared_extension_keys())
+        report.declared.push_back({key.name, key.module, key_value(&report.keys, key.name).has_value()});
+}
+
+// Copies a checked keys file to <home>/keys.txt.
+bool copy_to_home(const std::filesystem::path &source, KeysReport &report) {
+    std::error_code ec;
+    std::filesystem::create_directories(keys_file_path().parent_path(), ec);
+    std::filesystem::copy_file(source, keys_file_path(), std::filesystem::copy_options::overwrite_existing, ec);
+    if (ec) {
+        report.problems.push_back("Cannot copy it to " + path_text(keys_file_path()) + ": " + ec.message());
+        return false;
+    }
+    return true;
+}
+
 } // namespace
 
 KeysReport read_keys_file(const std::filesystem::path &path) {
     KeysReport report;
     report.path = path;
     std::error_code ec;
-    if (!std::filesystem::is_regular_file(path, ec)) return report;
+    if (!std::filesystem::is_regular_file(path, ec)) {
+        list_declared(report);
+        return report;
+    }
     report.file_found = true;
+    std::set<std::string> declared_found;  // names a module declares, kept as its keys
     const KeyValues values = read_key_values(path);
     for (const auto &[raw_name, value] : values) {
         const std::string name = lower(raw_name);
@@ -86,6 +130,25 @@ KeysReport read_keys_file(const std::filesystem::path &path) {
             report.keys.tags[static_cast<std::uint32_t>(tag)] = std::move(entry);
             continue;
         }
+        // A key an HLE extension module declares (extension_keys.hpp), unless
+        // it is one this program checks itself: that check stays.
+        if (const DeclaredKey *declared = find_declared_key(declared_extension_keys(), name);
+            declared != nullptr && expected_fingerprint(name) == nullptr) {
+            if (std::string problem = check_declared_key(*declared, *bytes, raw_name); !problem.empty()) {
+                report.problems.push_back(std::move(problem));
+                continue;
+            }
+            store_named_key(report.keys, name, *bytes);
+            declared_found.insert(name);
+            // A 16-byte one goes on, so a name this program also reads in its
+            // own way is kept that way as well.
+            if (bytes->size() != 16u) continue;
+        } else if (bytes->size() != 16u && expected_fingerprint(name) == nullptr && !name.starts_with("kirk.") &&
+                   !name.starts_with("savedata.")) {
+            // Not a name anything here reads: a warning, whatever its length.
+            report.warnings.push_back(raw_name + " is not a key this program or its extension modules use; it is ignored.");
+            continue;
+        }
         if (bytes->size() != 16u) {
             report.problems.push_back(raw_name + " must be 16 bytes.");
             continue;
@@ -108,6 +171,8 @@ KeysReport read_keys_file(const std::filesystem::path &path) {
             report.keys.savedata[std::atoi(name.substr(9).c_str())] = to_key(*bytes);
         }
     }
+    sort_out_unused_names(report, declared_found);
+    list_declared(report);
     report.tag_count = report.keys.tags.size();
     report.can_decrypt_executables = report.keys.kirk(0x5Du) != nullptr && report.keys.kirk_cmd1.has_value();
     report.can_encrypt_saves = true;
@@ -135,6 +200,7 @@ bool import_keys_file(const std::filesystem::path &source, KeysReport &report) {
         return false;
     }
     if (!report.problems.empty()) return false;
+    if (!report.keys.named.empty()) return copy_to_home(source, report);  // keys only modules use
     if (report.keys.kirk_aes.empty() && !report.keys.kirk_cmd1 && report.keys.savedata.empty() &&
         report.keys.tags.empty()) {
         report.problems.push_back("The file holds no keys.");
@@ -157,10 +223,13 @@ namespace portablekit {
 // The framework asks here for every key it uses (host/crypto_keys.hpp).
 const CryptoKeys *crypto_keys() {
     const app::KeysReport &report = app::active_keys();
+    if (!report.keys.named.empty()) return &report.keys;
     if (report.keys.kirk_aes.empty() && !report.keys.kirk_cmd1 && report.keys.savedata.empty() &&
         report.keys.tags.empty())
         return nullptr;
     return &report.keys;
 }
 
+bool keys_come_from_keys_file() { return true; }
+
 } // namespace portablekit

apps/portablekit/src/keys_file.hpp

@@ -12,6 +12,11 @@
 //   tag.C0CB167C = <16 or 0x90 bytes>   what an executable's tag selects
 //   tag.C0CB167C.slot = 5D         (optional) the KIRK slot of that tag
 //
+// HLE extension modules may declare more names (Registry::declare_key), each
+// with its length and optionally a fingerprint; they are read and checked the
+// same way. A name nothing declares is a warning and is left out, but does not
+// refuse the file.
+//
 // The fixed keys are checked against SHA-256 fingerprints compiled into the
 // program, so a mistyped key is named instead of producing garbage; a
 // fingerprint does not reveal the key. Tag keys have no fingerprint: an
@@ -30,6 +35,16 @@ struct KeysReport {
     std::filesystem::path path;
     CryptoKeys keys;
     std::vector<std::string> problems;  // one sentence each
+    // Names nothing reads (not this program, not an extension module): kept
+    // out, but they do not refuse the file.
+    std::vector<std::string> warnings;
+    // The keys HLE extension modules declare, and whether the file has them.
+    struct DeclaredStatus {
+        std::string name;
+        std::string module;
+        bool present{};
+    };
+    std::vector<DeclaredStatus> declared;
     // What the keys that passed allow.
     bool can_decrypt_executables{};     // kirk.aes.5D and kirk.cmd1
     bool can_encrypt_saves{};           // every save-data key

cmake/PortableKit.cmake

@@ -65,6 +65,7 @@ set(PORTABLEKIT_HOST_SOURCES
     host/hle/hle_adhoc.cpp
     host/hle/hle_extensions.cpp
     host/hle/hle_extension_host.cpp
+    host/extension_keys.cpp
     host/adhoc/client.cpp
     host/adhoc/discovery.cpp
     host/adhoc/host.cpp

docs/DESKTOP_APP.md

@@ -165,6 +165,7 @@ tag.C0CB167C.slot = 5D         # optional: the KIRK slot the tag's header uses
 - `portablekit keys import <file>` checks the file and copies it into the data folder. Every fixed key is checked against a SHA-256 fingerprint compiled into the program, so the player is told "`savedata.4` is not the right key" instead of getting garbage later. Tag keys have no fingerprint: an executable that decrypts to an ELF the runtime can load is the check.
 - What each group enables: `kirk.aes.5D` + `kirk.cmd1` + the game's `tag.*` decrypt that game's `EBOOT.BIN`; the nine save slots and `savedata.2`–`7` encrypt and decrypt saves. `portablekit keys status` says which of these the file provides.
 - Without keys, a game can still be added when its disc carries an unencrypted `BOOT.BIN` (LocoRoco 2 does), or when the player gives an executable they decrypted (`--executable`). The program never names a source of keys.
+- A name the program does not use is a warning ("... is not a key this program or its extension modules use; it is ignored.") and is left out; it does not refuse the file. HLE extension modules the build links may declare more names, which are read and checked like the program's own ([HLE_EXTENSIONS.md](HLE_EXTENSIONS.md#keys)); `keys status` lists them by module.
 
 Messages when something is missing, as the prototype prints them:
 

docs/HLE_EXTENSIONS.md

@@ -71,6 +71,10 @@ Everything a module may rely on is in `include/portablekit/hle_extension.hpp`. `
 | `now_us()` | Emulated time since boot. |
 | `log_once(key, message)` | Log a line the first time `key` is seen. |
 | `env(name)` | The port's variable `<PREFIX>_<name>`. |
+| `Registry::declare_key(name, sha256, size)` | Declares a key the module reads (see [Keys](#keys)). Since interface version 3. |
+| `key(name)`, `key_bytes(name)`, `kirk_key(slot)` | The program's keys by their keys-file name: PortableKit's own and declared ones; empty when missing. |
+| `read_open_file(fd, offset, bytes)` | The raw bytes of an open guest file, without moving its position. |
+| `set_file_filter(fd, filter)` | What the game reads from an open file instead of its bytes (see [Filtered files](#filtered-files)). |
 
 Handlers run on the emulation thread, one at a time. A module may include PortableKit's `host/` headers to reach further than this, but those change without notice; a need that comes up twice belongs in the header instead.
 
@@ -114,6 +118,25 @@ PORTABLEKIT_HLE_EXTENSION(my_module) {
 
 `previous` must be called with the caller's registers as the handler found them, and as its last act, in place of `finish()`.
 
+## Keys
+
+A module that needs keys the console's crypto uses declares each name it reads, at registration:
+
+```cpp
+registry.declare_key("vendor.table", "<sha256 of the value, lower-case hex>", 0x90);  // fingerprint and size optional
+```
+
+and reads it at run time with `ext::key_bytes("vendor.table")` (`ext::key` for 16-byte keys, `ext::kirk_key(slot)` for `kirk.aes.<slot>`). The names are the keys file's, not case-sensitive, and a lookup also finds PortableKit's own keys (`kirk.aes.5D`, `kirk.cmd1`, `savedata.2`, `tag.<tag>`).
+
+- **Desktop app.** The player's keys file accepts a declared name like one of PortableKit's own: it checks the length and, when one is declared, the fingerprint, and a wrong value is a problem that refuses the file on import. A declared `kirk.aes.<slot>` is also a KIRK key for PortableKit. A name nothing declares is a warning and is left out, but no longer refuses the file. `portablekit keys status` lists the declared keys by module, present or missing (`--json`: `extension_keys`). A name PortableKit checks itself keeps PortableKit's check.
+- **Ports.** PortableKit's own keys are compiled in. Declared keys come from an optional keys file in the same format: `<PREFIX>_KEYS`, else `keys.txt` in the data folder (`<PREFIX>_DATA_DIR`, else the port's own); names nothing declares are left alone there. Startup says `HLE extension keys: N of M declared keys present (<path>)`, with a line for each wrong one.
+
+An entry function may be called more than once (the keys are collected before the game starts); it only adds to its registry.
+
+## Filtered files
+
+`set_file_filter(fd, filter)` puts a `FileFilter` (its `size()` and `read(offset, destination, length)`) between the game and an open file, such as the plain text of an encrypted container. While it is set, PortableKit's own `sceIoRead`, `sceIoReadAsync`, `sceIoLseek`, `sceIoLseek32` (and their Async forms) and the disc seek ioctl `0x01010005` work on what the filter gives: positions and sizes count its bytes, and asynchronous results come with `sceIoWaitAsync`/`sceIoPollAsync` as usual. The position is left where it was when the filter is set. The filter reads the file's own bytes with `read_open_file`, which a filter does not change. A null filter restores the file; closing it (`sceIoClose`, `sceIoCloseAsync`) drops the filter. The raw disc device (`umd0:` and `sce_lbn` sector files, which count sectors) and directories take no filter. Writing to a filtered file writes the file's own bytes.
+
 ## What the log says
 
 ```text
@@ -134,7 +157,7 @@ HLE extensions: 1 module linked, ignored (PORTABLEKIT_NO_HLE_EXTENSIONS)
 
 ## Tests
 
-`portablekit_hle_extension_tests` builds the example module into a runtime with no kernel and checks a fill-in, an override, a fill-in left to the built-in, two modules adding one function, and the log lines.
+`portablekit_io_filter_tests` runs PortableKit's own IoFileMgrForUser handlers on a memory stick folder with a filter: synchronous and asynchronous reads and seeks, the raw bytes, removing it and closing. `portablekit_hle_extension_tests` builds the example module into a runtime with no kernel and checks a fill-in, an override, a fill-in left to the built-in, two modules adding one function, chained overrides, the log lines, and declared keys: collecting them, checking a value, looking keys up and reading a port's keys file.
 
 ## Licensing
 

examples/hle_extension/example_extension.cpp

@@ -1,5 +1,5 @@
-// An example HLE extension module: one function filled in, one overridden and
-// one wrapped (a chained override that passes every call on).
+// An example HLE extension module: one function filled in, one overridden, one
+// wrapped (a chained override that passes every call on) and one key declared.
 // Both behave as PortableKit already does, so building it into a program
 // changes nothing a game can see; it shows the mechanism and is what
 // tests/hle_extension_tests.cpp checks it with.
@@ -32,6 +32,9 @@ void watch_writeback_invalidate(ext::Runtime &rt, ext::AllegrexContext &ctx, con
 } // namespace
 
 PORTABLEKIT_HLE_EXTENSION(example) {
+    // A key the module would read with ext::key("example.demo"). Its value is
+    // not secret, it only shows the mechanism: 00112233445566778899AABBCCDDEEFF.
+    registry.declare_key("example.demo", "a8faed6abbf35c12a4b26e40f6feb19d736d90045c83b9f9a31f638d323e6811");
     registry.add("sceHttp", 0x87797BDDu, "sceHttpsLoadDefaultCert", load_default_cert);
     registry.override_builtin("UtilsForUser", 0x79D1C3FAu, "sceKernelDcacheWritebackAll", writeback_data_cache);
     registry.override_chained("UtilsForUser", 0xB435DEC5u, "sceKernelDcacheWritebackInvalidateAll",