HLE extension modules: system functions from a separately built library

CMakeLists.txt

@@ -127,6 +127,25 @@ if(PSPRECOMP_LTO AND PSPRECOMP_LTO_SUPPORTED)
     set_property(GLOBAL PROPERTY PORTABLEKIT_IPO ON)
 endif()
 
+# HLE extension modules (docs/HLE_EXTENSIONS.md): directories built apart from
+# the framework, each with a CMakeLists.txt that calls
+# portablekit_add_hle_extension(). Every program built here links them all.
+# Empty, the default, builds exactly what the framework is on its own.
+set(PORTABLEKIT_HLE_EXTENSION_DIRS "" CACHE STRING
+    "Directories of HLE extension modules to build into every program (a list)")
+set(_portablekit_extension_index 0)
+foreach(_extension_dir IN LISTS PORTABLEKIT_HLE_EXTENSION_DIRS)
+    get_filename_component(_extension_dir "${_extension_dir}" ABSOLUTE BASE_DIR "${CMAKE_SOURCE_DIR}")
+    if(NOT EXISTS "${_extension_dir}/CMakeLists.txt")
+        message(FATAL_ERROR "PORTABLEKIT_HLE_EXTENSION_DIRS: ${_extension_dir} has no CMakeLists.txt")
+    endif()
+    get_filename_component(_extension_leaf "${_extension_dir}" NAME)
+    string(MAKE_C_IDENTIFIER "${_extension_leaf}" _extension_leaf)
+    add_subdirectory("${_extension_dir}"
+        "${CMAKE_CURRENT_BINARY_DIR}/hle_extensions/${_portablekit_extension_index}_${_extension_leaf}")
+    math(EXPR _portablekit_extension_index "${_portablekit_extension_index} + 1")
+endforeach()
+
 # Keep the runtime as an object library so profile executables own its objects.
 # On Windows this also lets WINDOWS_EXPORT_ALL_SYMBOLS include runtime symbols
 # in the executable's import library for separately linked overlay modules.
@@ -333,6 +352,20 @@ if(PSPRECOMP_BUILD_TESTS)
     add_test(NAME portablekit_fast_loading_tests COMMAND portablekit_fast_loading_tests)
 endif()
 
+# HLE extension modules: filling in and overriding against a runtime with no
+# kernel, with the example module in examples/hle_extension/.
+if(PSPRECOMP_BUILD_TESTS)
+    add_executable(portablekit_hle_extension_tests
+        "${PORTABLEKIT_ROOT}/tests/hle_extension_tests.cpp"
+        "${PORTABLEKIT_ROOT}/tests/hle_extension_test_host.cpp"
+        "${PORTABLEKIT_ROOT}/host/hle/hle_extensions.cpp"
+        "${PORTABLEKIT_ROOT}/examples/hle_extension/example_extension.cpp")
+    target_include_directories(portablekit_hle_extension_tests PRIVATE "${PORTABLEKIT_ROOT}/host")
+    target_link_libraries(portablekit_hle_extension_tests PRIVATE psprecomp_core)
+    set_target_properties(portablekit_hle_extension_tests PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
+    add_test(NAME portablekit_hle_extension_tests COMMAND portablekit_hle_extension_tests)
+endif()
+
 # The system layer belongs to a game's target, so nothing above compiles
 # host/main.cpp or most of host/. That let two compile errors through until a
 # second game was built. This target builds the whole backend-independent
@@ -366,6 +399,7 @@ if(PSPRECOMP_BUILD_TESTS)
         "${PORTABLEKIT_ROOT}/host" "${PORTABLEKIT_ROOT}/third_party"
         "${CMAKE_CURRENT_BINARY_DIR}/host_check_nids")
     target_link_libraries(portablekit_host_check PRIVATE psprecomp_core ${CMAKE_DL_LIBS})
+    _portablekit_link_hle_extensions(portablekit_host_check)
     find_package(Threads REQUIRED)
     target_link_libraries(portablekit_host_check PRIVATE Threads::Threads)
     if(WIN32)

README.md

@@ -39,6 +39,7 @@ A port is its own repository: a file of constants, a short `CMakeLists.txt` and
 - [Writing a profile](docs/PROFILE_GUIDE.md)
 - [Building](docs/BUILDING.md), and [why the build keeps its incremental state the way it does](docs/BUILD_SYSTEM.md)
 - [Architecture](docs/ARCHITECTURE.md)
+- [HLE extension modules](docs/HLE_EXTENSIONS.md): system functions from a separately built library, filling in or overriding the framework's
 - [Instruction coverage](docs/INSTRUCTION_COVERAGE.md): which instructions a game has run, and which are implemented but unverified
 - [Where the code comes from, and third-party licences](docs/SOURCE_PROVENANCE.md)
 - [Contributing](CONTRIBUTING.md), and the [working notes](AGENTS.md) every change follows

apps/portablekit/src/app_main.cpp

@@ -32,7 +32,10 @@
 
 #include "app_paths.hpp"
 #include "corpus_abi.hpp"
+#include "hle/hle_extensions.hpp"
 #include "install/user_data.hpp"
+#include "portablekit_build_label.hpp"
+#include "portablekit_version.hpp"
 
 #include "psprecomp/interpreter.hpp"
 
@@ -72,6 +75,7 @@ const char *const kUsageText =
     "       portablekit keys status\n"
     "       portablekit cache clear <game>\n"
     "       portablekit toolchain\n"
+    "       portablekit version                 also --version\n"
     "  <game> is a game's id, its disc id (UCES01059 or UCES-01059), or a unique prefix.\n"
     "  Every command takes --json and then prints one JSON object on stdout, and\n"
     "  --data-dir <folder> to use another data folder than the one next to the program.\n";
@@ -466,6 +470,25 @@ int command_toolchain() {
     return toolchain.found ? 0 : 20;
 }
 
+// The build: its version, its label and the HLE extension modules linked in.
+int command_version() {
+    const auto modules = linked_hle_extensions();
+    if (g_json) {
+        std::vector<Json> items;
+        for (const HleExtensionModule &module : modules)
+            items.push_back(Json().field("name", module.name).field("title", module.title)
+                                .field("version", module.version).field("license", module.license));
+        std::cout << Json().field("ok", true).field("version", kBuildVersion).field("label", kBuildLabel)
+                         .field("abi", kCorpusAbi).array("hle_extensions", items).str() << "\n";
+        return 0;
+    }
+    std::cout << "PortableKit " << kBuildVersion;
+    if (*kBuildLabel != '\0') std::cout << " (" << kBuildLabel << ")";
+    std::cout << "\n  corpus ABI: " << kCorpusAbi << "\n";
+    print_hle_extension_modules(std::cout, modules);
+    return 0;
+}
+
 int dispatch(int argc, char **argv) {
     const Arguments args = parse(argc, argv, 1);
     g_json = args.has("json");
@@ -474,6 +497,8 @@ int dispatch(int argc, char **argv) {
         std::cout << kUsageText;
         return 0;
     }
+    if (args.has("version") || (!args.positional.empty() && args.positional[0] == "version"))
+        return command_version();
     if (args.positional.empty()) {
         activate_launcher_profile();
         set_environment("PORTABLEKIT_DATA_DIR", path_text(home_directory() / "launcher"));

apps/portablekit/src/launcher_ui.cpp

@@ -10,6 +10,7 @@
 #include "text_format.hpp"
 
 #include "hle/hle_common.hpp"
+#include "portablekit_build_label.hpp"
 #include "ui/file_browser.hpp"
 #include "ui/layer.hpp"
 #include "ui/widgets.hpp"
@@ -166,7 +167,8 @@ bool Launcher::frame() {
 void Launcher::library() {
     Layer &layer = Layer::get();
     if (layer.take_back()) quit_ = true;
-    begin_panel("##library", "PortableKit", "Your games", false);
+    begin_panel("##library", *kBuildLabel != '\0' ? std::string("PortableKit (") + kBuildLabel + ")" : "PortableKit",
+                "Your games", false);
     begin_content();
     if (first_) focus_next_row();
     first_ = false;

cmake/PortableKit.cmake

@@ -63,6 +63,8 @@ set(PORTABLEKIT_HOST_SOURCES
     host/hle/hle_utility.cpp
     host/hle/hle_savedata.cpp
     host/hle/hle_adhoc.cpp
+    host/hle/hle_extensions.cpp
+    host/hle/hle_extension_host.cpp
     host/adhoc/client.cpp
     host/adhoc/discovery.cpp
     host/adhoc/host.cpp
@@ -142,6 +144,101 @@ function(_portablekit_inherit_settings target)
     endif()
 endfunction()
 
+# portablekit_add_hle_extension(<name> LICENSE <licence> SOURCES ...
+#                               [TITLE <title>] [VERSION <version>]
+#                               [INCLUDE_DIRECTORIES ...] [LINK_LIBRARIES ...]
+#                               [COMPILE_DEFINITIONS ...])
+# declares an HLE extension module (docs/HLE_EXTENSIONS.md): a static library,
+# built from SOURCES against include/portablekit/hle_extension.hpp, that
+# defines PORTABLEKIT_HLE_EXTENSION(<name>). LICENSE is the licence the module
+# states (an SPDX identifier such as "MIT" where there is one), and it, TITLE
+# and VERSION are printed at startup and by --version. Every program portablekit_add_game()
+# makes afterwards links it and calls that entry at startup. <name> is a C
+# identifier and unique in the build. A module's directory is usually handed
+# to the build in PORTABLEKIT_HLE_EXTENSION_DIRS, and its CMakeLists.txt is
+# this one call; a port may also call it itself before portablekit_add_game().
+function(portablekit_add_hle_extension name)
+    cmake_parse_arguments(EXT "" "LICENSE;TITLE;VERSION"
+        "SOURCES;INCLUDE_DIRECTORIES;LINK_LIBRARIES;COMPILE_DEFINITIONS" ${ARGN})
+    if(NOT name MATCHES "^[A-Za-z_][A-Za-z0-9_]*$")
+        message(FATAL_ERROR "portablekit_add_hle_extension: '${name}' is not a C identifier")
+    endif()
+    if(NOT EXT_SOURCES)
+        message(FATAL_ERROR "portablekit_add_hle_extension(${name}): no SOURCES")
+    endif()
+    if(NOT EXT_LICENSE)
+        message(FATAL_ERROR "portablekit_add_hle_extension(${name}): no LICENSE")
+    endif()
+    if(NOT EXT_TITLE)
+        set(EXT_TITLE "${name}")
+    endif()
+    foreach(field IN ITEMS LICENSE TITLE VERSION)
+        string(FIND "${EXT_${field}}" "\"" quote)
+        string(FIND "${EXT_${field}}" "\\" backslash)
+        string(FIND "${EXT_${field}}" "\n" newline)
+        if(NOT quote EQUAL -1 OR NOT backslash EQUAL -1 OR NOT newline EQUAL -1)
+            message(FATAL_ERROR "portablekit_add_hle_extension(${name}): ${field} may not contain quotes, backslashes or new lines")
+        endif()
+    endforeach()
+    get_property(known GLOBAL PROPERTY PORTABLEKIT_HLE_EXTENSIONS)
+    if(name IN_LIST known)
+        message(FATAL_ERROR "portablekit_add_hle_extension: a module named '${name}' was already added")
+    endif()
+    set(target "portablekit_hle_extension_${name}")
+    add_library(${target} STATIC ${EXT_SOURCES})
+    target_compile_features(${target} PRIVATE cxx_std_20)
+    _portablekit_inherit_settings(${target})
+    # Headers only, as for an overlay module: the runtime's objects belong to
+    # the program the module is linked into. host/ is there for a module that
+    # reaches past the stable interface, at its own risk.
+    target_include_directories(${target} PRIVATE
+        "${PORTABLEKIT_ROOT}/include" "${PORTABLEKIT_ROOT}/host" ${EXT_INCLUDE_DIRECTORIES})
+    if(EXT_COMPILE_DEFINITIONS)
+        target_compile_definitions(${target} PRIVATE ${EXT_COMPILE_DEFINITIONS})
+    endif()
+    if(EXT_LINK_LIBRARIES)
+        target_link_libraries(${target} PRIVATE ${EXT_LINK_LIBRARIES})
+    endif()
+    set_property(GLOBAL APPEND PROPERTY PORTABLEKIT_HLE_EXTENSIONS "${name}")
+    set_property(GLOBAL PROPERTY PORTABLEKIT_HLE_EXTENSION_${name}_INFO
+        "\"${name}\", \"${EXT_TITLE}\", \"${EXT_VERSION}\", \"${EXT_LICENSE}\"")
+    message(STATUS "HLE extension module: ${EXT_TITLE} ${EXT_VERSION} (${EXT_LICENSE})")
+endfunction()
+
+# Links every HLE extension module added so far into <target> and generates
+# the list host/hle/hle_extensions.hpp's linked_hle_extensions() returns.
+function(_portablekit_link_hle_extensions target)
+    get_property(modules GLOBAL PROPERTY PORTABLEKIT_HLE_EXTENSIONS)
+    set(declarations "")
+    set(entries "")
+    foreach(module IN LISTS modules)
+        string(APPEND declarations "void portablekit_hle_extension_${module}(::portablekit::hle_extension::Registry &);\n")
+        get_property(info GLOBAL PROPERTY PORTABLEKIT_HLE_EXTENSION_${module}_INFO)
+        string(APPEND entries "        {${info}, &portablekit_hle_extension_${module}},\n")
+        target_link_libraries(${target} PRIVATE "portablekit_hle_extension_${module}")
+    endforeach()
+    if(modules)
+        set(body "    static const HleExtensionModule modules[] = {\n${entries}    };\n    return modules;")
+    else()
+        set(body "    return {};")
+    endif()
+    set(list_source "${CMAKE_CURRENT_BINARY_DIR}/generated_hle_extensions/${target}_hle_extensions.cpp")
+    file(CONFIGURE OUTPUT "${list_source}" CONTENT
+"// Generated by cmake/PortableKit.cmake: the HLE extension modules linked into ${target}.
+#include \"hle/hle_extensions.hpp\"
+
+${declarations}
+namespace portablekit {
+
+std::span<const HleExtensionModule> linked_hle_extensions() {
+${body}
+}
+
+} // namespace portablekit
+")
+    target_sources(${target} PRIVATE "${list_source}")
+endfunction()
+
 function(portablekit_add_game target)
     # Options a program that wraps the port uses (the desktop app,
     # apps/portablekit); a port passes none of them.
@@ -297,6 +394,29 @@ function(portablekit_add_game target)
     target_compile_features(${target} PRIVATE cxx_std_20)
     _portablekit_inherit_settings(${target})
     add_dependencies(${target} ${target}_version)
+    # What a build is called besides its version (PORTABLEKIT_BUILD_LABEL), in
+    # a header of its own, so that only a change of the label rebuilds what
+    # shows it.
+    set(PORTABLEKIT_BUILD_LABEL "" CACHE STRING
+        "A name for this build, shown beside the version in window titles, menus and --version")
+    string(FIND "${PORTABLEKIT_BUILD_LABEL}" "\"" label_quote)
+    string(FIND "${PORTABLEKIT_BUILD_LABEL}" "\\" label_backslash)
+    string(FIND "${PORTABLEKIT_BUILD_LABEL}" "\n" label_newline)
+    if(NOT label_quote EQUAL -1 OR NOT label_backslash EQUAL -1 OR NOT label_newline EQUAL -1)
+        message(FATAL_ERROR "PORTABLEKIT_BUILD_LABEL may not contain quotes, backslashes or new lines")
+    endif()
+    file(CONFIGURE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/generated_version/portablekit_build_label.hpp" CONTENT
+"#pragma once
+
+// Generated by cmake/PortableKit.cmake from PORTABLEKIT_BUILD_LABEL.
+namespace portablekit {
+// Empty when the build was given no label.
+inline constexpr const char *kBuildLabel = \"${PORTABLEKIT_BUILD_LABEL}\";
+}
+")
+    if(PORTABLEKIT_BUILD_LABEL)
+        message(STATUS "${target}: build label \"${PORTABLEKIT_BUILD_LABEL}\"")
+    endif()
     if(PORTABLEKIT_RENDERER)
         add_dependencies(${target} ${target}_shaders)
         # The sink opens its own SDL audio subsystem, so it only needs SDL to exist.
@@ -324,6 +444,7 @@ function(portablekit_add_game target)
         "${GAME_PROFILE_DIR}/host"
         "${GAME_PROFILE_DIR}/generated")
     target_link_libraries(${target} PRIVATE psprecomp_core ${CMAKE_DL_LIBS})
+    _portablekit_link_hle_extensions(${target})
     if(WIN32)
         # The ad hoc client's and server's sockets, and the list of interfaces.
         target_link_libraries(${target} PRIVATE ws2_32 iphlpapi)

docs/BRINGING_UP_A_GAME.md

@@ -76,6 +76,7 @@ Every one of these takes the profile's own prefix, so `TENKAWA_TRACE_KERNEL` for
 | --- | --- |
 | `<PREFIX>_NO_RENDER`, `<PREFIX>_NO_AUDIO` | No window, no sound: a boot check that runs anywhere |
 | `<PREFIX>_LIST_STUBS` | Name every import nothing implements, at start-up |
+| `<PREFIX>_NO_HLE_EXTENSIONS` | Ignore the [HLE extension modules](HLE_EXTENSIONS.md) the build links, for comparing with the framework's own HLE |
 | `<PREFIX>_STRICT_HLE` | Do not bind logging stubs for unimplemented imports, so the game stops at the first one instead of carrying on with a wrong answer |
 | `<PREFIX>_TRACE_KERNEL`, `_TRACE_IO`, `_TRACE_GE`, `_TRACE_SAVEDATA`, … | One subsystem each |
 | `<PREFIX>_TRACE_SYNC` | Every kernel object a thread waits on, which is how a deadlock is read |

docs/BUILDING.md

@@ -69,6 +69,8 @@ Measured on an Apple M1 with 8 GB, from a fresh clone with an empty compiler cac
 | `PORTABLEKIT_FFMPEG` | `bundled` | `bundled` builds FFmpeg with the project, `system` uses one found through pkg-config, `OFF` drops music and movies |
 | `PORTABLEKIT_RELEASE` | `OFF` | Build for distribution: no paths into the checkout, libraries from `lib/` next to the executable |
 | `PSPRECOMP_LTO` | `OFF` | Whole-program optimisation |
+| `PORTABLEKIT_BUILD_LABEL` | empty | A name for the build, shown beside the version: in the window title, the menu's About section, the desktop app's library header and `portablekit --version`. For telling a build made with other components (see below) from the plain one without patching sources |
+| `PORTABLEKIT_HLE_EXTENSION_DIRS` | empty | Directories of [HLE extension modules](HLE_EXTENSIONS.md) to link into every program |
 
 ## Dependencies
 

docs/DESKTOP_APP.md

@@ -355,6 +355,7 @@ portablekit keys import <file>
 portablekit keys status
 portablekit cache clear <game>
 portablekit toolchain
+portablekit version                  also --version: the build, its label, the corpus ABI, HLE extension modules
 ```
 
 `<game>` is the id (`UCES01059-e1075b96`), the disc id with or without its dash, or a unique prefix. `run` starts a background compile when -O2 is not ready yet, `tiered` by default: -O0 first, then -O2 from the same C++ (the recompiler runs once). `--seconds` bounds a run for tests; `PORTABLEKIT_MAX_OPT` caps the level loaded, for comparing levels. `--data-dir <folder>` works with every command.