diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c75cbd..8bdfa40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,12 +203,64 @@ add_executable(sfr_kinect_sensor_test tests/kinect_sensor_test.cpp src/kinect_se src/kinect_preview.cpp) target_include_directories(sfr_kinect_sensor_test PRIVATE src) add_test(NAME kinect_sensor COMMAND sfr_kinect_sensor_test) -add_executable(sfr_kinect_smoothing_test tests/kinect_smoothing_test.cpp src/kinect_smoothing.cpp) +add_executable(sfr_kinect_smoothing_test tests/kinect_smoothing_test.cpp src/kinect_smoothing.cpp src/nui_transform_smooth.cpp) target_include_directories(sfr_kinect_smoothing_test PRIVATE src) add_test(NAME kinect_smoothing COMMAND sfr_kinect_smoothing_test) -add_executable(sfr_kinect_calibration_test tests/kinect_calibration_test.cpp src/kinect_calibration.cpp) -target_include_directories(sfr_kinect_calibration_test PRIVATE src) -add_test(NAME kinect_calibration COMMAND sfr_kinect_calibration_test) +# The title's NuiTransformSmooth (827701A0) done natively, worked by hand. +add_executable(sfr_nui_transform_smooth_test tests/nui_transform_smooth_test.cpp src/nui_transform_smooth.cpp + src/nui_transform_smooth_guest.cpp src/guest_memory.cpp) +target_include_directories(sfr_nui_transform_smooth_test PRIVATE src) +add_test(NAME nui_transform_smooth COMMAND sfr_nui_transform_smooth_test) +if(WIN32) + # The same against the Kinect for Windows SDK 1.8's own NuiTransformSmooth + # (Kinect10.dll, loaded when there; no sensor is started). Skipped (77) + # without the SDK, or when it will not smooth without a sensor. + add_executable(sfr_nui_transform_smooth_sdk_test tests/nui_transform_smooth_sdk_test.cpp src/nui_transform_smooth.cpp) + target_include_directories(sfr_nui_transform_smooth_sdk_test PRIVATE src) + add_test(NAME nui_transform_smooth_sdk COMMAND sfr_nui_transform_smooth_sdk_test) + set_tests_properties(nui_transform_smooth_sdk PROPERTIES SKIP_RETURN_CODE 77) +endif() +# A Kinect v2's camera images and skeleton points in the preview's v1 sizes. +add_executable(sfr_kinect_v2_image_test tests/kinect_v2_image_test.cpp src/kinect_v2_image.cpp) +target_include_directories(sfr_kinect_v2_image_test PRIVATE src) +add_test(NAME kinect_v2_image COMMAND sfr_kinect_v2_image_test) +# Where the Kinect preview window puts its two picture panels. +add_executable(sfr_kinect_preview_layout_test tests/kinect_preview_layout_test.cpp src/kinect_preview_layout.cpp src/kinect_v2_image.cpp) +target_include_directories(sfr_kinect_preview_layout_test PRIVATE src) +add_test(NAME kinect_preview_layout COMMAND sfr_kinect_preview_layout_test) +# A Kinect v2's depth made into a Kinect v1's for the game. +# Its raw recording, and the depth made again from it offline through the same +# view state and synthesis as live (sfr_kinect_depth_resynth). +set(SFR_KINECT_V2_RESYNTH src/kinect_v2_depth.cpp src/kinect_ideal_diagram.cpp src/kinect_v2_image.cpp src/kinect_v2_resynth.cpp src/kinect_record_raw.cpp + src/kinect_record.cpp) +add_executable(sfr_kinect_v2_depth_test tests/kinect_v2_depth_test.cpp ${SFR_KINECT_V2_RESYNTH}) +target_include_directories(sfr_kinect_v2_depth_test PRIVATE src) +add_test(NAME kinect_v2_depth COMMAND sfr_kinect_v2_depth_test) +# The Recompiled view's top-down diagram (the preview window and the launcher). +add_executable(sfr_kinect_ideal_diagram_test tests/kinect_ideal_diagram_test.cpp src/kinect_ideal_diagram.cpp) +target_include_directories(sfr_kinect_ideal_diagram_test PRIVATE src) +add_test(NAME kinect_ideal_diagram COMMAND sfr_kinect_ideal_diagram_test) +# Recording a real Kinect (v1 or v2) and the race's detector answers to CSV, +# and the alone-or-both steadiness summary (docs/kinect-sensor.md). +add_executable(sfr_kinect_record_test tests/kinect_record_test.cpp src/kinect_record.cpp src/kinect_race_record.cpp src/kinect_nui_record.cpp) +target_include_directories(sfr_kinect_record_test PRIVATE src) +add_test(NAME kinect_record COMMAND sfr_kinect_record_test) +add_executable(sfr_kinect_record_summary tests/kinect_record_summary.cpp src/kinect_record.cpp) +target_include_directories(sfr_kinect_record_summary PRIVATE src) +# The depth images given to the game, recorded (SFR_KINECT_RECORD_DEPTH), the +# title's depth view features computed again from them, and a real Kinect +# v1's compared with a Kinect v2's (docs/kinect-sensor.md). +set(SFR_KINECT_DEPTH_TOOLS src/kinect_record_depth.cpp src/kinect_depth_features.cpp src/kinect_depth_compare.cpp + src/kinect_record.cpp src/kinect_race_record.cpp) +add_executable(sfr_kinect_depth_test tests/kinect_depth_test.cpp ${SFR_KINECT_DEPTH_TOOLS}) +target_include_directories(sfr_kinect_depth_test PRIVATE src) +add_test(NAME kinect_depth COMMAND sfr_kinect_depth_test) +add_executable(sfr_kinect_depth_features tests/kinect_depth_features.cpp ${SFR_KINECT_DEPTH_TOOLS}) +target_include_directories(sfr_kinect_depth_features PRIVATE src) +add_executable(sfr_kinect_depth_compare tests/kinect_depth_compare.cpp ${SFR_KINECT_DEPTH_TOOLS}) +target_include_directories(sfr_kinect_depth_compare PRIVATE src) +add_executable(sfr_kinect_depth_resynth tests/kinect_depth_resynth.cpp ${SFR_KINECT_V2_RESYNTH} src/kinect_record_depth.cpp) +target_include_directories(sfr_kinect_depth_resynth PRIVATE src) add_executable(sfr_nui_menu_players_test tests/nui_menu_players_test.cpp src/nui_skeleton.cpp src/guest_memory.cpp) target_include_directories(sfr_nui_menu_players_test PRIVATE src) add_test(NAME nui_menu_players COMMAND sfr_nui_menu_players_test) @@ -236,10 +288,12 @@ add_test(NAME nui_race_min_scale COMMAND sfr_nui_race_test 0.1) set_tests_properties(nui_race_min_scale PROPERTIES ENVIRONMENT "SFR_RACE_LEAN_SCALE=0") add_test(NAME nui_race_max_scale COMMAND sfr_nui_race_test 3.5) set_tests_properties(nui_race_max_scale PROPERTIES ENVIRONMENT "SFR_RACE_LEAN_SCALE=99") -add_executable(sfr_nui_race_players_test tests/nui_race_players_test.cpp src/nui_race.cpp src/guest_memory.cpp) +add_executable(sfr_nui_race_players_test tests/nui_race_players_test.cpp src/nui_race.cpp src/guest_memory.cpp + src/kinect_record.cpp src/kinect_race_record.cpp src/kinect_nui_record.cpp) target_include_directories(sfr_nui_race_players_test PRIVATE tests/race_hook_stub src) add_test(NAME nui_race_players COMMAND sfr_nui_race_players_test) -add_executable(sfr_nui_race_hooks_test tests/nui_race_hooks_test.cpp src/nui_race.cpp src/guest_memory.cpp) +add_executable(sfr_nui_race_hooks_test tests/nui_race_hooks_test.cpp src/nui_race.cpp src/guest_memory.cpp + src/kinect_record.cpp src/kinect_race_record.cpp src/kinect_nui_record.cpp) target_include_directories(sfr_nui_race_hooks_test PRIVATE tests/race_hook_stub src) add_test(NAME nui_race_hooks COMMAND sfr_nui_race_hooks_test) add_executable(sfr_shader_inputs_test tests/shader_inputs_test.cpp src/shader_inputs.cpp) @@ -364,10 +418,13 @@ else() set(SFR_CAMERA_CAPTURE src/camera_capture.cpp src/camera_capture_none.cpp) endif() if(WIN32) - set(SFR_KINECT_SENSOR src/kinect_sensor_win32.cpp src/kinect_v2_win32.cpp src/kinect_smoothing.cpp) + set(SFR_KINECT_SENSOR src/kinect_sensor_win32.cpp src/kinect_v2_win32.cpp src/kinect_v2_image.cpp src/kinect_v2_depth.cpp src/kinect_ideal_diagram.cpp src/kinect_smoothing.cpp + src/nui_transform_smooth.cpp src/kinect_record.cpp src/kinect_race_record.cpp src/kinect_nui_record.cpp src/kinect_record_depth.cpp src/kinect_record_raw.cpp + src/kinect_v2_resynth.cpp) set(SFR_VOICE src/voice_commands.cpp src/voice_commands_win32.cpp) else() - set(SFR_KINECT_SENSOR src/kinect_sensor_none.cpp) + set(SFR_KINECT_SENSOR src/kinect_sensor_none.cpp src/kinect_ideal_diagram.cpp src/nui_transform_smooth.cpp src/kinect_record.cpp src/kinect_race_record.cpp src/kinect_nui_record.cpp src/kinect_record_depth.cpp + src/kinect_record_raw.cpp) set(SFR_VOICE src/voice_commands.cpp src/voice_commands_none.cpp) endif() # The Kinect's voice commands, through the host's speech recognizer. @@ -566,6 +623,9 @@ add_test(NAME launcher_settings COMMAND sfr_launcher_settings_test) add_executable(sfr_aspect_settings_test tests/aspect_settings_test.cpp) target_include_directories(sfr_aspect_settings_test PRIVATE src) add_test(NAME aspect_settings COMMAND sfr_aspect_settings_test) +add_executable(sfr_consecutive_failures_test tests/consecutive_failures_test.cpp) +target_include_directories(sfr_consecutive_failures_test PRIVATE src) +add_test(NAME consecutive_failures COMMAND sfr_consecutive_failures_test) add_executable(sfr_launcher_screen_fit_test tests/launcher_screen_fit_test.cpp) target_include_directories(sfr_launcher_screen_fit_test PRIVATE src) add_test(NAME launcher_screen_fit COMMAND sfr_launcher_screen_fit_test) @@ -646,7 +706,7 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/imgui/imgui.cpp") # The launcher lists the host cameras and controllers. set(SFR_LAUNCHER_SOURCES src/launcher_main.cpp src/launcher_settings.cpp src/mod_loader.cpp src/input_bindings.cpp src/pad_devices.cpp src/pad_assignment.cpp src/launcher_sound.cpp src/launcher_art.cpp src/sony_gamepad.cpp ${SFR_CAMERA_CAPTURE} src/kinect_sensor.cpp ${SFR_KINECT_SENSOR} ${SFR_VOICE} - src/kinect_preview.cpp src/kinect_preview_window.cpp src/camera_preview_window.cpp src/kinect_calibration.cpp src/camera_debug.cpp src/text_wrap.cpp src/touch_scroll.cpp + src/kinect_preview.cpp src/kinect_preview_window.cpp src/kinect_preview_layout.cpp src/camera_preview_window.cpp src/camera_debug.cpp src/text_wrap.cpp src/touch_scroll.cpp src/live_connect.cpp ${SFR_IMGUI}/imgui.cpp ${SFR_IMGUI}/imgui_draw.cpp ${SFR_IMGUI}/imgui_tables.cpp ${SFR_IMGUI}/imgui_widgets.cpp) if(WIN32) @@ -1072,6 +1132,31 @@ if(SFR_BUILD_DIAGNOSTIC) message(FATAL_ERROR "Run scripts/generate_diagnostic.py first; raw output is not safe to execute.") endif() file(GLOB SFR_DIAGNOSTIC_SOURCES CONFIGURE_DEPENDS "${SFR_DIAGNOSTIC_DIR}/ppc_recomp.*.cpp") + # The title's depth view (82439998, 82439D80) as generated, copied out of + # those sources into the build directory and run on test images against + # the offline extractor (kinect_depth_features.h): bit for bit. + set(SFR_DEPTH_VIEW_GUEST "${CMAKE_CURRENT_BINARY_DIR}/kinect_depth_view_guest.inc") + add_custom_command(OUTPUT "${SFR_DEPTH_VIEW_GUEST}" + COMMAND ${CMAKE_COMMAND} "-DDIR=${SFR_DIAGNOSTIC_DIR}" "-DOUT=${SFR_DEPTH_VIEW_GUEST}" "-DFUNCTIONS=82439998,82439D80" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/extract_guest_functions.cmake" + DEPENDS ${SFR_DIAGNOSTIC_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/tests/extract_guest_functions.cmake" + VERBATIM) + add_executable(sfr_kinect_depth_view_guest_test tests/kinect_depth_view_guest_test.cpp src/kinect_depth_features.cpp + "${SFR_DEPTH_VIEW_GUEST}") + set_source_files_properties("${SFR_DEPTH_VIEW_GUEST}" PROPERTIES HEADER_FILE_ONLY TRUE) + target_include_directories(sfr_kinect_depth_view_guest_test PRIVATE src "${CMAKE_CURRENT_BINARY_DIR}" "${SFR_DIAGNOSTIC_DIR}" + tools/XenonRecomp/thirdparty/simde) + # The generated code as the diagnostic compiles it; the extractor as the tools do. + if(MSVC) + set(SFR_DEPTH_VIEW_GUEST_OPTIONS -fno-strict-aliasing -Wno-unused-variable -Wno-unused-value /fp:strict) + else() + set(SFR_DEPTH_VIEW_GUEST_OPTIONS -fno-strict-aliasing -Wno-unused-variable -Wno-unused-value -ffp-model=strict) + endif() + if(CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64|amd64|x86_64") + list(APPEND SFR_DEPTH_VIEW_GUEST_OPTIONS -march=sandybridge) + endif() + set_source_files_properties(tests/kinect_depth_view_guest_test.cpp PROPERTIES COMPILE_OPTIONS "${SFR_DEPTH_VIEW_GUEST_OPTIONS}") + add_test(NAME kinect_depth_view_guest COMMAND sfr_kinect_depth_view_guest_test) if(ANDROID) # The game is libmain.so; SDL's activity calls its SDL_main # (src/android_main.cpp), which sets up and calls the usual main. @@ -1127,7 +1212,7 @@ if(SFR_BUILD_DIAGNOSTIC) ${SFR_CAMERA_CAPTURE} ${SFR_POSE_SOURCES}) # A real Kinect (SFR_CAMERA=kinect), through the Kinect for Windows runtime. target_sources(sfr_cpu_diagnostic PRIVATE src/kinect_sensor.cpp ${SFR_KINECT_SENSOR} ${SFR_VOICE} - src/kinect_preview.cpp src/kinect_preview_window.cpp src/kinect_calibration.cpp) + src/kinect_preview.cpp src/kinect_preview_window.cpp src/kinect_preview_layout.cpp) if(SFR_POSE_AVAILABLE) target_link_libraries(sfr_cpu_diagnostic PRIVATE sfr_onnxruntime) sfr_copy_onnx_dlls(sfr_cpu_diagnostic) @@ -1149,6 +1234,8 @@ if(SFR_BUILD_DIAGNOSTIC) target_link_libraries(sfr_cpu_diagnostic PRIVATE sfr_wait_trace) target_link_libraries(sfr_cpu_diagnostic PRIVATE sfr_native_timer_resolution) target_sources(sfr_cpu_diagnostic PRIVATE src/guest_graphics_hooks.cpp src/game_patches.cpp src/nui_hooks.cpp src/nui_race_hooks.cpp src/pause_gesture.cpp) + # The title's NuiTransformSmooth and the SFR_NUI_ANALYSIS_TRACE observations (docs/kinect-sensor.md). + target_sources(sfr_cpu_diagnostic PRIVATE src/nui_smooth_hooks.cpp src/nui_transform_smooth_guest.cpp src/nui_analysis_hooks.cpp) # The title's heap on o1heap (SFR_HOST_HEAP=1; docs/architecture-migration.md, phase 2). target_sources(sfr_cpu_diagnostic PRIVATE src/heap_hooks.cpp src/guest_heap.cpp) target_link_libraries(sfr_cpu_diagnostic PRIVATE sfr_o1heap) diff --git a/README.md b/README.md index cf509fe..d9dcc1b 100644 --- a/README.md +++ b/README.md @@ -370,15 +370,25 @@ require a sensor or its SDK. | Kinect for Xbox 360 / Kinect for Windows v1 | SDK 1.8 and a powered USB connection; the Xbox 360 sensor requires the full SDK, while Kinect for Windows v1 can use Runtime 1.8 | Xbox 360 Kinect tested through races on Windows 10 | | Kinect v2 / Kinect for Xbox One | SDK 2.0, USB 3.0 and a compatible powered PC adapter | Experimental; skeleton tracking is implemented, but hardware gameplay validation is still pending | -Install the matching SDK, connect the sensor, then select **Kinect** in the -launcher's **Motion input > Camera** setting and choose **Open preview** to check tracking. -Both generations use the same option: the runtime tries v1 first, then v2. +Install the matching SDK, connect the sensor, then select **Kinect (Xbox 360)** or +**Kinect v2 (Xbox One)** in the launcher's **Motion input > Camera** setting and choose +**Open preview** to check tracking. Each choice opens its own generation, so with both sensors +plugged in you can pick the v2. An older `settings.ini` with `camera=kinect` and only a v2 +connected still works: when no Xbox 360 Kinect opens, the v2 is used and `game.log` says so. +There is no smoothing option any more: the game smooths a real Kinect's skeletons (v1 and v2) once, +with the filter and settings the original game asks its Kinect library for. +A Kinect v2 always hands the game a Kinect v1-style depth image made from its own depth. +As on the console, a Kinect's race input always comes from the game's own detectors: the lean, +crouch and jump read the depth image (crouch and jump the skeleton's hips as well), and the +other gestures and the menus read the skeleton; nothing on the PC stands in for them. If the +depth cannot be opened the game gets none, and `game.log` and the launcher's Kinect preview say why. Keep your whole body visible and allow enough space to move. -Kinect v1 can supply skeleton, depth and color data. The current v2 backend -supplies skeletons only, with additional skeleton-based steering and crouch/jump -handling. Physical Kinect input is Windows-only; Linux and Android releases +Kinect v1 can supply skeleton, depth and color data. A Kinect v2 gives the game skeletons and a depth image made +from its own depth; +its color and depth images also appear in the launcher's preview (not yet +verified on hardware). Physical Kinect input is Windows-only; Linux and Android releases do not provide it. SDK links, adapter requirements and troubleshooting: diff --git a/README.zh-TW.md b/README.zh-TW.md index 838dc5b..0f0d980 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -167,12 +167,18 @@ Kinect 為選用功能；使用手把或鍵盤遊玩，不需要感測器，也 | Xbox 360 版 Kinect／Kinect for Windows 第一代 | SDK 1.8 與供電 USB 連接；Xbox 360 版需完整 SDK，Kinect for Windows 第一代可用 Runtime 1.8 | 已在 Windows 10 上用 Xbox 360 版 Kinect 實際跑完比賽 | | Kinect v2／Xbox One 版 Kinect | SDK 2.0、USB 3.0 與相容的 PC 電源轉接器 | 實驗性支援；已實作骨架追蹤，仍待實機遊玩驗證 | -安裝對應 SDK 並連接感測器後，在啟動器的 **體感與攝影機 → 攝影機** 設定選擇 **Kinect**， -再按 **開啟預覽** 確認追蹤狀態。兩代共用同一個選項：程式先嘗試 v1，再嘗試 v2。 +安裝對應 SDK 並連接感測器後，在啟動器的 **體感與攝影機 → 攝影機** 設定選擇 **Kinect（Xbox 360）** 或 +**Kinect v2（Xbox One）**，再按 **開啟預覽** 確認追蹤狀態。兩個選項各自只開對應的那一代， +兩種感測器都接上時可指定要用 v2。舊的 `settings.ini`（`camera=kinect`）若只接了 v2 仍可使用： +找不到 Xbox 360 版時會改用 v2，`game.log` 會註明。「平滑」選項已移除：遊戲會照原版要求 Kinect 函式庫的濾波器與參數， +把實體 Kinect（v1 與 v2）的骨架平滑一次。 +Kinect v2 一律把自己的深度換成 Kinect v1 格式的深度影像交給遊戲。跟主機一樣，Kinect 的比賽輸入 +永遠來自遊戲自己的判斷器：傾身、蹲下、跳躍讀深度影像（蹲下與跳躍也讀骨架的髖部），其他動作與選單讀骨架， +電腦端沒有任何替代判定；深度打不開時遊戲就沒有深度， +`game.log` 與 launcher 的 Kinect 預覽會說明原因。 請讓全身進入感測器範圍，並預留足夠的活動空間。 -Kinect v1 可提供骨架、深度與彩色影像；目前的 v2 支援只提供骨架，並以骨架補足 -轉向與蹲跳判定。實體 Kinect 輸入僅支援 Windows，Linux 與 Android 發行包尚未提供。 +Kinect v1 可提供骨架、深度與彩色影像；v2 交給遊戲的有骨架與由自身深度換算的深度影像；v2 的彩色與深度影像會顯示在 launcher 的預覽視窗（尚未實機驗證）。實體 Kinect 輸入僅支援 Windows，Linux 與 Android 發行包尚未提供。 SDK 下載、轉接器需求與疑難排解見 [Kinect 感測器使用說明](docs/kinect-sensor.md)。 若要使用一般攝影機進行體感操作，請看下一節。 diff --git a/docs/camera-input.md b/docs/camera-input.md index a1827cf..382300e 100644 --- a/docs/camera-input.md +++ b/docs/camera-input.md @@ -7,16 +7,19 @@ ## 三種設定 -launcher「體感與攝影機」分頁的「攝影機」（`SFR_CAMERA`）有三種 webcam 設定，Windows 另有實體 Kinect： +launcher「體感與攝影機」分頁的「攝影機」（`SFR_CAMERA`）有三種 webcam 設定，Windows 另有兩種實體 Kinect： | 設定 | `SFR_CAMERA` | 畫面 | 誰在操作 | | --- | --- | --- | --- | | 關閉 | 空白 | 不開攝影機 | 手把 | | 畫面 | `picture` | 開攝影機，交給遊戲顯示 | 手把 | -| 體感 | `motion` | 開攝影機 | 攝影機姿勢，手把操作時優先接手 | -| Kinect | `kinect` | 實體 Kinect（Windows） | Kinect 追蹤到的身體，見 [實體 Kinect](kinect-sensor.md) | +| Webcam | `motion` | 開攝影機 | 攝影機姿勢，手把操作時優先接手 | +| Kinect（Xbox 360） | `kinect` + `SFR_KINECT_MODEL=1` | 實體 Xbox 360 版 Kinect（Windows）；找不到時改用接上的 v2 | Kinect 追蹤到的身體，見 [實體 Kinect](kinect-sensor.md) | +| Kinect v2（Xbox One） | `kinect` + `SFR_KINECT_MODEL=2` | 實體 Kinect v2（Windows），只開 v2 | 同上 | -「體感」保留原本的手把與鍵盤操作。每次提交完整骨架時只選一個來源， +launcher 設定檔裡這兩項分別存成 `camera=kinect` 與 `camera=kinect_v2`；遊戲端一律以 `SFR_CAMERA=kinect` 判斷是否選了實體 Kinect。實體 Kinect 的比賽輸入（傾身、蹲下、跳躍、側移）永遠來自遊戲自己的深度判斷器，沒有骨架備援；這裡的 `CameraRaceMotion` 只用在 webcam。 + +「Webcam」保留原本的手把與鍵盤操作。每次提交完整骨架時只選一個來源， 避免攝影機與手把同時拉動同一位玩家；接手規則見下方「兩位玩家」。 ## 從畫面到骨架 @@ -75,7 +78,7 @@ launcher「體感與攝影機」分頁的「攝影機」（`SFR_CAMERA`）有三 ## 骨架 Debug 視窗（Windows） -launcher 選擇「攝影機 → 體感」後，可開啟「骨架 Debug 視窗」 +launcher 選擇「攝影機 → Webcam」後，可開啟「骨架 Debug 視窗」 （英文介面為 **Skeleton debug window**）。預設關閉，設定存為 `settings.ini` 的 `camera_debug=0/1`，會在下次啟動遊戲時套用；遊戲已經 執行中時，必須重新啟動才會套用變更。 @@ -266,7 +269,8 @@ Camera 持續動作以主機單調時鐘計時；遊戲計時欄位實際可為 - 攝影機的畫面還沒接到遊戲自己的 NUI 影像串流（`82768C40`），所以「畫面」 目前只是把相機打開。 -- 真正的 Kinect 感測器見 [實體 Kinect](kinect-sensor.md)（Windows，`SFR_CAMERA=kinect`）。 +- 真正的 Kinect 感測器見 [實體 Kinect](kinect-sensor.md)（Windows，`SFR_CAMERA=kinect`）。Kinect v2 的預覽視窗會提示身體被畫面哪一邊切到 + （只顯示與寫 log／錄製，不影響遊戲輸入），見該文件「預覽視窗」。 - Android 的遊戲還沒把姿勢模型打包進 APK（手機鏡頭的擷取已經有了，見下方）。 ## Android：手機自己的鏡頭 diff --git a/docs/kinect-sensor.md b/docs/kinect-sensor.md index 18191f9..a22c4ae 100644 --- a/docs/kinect-sensor.md +++ b/docs/kinect-sensor.md @@ -3,7 +3,8 @@ 日期：2026-09-27。接續 [攝影機體感輸入](camera-input.md)。 遊戲原本就是 Kinect 專用的：Xbox 360 上的 NUI 函式庫回報玩家骨架，遊戲自己的 -約 25 個手勢判斷器（跳躍、蹲下、踢地、傾斜……）讀這副骨架。沒有感測器時， +約 30 個手勢判斷器（跳躍、蹲下、踢地、傾斜……）讀這副骨架（遊戲先用函式庫的 `NuiTransformSmooth` 平滑它）， +傾斜、蹲下、跳躍與賽前量測另外讀深度影像（見「遊戲怎麼用骨架」）。沒有感測器時， [NUI 模擬](main-menu.md) 用手把假裝出一副骨架，[比賽](race-controls.md) 更直接 把判斷器換成讀手把。 @@ -26,7 +27,7 @@ 底下的 Audio、Camera、Device 等項目。綠燈亮起表示有電。 3. （選用）SDK 附的 *Developer Toolkit Browser* 裡的 *Skeleton Basics* 範例可以先確認 骨架追蹤正常。 -4. launcher → 體感與攝影機 →「攝影機」選 **Kinect**，按「開啟預覽」： +4. launcher → 體感與攝影機 →「攝影機」選 **Kinect（Xbox 360）**，按「開啟預覽」： - 「Kinect 運作中」：可以玩了。 - 「尚未安裝 Kinect SDK 1.8」：按「下載 SDK」回到步驟 1。 - 「沒有連接 Kinect」：檢查電源轉接線與 USB。 @@ -42,9 +43,15 @@ ## Kinect v2（Xbox One 版） -v2 目前只提供骨架，會自動啟用骨架轉向與蹲跳，不必額外設定 `SFR_KINECT_DEPTH=0`。 -1P／2P 各自校正站姿與計時；遊戲最多接收兩位玩家，追蹤身分與原始感測器槽位保持穩定。 -`SFR_KINECT_BODY_LEAN`、`SFR_KINECT_BODY_GESTURES` 仍可用 `1`／`0` 強制開關。 +v2 一律把自己的深度換成 Kinect v1 格式的深度加玩家編號影像交給遊戲（見下方「Kinect v2 深度」）， +遊戲的深度檢視與判斷器照 v1 的方式使用它；沒有開關，也沒有主機端的替代判定： +跟主機一樣，比賽的輸入永遠來自遊戲自己的判斷器。它們讀**深度與骨架兩者**：深度只用在傾斜那一對數值、蹲下、跳躍與賽前量測， +其他約 20 個判斷器（舉手、側移、煞車、站姿、踢地、招式、道具、抓取、特技、加速、蓄力……）與轉向的身體傾角讀的是骨架關節（見下方「遊戲怎麼用骨架」）； +選單的游標與玩家身分也是骨架。 +深度讀取器打不開時，遊戲就拿不到深度（log 會寫一行 `NATIVE_KINECT_DEPTH unavailable model=… reason=…`， +launcher 的 Kinect 預覽也會顯示原因），不會有替代資料。 +launcher 的預覽視窗（與遊戲中的「Kinect 調整」視窗）會顯示 v2 的彩色與深度影像並疊上骨架（見下方「預覽視窗」）。 +遊戲最多接收兩位玩家，追蹤身分與原始感測器槽位保持穩定。 這條路徑有自動測試，尚未完成 Kinect v2 實機遊玩驗證。 v2 的資料流程、#97／#132／#165 的 log 分析、待實機確認的項目，以及啟動器「體感追蹤紀錄」 開關（`SFR_CAMERA_RACE_TRACE=1`，只寫數字、不含影像）的用法，見 [kinect-v2.md](kinect-v2.md)（英文）。 @@ -53,31 +60,36 @@ v2 的資料流程、#97／#132／#165 的 log 分析、待實機確認的項目 電腦上的 **USB 3.0** 連接埠（Intel 或 Renesas 晶片的控制器較穩，部分 AMD 控制器有相容性問題）。 2. 安裝 [Kinect for Windows SDK 2.0](https://www.microsoft.com/download/details.aspx?id=44561)， 可以先用 SDK Browser 裡的 *Body Basics* 確認骨架正常。 -3. launcher 一樣選 **Kinect**：先找 Xbox 360 版，找不到再找 v2，會用第一台找到的。 - log 會寫 `NATIVE_KINECT started=1 model=Kinect v2`。 +3. launcher 選 **Kinect v2（Xbox One）**：只開 v2，不會去找 Xbox 360 版，所以兩種都接上時也能指定 v2 + （launcher 以 `SFR_CAMERA=kinect` 加 `SFR_KINECT_MODEL=2` 交給遊戲）。log 會寫 + `NATIVE_KINECT started=1 model=Kinect v2`。 + 舊的 `camera=kinect`（`SFR_KINECT_MODEL=1` 或沒有設定）仍先找 Xbox 360 版；找不到、但 v2 開得起來時 + 改用 v2，並寫一行 `NATIVE_KINECT no Kinect (Xbox 360) opened, using the Kinect v2 found instead`， + 讓只有 v2 的舊設定繼續能用。 +4. launcher 的「深度視角」（`kinect_v2_view`，見下方「Kinect v2 深度」的三種視角）依序是 **Recompiled**（`ideal`，launcher 的預設）、 + **模擬**（`fit`）、**原始**（`sensor`）。選 Recompiled 時下面有距離、高度、跟隨速度、死區、骨架距離五個滑桿， + 再下面是「俯視位置」圖（見「預覽視窗」的俯視圖；預覽沒開時只畫框與兩台相機，並提示「開啟預覽即可看到玩家位置」）。 + 舊 `settings.ini` 已存的視角照舊；**遊戲本身**沒有設 `SFR_KINECT_V2_DEPTH_VIEW` 時仍是 `fit`，只有 launcher 的預設改成 Recompiled。 v2 有 25 個關節，前 20 個與第一代同名同順序，只有一處不同：第一代的「肩中心」 對應 v2 的 SpineShoulder（20），而不是位置更高的 Neck（2）；指尖與拇指（21～24）不用。 身體中心用 SpineBase。座標系與第一代相同（公尺、+x 朝玩家右手、+z 離開感測器）， -所以不用換算。v2 SDK 沒有第一代的 `NuiTransformSmooth`，玩家回報關節在比賽中會抖（#97）， -所以 v0.7.2 起 v2 的關節預設套用同一種濾波（Holt 雙指數平滑，參數與第一代相同：0.5／0.5／0.5、 -抖動半徑 5 公分、最大偏差 4 公分，推測的關節半徑加倍；[`kinect_smoothing.cpp`](../src/kinect_smoothing.cpp)）， -約多一格延遲；log 會寫 `NATIVE_KINECT model=v2 smooth=1`。 +所以不用換算。v2 SDK 沒有第一代的 `NuiTransformSmooth`，但**遊戲自己會平滑骨架**（見下方「骨架平滑」）： +v1 與 v2 的關節都照原樣交給遊戲，由遊戲的 `NuiTransformSmooth`（現在以原生方式實作）平滑一次。 +v0.7.2 起主機端對 v2 的平滑（`KinectJointSmoother`，當時是為了比賽中關節會抖，#97）只是「碰巧」補上了遊戲沒做成的平滑， +現在預設關閉（`NATIVE_KINECT model=v2 smooth=0`），否則同一副骨架會被平滑兩次。 -launcher 的 **Kinect** 設定多了三項（下次啟動遊戲時生效；預設值等於原本行為，不會傳任何設定）： - -| 設定 | 範圍（預設） | 對應 | -| --- | --- | --- | -| 平滑 Kinect v2 追蹤 | 開／關（開） | 關閉時 `SFR_KINECT_SMOOTH=0` | -| 傾身靈敏度 | 10～100%（100%） | `SFR_KINECT_LEAN_SCALE` = 3.5 × 百分比 | -| 蹲下深度 | 5～35 公分（15） | `SFR_KINECT_CROUCH_DEPTH`（公尺） | +launcher 的「平滑 Kinect v2 追蹤」已移除；舊 `settings.ini` 裡的 `kinect_smooth` 會被忽略，也不會再寫回，launcher 不再傳 `SFR_KINECT_SMOOTH`。 +「選單游標穩定」與預覽視窗兩種都有。傾身靈敏度、蹲下深度與預覽的「校正」都已移除（遊戲的深度判斷器自己決定這些）。舊 `settings.ini` 裡的 `kinect_lean`、`kinect_crouch_cm`、`kinect_v2_depth` 會被忽略，也不會再寫回。 程式在 [`kinect_v2_win32.cpp`](../src/kinect_v2_win32.cpp)：執行時才載入 `Kinect20.dll`， COM 介面依 SDK 2.0 的 `Kinect.h` 宣告到用得到的最後一個方法為止（vtable 只看順序）。 關節對應與「兩種感測器都失敗時回報哪個原因」在 [`kinect_sensor.cpp`](../src/kinect_sensor.cpp)， 有單元測試。 -不開遊戲也可以檢查：`sfr_kinect_probe [秒數]` 會印出追蹤到的人與右手位置。 +不開遊戲也可以檢查：`sfr_kinect_probe [秒數]` 會印出追蹤到的人與右手位置（加 `--smooth` 時先用和遊戲相同的濾波平滑，等於 `SFR_KINECT_SMOOTH=1`）； +`sfr_kinect_probe [秒數] --images [檔名前綴]` 另外把第一張彩色與深度影像（與預覽相同的影像大小：v1 為 640×480／320×240，v2 依畫面長寬比） +存成 `kinect_colour.bmp`、`kinect_depth.bmp`（玩家像素為紅色），並印出頭部落在兩張影像的哪個位置。 每 5 秒另有一行 `NATIVE_KINECT model=v1 frames=… with_body=… empty_waits=… failed=…`： `frames=0` 且 `failed` 一直增加（`last_error=0x83010001`，`E_NUI_FRAME_NO_DATA`）表示 感測器有通知新畫面、卻交不出骨架。一台 Kinect for Xbox 360（SDK 1.8、Windows 10） @@ -102,38 +114,31 @@ COM 介面依 SDK 2.0 的 `Kinect.h` 宣告到用得到的最後一個方法為 - [`kinect_sensor_win32.cpp`](../src/kinect_sensor_win32.cpp)：執行時才 `LoadLibrary` 載入 `Kinect10.dll`（建置不需要 SDK，沒有 Kinect 的人也不需要那個 DLL）， - 開啟骨架追蹤，在自己的執行緒上讓 `NuiSkeletonGetNextFrame` 等待新框架，套用 SDK - 建議的 `NuiTransformSmooth` 平滑參數（約一格延遲）。 -- 感測器仰角（[`kinect_level`](../src/kinect_sensor.h)）：骨架座標以感測器為準，放得低、 - 鏡頭往上仰時，站直的人看起來往後倒，遊戲的站姿校正就量到歪的站姿。每格依感測器加速度計 - 給的重力方向，把骨架和地板繞感測器轉正，使重力朝正下方（超過 30 度或沒有重力資料就不動； - `SFR_KINECT_LEVEL=0` 關閉）。`sfr_kinect_probe` 與遊戲的 `NATIVE_KINECT_BODY`（每 3 秒） - 都會印出 `level=`（轉掉的度數），`NATIVE_KINECT_BODY` 另有髖、肩、頭、雙手的位置與雙手的 - 追蹤狀態（2 追蹤到、1 推測、0 沒有）。 + 開啟骨架追蹤，在自己的執行緒上讓 `NuiSkeletonGetNextFrame` 等待新框架，原樣交出（不平滑：遊戲自己平滑， + 見「骨架平滑」；`SFR_KINECT_SMOOTH=1` 才在這裡用 SDK 的 `NuiTransformSmooth`）。 +- `NATIVE_KINECT_BODY`（每 3 秒）印出 `gravity_y=`、髖、肩、頭、雙手的位置與雙手的追蹤狀態 + （2 追蹤到、1 推測、0 沒有）。骨架座標以感測器為準，原樣交給遊戲，不做水平校正。 - 比賽轉向（[`nui_race_hooks.cpp`](../src/nui_race_hooks.cpp)）：遊戲的傾斜判斷讀身體紀錄 - +640／+644，在主機上是深度影像裡玩家左右兩側的像素數，不是關節。成功開啟深度串流時保留 - 原版判定；v2、停用深度或深度開啟失敗時，1P／2P 各自由骨架計算：比賽開始先保持站姿約 0.6 秒 - 校正，之後以肩膀相對髖部的左右傾角換成那一對數值（`NUI_RACE_SENSOR_LEAN active=1`， - `CAMERA_RACE_CALIBRATION ready=1`；`SFR_CAMERA_RACE_TRACE=1` 每 30 格印出 `lean=`）。 - 傾角約 4° 起算、30° 算傾到底（`lean=±1`）；傾到底時寫入遊戲可讀的最大值 3.5（兩數相差 - 4.5 倍），`SFR_KINECT_LEAN_SCALE`（0.1–3.5）可調小。原本傾到底只給 1，彎到極限也只轉一點點。 - 其他動作仍由遊戲自己的判斷器讀骨架。沒有深度串流時，**蹲下**（連帶蓄力與起跳） - 原版判斷「沒有」時，改看該玩家的骨架：髖部比校正時的 - 站姿低一段距離持續 0.12 秒算蹲下，再明顯站起來算起跳（同 webcam 體感）。感應器放高時常 - 看不到腳，猜出來的腳踝會讓「髖部離腳踝多高」忽大忽小，所以 Kinect 改量髖部比校正時低了 - 多少（地板在校正時固定）：預設降 0.15 公尺算蹲下，`SFR_KINECT_CROUCH_DEPTH`（0.05–0.35）可調。 - `SFR_KINECT_BODY_GESTURES=0` 關掉；`SFR_CAMERA_RACE_TRACE=1` 時每次結果改變印出 - `KINECT_GESTURE address= original= result=`（822C8778 蹲、822CB840 蓄力、822C9050 跳、 - 822CA6B0 Side）。1P／2P 分別校正，追蹤身分改變時會清除舊的動作狀態。 + +640／+644，在主機上是深度影像裡玩家左右兩側的像素數，不是關節；但轉向還另外混入一個骨架的傾角 + （`822C6200` 的 head − spine 傾角、`822C8958` 的肩中心 − 髖中心側傾）。Kinect（v1 與 v2）的比賽**一律**由遊戲自己的判斷器決定 + （深度用於傾斜、蹲下、跳躍；蓄力、側移與其他約 20 個判斷器讀骨架關節），主機端沒有替代或補充判定， + 也沒有開關（已移除 `SFR_KINECT_DEPTH`、`SFR_KINECT_BODY_LEAN`、`SFR_KINECT_BODY_GESTURES`、 + `SFR_KINECT_LEVEL`）。深度影像超過 0.5 秒沒到遊戲時，只在影像恢復時記一行 + `NUI_KINECT_DEPTH gap ms=…`，行為不變。webcam 的「體感」另有自己的動作判斷 + （[camera-input.md](camera-input.md)），不受影響。 - [`KinectPlayerSlots`](../src/kinect_sensor.h)：遊戲最多接收兩副完整骨架。 每個人只要還被追蹤就保留其身分與感測器槽位，新走進來的人補上離開者的位置。 + 這是硬性限制：遊戲（`82438EB0`）把**每一個** TRACKED 槽位依序寫進管理器 +7740 起只有兩格的陣列，第 3 副會寫到 +7748， + 第 4 副會蓋掉骨架執行緒的停止旗標（+7752）。單元測試（`kinect_sensor`）讓 v2 一直有 3–6 個人進出、換位置、順序打亂， + 每一格都確認交給遊戲的骨架（寫進遊戲框架後的 TRACKED 數）不超過兩副、每人一次、各在自己的槽位，而且還在場的玩家不會被換掉。 + 這個測試找到一個漏洞：感測器在同一格給出兩個相同的 tracking id 時（v2 的 64 位元 id 縮成遊戲用的 32 位元），兩個都會留下， + 加上另一位玩家就是三副；現在每個玩家只留第一個。 - [`nui_hooks.cpp`](../src/nui_hooks.cpp) 的 `NuiSkeletonGetNextFrame`： `SFR_CAMERA=kinect` 時，框架表頭的地板平面與重力方向用感測器的，兩個欄位寫入 真實的關節、每個關節的追蹤狀態（追蹤／推測）與身體中心。沒有人站在前面就是空框架， 遊戲會像在主機上一樣請玩家站到感測器前。 - [`nui_race_hooks.cpp`](../src/nui_race_hooks.cpp)：有實體感測器時 - （`nui_body_from_sensor()`），比賽保留原生身體紀錄與「On your Gear!」身體量測。 - 深度串流可用時沿用原版判定，否則套用上述骨架轉向與蹲跳補充判定。 + （`nui_body_from_sensor()`），比賽保留原生身體紀錄與「On your Gear!」身體量測，原版深度判定照常。 - **Pause Gesture／暫停手勢**：Xbox 360 Kinect 的系統暫停手勢（universal pause）：右手垂在身側、 左手伸直往身體外側斜下約 45 度舉著。主機上這是系統的功能，不是遊戲的：舉著時左下角出現進度圖示， 然後叫出遊戲的暫停選單。這裡由 [`pause_gesture.h`](../src/pause_gesture.h) 判斷姿勢，舉滿 @@ -166,6 +171,297 @@ COM 介面依 SDK 2.0 的 `Kinect.h` 宣告到用得到的最後一個方法為 （409 英文、404 繁中），預設用系統的；辨識器說不出的詞（例如中文辨識器遇到英文）會被略過。 log：`NATIVE_VOICE started phrases=…`、`NUI_VOICE heard=… word=…`。尚未實機測試。 +## 錄製（離線分析用） + +把感測器的每一格骨架與比賽中遊戲判斷器的回答存成 CSV，之後不開遊戲、不接感測器也能分析 +（[`kinect_record.h`](../src/kinect_record.h)、[`kinect_race_record.h`](../src/kinect_race_record.h)，有單元測試 `kinect_record`）。 +v1 與 v2 用同一種欄位格式。 + +- **遊戲中**：設 `SFR_KINECT_RECORD=<前綴>`（例如 `D:\rec\run1`）。 + - `<前綴>-skeleton.csv`：感測器每一格一列（每個追蹤到的人一列；沒人時也有一列，`bodies=0`）， + 在平滑與水平校正**之前**原樣記下：主機 `steady_clock` 微秒（`steady_us`）、感測器自己的編號與時間戳 + （v1 `dwFrameNumber`／`liTimeStamp` 毫秒；v2 為後端計數／`RelativeTime` 100 ns）、地板平面、重力（v1）、 + 深度影像無效像素比例（`depth_invalid`，有開深度時）、骨架槽位、完整 tracking id、v1 的骨架中心、v2 的手部狀態， + 以及 25 個關節（v2 `JointType` 的順序與名稱）的位置與狀態（0 未追蹤、1 推測、2 追蹤；v1 沒有的 Neck 與指尖、拇指留空， + v1 的 ShoulderCenter 寫在 `spine_shoulder`），最後一欄 `clipped_edges` 是 v2 的 `IBody::get_ClippedEdges` 位元遮罩 + （1 右、2 左、4 上、8 下，0 無；v1 不提供，留空）。沒有這欄的舊錄製檔照樣讀得進來，可拿它對照比賽不準時玩家是否被畫面切到。 + - `<前綴>-race.csv`：比賽中每次更新、每個由感測器驅動的玩家紀錄一列：遊戲深度檢視的傾斜那一對數值 + （`depth_right`／`depth_left`），以及遊戲原本的蹲 822C8778、蓄力 822CB840、跳 822C9050、側移 822CA6B0 + 判斷器這次回答的 `*_original`（遊戲自己的）與 `*_result`（比賽實際採用的，兩者相同，因為沒有任何東西取代它）。 + v1 的 `*_original` 與深度傾斜就是主機版的深度判斷，可當標準答案，拿來和 v2 合成深度的結果比較。 + - `<前綴>-nui.csv`（[`kinect_nui_record.h`](../src/kinect_nui_record.h)）：遊戲自己的骨架消費者這一次做了什麼。 + 實體 Kinect（v1／v2）時，每次輸入更新（`824920F8`，每個遊戲畫格一次，選單與比賽都有）每位有身體紀錄的玩家一列（210 欄，有標頭）： + - `steady_us`、`update`（輸入更新的計數）、`player`（0、1）、`tracking_id`、`body_slot`（rec +784）、`racing`。 + - 兩隻手的選單游標（`left_*`、`right_*`；`82492F30` 剛算完的值，手部封包 P = input + 32 + 2544·玩家 + 16 + 400·手）： + `flags`（P+0）、`theta`（P+60，度）、`r`（P+148）、`target_x/_y`（P+96/+100，像素）、`cursor_x/_y`（P+80/+84，像素，選單讀的就是這個）。 + - 身體紀錄的關節（`hand_l hand_r shoulder_l shoulder_r shoulder_c head spine hip_c` 各 `_x _y _z`）：遊戲**平滑之後**、而且是它的鏡像座標 (−x, y, −z)。 + - `lean_depth`、`lean_tilt`：比賽中 `822C6200` 兩個濾波器的平均（source +44 深度傾斜、+88 身體傾角）。 + - 研究列出的每一個比賽判斷器（附錄 A 與 3.3 節，共 32 個：`jump crouch charge arms_up catch_lr catch flying accel accel2 side brake grab kick kick2 + power power2 stance trick hammer over_throw under_swing under_throw inflator shake wipe lever handle front_curve swim hold_hands steer steer_alt`）， + 各 4 欄：`_result`（回傳值：1 認出、2 相反狀態、0 其他）、`_primary`、`_secondary`（這個判斷器這次在結果 +4／+20 **新加上的**位元，十進位）、 + `_value`（有自己數值的：煞車與抓取的強度 +80、站姿 +72、特技轉角 +56、拉桿 +82、把手 +83、前彎第二筆結果 +81、擦螢幕的 x（detector +76，y 在 `wipe_y`）、 + 轉向 +76）。上一列之後沒被問到的判斷器是空的；同一次更新被問兩次時記最後的回傳值與所有位元。`hold_hands` 問的是整隊，記在玩家 0。 + - `calib_step`（`arms_forward`＝`822C65B0`、`body`＝`822C6AD8`）與 `calib_0`…`calib_132`：該玩家的賽前量測在上一列之後完成時，校正物件 `[source+92]` + 的 +0..+132（+112、+116、+120 是整數像素，其他是浮點）；沒有完成時空白。 + 判斷器與量測的值在它們的 hook 裡收集，下一次輸入更新時寫成該玩家的一列，所以一列是「上一列之後到這次輸入更新為止」。 + 和其他錄製檔一樣經過 `KinectRecordQueue`，遊戲不等硬碟；60 fps、兩位玩家約每秒 120 列（約 60–100 KB）。 + - 每個檔案最多 `SFR_KINECT_RECORD_MB` MB（預設 512），滿了就停並在 log 說一次（`NATIVE_KINECT_RECORD full`）。 + - **錄製不會等硬碟**（[`KinectRecordQueue`](../src/kinect_record.h)）：所有錄製檔（兩個 CSV、深度檔、原始 v2 檔；遊戲與探針都一樣） + 以前都在呼叫者的執行緒上同步 `fwrite`＋`fflush`，遊戲的 hook 因此會被慢的硬碟卡住（hook 擋住時其他客體執行緒也停，畫面就頓一下）。 + 現在呼叫者只把一份自己的緩衝區交給有上限的佇列（`SFR_KINECT_RECORD_QUEUE_MB`，預設 256），每個程式一條背景執行緒寫入並在每批之後 + `fflush`（程式被砍掉時只少還在佇列裡的）。佇列滿了就**丟掉**那一項並計數，不等；log 每秒最多一行 + `NATIVE_KINECT_RECORD dropped=N (disk too slow)`，每 5 秒的 `NATIVE_KINECT model=v1|v2` 行多 `record_dropped= record_queued_mb=`。 + 被丟掉的深度或原始影像在檔案裡就是影像編號與 `steady_us` 的空缺，工具照一般的空缺處理。結束時每個檔案最多等 10 秒把佇列寫完再關檔。 + 檔案格式與「第一列／第一張到時才開檔」不變。離線工具（`sfr_kinect_depth_resynth`）改成佇列滿時等待，不丟。 + 單元測試（`kinect_record`）：順序、預算與丟棄計數、解構時寫完、硬碟卡住時呼叫者不等（假的寫入端卡住時最慢一次約 50 µs）。 + - **錄到 SSD**：A: 是同時放遊戲資料的傳統硬碟，錄製每秒十幾 MB 會和遊戲讀資料搶；雙錄製的腳本預設錄到 `C:\SfrRec`（見 KINECT_DUAL_RECORDING.md）。 +- **不開遊戲**：`sfr_kinect_probe <秒數> --v1|--v2|--both --record <前綴>`。`--both` 在同一個程式裡同時開 v1 與 v2， + 先印出各自是否開成功（`Kinect v1: open`／`Kinect v2: not open (no-sensor)`），只開成功一台也照樣錄； + 兩台共用同一個 `steady_clock`。錄製時 v2 的深度讀取器也會打開，以記錄 `depth_invalid`。 + 兩個 SDK 的執行環境能否在同一個程式裡同時運作，還沒有實機試過。 +- **遊戲＋探針同時**：Windows 的 `steady_clock` 是全機共用的（QueryPerformanceCounter），所以可以讓遊戲用 v1 錄 + （有標準答案），另開 `sfr_kinect_probe <秒數> --v2 --record <另一個前綴>` 錄 v2，事後用 `steady_us` 對齊。 +- **干擾 A/B**：`sfr_kinect_record_summary <單開>-skeleton.csv <雙開>-skeleton.csv` 比較站著不動時臀部、肩膀中心、雙手的抖動 + （逐格差的 RMS 除以 √2）、關節追蹤比例、深度無效比例與失去追蹤次數，並對照「抖動增加 < 20%、追蹤比例下降 < 5 個百分點、 + 深度破洞增加 < 10 個百分點、失去追蹤 0 次」。 +- **深度影像**：`SFR_KINECT_RECORD_DEPTH=1`（探針用 `--depth`）另存深度影像，`<前綴>-race.csv` 也多了遊戲深度檢視的全部特徵； + 格式、記憶體配置與工具見下一節。 +- **原始 v2 資料**：`SFR_KINECT_RECORD_RAW=1`（探針用 `--raw`）另存 v2 合成每張影像所用的原始深度、身體索引、身體與地板（`<前綴>-v2raw.bin`）， + 可以離線用任何視角重做（見下一節「原始 v2 錄製」）。 + +## 深度錄製與分析工具 + +目的：判斷 Kinect v2 合成的 v1 格式深度（`fit` 視角）交給遊戲自己的深度判斷器時，和真正的 Kinect v1 差多少。做法是 +**同一個人、同一個動作、同一時間**：遊戲接真正的 v1（launcher 攝影機選「Kinect（Xbox 360）」），遊戲的深度檢視從 v1 深度 +算出的特徵與判斷結果就是標準答案；同時另一個程式 `sfr_kinect_probe --v2` 錄 v2（兩台是不同的 USB 裝置，SDK 1.8 與 2.0 +的執行環境各管各的）。兩個程式都用主機的 `steady_clock`（`steady_us`）。分開兩次錄的資料無法逐格比較（人每次動作不同）， +所以不做那種比較。 + +### 檔案 + +| 檔案 | 誰寫 | 內容 | +| --- | --- | --- | +| `<前綴>-skeleton.csv` | 遊戲（v1）、探針（v1／v2） | 骨架，見上一節 | +| `<前綴>-race.csv` | 遊戲 | 比賽中每次更新一列：舊的 14 欄＋深度檢視的 19 欄（見下方） | +| `<前綴>-nui.csv` | 遊戲 | 每次輸入更新、每位玩家一列：游標、平滑後的關節、全部 32 個比賽判斷器、賽前量測（見上一節） | +| `<前綴>-depth.bin` | 遊戲 | 遊戲經 `NuiImageStreamGetNextFrame`（`82767148`）**實際拿到**的每張深度影像（v1 原樣，或 v2 合成） | +| `<前綴>-v1-depth.bin`、`<前綴>-v2-depth.bin` | 探針 `--record <前綴> --depth` | 各感測器交給遊戲時會是的影像：v1 原始深度、v2 以 `fit` 合成（`game_image()`）；一台一個檔，`--v1`／`--v2`／`--both` 都一樣 | +| `<前綴>-v2raw.bin` | 探針 `--record <前綴> --raw`，或遊戲用 v2 時 `SFR_KINECT_RECORD_RAW=1` | v2 合成每一張影像時**用到的原始資料**（格式 `SFRKRAW1`，見下方「原始 v2 錄製」），可離線用任何視角重做 | + +開關：`SFR_KINECT_RECORD=<前綴>` 加 `SFR_KINECT_RECORD_DEPTH=1`（探針的 `--depth` 等於後者）。每個深度檔最多 +`SFR_KINECT_RECORD_DEPTH_MB` MB（預設 4096，滿了停止並在 log 寫一次 `NATIVE_KINECT_RECORD depth full <檔名> after … bytes (… images): stopped`）； +`SFR_KINECT_RECORD_DEPTH_EVERY=N` 只留每第 N 張（預設 1）。第一張影像到時才開檔，log 寫 +`NATIVE_KINECT_RECORD writing <檔名> (depth 320x240, at most 4096 MB, every 1st image, … MB free)`；該磁碟可用空間少於上限的兩倍時多一行 +`NATIVE_KINECT_RECORD depth low disk: … MB free for …, less than twice the cap (… MB)`。CSV 仍受 `SFR_KINECT_RECORD_MB` 限制。 + +### 深度檔格式 `SFRKDEP2`（[`kinect_record_depth.h`](../src/kinect_record_depth.h)，全部小端序） + +檔頭 32 bytes： + +| 位移 | 型別 | 內容 | +| --- | --- | --- | +| 0 | char[8] | `SFRKDEP2` | +| 8 | uint32 | 版本 2 | +| 12 | uint32 | 檔頭大小 32 | +| 16 / 20 | uint32 | 寬 320、高 240 | +| 24 | uint32 | 像素格式 1：uint16，`毫米 << 3 \| 玩家編號`（0 無、1–6），即遊戲收到的值 | +| 28 | uint32 | 每張的表頭大小 68 | + +每張影像：68 bytes 表頭，接著資料。 + +| 位移 | 型別 | 內容 | +| --- | --- | --- | +| 0 | char[4] | `KDF2` | +| 4 | uint32 | 資料 bytes 數 | +| 8 | int64 | `steady_us`：感測器後端收到（v1）或合成完成（v2）的主機時間，對齊用的時鐘 | +| 16 | int64 | `handed_us`：交給遊戲（遊戲）或探針讀到（探針）的主機時間 | +| 24 | int64 | `sdk_time`：v1 的 `liTimeStamp`（毫秒），v2 深度幀的 `RelativeTime`（100 ns）；0 不明 | +| 32 | uint64 | `number`：影像編號，就是遊戲在 `NUI_IMAGE_FRAME` +8 看到的值（每個程式、每台感測器從 1 起算） | +| 40 | uint64 | `sdk_frame`：v1 的 `dwFrameNumber`；v2 沒有，為 0 | +| 48 | uint8 | 來源：1 真正的 v1、2 v2 合成 | +| 49 | uint8 | 編碼：0 原樣、1 零值遊程 | +| 50 | uint16 | 0 | +| 52 | float[3] | 虛擬 v1 的位移（v2 的 `KinectDepthView`；v1 為 0） | +| 64 | float | 深度倍數 `z_scale`（v1 為 1） | + +資料：320×240 個 uint16（原樣，列由上而下），或零值遊程（非 0 的 token 是一個像素；0 之後跟一個 1–65535 的個數，表示那麼多個 0 像素）， +兩者取小。檔尾被截斷的一張（遊戲在寫入中結束）讀取時略過並標示。 + +大小估計（原樣每張 153,600 bytes，零值遊程視背景約省 0–50%）：遊戲每 4 個遊戲畫格要一張（60 fps 時約 15 張／秒）約 2.2 MB/s、 +每分鐘約 130 MB；探針 v2 每秒 30 張約 4.4 MB/s、每分鐘約 265 MB。預設上限 4096 MB 約可錄遊戲 30 分鐘、探針 v2 15 分鐘。 + +### 原始 v2 錄製 `SFRKRAW1`（[`kinect_record_raw.h`](../src/kinect_record_raw.h)，全部小端序） + +目的：同一次錄影可以離線換視角、換參數重做 v2 交給遊戲的深度（`sfr_kinect_depth_resynth`），不必再錄一次。 +寫入的是 v2 感應器執行緒**合成那一張影像時**用的東西：依 `RelativeTime` 配成一對的深度幀與身體索引幀、視角當時保存的最新骨架幀 +（身體與地板，`KinectDepthViewer::bodies`）與兩個時鐘，所以和合成看到的是同一批幀。開關：`SFR_KINECT_RECORD=<前綴>` 加 +`SFR_KINECT_RECORD_RAW=1`（探針的 `--raw` 等於後者；不加 `--raw` 時探針的輸出和以前一樣）。`SFR_KINECT_RECORD_RAW_MB` 上限（預設 16384， +滿了停止並 log 一次 `NATIVE_KINECT_RECORD raw full …`），`SFR_KINECT_RECORD_RAW_EVERY=N` 只留每第 N 張（預設 1；`ideal` 與 `player` 的狀態依收到的幀前進， +只有每張都留才能完全重現）。檔案在感應器的 mapper 第一次給出點（校正已到、表格也就有了）時才開，或約 5 秒都沒有時改用鏡頭參數；log +`NATIVE_KINECT_RECORD writing <前綴>-v2raw.bin (v2 raw depth 512x424, rays sensor-table, at most 16384 MB, every frame, … MB free)`， +可用空間少於上限兩倍時另有 `NATIVE_KINECT_RECORD raw low disk: …`；每 5 秒的 `NATIVE_KINECT model=v2` 行多 `raw_frames=`。 + +檔頭（64 + 8·寬·高 bytes）： + +| 位移 | 型別 | 內容 | +| --- | --- | --- | +| 0 | char[8] | `SFRKRAW1` | +| 8 | uint32 | 版本 1 | +| 12 | uint32 | 檔頭大小 64 + 8·512·424 = 1,736,768 | +| 16 / 20 | uint32 | 深度寬 512、高 424 | +| 24 | uint32 | 射線來源：1 感應器自己的表（`ICoordinateMapper::GetDepthFrameToCameraSpaceTable`，第 18 格）、2 感應器的鏡頭（`GetDepthCameraIntrinsics`，不含畸變）、3 典型 v2 鏡頭 | +| 28 | float[7] | 感應器的深度鏡頭：焦距 x、y，中心 x、y（像素），徑向 2、4、6（不知道時 0） | +| 56 | uint32 | 每張的表頭大小 208 | +| 60 | uint32 | 0 | +| 64 | float[2·寬·高] | 每個深度像素一條射線（x/z、y/z，攝影機座標 +y 朝上），列由上而下 | + +每張：208 bytes 表頭，接著資料。 + +| 位移 | 型別 | 內容 | +| --- | --- | --- | +| 0 | char[4] | `KRW1` | +| 4 | uint32 | 資料 bytes 數 | +| 8 | int64 | `steady_us`：合成完成的主機時間（與深度檔相同的時鐘） | +| 16 | int64 | 深度幀的 `RelativeTime`（100 ns；身體索引幀與它相差不到半格） | +| 24 | int64 | 最新骨架幀的 `RelativeTime`（100 ns；還沒有時 0） | +| 32 | uint64 | 合成影像的編號（與深度檔的 `number` 相同） | +| 40 | uint32 | 旗標：bit 0 = 即時合成用的是感應器 mapper 的點（否則是典型鏡頭，`kinect_v2_depth_points_from_lens`） | +| 44 | uint32 | 身體數（0–6） | +| 48 | float[4] | 該骨架幀的地板平面（全 0：沒有） | +| 64 | 6 × 24 bytes | 前「身體數」個有效：uint64 tracking id（遊戲拿到的 32 位元，`kinect_v2_short_tracking_id`）、uint8 body index（0–5，身體索引影像的值）、uint8 追蹤中（1）、uint8 SpineBase 關節狀態（0／1／2）、uint8 0、float[3] SpineBase（公尺） | + +資料：512×424 個 uint16 深度（公釐，0 沒有），接著身體索引的遊程：uint32 遊程數，每段 uint8 值（0–5，255 沒有人）、uint8 0、uint16 個數（1–65535）。 +檔尾被截斷的一張讀取時略過。 + +大小：每張約 434 KB 深度 + 幾 KB 遊程，約 0.44 MB；每秒 30 張約 13 MB/s、每分鐘約 0.79 GB，10 分鐘約 7.4 GiB；預設上限 16384 MB 約 22 分鐘。 + +**離線重做 `sfr_kinect_depth_resynth --view fit|sensor|player|ideal [--distance m] [--height m] [--follow s] [--deadband m] [--fit-scale k] --out `** +（[`kinect_v2_resynth.h`](../src/kinect_v2_resynth.h)）：寫出標準的 `SFRKDEP2` 深度檔（來源 2），`sfr_kinect_depth_features`、`sfr_kinect_depth_compare` 照常讀。 +用的是**即時路徑同一套**視角狀態（`KinectDepthViewer`：`fit`／`sensor`／`player` 的鎖定、`ideal` 的跟隨）與合成（`kinect_v2_synthesize_depth`）： +每張先把它帶的骨架幀（是新的才）交給 `body_frame`，再以深度幀的時間 `depth_frame`，然後合成。點：旗標說即時用了 mapper 的幀用檔頭的射線 +（射線朝向不對時翻正並提示），其他幀用典型鏡頭（與即時相同）。沒指定的參數用環境變數的預設；`--fit-scale` 沒指定時用檔頭的鏡頭焦距（即時也是用感應器的焦距）。 +單元測試（`kinect_v2_depth`）以合成資料確認四種視角的「即時合成」與「錄成原始檔再離線重做」逐位元相同。 +即時路徑的點來自 `MapDepthFrameToCameraSpace`，離線來自表格乘深度；兩者在實機上是否逐位元相同尚未驗證（應該只差浮點捨入）。 + +### 深度檢視的特徵（記憶體配置，由反組譯推得） + +深度檢視 `82439530`（Kinect 管理器的工作執行緒）在 `82438268` 取到新影像後，對管理器 +7740、+7744 的兩個玩家槽位 +（骨架槽位 0–5，負數為無；`82439578`–`824395D8`）各跑： + +- `82439998(slot, manager+7584, buffer=view+72, width=view+48, height=view+52)`（`8243959C`） +- `82439D80(slot, record+640, manager+7584+8*slot, buffer, width, height)`（`824395CC`；`824395A0`–`824395C4` 算出 + `record = manager + 2784 + 800*slot` 的 +640 與 `(slot+948)*8 = 7584+8*slot`） + +身體紀錄 `record = manager + 2784 + 800*slot`：建構函式 `82438A98` 在 `manager+3552`（record0 +768）寫骨架 +64、`manager+3568`（+784）寫 0， +`manager+4368`（record1 +784）寫 1……，所以 record +768 是骨架、+784 是槽位。比賽的判斷器讀的就是這份紀錄（+640／+644 即 `depth_right`／`depth_left`）。 + +玩家像素：`(pixel & 7) == slot + 1`（`vandc` 去掉 0xFFF8，`82439BA4`／`82439F80`／`8243A430`；與 1–6 的表比對，`82439BC4`／`82439FA0`／`8243A434`）。 +深度是 `pixel & 0xFFF8`（毫米 << 3，不右移）。影像當成連續的 uint16 陣列，每列 width/8 個 8 像素向量（行 c 在第 c % 8 道）。 + +**第一段 `82439998`**（寫在管理器，每槽位 8 bytes 一組）： + +| 位址 | 內容 | 依據 | +| --- | --- | --- | +| manager+7584+8·slot | 重心欄 = 欄總和 ÷ 像素數（無號除法，捨去） | `divwu` `82439CB4`、`stw` `82439CB8` | +| manager+7588+8·slot | 重心列 = 列總和 ÷ 像素數 | `divwu` `82439CDC`、`stw` `82439CE0` | +| manager+7632+8·slot | 最小欄 | `stwx` `82439D50`（各道 `vminuh` `82439BB4`） | +| manager+7636+8·slot | 最小列（第一個有玩家像素的列） | `stw r31,52` `82439D28`（`82439C08`–`82439C28`） | +| manager+7680+8·slot | 「最大欄」：各道最大欄先把第 e 道與第 e+4 道（欄 % 8）用 `vminuw`（`82439D30`）取**較小**者，再取四者最大（`vmaxuw` `82439D60`、`82439D6C`）；所以不一定是真正的最右欄（例：欄 100–139 的方塊得 135），遊戲就是這個值 | `stwx` `82439D78`（各道 `vmaxuh` `82439BBC`） | +| manager+7684+8·slot | 最大列 | `stw r8,100` `82439D64`（`82439C2C`–`82439C34`） | + +沒有像素時重心是 (width/2, height/2)（`82439CE8`–`82439CF4`），最小欄＝width、最小列＝height、最大欄列＝0。 +像素數本身只在暫存器，不寫入記憶體。各道的欄、列、個數是 16 位元飽和加法（`vadduhs` `82439BDC`–`82439BE4`），每列結束時併入 32 位元字（`82439BFC`–`82439C20`）；320×240 不會飽和。 + +**第二段 `82439D80`**（寫在身體紀錄 +640–+679），用第一段的重心 (cc, cr)：列 0..cr−1 是上半（`82439F68`–`82439FF0`）， +列 cr..height−1 是下半（`8243A010`–`8243A098`）；欄 < cc 算左（`vcmpgtuh` cc > 欄，`82439F90`／`8243A038`），其餘算右。 + +| 位址 | 內容 | 依據 | +| --- | --- | --- | +| record+640 | 下左像素數（float）＝畫面左下＝玩家右側；`depth_right` | `stvx v63` `8243A2D4` | +| record+644 | 下右像素數；`depth_left` | 同上 | +| record+648 / +652 | 上左 / 上右像素數 | 同上 | +| record+656..+668 | 四區（同順序）的平均深度（毫米<<3，float）＝總和 × 1/個數（`vrefp` 加兩次牛頓法）；空的區為 65528.0（`821AC140` 的常數） | `8243A300`–`8243A3A0`、`stvx v12` `8243A3A4` | +| record+672 | near：玩家像素中深度距「平均深度取整再 & 0xFFF8」小於 600（75 mm）的個數；以有號 16 位元比較（`vcmpgtsh`） | 迴圈 `8243A42C`–`8243A468`（`8243A454`）、`stw` `8243A4A0` | +| record+676 | 平均深度 = (((上右 + 上左) + 下右) + 下左) × 0.25（`8218FCE4` 的 0.25） | `8243A3A8`–`8243A3C4`、`stfs` `8243A3C8` | + +門檻：總數 `((上右 + 上左) + 下右) + 下左`（float）小於 `(width·height) >> 9`（320×240 為 150；`mullw` `8243A1D8`、`rlwinm` `8243A1E4`、`fcmpu` `8243A294`）時， +只把 +640–+655 清成 0（`stvx v0` `8243A29C`）就返回：**+656–+679 保留上一張影像的值**。 + +`<前綴>-race.csv` 新增的 19 欄（舊的 14 欄不變，舊檔照樣讀得進來）：`view_frame`（深度檢視這次算的影像編號）、`view_us`、`slot`、 +`centroid_col`、`centroid_row`、`min_col`、`min_row`、`max_col`、`max_row`、`count_lower_left`、`count_lower_right`、`count_upper_left`、 +`count_upper_right`、`mean_lower_left`…`mean_upper_right`、`near_count`、`mean_depth`。這些值在 `82439530` 的 hook 裡、深度檢視剛跑完 +**同一張影像**時，在同一個執行緒從記憶體讀出（[`nui_hooks.cpp`](../src/nui_hooks.cpp)），再由比賽每次更新的那一列帶出 +（[`kinect_race_record.h`](../src/kinect_race_record.h)）；影像很少時 count 為 0、其餘是舊影像的值，這和遊戲記憶體一致。 +`view_frame` 與 `<前綴>-depth.bin` 的 `number` 是同一個計數器，所以可以逐張對應。 + +### 工具 + +- **`sfr_kinect_depth_features [--csv out.csv] [--slot 0..5] [--summary] [--validate <前綴>-race.csv] [--show N]`** + （[`kinect_depth_features.h`](../src/kinect_depth_features.h)）：照上面的演算法離線重算每張影像、每個有像素的槽位的特徵， + `--csv` 每列：影像的時鐘與編號、無效像素比例、玩家像素數、槽位、像素數、重心、範圍、四區個數、是否寫入、四區平均、near、平均深度； + 最後一定印出摘要（張數、時間長度、每秒張數、平均無效像素比例、平均玩家像素、有玩家的比例）。 + `--validate` 拿遊戲的 race CSV：每個（影像編號, 槽位）只比一次（比賽每次更新都寫一列，同一張影像會重複）， + 以**影像編號**對應（兩個檔用同一個計數器，不靠時鐘），逐欄完全比對（float 逐位元），印出每欄與全部欄位的完全相符率和前 N 個差異； + 影像太少時遊戲沒寫的平均、near、平均深度不比。全部相符時結束碼 0，否則 3。 +- **`sfr_kinect_depth_compare [--skeleton ] [--csv aligned.csv] [--report report.txt] [--max-lag-ms 1000] [--gap-ms 100]`** + （[`kinect_depth_compare.h`](../src/kinect_depth_compare.h)）：每張影像取像素最多的槽位當玩家（v1 的骨架槽位和 v2 的身體索引不必相同）。 + 先用主機時鐘對齊，再以玩家重心列（`centroid_row`）每 10 ms 重新取樣做互相關，在 ±`--max-lag-ms` 內找出 v2 比 v1 晚多少（拋物線內插到小於一步）， + 印出時間差與 r；`--skeleton` 另以兩個骨架檔的 SpineBase 高度做同樣的互相關當對照。之後在每張 v1 影像的時刻（加上時間差） + 線性內插 v2（兩側影像相距不超過 `--gap-ms`、都有玩家），每個特徵印出 n、Pearson r、最小平方法 `v1 = slope·v2 + intercept`、 + RMSE（v1 − v2）、擬合後殘差 RMSE、該特徵自己的最佳時間差、兩邊平均；另印兩台的有效像素比例、平均玩家像素、失去玩家的比例（v1、v2、任一台）。 + 特徵：重心、範圍、玩家像素數、四區個數、`lean`（遊戲由 +640/+644 算的傾斜，最多 ±3.5）、四區平均（空區不算）、near、平均深度、無效像素比例。 + 報告用繁體中文標籤，`--report` 另存 UTF-8 文字檔，`--csv` 存對齊後的序列。 + 另印輪廓大小比（`player_pixels` v2/v1）與平均深度比（`mean_depth` v2/v1），都是兩邊都有玩家的配對影像的平均。 + `--events `：遊戲自己判定的每一次蹲下與跳（race CSV 的 `crouch_original`／`jump_original` 每段連續 1 的第一列，依身體紀錄分開，沒問的列略過）， + 在對齊後的序列上算 v1 與 v2 的 `centroid_row` 與 `min_row` 幅度：基準 = 事件前 2.0–0.5 秒的平均，反應 = 之後 1.2 秒內的最大（蹲）或最小（跳）減基準； + 印出兩邊的平均幅度、比值 v2/v1 與同方向的次數（`kinect_event_amplitudes`，有單元測試）。用 run2 驗證：12 次蹲（11 次兩邊都有資料）、11 次跳， + 蹲 `centroid_row` 34.0 對 25.5（0.751）、`min_row` 58.7 對 52.0（0.886）；跳 `centroid_row` −38.4 對 −32.0（0.833）、`min_row` −54.4 對 −71.1（1.307），與先前手算相同。 + 報告最後一行 `SUMMARY key=value …` 收集重點數字給腳本用（`lean_r`、`lean_slope`、`centroid_row_r`／`_slope`、`mean_depth_r`／`_slope`、`pixel_ratio`、`depth_ratio`、 + `crouch_centroid_row_ratio` 等）。 +- **`sfr_kinect_depth_resynth`**：由原始 v2 錄製用任何視角重做深度檔，見上方「原始 v2 錄製」。 +- **`sfr_kinect_record_summary`**：骨架的干擾 A/B（見上一節），不變。 + +驗證：`sfr_kinect_depth_view_guest_test`（只在 `-Diagnostic` 建置時有；建置時用 `tests/extract_guest_functions.cmake` 從 +`out/recomp/diagnostic` 複製出 `82439998` 與 `82439D80` 的生成程式碼到建置目錄，放在自己的 4 GB 客體記憶體裡跑；兩個函式讀的四個 +常數照解碼後映像的值放進去：`8217D100`／`8217D110` 的 vperm 表、`8218FCE4` 的 0.25、`821AC140` 的 65528.0×4，複製到 `83E54520`） +以 600 張各式測試影像（多位玩家、深度斜坡、雜訊、破洞、沒有深度的玩家像素、貼邊、全畫面最深值的飽和情況）×6 個槽位比對離線演算法： +3600 組全部欄位逐位元相同（677 組寫入完整特徵、2923 組像素太少只清零）。`sfr_kinect_depth_test` 有手算的合成影像、檔案格式、 +互相關與擬合的單元測試。還沒有用真實錄影做過 `--validate`（需要實機）。 + +## 觀察用 trace（`SFR_NUI_ANALYSIS_TRACE`） + +為了驗證遊戲怎麼用骨架的研究（見「遊戲怎麼用骨架」）裡還沒在執行中確認的推論，`SFR_NUI_ANALYSIS_TRACE=1` 讓遊戲在 +`game.log` 多寫幾種觀察行（[`nui_analysis_trace.h`](../src/nui_analysis_trace.h)）。它們**不改變**遊戲的任何行為：每個 hook 照常跑原本的函式， +之後只讀客體記憶體。每一種都有自己的上限（平均分散在每一秒裡、總行數也有上限，到了上限寫一行 `NUI_TRACE_<種類> stopped after N lines (cap)` +就不再寫），所以玩很久 `game.log` 也不會變得很大。所有行都以 `NUI_TRACE_<種類>` 開頭： + +| 種類 | 在哪裡 | 每秒最多 | 總數上限 | 內容 | +| --- | --- | --- | --- | --- | +| `SMOOTH` | `NuiTransformSmooth`（`827701A0`，[`nui_smooth_hooks.cpp`](../src/nui_smooth_hooks.cpp)） | 1 | 4000 | `mode=`（`native` 原生平滑、`emulated` 模擬骨架不平滑、`host` 後端已平滑、`original` 跑原版）、`result=`（回傳值）、`initialized=`（`[82B5D2AC]`，函式庫的「已初始化」旗標）、`params=` 與五個參數、`frame_number=`、`tracked=`（框架裡 TRACKED 的骨架數） | +| `CURSOR` | 輸入更新 `824920F8` 之後（[`nui_hooks.cpp`](../src/nui_hooks.cpp)） | 10 | 36000 | 每位有身體的玩家一行：`left[...]`／`right[...]` 是該手的 `flags=`（P+0：0x8 舉起、0x10 放下）、`theta=`（P+60）、`r=`（P+148）、`target=`（P+96/+100）、`cursor=`（P+80/+84，選單讀的像素）與 `formula=`（由 θ、r 照研究的公式算的 x、y）；之後是輸入物件裡遊戲看到的關節 `hand_l= hand_r= shoulder_l= shoulder_r= head= spine=`（玩家區塊 +1264，平滑與鏡像 (−x, y, −z) 之後） | +| `TURN` | `822C6200`、`822C8958` 之後（[`nui_race_hooks.cpp`](../src/nui_race_hooks.cpp)），只在比賽中 | 5＋5 | 各 18000 | `stage=lean`：深度傾斜的 10 筆歷史（`depth_lean=` 新的在前，`depth_lean_mean=` 是 `822C63E8` 的加權平均，source +44）與身體傾角的 10 筆歷史（`tilt=`、`tilt_mean=`，source +88），以及 `depth_right=`／`depth_left=`（rec +640/+644）。`stage=steer`：讀到的 `lean_in=`（source +44）、自己的濾波 `filter_mean=`（detector +112）、寫出的 `steer=`（結果 +76，有號位元組）、`steer78=`、方向位元 `bits=`、`result=` | +| `CALIB` | `822C65B0`（雙臂往前）、`822C6AD8`（量身體）之後（[`nui_analysis_hooks.cpp`](../src/nui_analysis_hooks.cpp)） | 20 | 2000 | 每次回傳由「不是 1」變成 1 時一行：`step=arms_forward|body`、`calib=`（`[source+92]`）、`record=`、`tracking_id=`、`slot=`，以及 `+0=` 到 `+132=` 全部 34 個值（+112、+116、+120 是整數像素，其他是浮點） | +| `BIND` | 綁定 `82222B80` 之後（[`nui_analysis_hooks.cpp`](../src/nui_analysis_hooks.cpp)） | 10 | 2000 | 模式（`mode=`，box +176）、`team_mode=`（`83E515E7`，3 是接力）、兩位玩家的 rec（box +120/+124）與它們的 tracking id、enrollment（box +136/+140）、候選數（+180/+184）；其中任一個（候選數除外）改變時一行 | + +**怎麼開**：遊戲從 launcher 繼承環境變數，所以在 PowerShell 裡先設再開 launcher： + +```powershell +$env:SFR_NUI_ANALYSIS_TRACE = '1' +Start-Process .\FreeRidersRecompiled.exe -WorkingDirectory . +Remove-Item env:SFR_NUI_ANALYSIS_TRACE +``` + +**拿到有用資料要做什麼**（v1 與 v2 各跑一次，比較兩份 `game.log`）： + +1. 進到主選單後，舉起右手讓游標出現；站著不動，手臂伸直，**手繞著肩膀左右慢慢掃過整個畫面**（約 90°），再上下移動。 + `NUI_TRACE_CURSOR` 的 `cursor=` 應該慢慢追上 `formula=`；水平位置只跟 θ 有關，所以同一個掃法在 v1、v2 上的 `theta=` 範圍應該一樣。 +2. 看一行 `NUI_TRACE_SMOOTH`：預設是 `mode=native result=0x0`，參數應該是遊戲自己的 0.5／0.5／0.5／0.05／0.04； + 加 `SFR_NUI_SMOOTH=0` 時跑原版，預期 `mode=original result=0x8000ffff initialized=0`（函式庫沒初始化，什麼都不做）。 + 兩種各玩一段，比較 `NUI_TRACE_CURSOR` 的游標抖動，就是平滑的效果（見「骨架平滑」）。 +3. 開一場 Free Race（單人），在「On your Gear!」照畫面做完量測：每一步完成時應該各有一行 `NUI_TRACE_CALIB`。 +4. 比賽中左右傾身、站直、再傾向另一邊：`NUI_TRACE_TURN stage=lean` 與 `stage=steer` 顯示深度傾斜與身體傾角兩條歷史怎麼合成最後的轉向。 +5. 有兩個人時再開 2P、Tag 與 Relay（接力換人一次）：`NUI_TRACE_BIND` 記下每次綁定改變。 + ## 其他平台 - **Linux**：Kinect 第一代沒有骨架追蹤器可用，但核心的 `gspca_kinect` 驅動會把 @@ -179,6 +475,11 @@ COM 介面依 SDK 2.0 的 `Kinect.h` 宣告到用得到的最後一個方法為 mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架畫面與地板；v2 還沒在 實機上試過。v2 的 COM 方法順序若與 SDK 不符，`sfr_kinect_probe` 會讀不到骨架或當掉，那是最先要看的地方。 +v2 預覽影像用到的介面（彩色、深度、身體索引的 frame source／reader／frame） +同樣是照 `Kinect.h` 的順序手寫，尚未在實機上驗證；`sfr_kinect_probe 5 --images` 存下的兩張圖是最快的檢查方式。 +所有手寫的 v2 介面（`IKinectSensor`、各 frame source／reader／frame、`IBody`、`ICoordinateMapper`） +的方法槽位已逐一對照 SDK 2.0（v2.0_1409）`Kinect.h` 的 C 風格 `*Vtbl` 宣告，槽位編號寫在 +[`kinect_v2_win32.cpp`](../src/kinect_v2_win32.cpp) 的註解裡；建置仍不需要 SDK（`Kinect20.dll` 執行時才載入）。 特別需要確認的： - 選單的手部游標位置是否對得上（模擬玩家的右手在 x=0.175 為畫面中央）。 @@ -195,7 +496,7 @@ mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架 一直沒有深度資料（`NUI_DEPTH_VIEW_SKIPPED`）。 遊戲也用同一個類別（函式表 `0x821A8768`）開彩色串流（類型 1、解析度 2，640×480、每像素 4 bytes）。 -`SFR_KINECT_DEPTH=1`（且 `SFR_CAMERA=kinect`）接上兩條串流： +`SFR_CAMERA=kinect` 時接上兩條串流： - 感應器端（[`kinect_sensor_win32.cpp`](../src/kinect_sensor_win32.cpp)）：`NuiInitialize` 多加深度與彩色 旗標，開 SDK 的深度加玩家編號（320×240）與彩色（640×480）串流，讀骨架的迴圈順便取兩者最新一幀 @@ -207,16 +508,164 @@ mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架 交給遊戲。之後的上色（`82438328`）、+72 緩衝區、輪廓計算全是遊戲自己的程式。 `NUI_IMAGE_STREAM_OPEN ... backend=kinect texture=`、前 4 幀的 `NUI_IMAGE_FRAME type= number=` 可確認。 -有了深度影像，遊戲自己的深度檢視會算出傾斜那一對數值；`SFR_CAMERA_RACE_TRACE=1` 的 -`CAMERA_RACE_MOTION` 行以 `depth_pair=右/左` 印出它（在骨架傾斜寫入之前）。 -`SFR_KINECT_BODY_LEAN=0` 不寫骨架傾斜，直接用遊戲原本由深度算的轉向。 - -### 預設：全部交給遊戲 - -選 Kinect v1 時預設嘗試開啟深度與彩色串流。深度成功開啟時，傾斜由遊戲的深度檢視計算、 -蹲下與起跳沿用遊戲判斷器，也不做水平校正。Kinect v2 尚未提供深度串流；它與深度開啟失敗、 -`SFR_KINECT_DEPTH=0` 的情況，都會自動使用上面「比賽轉向」「蹲下」「感測器仰角」的骨架備援。 -`SFR_KINECT_BODY_LEAN`、`SFR_KINECT_BODY_GESTURES`、`SFR_KINECT_LEVEL` 設 1 或 0 可以個別強制。 +有了深度影像，遊戲自己的深度檢視會算出傾斜那一對數值；`SFR_KINECT_RECORD` 的 `<前綴>-race.csv` +以 `depth_right`／`depth_left` 記下它。 + +### 全部交給遊戲 + +選 Kinect v1 時一定會開啟深度與彩色串流，選 Kinect v2 時一定會合成深度串流（見下一節）。傾斜由遊戲的深度檢視計算， +蹲下與起跳沿用遊戲判斷器，也不做水平校正。**沒有骨架備援**：v1 的深度串流打不開、或 v2 的深度與身體索引讀取器打不開時， +遊戲的 `NuiImageStreamOpen` 照常失敗（交給原本的函式，在沒有初始化的函式庫上就是失敗，與沒有深度的主機一樣）， +log 寫 `NATIVE_KINECT_DEPTH unavailable model=Kinect v1|Kinect v2 reason=…`（`reason`：`no-image-api`、`stream-0x…`、 +`depth-0x…`、`body-index-0x…`），launcher 的 Kinect 預覽也會寫「遊戲沒有深度影像」與原因。 + +### Kinect v2 深度 + +（沒有骨架備援：讀取器打不開時遊戲沒有深度，只記 log 並在 launcher 預覽顯示；影像中斷超過 0.5 秒只在恢復時記一行 `NUI_KINECT_DEPTH gap ms=…`。） + +主機端用骨架猜的傾斜與蹲跳是 v2 玩家轉向、跳躍不準的原因（已移除）：主機上傾斜的那一對數值與蹲、跳的深度部分來自遊戲自己的深度檢視 +（蹲、跳另外也讀骨架的髖中心；其他判斷器讀骨架，見「遊戲怎麼用骨架」）。 +Kinect v2 一律（沒有開關）交給遊戲一條和 v1 一樣的 +深度加玩家編號串流，遊戲原本的深度檢視與判斷器照 v1 的方式運作 +（[`kinect_v2_depth.h`](../src/kinect_v2_depth.h)，有單元測試 `kinect_v2_depth`）。Kinect v1 不受影響。 + +- **怎麼做**：v2 的感應器執行緒另開一組深度與身體索引讀取器（與預覽的分開，互不搶畫面），兩者 `RelativeTime` 相同時， + 用感應器自己的 `ICoordinateMapper::MapDepthFrameToCameraSpace`（vtable 第 14 格，對照 SDK 2.0 `Kinect.h`）把每個 + 512×424 的深度像素換成攝影機座標（拿不到時用 v2 典型鏡頭，log 的 `depth_lens_frames=`），移到虛擬 v1 相機， + 再用 v1 的標稱鏡頭投影成 320×240：`x = 160 + X·285.63/Z`、`y = 120 − Y·285.63/Z`（`NuiImageCamera.h` 的 + `NUI_CAMERA_DEPTH_NOMINAL_FOCAL_LENGTH_IN_PIXELS`，`NuiSkeleton.h` 的 `NuiTransformSkeletonToDepthImage`；像素 n 在 n）。 + 同一像素取最近的點（z-buffer），玩家編號是那一點的身體索引 + 1（與骨架槽位 + 1 相同）。只填一像素寬的洞， + 而且左右或上下兩鄰居都有值、相差 5 公分內、同一位玩家；其他空的都是「沒有值」。 +- **格式**：與 v1 路徑交給遊戲的相同：`毫米 << 3 | 玩家（1–6）`。v1 的串流以旗標 0 開啟（沒有 + `NUI_IMAGE_STREAM_FLAG_DISTINCT_OVERFLOW_DEPTH_VALUES`），太近、太遠、未知都回報 `NUI_IMAGE_DEPTH_NO_VALUE`（0）， + 所以合成的影像也是：虛擬相機看到的距離不在 800–4000 公釐（`NUI_IMAGE_DEPTH_MINIMUM`／`_MAXIMUM`）內，或沒有點，就是 0。 +- **虛擬相機放哪**（`SFR_KINECT_V2_DEPTH_VIEW`，開發者調校用，三種）： + - `fit`（預設）：虛擬 v1 就在 v2 的位置，但把每個攝影機座標點的深度乘上 k（`(X, Y, k·Z)`）再用 v1 鏡頭投影， + v2 的整個視野就收進 320×240（左右各留約 15 欄空白）。k = 285.63 ÷（v2 深度焦距 × 240/424）：焦距用感應器自己的 + `ICoordinateMapper::GetDepthCameraIntrinsics`（vtable 第 19 格，`FocalLengthY`；感應器送出校正前先用典型的 366，k ≈ 1.378）， + `SFR_KINECT_V2_DEPTH_FIT_SCALE`（1.0–2.0）可直接指定，1.0 就等於 `sensor`。這是 z 方向的拉長，不是把影像縮放： + 真實距離 d 的人在遊戲看來和 v1 前 k·d 處同樣大小的人一樣，深度值也是 k·d，所以畫面大小與深度一致。 + 換算後不在 800–4000 公釐的仍是「沒有值」，所以實際可用的距離約是 0.58–2.90 公尺（k = 1.378）。沒有任何狀態或鎖定， + 一人、兩人、接力換人都交給遊戲自己處理。**取捨**：沿 z 的深度差也被放大 k 倍（往前、往後的動作在遊戲看來大 k 倍）， + 畫面角度被壓縮成 1/k（左右移動在畫面上變小）；比 v1 能站得近，但最遠只到約 2.9 公尺。 + - `player`：換 v2 往往就是要站得比 v1 允許的近，所以虛擬 v1 不放在 v2 的位置，而是沿 v2 的 z 移動，讓玩家的參考點 + （身體中心，v2 的 SpineBase）落在 `SFR_KINECT_V2_DEPTH_DISTANCE` 公尺（預設 2.5，1–4）；知道地板時再沿 v2 的 y 移動， + 讓相機像放在地板上方 `SFR_KINECT_V2_DEPTH_HEIGHT` 公尺（預設 0.8，0.2–2）的 v1 那樣看到參考點。只平移，不轉向，也不左右移。 + - `sensor`：虛擬 v1 就在 v2 的位置（只換鏡頭與解析度，看到的是 v2 視野中間 v1 那麼寬的部分）。 + - `ideal`（launcher 的預設「Recompiled」；遊戲沒有設這個變數時仍是 `fit`）：v2 量到的是真實的 3D 場景，所以**模擬一台擺在理想位置的真正 v1**：把 v2 的點雲從一台 + **水平**、v1 鏡頭、**不拉長 z** 的虛擬相機重新投影，輪廓大小、深度值、動作幅度都是真正的 v1 在參考距離會給的 + （[`KinectIdealState`](../src/kinect_v2_depth.h)，有單元測試）。每張深度影像做一次（深度幀自己的 `RelativeTime` 當時間）， + 是「前一個狀態、這一格的時間、最新骨架幀的身體與地板」的純函式，所以原始錄製（見「原始 v2 錄製」）離線重做會得到一樣的影像。 + 1. **水平**：由 v2 的 `FloorClipPlane`（法向量朝上，與其他地方的重力慣例相同：v2 的重力就是地板法向量）算出旋轉 + `R = Rx(俯仰)·Rz(側傾)`，先繞相機 z 去掉法向量的 x，再繞相機 x 去掉 z，使 `R·n = (0, 1, 0)`；光軸方向不變（沒有偏航）。 + 每個攝影機座標點都先轉 `R`。地板平面先平滑（時間常數 1 秒；第一個有效的直接採用，無效的那格不更新）。 + 法向量與距離分開處理：v2 很早就給出法向量（重力），但還沒真正看到地板時（或一直看不到時）`FloorClipPlane` 的 W（感應器離地高度）是 0。 + 以前把這個 0 當成高度，相機就被放在感應器上方 `HEIGHT` 公尺（離地約 1.7 公尺），玩家在模擬 V1 畫面裡偏下、腳被切掉； + 現在 W 至少 0.05 公尺才算知道高度（第一個直接採用、之後平滑、W 為 0 的那格不更新），不知道高度時只做水平、相機留在感應器自己的高度。 + log 會在高度出現或消失時寫一行 `NATIVE_KINECT_V2_DEPTH ideal floor height found …`／`… floor without a height (W=0) …`（每秒最多一行）， + 每 5 秒的狀態行也多 `floor_given=a,b,c,d sensor_height= camera_height= height=`（感應器給的地板、感應器離地高度、相機離地高度、設定的高度）。 + 一直沒有有效地板時不轉、相機在感應器自己的高度，log 一次 `NATIVE_KINECT_V2_DEPTH ideal no floor plane yet`；找到時 log 一次 `... ideal floor found: levelled`。 + 2. **位置**（轉正之後只平移）：參考點是**所有**被追蹤的人 SpineBase 的平均（一人、兩人都行），在轉正後的座標； + 相機在參考點正前方 `SFR_KINECT_V2_DEPTH_DISTANCE` 公尺（預設 2.5，1–4）、左右對齊參考點、高度是地板上方 + `SFR_KINECT_V2_DEPTH_HEIGHT` 公尺（`ideal` 預設 1.0，0.2–2；`player` 仍是 0.8）。高度只跟地板，**永遠不跟玩家**。 + 參考點的 x、z 各自以時間常數 `SFR_KINECT_V2_DEPTH_FOLLOW` 秒（預設 2.0，0.2–20）的指數平滑**慢慢跟**， + 而且只跟超出死區 `SFR_KINECT_V2_DEPTH_DEADBAND` 公尺（預設 0.15，0–1）的部分：死區內（傾身、小步側移、跳）相機不動， + 動作照樣留在影像裡；長時間的漂移、有人加入或離開則平順滑過去（相機最後停在離參考點一個死區寬的地方）。 + 沒有人時保持最後的位置；沒人之後第一個出現的人（以及第一次找到地板時）直接跳到位，不滑。 + 3. **投影**：`x = 160 + X·285.63/Z'`、`y = 120 − Y·285.63/Z'`（Z' 是到虛擬相機的距離），深度值就是到虛擬相機的公釐， + z-buffer、一像素補洞、打包與「800–4000 公釐以外是沒有值」都和其他視角相同，玩家編號仍是 body index + 1。 + 因為 D = 2.5，玩家本來就在 2.5 公尺；離虛擬相機 0.8–4 公尺的東西會保留（log 的 `true_range=0.80-4.00m from the virtual camera`）。 + 4. 交給遊戲的骨架（關節、身體中心、指尖拇指；全 0 的點維持 0）、地板平面與重力做同樣的旋轉＋平移（不拉長），骨架與深度一致； + 轉正後地板是 `(0, 1, 0, H)`、重力是 `(0, 1, 0)`。預覽視窗疊骨架時 `image_point` 先把旋轉與平移還原再對到 v2 的影像。 + - **取捨**：v2 沒看到的地方（虛擬相機的視野超出 v2 視野的部分）一律空著；換了視點之後，被擋住的表面只有 v2 當初看到的部分會出現， + 其餘是洞（不補）；v2 看到的角度不變，所以近距離時輪廓的側面、底面不會像真正擺在那裡的 v1 那樣看得到。 + 跟隨慢，所以大步側移的開頭仍會在影像裡移動，之後才慢慢回到中間（超出死區的部分）。 + `SFR_KINECT_RECORD_DEPTH` 的深度檔只記錄平移（`view_origin`）與 z_scale，不記錄旋轉。 + - **骨架與深度分開調**：遊戲從不把骨架投影到深度影像，兩者在幾何上不必一致，所以可以各自調整。注意骨架不只給選單： + 游標、玩家身分、轉向的身體傾角、蹲跳的髖中心、約 20 個比賽判斷器與賽前量測都讀它（深度只給傾斜那一對數值、蹲跳的輪廓與賽前量測的輪廓）， + 所以骨架的轉換也會改變這些判斷（`fit` 的 z 拉長會讓含 z 的部位長度與往前伸的動作都變大，這也是預設改成 1.0 的原因之一）： + - `SFR_KINECT_V2_SKELETON_STRETCH`（只對 `fit`，1.0–2.0；**不設＝1.0**，`depth`＝與深度相同的 k，也就是 2026-10-19 以前的預設）： + 只改交給遊戲的骨架與地板的 z 拉長，深度不變。1.0 就是感應器真實的座標（公尺），所以預設交給遊戲的骨架和真正的 v1 一樣是真實距離； + 深度照樣拉長 k 倍。遊戲從不把骨架投影到深度影像（不呼叫 `NuiTransformSkeletonToDepthImage`），兩者不必一致。 + - `SFR_KINECT_V2_SKELETON_DISTANCE`（只對 `ideal`，1–4 公尺；不設＝深度的 `SFR_KINECT_V2_DEPTH_DISTANCE`）：骨架用的虛擬相機 + 沿 z 移到離參考點這麼遠（同樣水平、同樣左右與高度），深度不變。 + - 對游標的影響：游標的水平位置是「肩→手」方向和 X 軸的 **3D 夾角**（見「選單游標的公式」），不除以 Z。骨架的 z 拉長 k 倍時， + 手往前伸的 z 分量也變成 k 倍，單位向量的 x 分量變小：手臂直直往前伸時水平靈敏度最多只剩 1/k（k = 1.378 時約 0.73，要多移動約 38%）。 + 預設的 1.0 讓靈敏度回到真正 v1 的大小；垂直只用 y 的比值，不受影響。`ideal` 的骨架距離只是沿 z 平移，夾角不變，所以不影響游標。 + - 兩個值都寫在開始的 `NATIVE_KINECT_V2_DEPTH on …` 行（`skeleton_stretch=1.000 (real metres)`、`skeleton_stretch=depth's (1.378)`； + `skeleton_distance=`，不設時為 `depth's`）。`fit` 時每 5 秒的狀態行一定有 `skeleton_view=x,y,z skeleton_z_scale=`（實際交給遊戲的骨架倍數）， + `ideal` 設了骨架距離時也有。預覽視窗疊骨架時照骨架自己的轉換還原。 + - **參考點是鎖定的**，不會一直跟著玩家：第一次有人被追蹤時，取最靠中間的人，前 15 格（0.5 秒）平均後鎖定 + （這 0.5 秒先用第一格的位置），之後只要還有人被追蹤就不動；所有人離開 1 秒後才放開，下一個人走進來再重新鎖定。 + 所以側移、蹲下、跳起在影像裡仍是移動，不會被抵銷。 + - 交給遊戲的骨架（關節、身體中心、指尖拇指）與地板平面做同樣的平移（`player`、`ideal`），`fit` 預設不拉長（真實公尺，見上方 + `SFR_KINECT_V2_SKELETON_STRETCH`；設成 `depth` 才和深度一樣拉長，地板法向量重新正規化，重力跟著地板法向量）。遊戲不把骨架投到深度像素， + 所以兩者不一致也沒關係；預覽視窗疊骨架時會換回 v2 自己的座標。深度模式下不做水平校正（同 v1）。 + - **取捨**：虛擬相機移到 v2 後面時，v2 的視野（約 70°×60°）在 2.5 公尺處比 v1 的 58.5°×45.6° 窄，影像邊緣可能空著， + 往旁邊大步移動會先出 v2 的畫面；被身體擋住的背景本來就量不到，平移不轉向的情況下新露出的洞很少，留空不補。 + 放開鎖定前若換了站位，玩家在虛擬影像裡會偏前或偏後，離開畫面 1 秒再回來就會重新鎖定。 +- **比賽**：完全由遊戲自己的判斷器決定（深度與骨架，見「遊戲怎麼用骨架」），主機端沒有替代判定，也沒有任何開關。讀取器打不開時遊戲就沒有深度 + （log 寫 `NATIVE_KINECT_DEPTH unavailable model=Kinect v2 reason=depth-0x…` 或 `body-index-0x…`，launcher 預覽也會顯示）； + 超過 0.5 秒沒有新深度影像（例如暫停或感應器停送）只在影像恢復時 log 一行 `NUI_KINECT_DEPTH gap ms=…`，行為不變。 + 骨架平滑、選單游標穩定、暫停手勢不變。 +- **log**：開始時 `NATIVE_KINECT_V2_DEPTH on view=fit scale=1.378 focal=366.0 (typical) true_range=0.58-2.90m depth=0x00000000 body_index=0x00000000` + （`player` 是 `distance= height=`；`ideal` 是 `distance=2.50 height=1.00 follow=2.00 deadband=0.15 true_range=0.80-4.00m from the virtual camera (the players at 2.50m)`， + 每 5 秒的狀態行另有 `depth_camera=x,y,z level=0|1 pitch= roll= floor_d= reference=x,z reference_bodies=`（虛擬相機在轉正後座標的位置、去掉的俯仰與側傾角度、地板距離、跟隨中的參考點）；讀取器打不開時是 `off`；fit 等到感應器給出鏡頭參數時另有一行 `NATIVE_KINECT_V2_DEPTH fit scale=… focal=… (sensor) true_range=…`）、`NATIVE_KINECT started=1 model=Kinect v2 depth=1`、遊戲開啟串流時 + `NUI_IMAGE_STREAM_OPEN type=0 resolution=1 ... backend=kinect`；每 5 秒的 `NATIVE_KINECT model=v2` 行多 + `depth_synth_frames=`（約 150）、`depth_valid=`（最新一張有深度的比例）、`depth_player_pixels=`、`depth_view=`（虛擬相機的位移；fit 時為 0） + 與 `depth_latched=`；`NUI_IMAGE_FRAMES depth=給/要` 是遊戲實際拿到幾張。 +- **錄製**：`SFR_KINECT_RECORD=<前綴>` 加 `SFR_KINECT_RECORD_DEPTH=1` 把**遊戲實際拿到**的每張深度影像存成 `<前綴>-depth.bin` + （格式 `SFRKDEP2`，含虛擬相機位移與深度倍數 k；見下方「深度錄製與分析工具」），之後可以和真正 v1 的影像比較。 + 舊的 `SFRKDEP1`（v2 合成的每一張，不論遊戲有沒有拿）已移除，工具讀到會說明不支援。 +- **不開遊戲**：`sfr_kinect_probe 10 --v2 --game-depth [--images]` 讀同一條合成路徑（v2 一律合成，旗標只決定要不要讀取並印出），每半秒印出 `game_depth_frames= valid= player_pixels=`， + `--images` 另存第一張為 `kinect_game_depth.bmp`。 +- **兩位玩家**：遊戲的深度檢視（生成的程式碼 `82439998`、`82439D80`）以玩家槽位查一張 1–6 的表，每個像素先去掉 `0xFFF8` + 只留低 3 位元的玩家編號，只累加編號等於「槽位 + 1」的像素：`82439998` 算該玩家的重心與範圍，`82439D80` 以**該玩家自己的** + 重心欄／列把他的像素分成左右（上下）再加總深度與像素數，就是 +640／+644 的來源。沒有用影像左右兩半分人，所以兩人只要 + 身體都在影像內、像素帶著各自的玩家編號（這裡就是 body index + 1，與骨架槽位一致）即可，站哪裡都行。 +- **接力賽**：遊戲等上一棒走出感應器、下一棒走進來（[Relay Race](relay-race.md)；手把模擬時是 45 格沒有骨架），所以實機交棒 + 中間通常有一段沒有人被追蹤的時間。`fit` 沒有狀態，不受影響；`player` 的鎖定在所有人離開 1 秒後放開，下一棒會重新鎖定； +`ideal` 在沒人時保持相機，下一棒一出現就直接跳到他前面。 + +**還沒驗證的**：遊戲的深度判斷器在合成影像上的反應（輪廓的大小、邊緣、相機高度是否像真正的 v1）；主機的 NUI 是否用不同的特殊值； +`MapDepthFrameToCameraSpace` 在實機上的結果；兩位玩家同時時的表現。`ideal` 只有單元測試與合成資料，沒有實機或真實錄影 +（run1、run2 沒有原始檔）；v2 `FloorClipPlane` 的雜訊、`GetDepthFrameToCameraSpaceTable` 在實機上的內容與朝向（離線工具會檢查並翻正）， +以及即時（mapper）與離線（表格 × 深度）的點是否逐位元相同，都還沒在實機上確認。 + +### 遊戲怎麼用骨架 + +（詳細的研究與位址在 repo 外的 `NUI_SKELETON_ANALYSIS.md`；這裡是結論。）以前這份文件說「比賽只靠深度、骨架只負責選單」，那是錯的： + +- **骨架**：遊戲每格把 NUI 框架（平滑後，見「骨架平滑」）的 20 個關節乘上鏡像 diag(−1, 1, −1)（`82438F80`，不用框架的重力，所以不做水平校正） + 寫進每個槽位的身體紀錄（管理器 +2784 + 800·槽位）。讀它的有：選單游標（`82492F30`）、舉手／放下、面向感應器、玩家身分與綁定 + （`82222B80`），以及比賽中約 20 個判斷器：舉手 `822C8650`、蓄力 `822CB840`、側移 `822CA6B0`、煞車 `822C9BF0`、抓取 `822CB0B8`、 + 踢地 `822CA518`／`822CBD28`、招式 `822CA810`／`822CBF30`、站姿 `822CAC90`、特技 `822C9938`、加速 `822C9A80`／`822CAF48`、飛行 `822C9180`、 + 道具（上投、下揮、充氣、擦螢幕）、拉桿、前彎、把手、牽手，還有轉向的身體傾角（`822C6200` 的 head − spine、`822C8958` 的肩中心 − 髖中心）。 +- **深度**：只用在四個地方：傾斜那一對數值（rec +640／+644）、蹲下（重心列，加上髖中心 z）、跳（重心列與最小列，加上髖中心 y）、 + 賽前量測（重心、輪廓高度）。蹲與跳都是相對於每位玩家自己在「On your Gear!」量到的基準（`calib`）。 +- 遊戲從不呼叫 `NuiTransformSkeletonToDepthImage`，骨架與深度在幾何上不必一致。 +- 遊戲最多兩副 TRACKED 骨架（見 `KinectPlayerSlots`）。 + +`SFR_KINECT_RECORD` 的 `<前綴>-nui.csv` 把上面這些消費者每一次的結果都記下來（見「錄製」），`SFR_NUI_ANALYSIS_TRACE=1` 則在 log 裡觀察（見「觀察用 trace」）。 + +### 選單游標的公式 + +寫游標的是 `82492F30`（輸入物件 `[83E52FB8]`，每個遊戲畫格、每位玩家一次；骨架只在新框架時改變）。每隻手的封包 +P = 輸入物件 + 32 + 2544·玩家 + 16 + 400·手（左手用關節 4、5、7，右手 8、9、11），坐標都是遊戲的鏡像座標： + +- **水平**：θ = 「肩 − 手」單位向量和 X 軸的 **3D 夾角**（度，P+60），目標 x = 1280·(135° − θ)/90°（P+96）。θ = 135° 在左緣、90° 在中央、45° 在右緣： + **手繞著肩膀轉 90° 就掃過整個畫面寬**，和距離、身高都無關（不除以 Z，所以以前猜的「透視投影」不對）。 +- **垂直**：r = (head − hand).y ÷ (head − spine).y（P+148），目標 y = 720·(r − 0.23)（P+100）：畫面的高度等於頭到 spine 的長度， + 手和頭同高時在上緣以上（被夾住），r = 1.23 在下緣。 +- **遊戲自己的游標平滑**：每個遊戲畫格 P+80 += (P+96 − P+80)·0.065（`82196FCC`；P+8 的 0x2 位元設起時直接跳到目標）， + 時間常數約 15 格（60 fps 時約 0.25 秒，fps 低時更慢）；之後 `822448B0` 把 x 夾在 0..1256、y 在 0..708，交給選單的游標物件。 + 這是骨架平滑（`NuiTransformSmooth`）**之外**的第二層；主機端的「選單游標穩定」（見下一節）是第三層，只在選單、只對手部。 +- **出現與收起**：前臂（肘 − 手）與 +Y 的夾角 ≥ 90°（手不低於手肘）連續 20 格舉起游標（P+0 |= 8）；≤ 30° 連續 10 格放下（|= 16）。 +- 和模擬器的兩次實測吻合（水平約 2600 px/m、垂直 1376 px/m，見 `nui_hooks.cpp`）。 +- **對 Kinect v2 的影響**：水平用的是含 z 的 3D 夾角，`fit` 的骨架 z 若拉長 k 倍，手往前伸時單位向量的 x 分量變小，中央附近的水平靈敏度是 + `1/√(dy² + (k·dz)²)` 對 `1/√(dy² + dz²)`，手臂直直往前伸時降成 1/k ≈ 0.73。所以 `fit` 的骨架預設改成真實公尺（`SFR_KINECT_V2_SKELETON_STRETCH` + 預設 1.0）；垂直只用 y 的比值，不受影響。v2 的 Head 比 v1 高、SpineMid 和 v1 的 Spine 高度不同，r 的尺度可能還要校準（尚未實機比較）。 +- 驗證：`SFR_NUI_ANALYSIS_TRACE=1` 的 `NUI_TRACE_CURSOR` 每行有 θ、r、目標、游標與照公式算的 `formula=`；`<前綴>-nui.csv` 有每一格的值。 ### 骨架原樣交給遊戲 @@ -240,15 +689,52 @@ mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架 - 感應器角度在預覽視窗裡調整（見下方「預覽視窗」）：「往上」「往下」或鍵盤 ↑↓，每次用 Kinect 的馬達 轉 5 度（SDK 的 `NuiCameraElevationSetAngle`，−27～27 度；SDK 要求馬達一秒最多轉一次，轉動時按鈕停用）。 感應器斷電前會維持角度。 -- 骨架平滑：之前每一幀都先用 SDK 的 `NuiTransformSmooth` 平滑。主機的執行環境交給遊戲的是 - 未平滑的骨架，要不要平滑由遊戲自己決定（`NuiTransformSmooth` 也在它連結的 NUI 函式庫裡）， - 所以現在預設不平滑（`NATIVE_KINECT_OPEN ... smooth=0`）；`SFR_KINECT_SMOOTH=1` 恢復。 - 遊戲是否自己呼叫平滑，可以用 - `scripts/analyse_detectors.py ... --calls-into 82760000-82780000` 列出遊戲呼叫 NUI 函式庫的每一處來查。 +- 骨架平滑：主機的執行環境交給遊戲的是未平滑的骨架，**遊戲自己呼叫 `NuiTransformSmooth`**，見下一節。 + +### 骨架平滑（遊戲自己的 `NuiTransformSmooth`） + +遊戲每拿到一張新骨架框架，就在 Kinect 管理器的更新（`82438D80`，呼叫處 `82438E58`）對自己那份框架（管理器 +16） +呼叫 `NuiTransformSmooth`（`827701A0`），參數在管理器 +2752，是建構函式（`82438A98`）寫入的 SDK 預設值 +0.5／0.5／0.5／0.05／0.04，之後才把關節轉成身體紀錄。所以選單游標、比賽中約 20 個讀關節的判斷器與賽前量測，原本拿到的都是平滑過的骨架。 + +**之前的問題**：`827701A0` 一開頭就檢查函式庫的「已初始化」旗標 `[82B5D2AC]`，這個旗標只有原版 `NuiInitialize`（`8276FD10`）會設， +而我們的 `NuiInitialize` hook 不跑原版，所以它一直回傳 `0x8000FFFF`、什麼都不做，遊戲也不看回傳值：v1 的骨架一直沒有被平滑； +v2 則是主機端自己的平滑碰巧補上。`SFR_NUI_ANALYSIS_TRACE=1` 的 `NUI_TRACE_SMOOTH`（加 `SFR_NUI_SMOOTH=0`）可以看到 `result=0x8000ffff initialized=0`。 + +**現在**（[`nui_smooth_hooks.cpp`](../src/nui_smooth_hooks.cpp)、[`nui_transform_smooth.h`](../src/nui_transform_smooth.h)，有單元測試 `nui_transform_smooth`）： +hook `827701A0`，以原生方式做函式庫的濾波，用遊戲傳進來的參數（r4 指向五個大端序 float；r4 為 0 時用函式庫預設），直接改遊戲的大端序框架，回傳 `S_OK`。 +演算法照遊戲連結的函式庫（`827701A0` 對每個槽位呼叫 `82779700`）逐步讀出： + +- 6 個骨架槽位各有 20 個關節的歷史（原始值、濾波值、趨勢、已看過幾格）。槽位不是 TRACKED（2）就清掉歷史、不濾波； + 我們另外在槽位的 tracking id 改變時清掉（換了一個人；主機的函式庫沒有檢查這個）。 +- 關節位置剛好是 (0, 0, 0) 時重新開始，原樣交出；因為它成了新歷史的第一格，下一格是它和原點的平均（再被最大偏差拉回量測值 4 公分內），這是函式庫（與 SDK 範例）原本的行為。 +- 關節的追蹤狀態只影響半徑：推測（1）的抖動半徑與最大偏差加倍；「沒追蹤」（0）但有位置的關節照樣濾波。 +- 抖動半徑最小 0.0001 公尺；骨架的位置（中心，+16）不濾波；每個關節的 w 改成 1。 +- 第一格：量測值、趨勢 0。第二格：兩次量測的平均，趨勢往差值修正。之後：離上一個濾波值不到抖動半徑時，依移動距離比例拉向上一個輸出； + 濾波值 =（1 − smoothing）·量測 + smoothing·（上一個濾波值 + 上一個趨勢）；趨勢 = correction·（濾波值 − 上一個濾波值）+（1 − correction）·上一個趨勢； + 輸出 = 濾波值 + prediction·趨勢，離量測值超過最大偏差時拉回到最大偏差上。 +- 骨架 hook 在感測器沒有新框架時會把上一張再交給遊戲（框架編號 +8 相同）；同一個框架編號不會平滑第二次，而是再寫一次上一次的輸出。 + +**只平滑實體 Kinect（v1、v2）的骨架**。手把模擬與 webcam 體感的骨架原樣保留，但一樣回傳 `S_OK`：主機上執行環境追蹤到的骨架全部會被平滑， +但模擬的骨架不是感測器量的——手把的骨架是精確的、只隨手把移動，webcam 的骨架已經過 `pose_smoothing`——再平滑只會讓每個選單與比賽動作多一格左右的延遲。 + +**一副骨架只平滑一次**：v2 主機端的平滑（`KinectJointSmoother`）與 v1 的 SDK 平滑預設都關閉；`SFR_KINECT_SMOOTH=1` 時改由後端平滑（給 +`sfr_kinect_probe --smooth` 與測試用），遊戲這邊就不再平滑。`KinectJointSmoother` 現在用的就是同一個濾波步驟（單元測試確認兩者逐位元相同）。 +錄製（`<前綴>-skeleton.csv`）一律記錄平滑**之前**的骨架。`SFR_NUI_SMOOTH=0` 改跑原版（什麼都不做，和以前一樣），方便比較。 + +log：第一次與模式改變時一行 `NUI_TRANSFORM_SMOOTH mode=native|emulated|host|original result=0x…`；`SFR_NUI_ANALYSIS_TRACE=1` 時每秒最多一行 +`NUI_TRACE_SMOOTH mode=… result=… initialized=… smoothing=0.5 correction=0.5 prediction=0.5 jitter_radius=0.05 max_deviation_radius=0.04 frame_number=… tracked=…`。 + +**和真正的 SDK 比對**：`sfr_nui_transform_smooth_sdk_test`（Windows，CTest `nui_transform_smooth_sdk`）載入 SDK 1.8 的 `Kinect10.dll`， +把同一串合成的骨架框架（兩個人、晃動加雜訊、推測與沒追蹤的關節、遺失成原點的關節、離開再回來；預設參數與另外五組參數） +同時交給 SDK 的 `NuiTransformSmooth` 與我們的實作，比對每個關節（容許 0.1 公釐）。**它不呼叫 `NuiInitialize`、不啟動任何感測器**。 +在這台開發機上，沒有初始化時 SDK 的 `NuiTransformSmooth` 對空框架與有骨架的框架都回傳 `0x8007000E`（E_OUTOFMEMORY，和主機函式庫 +配置歷史緩衝區失敗時回的一樣），不做任何事，所以這個測試回報 `SKIPPED` 並以 77 結束（CTest 記為略過）；演算法的正確性改由 +`nui_transform_smooth_test` 依函式庫的程式碼逐步手算確認。在已初始化感測器的程式裡比對（例如 `sfr_kinect_probe`）還沒有做。 ### 選單游標 -骨架照原樣交給遊戲，只有選單游標和主機不同（[`kinect_stability.h`](../src/kinect_stability.h)，和 webcam 共用 +骨架照原樣交給遊戲（遊戲自己再平滑骨架、再平滑游標，見「骨架平滑」與「選單游標的公式」），只有選單游標和主機不同（[`kinect_stability.h`](../src/kinect_stability.h)，和 webcam 共用 [`pose_stability.h`](../src/pose_stability.h) 的 `MenuHandSteadying`；尚未實機確認效果）：選單中（不在比賽時）雙手以肩膀為準， 靜止時小幅晃動不移動游標，慢慢移動立刻跟上，快速移動完全不延遲，單一格的突跳忽略。比賽中不作用。每個被追蹤的人各自記憶， 依 SDK 追蹤編號區分。靜止圈半徑在啟動器「體感」頁的「選單游標穩定」設定（1–30 公釐，預設 8，0 關閉；`SFR_MENU_STEADY_RADIUS`、`SFR_MENU_HAND_STEADY=0`）。遊戲視窗的標題列每半秒顯示 `[FPS 59.8 | 16.7/41.0 ms] [Kinect 30] [Vulkan 100%]`：FPS、平均／最長畫格時間、遊戲每秒讀到幾格新骨架、繪圖後端與渲染倍率；啟動器「畫面與效能」頁的「標題顯示效能數字」可以關掉（`SFR_TITLE_STATS=0`）。 @@ -266,6 +752,35 @@ mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架 僅 Windows）：左邊彩色影像、右邊深度影像（玩家依骨架位置上色），兩者都疊上追蹤到的骨架 （深度照 SDK 的 `NuiTransformSkeletonToDepthImage` 投影，精確；彩色用標稱焦距，兩顆鏡頭相距幾公分， 是近似對齊），上方列出每個人的編號、骨架位置、距離與「腳是否在畫面內」（腳關節被追蹤到才算）。 +Kinect v2 的兩顆鏡頭視野不同（彩色約 84°×54°、深度約 71°×60°），預設把兩個畫面都裁成相同範圍（兩者視野的交集，不再硬切成 4:3，約 1.373:1；以彩色標稱焦距 1060 px、深度 366 px 算出：彩色取約 x 219–1701（1482 px）、y 全部 1080；深度保留完整寬度 x 0–512，只取約 y 25.7–398.3，約 373 列，上下各少約 26 列，約 3°）。兩個面板大小與長寬比相同，沒有黑邊；預覽影像的長邊上限為彩色 640、深度 512，因此彩色約 640×466（每個預覽像素取平均）、深度約 512×373（取最近的像素；玩家依 v2 的身體索引上色）。視窗的面板大小不再固定，而是依畫面的真實長寬比算出（`kinect_preview_layout`，一個純函式，繪圖與骨架座標共用），視窗縮小時兩個面板等比例縮小。彩色的焦距只是標稱值（沒有彩色內參的 API），兩顆鏡頭又相距幾公分（2 公尺處約 1°），所以兩個畫面只是近似對齊。深度畫面上下各約 3° 的範圍遊戲仍然看得到，但預覽不顯示，因此真正的畫面邊緣要看「身體被畫面切到」的警告（被切的邊）。預覽視窗右上角（「往上」左邊）的下拉選單選深度視角，選項與 launcher 的「深度視角」相同、順序也相同：Recompiled、模擬、原始（Kinect v1 時反灰）。開啟時選的是 launcher 目前的視角與 Recompiled 的五個值；在預覽裡改選只影響這個視窗，**不會寫回 launcher 的設定**。launcher 的滑桿在預覽開著時一動，預覽的感應器就改用新值（launcher 每一格把設定交給預覽，`KinectPreviewWindow::set_depth_choice`）；launcher 改了視角時預覽也跟著換。預覽自己開的 v2 用 `KinectSensor::set_depth_settings` 換掉環境變數的視角與調校（在感應器自己的執行緒、下一張深度影像前套用；遊戲不會呼叫），所以預覽裡的骨架與「遊戲拿到的深度」就是那個視角做出來的。遊戲中的「Kinect 調整」視窗借用遊戲的感應器，**永遠不改遊戲的視角**：選單從遊戲自己的視角（`SFR_KINECT_V2_DEPTH_VIEW`）開始，改選只換畫面的裁切，只有遊戲本身是 `ideal` 時才有下面 Recompiled 的四格畫面。各選項顯示： +- **模擬**：兩顆鏡頭共同的範圍（上述裁切），彩色＋深度兩格。 +- **原始**：等於以前的「原始」按鈕，關掉裁切，改顯示各鏡頭完整畫面（見下）。 +- **Recompiled**：左邊 2×2、右邊一格俯視圖（`kinect_preview_recompiled_layout`，純函式，有單元測試；兩欄同寬，每張畫面都和欄一樣寬、照真實長寬比、靠格子上緣，所以彩色正好在彩色下方、深度正好在深度下方；每列上面一行標題，標題上方留約 8 像素；畫面依視窗高度與三分之二的寬度能放多大就放多大，俯視圖佔其餘寬度（至少三分之一）、與兩列同高；視窗預設 1360×820，可縮放）。 + 上排「V2 彩色影像」「V2 深度影像」是實體 v2 的完整畫面（不裁切）＋骨架；下排「模擬 V1 彩色影像」與「模擬 V1 深度影像」**逐像素對齊**， + 深度是**遊戲實際拿到**的 320×240 影像（`game_image()` 的複本，不影響遊戲自己讀的），兩格的骨架都用 v1 鏡頭 + （`NuiTransformSkeletonToDepthImage`）投影，有骨架距離時先換回深度的視角，所以與深度精確對齊。 + 模擬 V1 彩色是逐像素重投影（[`kinect_v1_depth_points`](../src/kinect_v2_depth.h)，有單元測試）：深度影像的每個像素以它的深度經 v1 標稱鏡頭反投影到虛擬相機座標， + 再用做出這張影像的同一個視角（影像本身記著平移、z 倍數與水平旋轉）反轉回 v2 感應器座標，整批交給感應器的 `ICoordinateMapper::MapCameraPointsToColorSpace`（拿不到時用典型鏡頭）放進 v2 彩色畫面（`KinectSensor::colour_image_positions`），取最近像素的顏色；沒有深度的像素是黑的。 + 所以彩色與深度看到的是同一批點：深度精確、左右在半個像素內；彩色取自 640×360 的預覽影像，兩顆鏡頭的對齊精度就是 mapper 的精度。沒有人時虛擬相機就在 v2 的位置（左右、前後都是 0）、水平、離地 `HEIGHT`，看 v2 視野的正中間；離它超過 4.0 公尺（或不到 0.8 公尺）的地方沒有深度，所以是黑的。 + 右邊「俯視位置」是 Recompiled 的俯視圖（[`kinect_ideal_diagram`](../src/kinect_ideal_diagram.h)，純函式，有單元測試 `kinect_ideal_diagram`； + launcher 的「俯視位置」用同一個函式畫）：實體 v2 在下方往上看，藍色實線、淡藍填色的三角形是它 70.6° 的水平視野，頂點標「實體 v2」； + 橘色虛線、淡橘填色的三角形是虛擬 v1 的 57° 視野，頂點標「虛擬 v1」，兩條橘色點線是 v1 量得到的 0.8 與 4.0 公尺； + 點線橫線是玩家的深度線（虛擬 v1 就在它後面 `DISTANCE` 公尺）；1P 紫、2P 綠（依遊戲的玩家槽位規則 `KinectPlayerSlots`，其他人灰色）的圓在實際位置， + 後面淡色橫帶是那個深度兩台相機都看得到的左右範圍；旁邊寫「v1 到虛擬 v1 的距離（遊戲看到的深度）、v2 到實體 v2 的距離」， + 出了 v1 視野、v1 的 0.8–4.0 公尺或 v2 視野時多一行紅字。左右與深度畫面相同（+x 畫在右邊），所以和旁邊的畫面一致。左邊是從實體 v2 量的深度尺、下方是左右尺（0 在 v2 的中線），每 0.5 公尺一個刻度、每 1 公尺標字並畫一條很淡的橫線；虛擬 v1 的中線上另標 0.8 m 與 4.0 m（從虛擬相機量，遊戲讀得到的範圍）。圖的比例一定包含 0.5–4.5 公尺深、左右各 2.5 公尺。 + 虛擬相機的位置**就是**深度合成用的那一個：感應器每做一張深度影像就把 `KinectDepthViewer` 的 ideal 狀態 + （`kinect_ideal_snapshot`：相機 = `kinect_ideal_view` 的原點，玩家 = SpineBase 轉到同一個水平座標）存一份給 `KinectSensor::ideal_snapshot`。 + `KinectDepthViewer` 現在**每種視角都會推進 ideal 狀態**（只有 `ideal` 用它做影像，其他視角的輸出不變），所以預覽或 launcher 畫的永遠是 Recompiled 會放的位置。 + 預覽視窗裡「與感應器距離」那行在 Recompiled 時是交給遊戲的骨架距離（虛擬相機），實體 v2 的距離看俯視圖。 + +原始視角關掉裁切，改顯示各鏡頭完整畫面，面板各自照畫面的長寬比：彩色 1920×1080（16:9）縮成 640×360（每 3×3 像素取平均）；深度 512×424（約 1.208:1）維持原大小，兩個面板同高、靠上對齊，寬度依長寬比（放不下時等比例縮小），沒有黑邊。這只影響預覽的顯示，只保留在該視窗開著的期間，不影響遊戲拿到的深度、骨架或錄影。裁切的計算只在一個純函式（`kinect_v2_crop`）裡，像素轉換、骨架座標與畫面矩形都用它。骨架位置由感測器自己的 +`ICoordinateMapper`（`MapCameraPointsToColorSpace`／`MapCameraPointsToDepthSpace`，用感測器出廠的兩顆鏡頭校正）換算到 +彩色與深度影像，再照裁切與縮放（原始模式則只有縮放）移進預覽；拿不到 mapper，或 mapper 回答 -infinity（放不進影像）時，改用 v2 的典型鏡頭參數估計（近似對齊）。 +log 的 `NATIVE_KINECT_MAPPER model=v2 result=…` 說明有沒有拿到 mapper，每 5 秒的 `NATIVE_KINECT model=v2` 行在預覽開著時多 +`preview_mapped_points=`（mapper 放的點）與 `preview_estimated_points=`（鏡頭估計的點）。影像串流在預覽第一次要影像時 +才開啟，預覽約 3 秒沒再要就關閉（log：`NATIVE_KINECT_IMAGE model=v2 open …`／`closed`；開著時每 5 秒的 +`NATIVE_KINECT model=v2` 行會多 `preview_colour_frames=`、`preview_depth_frames=`），遊戲本身不會開啟它們 +（[`kinect_v2_image.cpp`](../src/kinect_v2_image.cpp)，有單元測試 `kinect_v2_image`）。 還沒能玩的時候，最上面一行依序列出四個檢查：執行環境（SDK）、感應器、資料傳送、人體追蹤， 各標 ✓（通過）、✗（失敗）或 …（等待中），下一行說明第一個失敗的步驟要怎麼處理 （[`kinect_readiness`](../src/kinect_preview.h)）。每一步都要真的在運作才算通過：感應器開了但 @@ -274,18 +789,12 @@ mingw 編譯連結過。Kinect for Xbox 360（v1）已在實機上讀到骨架 右上角的「往上」「往下」（或 ↑↓）轉動感應器，邊看邊調到全身入鏡。SDK 一個程式只能開一次 Kinect，所以 按「開始遊戲」會先關閉預覽，把 Kinect 讓給遊戲。 -#### 校正 - -預覽視窗的「校正」（只有啟動器的預覽有）照主機版 Kinect 調整的方式一步一步帶玩家做，量出比賽讀的、 -每個人不一樣的數值（[`kinect_calibration.h`](../src/kinect_calibration.h)，有單元測試）： - -1. 站直不動 1.5 秒：量站姿的臀部高度（Kinect 回報的地板平面上方；沒有地板平面時用臀部本身的高度）。 -2. 像比賽時那樣蹲下再站起來：量最低點比站姿低多少。 -3. 跳一下：量臀部高出站姿多少（10 秒內沒看到就跳過，結果寫「沒有偵測到」）。 -4. 往一邊傾、再站直；5. 往另一邊傾、再站直：量上身（臀部到肩膀）左右各傾了幾度。比賽在 30° 轉到底。 - -完成後「蹲下深度」設成量到的蹲下深度的 70%（5～35 公分）並存檔：比賽中的蹲通常比校正時淺一點， -仍然要算得到。Issue #97 的玩家蹲 12～14 公分，預設的 15 公分永遠到不了；校正後是 8～10 公分。 +**身體被畫面切到（Kinect v2）**：v2 對每個追蹤到的人回報被畫面哪幾個邊切到（`IBody::get_ClippedEdges`，即 SDK 的 `FrameEdges`）。 +預覽（含遊戲中的調整視窗）在每個被切到的人下面多一行橘紅警告，例如「身體被畫面切到：上方，請往後退，或把感應器角度往上」，沒被切到就不顯示。 +邊是**畫面**的邊，和預覽畫出的影像一致（未鏡像，也是 Microsoft 範例 BodyBasics 的畫法）：影像左邊是感應器的左邊，玩家面對感應器時就是玩家的右邊， +所以畫面左邊被切到要請玩家往**自己的左邊**移動、右邊被切到往自己的右邊（上、下被切到請往後退或調感應器角度；左右同時或上下同時被切到只能往後退）。 +log 在被切到的邊改變時寫 `NATIVE_KINECT_CLIPPED model=v2 body=<槽位> edges=top+left|…|none mask=<位元遮罩>`（每個人每秒最多一行）， +每 5 秒的 `NATIVE_KINECT model=v2` 行有人被切到時多 `clipped<槽位>=<邊>`。這只供顯示與記錄，不改變遊戲收到的骨架、深度或任何偵測結果。 遊戲中也開得到同一個視窗（[`kinect_tuner.h`](../src/kinect_tuner.h)）：選單裡調整 Kinect 的選項會向系統要 Kinect 疑難排解（`XamShowNuiTroubleshooterUI`）或 Kinect Guide（`XamShowNuiGuideUI`，結算畫面的 Guide 按鈕）， diff --git a/scripts/build_tools.ps1 b/scripts/build_tools.ps1 index a2ead3b..a50a127 100644 --- a/scripts/build_tools.ps1 +++ b/scripts/build_tools.ps1 @@ -30,6 +30,7 @@ $diagnosticPath = [IO.Path]::GetFullPath((Join-Path $repoRoot $DiagnosticDirecto if ($LASTEXITCODE -ne 0) { throw 'Xenon tools configure failed.' } $targets = @('XenonAnalyse', 'XenonRecomp', 'sfr_image_dump', 'sfr_memory_test', 'sfr_xex_module_test', 'sfr_virtual_memory_test', 'sfr_critical_section_test', 'sfr_hardware_info_test', 'sfr_thread_local_storage_test', 'sfr_system_time_test', 'sfr_guest_clock_test', 'sfr_timestamp_bundle_test') $targets += 'sfr_vector_memory_test' +$targets += 'sfr_consecutive_failures_test' $targets += 'sfr_guest_entry_state_test' $targets += 'sfr_guest_registers_test' $targets += 'sfr_guest_function_table_test' @@ -146,7 +147,22 @@ $targets += 'sfr_host_core_plan_test' $targets += 'sfr_host_thread_stats_test' $targets += 'sfr_vector_unpack_test' $targets += 'sfr_kinect_smoothing_test' -$targets += 'sfr_kinect_calibration_test' +# The title's NuiTransformSmooth done natively; and against the SDK 1.8's own (skipped without it). +$targets += 'sfr_nui_transform_smooth_test' +$targets += 'sfr_nui_transform_smooth_sdk_test' +$targets += 'sfr_kinect_v2_image_test' +$targets += 'sfr_kinect_preview_layout_test' +$targets += 'sfr_kinect_v2_depth_test' +$targets += 'sfr_kinect_ideal_diagram_test' +$targets += 'sfr_kinect_record_test' +# Reading a Kinect recording back without the game (docs/kinect-sensor.md). +$targets += 'sfr_kinect_record_summary' +# Depth recordings: their format, the title's depth view features again, v1 against v2. +$targets += 'sfr_kinect_depth_test' +$targets += 'sfr_kinect_depth_features' +$targets += 'sfr_kinect_depth_compare' +# A Kinect v2's raw recording made into the game's depth again, through any view. +$targets += 'sfr_kinect_depth_resynth' $targets += 'sfr_native_timer_resolution_test' $targets += 'sfr_native_constant_upload_test' $targets += 'sfr_pipeline_manifest_test' @@ -180,6 +196,6 @@ $targets += 'sfr_guest_texture_binding_test' $targets += 'sfr_native_notification_event_test' $targets += 'sfr_native_notifications_test' $targets += 'sfr_notification_placement_test' -if ($Diagnostic) { $targets += 'sfr_cpu_diagnostic'; $targets += 'sfr_native_winsock_test' } +if ($Diagnostic) { $targets += 'sfr_cpu_diagnostic'; $targets += 'sfr_native_winsock_test'; $targets += 'sfr_kinect_depth_view_guest_test' } & cmake --build $build --target $targets --parallel $Jobs if ($LASTEXITCODE -ne 0) { throw 'Xenon tools build failed.' } diff --git a/src/consecutive_failures.h b/src/consecutive_failures.h new file mode 100644 index 0000000..b1cca85 --- /dev/null +++ b/src/consecutive_failures.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace sfr { + +// How many failures in a row a caller skips a frame for (a window being +// dragged to a new size can refuse an acquire, a present or a resize now and +// then) before it gives up. A success resets the count. +inline constexpr uint32_t consecutive_failure_limit = 8; + +// Counts one more failure; true while it is still within the limit. +[[nodiscard]] inline bool failure_within_limit(uint32_t& failures, uint32_t limit = consecutive_failure_limit) noexcept { + return ++failures <= limit; +} + +} // namespace sfr diff --git a/src/diagnostic_hooks.h b/src/diagnostic_hooks.h index 13ec5bd..74db661 100644 --- a/src/diagnostic_hooks.h +++ b/src/diagnostic_hooks.h @@ -405,8 +405,6 @@ std::optional nui_second_gamepad(uint32_t user, bool racing); // instead of the pad or the camera's motion standing in for them // (nui_race_hooks.cpp). bool nui_body_from_sensor(); -// True only if the selected backend successfully opened its depth stream. -bool nui_sensor_has_depth(); // Advances with each new frame or expired tracking update (0 without one): the race // steers from its bodies only when a frame is new (nui_race_hooks.cpp). uint64_t kinect_frame_generation(); diff --git a/src/kinect_calibration.cpp b/src/kinect_calibration.cpp deleted file mode 100644 index a2fbb1f..0000000 --- a/src/kinect_calibration.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include "kinect_calibration.h" -#include "kinect_sensor.h" -#include "nui_skeleton.h" - -#include -#include - -namespace sfr { -namespace { -// Standing still: the hips within this band for this long. -constexpr float still_band = 0.03f; -constexpr double still_seconds = 1.5; -// A crouch or jump: the hips this far from standing, then back within the -// return band of it. -constexpr float crouch_least = 0.05f, jump_least = 0.05f, return_band = 0.025f; -// A lean: at least this far from upright, then back within the return lean. -constexpr float lean_least = 10.0f, lean_return = 5.0f; -// After this long without the motion, the window says to try it again. -constexpr double motion_wait = 10.0; -} - -uint32_t KinectCalibration::crouch_cm_for(float drop_metres) { - const long cm = std::lround(double(drop_metres) * 100.0 * crouch_share); - return uint32_t(std::clamp(cm, 5L, 35L)); -} - -void KinectCalibration::start() { - *this = KinectCalibration{}; - step_ = KinectCalibrationStep::stand; - step_started_ = -1; // set by the first frame -} - -void KinectCalibration::advance(KinectCalibrationStep next, double seconds) { - step_ = next; - step_started_ = seconds; - progress_ = 0; - deepest_ = highest_ = leaned_ = 0; - moved_ = waiting_long_ = false; -} - -void KinectCalibration::observe(const KinectBody* body, const std::array& floor, double seconds) { - if (step_ == KinectCalibrationStep::idle || step_ == KinectCalibrationStep::done) return; - if (step_started_ < 0) step_started_ = seconds; - tracked_ = body && body->joint_states[nui_joint::hip_center] && body->joint_states[nui_joint::shoulder_center]; - if (!tracked_) { - still_since_ = -1; - if (step_ == KinectCalibrationStep::stand) progress_ = 0; - return; - } - const auto& hip = body->joints[nui_joint::hip_center]; - const auto& shoulders = body->joints[nui_joint::shoulder_center]; - const bool has_floor = floor[0] != 0.0f || floor[1] != 0.0f || floor[2] != 0.0f; - const float height = has_floor ? floor[0] * hip[0] + floor[1] * hip[1] + floor[2] * hip[2] + floor[3] : hip[1]; - // The trunk's tilt to the side, from the hips up to the shoulders. - const float tilt = std::atan2(shoulders[0] - hip[0], shoulders[1] - hip[1]) * 57.29578f; - waiting_long_ = seconds - step_started_ > motion_wait; - - switch (step_) { - case KinectCalibrationStep::stand: { - // Standing still: every sample since still_since_ within the band. - // One outside it starts the stillness again from itself. - const float low = std::min(still_low_, height), high = std::max(still_high_, height); - if (still_since_ < 0 || high - low > still_band) { - still_since_ = seconds; - still_low_ = still_high_ = height; - still_sum_ = still_lean_sum_ = 0; - still_count_ = 0; - } else { - still_low_ = low; - still_high_ = high; - } - still_sum_ += height; - still_lean_sum_ += tilt; - ++still_count_; - progress_ = float(std::min(1.0, (seconds - still_since_) / still_seconds)); - drop_ = 0; - lean_ = 0; - if (seconds - still_since_ >= still_seconds && still_count_ > 0) { - result_.standing_height = still_sum_ / float(still_count_); - neutral_lean_ = still_lean_sum_ / float(still_count_); - advance(KinectCalibrationStep::crouch, seconds); - } - return; - } - default: break; - } - drop_ = result_.standing_height - height; - lean_ = tilt - neutral_lean_; - switch (step_) { - case KinectCalibrationStep::crouch: - deepest_ = std::max(deepest_, drop_); - moved_ = moved_ || deepest_ >= crouch_least; - progress_ = std::min(1.0f, deepest_ / 0.15f); - if (moved_ && drop_ < return_band) { - result_.crouch_drop = deepest_; - result_.crouch_cm = crouch_cm_for(deepest_); - advance(KinectCalibrationStep::jump, seconds); - } - break; - case KinectCalibrationStep::jump: - highest_ = std::max(highest_, -drop_); - moved_ = moved_ || highest_ >= jump_least; - progress_ = std::min(1.0f, highest_ / 0.10f); - if (moved_ && -drop_ < return_band) { - result_.jump_rise = highest_; - advance(KinectCalibrationStep::lean_left, seconds); - } else if (!moved_ && waiting_long_) { - advance(KinectCalibrationStep::lean_left, seconds); // no jump seen: the result says so - } - break; - case KinectCalibrationStep::lean_left: - case KinectCalibrationStep::lean_right: { - // The two leans go opposite ways; which sign is left depends on the - // sensor, so the first lean decides it. - const bool second = step_ == KinectCalibrationStep::lean_right; - const float toward = second ? -leaned_sign_ : 0.0f; - const float amount = toward == 0.0f ? std::fabs(lean_) : std::max(0.0f, lean_ * toward); - if (amount > std::fabs(leaned_)) leaned_ = toward == 0.0f ? lean_ : amount * toward; - moved_ = moved_ || std::fabs(leaned_) >= lean_least; - progress_ = std::min(1.0f, std::fabs(leaned_) / 30.0f); - if (moved_ && std::fabs(lean_) < lean_return) { - if (!second) { - result_.lean_left = std::fabs(leaned_); - leaned_sign_ = leaned_ < 0 ? -1.0f : 1.0f; - advance(KinectCalibrationStep::lean_right, seconds); - } else { - result_.lean_right = std::fabs(leaned_); - advance(KinectCalibrationStep::done, seconds); - } - } - break; - } - default: break; - } -} -} diff --git a/src/kinect_calibration.h b/src/kinect_calibration.h deleted file mode 100644 index 2fd0cd4..0000000 --- a/src/kinect_calibration.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once -#include -#include - -namespace sfr { -struct KinectBody; - -// The Kinect calibration a player walks through in the Kinect preview, like -// the console's own: stand where the sensor sees you, stand still, crouch as -// you would in a race, jump, lean each way. It measures the two things the -// race reads that differ from body to body - how far the hips drop in a -// crouch (the launcher's crouch depth, SFR_KINECT_CROUCH_DEPTH: Issues #97 -// and #132, where a 12 cm crouch never reached the 15 cm default) and how far -// the player leans - and suggests the crouch depth to set. -// -// Heights are of the hip centre above the floor the sensor reports, or the -// hips' own height when it reports none (only differences are used). -enum class KinectCalibrationStep : uint8_t { idle, stand, crouch, jump, lean_left, lean_right, done }; - -struct KinectCalibrationResult { - float standing_height = 0; // metres above the floor - float crouch_drop = 0; // metres, the deepest of the crouch - float jump_rise = 0; // metres above standing, 0 when no jump was seen - float lean_left = 0, lean_right = 0; // degrees from upright, each way - uint32_t crouch_cm = 0; // the crouch depth to set -}; - -class KinectCalibration { -public: - // Begins again from the first step. - void start(); - void cancel() { step_ = KinectCalibrationStep::idle; } - // One sensor frame: the player's body (null when nobody is tracked), the - // floor plane (ax + by + cz + d, all zero when unknown) and the time. - void observe(const KinectBody* body, const std::array& floor, double seconds); - - KinectCalibrationStep step() const { return step_; } - // How far the current step is, 0..1 (standing still, holding a lean). - float progress() const { return progress_; } - // Live values for the window: the hips' drop below standing (negative - // above it), and the lean from upright in degrees (left negative). - float drop() const { return drop_; } - float lean() const { return lean_; } - bool tracked() const { return tracked_; } - // The step timed out waiting for the motion (a crouch or a lean it did - // not see); the step stays, so the player can try again. - bool waiting_long() const { return waiting_long_; } - const KinectCalibrationResult& result() const { return result_; } - - // A crouch counts when the hips drop this much below standing; the - // crouch depth suggested is this share of the player's own crouch, so a - // race's crouch, a little shallower than the calibration's, still counts. - static constexpr float crouch_share = 0.7f; - static uint32_t crouch_cm_for(float drop_metres); - -private: - void advance(KinectCalibrationStep next, double seconds); - KinectCalibrationStep step_ = KinectCalibrationStep::idle; - KinectCalibrationResult result_; - double step_started_ = 0, still_since_ = -1; - float still_low_ = 0, still_high_ = 0, still_sum_ = 0, still_lean_sum_ = 0; - int still_count_ = 0; - float neutral_lean_ = 0; - float progress_ = 0, drop_ = 0, lean_ = 0; - float deepest_ = 0, highest_ = 0, leaned_ = 0; - float leaned_sign_ = 0; // which way the first lean went - bool tracked_ = false, waiting_long_ = false, moved_ = false; -}; -} diff --git a/src/kinect_depth_compare.cpp b/src/kinect_depth_compare.cpp new file mode 100644 index 0000000..2d99b41 --- /dev/null +++ b/src/kinect_depth_compare.cpp @@ -0,0 +1,312 @@ +#include "kinect_depth_compare.h" + +#include +#include +#include +#include +#include +#include + +namespace sfr { + +namespace { +constexpr double none = std::numeric_limits::quiet_NaN(); +} + +double kinect_series_at(const KinectSeries& series, int64_t time, int64_t max_gap_us) { + const auto& t = series.t; + if (t.empty() || time < t.front() || time > t.back()) return none; + const size_t after = size_t(std::lower_bound(t.begin(), t.end(), time) - t.begin()); + if (t[after] == time) return series.v[after]; + const size_t before = after - 1; + const double a = series.v[before], b = series.v[after]; + if (!std::isfinite(a) || !std::isfinite(b) || t[after] - t[before] > max_gap_us) return none; + const double f = double(time - t[before]) / double(t[after] - t[before]); + return a + (b - a) * f; +} + +namespace { +// Pearson correlation of the pairs where both are present. +double correlation(const std::vector& a, const std::vector& b, size_t& pairs) { + double sa = 0, sb = 0, saa = 0, sbb = 0, sab = 0; + pairs = 0; + for (size_t i = 0; i < a.size() && i < b.size(); ++i) { + if (!std::isfinite(a[i]) || !std::isfinite(b[i])) continue; + ++pairs; + sa += a[i]; + sb += b[i]; + saa += a[i] * a[i]; + sbb += b[i] * b[i]; + sab += a[i] * b[i]; + } + if (pairs < 2) return none; + const double n = double(pairs); + const double va = saa - sa * sa / n, vb = sbb - sb * sb / n; + // A constant series (to rounding) correlates with nothing. + if (!(va > 1e-12 * saa) || !(vb > 1e-12 * sbb)) return none; + return (sab - sa * sb / n) / std::sqrt(va * vb); +} +} + +KinectLag kinect_cross_correlate(const KinectSeries& a, const KinectSeries& b, int64_t max_lag_us, int64_t step_us, + size_t min_pairs) { + KinectLag out; + if (a.t.empty() || b.t.empty() || step_us <= 0) return out; + const int64_t start = a.t.front(), end = a.t.back(); + // Samples close enough to stand for the series between them: three + // steps, or a tenth of a second, whichever is more. + const int64_t gap = std::max(3 * step_us, 100000); + std::vector sa; + for (int64_t t = start; t <= end; t += step_us) sa.push_back(kinect_series_at(a, t, gap)); + const int64_t steps = max_lag_us / step_us; + std::vector rs(size_t(2 * steps + 1), none); + std::vector sb(sa.size()); + double best = -2; + int64_t best_k = 0; + for (int64_t k = -steps; k <= steps; ++k) { + for (size_t i = 0; i < sa.size(); ++i) sb[i] = kinect_series_at(b, start + int64_t(i) * step_us + k * step_us, gap); + size_t pairs = 0; + const double r = correlation(sa, sb, pairs); + if (pairs < min_pairs || !std::isfinite(r)) continue; + rs[size_t(k + steps)] = r; + if (r > best) { + best = r; + best_k = k; + out.pairs = pairs; + } + } + if (best < -1) return out; + out.found = true; + out.r = best; + double lag = double(best_k); + const size_t at = size_t(best_k + steps); + if (at > 0 && at + 1 < rs.size() && std::isfinite(rs[at - 1]) && std::isfinite(rs[at + 1])) { + const double l = rs[at - 1], c = rs[at], r = rs[at + 1]; + const double denominator = l - 2 * c + r; + if (denominator < 0) lag += std::clamp(0.5 * (l - r) / denominator, -0.5, 0.5); + } + out.lag_us = lag * double(step_us); + return out; +} + +KinectFit kinect_fit(const std::vector& truth, const std::vector& other) { + KinectFit out; + double st = 0, so = 0, stt = 0, soo = 0, sto = 0, sd = 0; + for (size_t i = 0; i < truth.size() && i < other.size(); ++i) { + const double y = truth[i], x = other[i]; + if (!std::isfinite(x) || !std::isfinite(y)) continue; + ++out.n; + st += y; + so += x; + stt += y * y; + soo += x * x; + sto += y * x; + sd += (y - x) * (y - x); + } + if (!out.n) { + out.r = out.slope = out.intercept = out.rmse = out.residual_rmse = none; + return out; + } + const double n = double(out.n); + out.mean_truth = st / n; + out.mean_other = so / n; + out.rmse = std::sqrt(sd / n); + const double vo = soo - so * so / n, vt = stt - st * st / n, c = sto - st * so / n; + // Constant to rounding: no correlation, no slope. + const bool other_moves = vo > 1e-12 * soo, truth_moves = vt > 1e-12 * stt; + out.r = other_moves && truth_moves ? c / std::sqrt(vo * vt) : none; + if (other_moves) { + out.slope = c / vo; + out.intercept = out.mean_truth - out.slope * out.mean_other; + } else { + out.slope = none; + out.intercept = out.mean_truth - out.mean_other; + } + double residual = 0; + for (size_t i = 0; i < truth.size() && i < other.size(); ++i) { + const double y = truth[i], x = other[i]; + if (!std::isfinite(x) || !std::isfinite(y)) continue; + const double fitted = std::isfinite(out.slope) ? out.slope * x + out.intercept : x + out.intercept; + residual += (y - fitted) * (y - fitted); + } + out.residual_rmse = std::sqrt(residual / n); + return out; +} + +KinectDepthSample kinect_depth_sample(const KinectDepthRecordFrame& frame) { + KinectDepthSample out; + out.steady_us = frame.steady_us; + out.number = frame.number; + const KinectDepthImageStats stats = kinect_depth_image_stats(frame.pixels.data(), frame.pixels.size()); + out.invalid = stats.invalid; + uint32_t best = 0, slot = 0; + for (uint32_t index = 1; index <= 6; ++index) { + out.player_pixels += stats.players[index]; + if (stats.players[index] > best) { + best = stats.players[index]; + slot = index - 1; + } + } + out.player = best > 0; + if (out.player) out.features = kinect_depth_view_features(frame.pixels.data(), frame.width, frame.height, slot); + return out; +} + +double kinect_depth_lean(float lower_left, float lower_right) { + if (lower_left == 0.0f) return 1.0; + if (lower_left == lower_right) return 0.0; + if (lower_left > lower_right) + return lower_right > 0 ? std::min(3.5, double(lower_left) / double(lower_right) - 1.0) : 3.5; + return -std::min(3.5, double(lower_right) / double(lower_left) - 1.0); +} + +const std::vector kinect_depth_compare_features = { + "centroid_col", "centroid_row", "min_col", "min_row", "max_col", "max_row", "player_pixels", + "count_lower_left", "count_lower_right", "count_upper_left", "count_upper_right", "lean", + "mean_lower_left", "mean_lower_right", "mean_upper_left", "mean_upper_right", "near_count", "mean_depth", + "invalid_ratio", +}; + +double kinect_depth_feature(const KinectDepthSample& s, size_t feature) { + if (feature == 18) return double(s.invalid); + if (!s.player) return none; + const KinectDepthCentroid& c = s.features.centroid; + const KinectDepthSplit& p = s.features.split; + switch (feature) { + case 0: return c.column; + case 1: return c.row; + case 2: return c.min_column; + case 3: return c.min_row; + case 4: return c.max_column; + case 5: return c.max_row; + case 6: return c.pixels; + case 7: case 8: case 9: case 10: return double(p.counts[feature - 7]); + case 11: return kinect_depth_lean(p.counts[lower_left], p.counts[lower_right]); + // An empty part's mean is the title's 65528: not a depth to compare. + case 12: case 13: case 14: case 15: + return p.written && p.counts[feature - 12] > 0 ? double(p.means[feature - 12]) : none; + case 16: return p.written ? double(p.near_count) : none; + case 17: return p.written ? double(p.average) : none; + default: return none; + } +} + +std::vector kinect_detector_event_starts(const std::vector& rows, size_t detector) { + std::vector starts; + if (detector >= kinect_race_detector_count) return starts; + std::vector> previous; // per body record, its last answer + for (const KinectRaceRecord& row : rows) { + const int value = row.original[detector]; + if (value < 0) continue; // not asked on this update + auto it = std::find_if(previous.begin(), previous.end(), [&](const auto& p) { return p.first == row.record; }); + if (it == previous.end()) it = previous.insert(previous.end(), {row.record, -1}); + if (value == 1 && it->second != 1) starts.push_back(row.steady_us); + it->second = value; + } + std::sort(starts.begin(), starts.end()); + return starts; +} + +KinectEventAmplitudes kinect_event_amplitudes(const std::vector& times, const std::vector& truth, + const std::vector& other, const std::vector& starts, + bool upward, int64_t baseline_from_us, int64_t baseline_to_us, + int64_t response_us) { + KinectEventAmplitudes out; + out.events = starts.size(); + double sum_truth = 0, sum_other = 0; + const auto amplitude = [&](const std::vector& v, int64_t t) { + double base = 0, peak = upward ? -INFINITY : INFINITY; + size_t n = 0, m = 0; + const size_t first = size_t(std::lower_bound(times.begin(), times.end(), t + baseline_from_us) - times.begin()); + for (size_t i = first; i < times.size() && i < v.size() && times[i] <= t + response_us; ++i) { + if (!std::isfinite(v[i])) continue; + if (times[i] <= t + baseline_to_us) { + base += v[i]; + ++n; + } + if (times[i] >= t) { + peak = upward ? std::max(peak, v[i]) : std::min(peak, v[i]); + ++m; + } + } + return n && m ? peak - base / double(n) : none; + }; + for (const int64_t t : starts) { + const double a = amplitude(truth, t), b = amplitude(other, t); + if (!std::isfinite(a) || !std::isfinite(b)) continue; + ++out.measured; + sum_truth += a; + sum_other += b; + out.same_direction += (a > 0) == (b > 0); + } + out.truth = out.measured ? sum_truth / double(out.measured) : none; + out.other = out.measured ? sum_other / double(out.measured) : none; + return out; +} + +const std::array kinect_depth_view_fields = { + "centroid_col", "centroid_row", "min_col", "min_row", "max_col", "max_row", + "count_lower_left", "count_lower_right", "count_upper_left", "count_upper_right", + "mean_lower_left", "mean_lower_right", "mean_upper_left", "mean_upper_right", "near_count", "mean_depth", +}; + +bool KinectDepthViewCheck::all_equal() const { + for (size_t i = 0; i < kinect_depth_view_field_count; ++i) + if (compared[i] && !equal[i]) return false; + return true; +} + +namespace { +std::string text(uint32_t value) { return std::to_string(value); } +std::string text(float value) { + char out[32]; + std::snprintf(out, sizeof out, "%.9g", double(value)); + return out; +} +// Bit for bit (a NaN the CSV could not hold is never equal). +bool same(float a, float b) { + uint32_t x = 0, y = 0; + std::memcpy(&x, &a, 4); + std::memcpy(&y, &b, 4); + return x == y; +} +} + +KinectDepthViewCheck kinect_depth_view_check(const KinectDepthViewRecord& game, const KinectDepthFeatures& tool) { + KinectDepthViewCheck out; + const KinectDepthCentroid& g = game.centroid; + const KinectDepthCentroid& t = tool.centroid; + const std::array, 6> integers = {{{g.column, t.column}, {g.row, t.row}, + {g.min_column, t.min_column}, {g.min_row, t.min_row}, {g.max_column, t.max_column}, {g.max_row, t.max_row}}}; + for (size_t i = 0; i < 6; ++i) { + out.compared[i] = true; + out.equal[i] = integers[i].first == integers[i].second; + out.game[i] = text(integers[i].first); + out.tool[i] = text(integers[i].second); + } + const KinectDepthSplit& s = tool.split; + for (size_t i = 0; i < 4; ++i) { + out.compared[6 + i] = true; + out.equal[6 + i] = same(game.counts[i], s.counts[i]); + out.game[6 + i] = text(game.counts[i]); + out.tool[6 + i] = text(s.counts[i]); + out.game[10 + i] = text(game.means[i]); + out.tool[10 + i] = s.written ? text(s.means[i]) : "(not written)"; + } + out.game[14] = text(game.near_count); + out.tool[14] = s.written ? text(s.near_count) : "(not written)"; + out.game[15] = text(game.average); + out.tool[15] = s.written ? text(s.average) : "(not written)"; + if (s.written) { + for (size_t i = 0; i < 4; ++i) { + out.compared[10 + i] = true; + out.equal[10 + i] = same(game.means[i], s.means[i]); + } + out.compared[14] = out.compared[15] = true; + out.equal[14] = game.near_count == s.near_count; + out.equal[15] = same(game.average, s.average); + } + return out; +} + +} diff --git a/src/kinect_depth_compare.h b/src/kinect_depth_compare.h new file mode 100644 index 0000000..4e69559 --- /dev/null +++ b/src/kinect_depth_compare.h @@ -0,0 +1,124 @@ +#pragma once +#include "kinect_depth_features.h" +#include "kinect_race_record.h" +#include "kinect_record_depth.h" + +#include +#include +#include +#include + +namespace sfr { + +// Comparing two depth recordings of the same movement at the same time +// (sfr_kinect_depth_compare; docs/kinect-sensor.md): a real Kinect v1's +// images, which the game's depth view reads on a console, against the ones a +// Kinect v2 makes for the game (kinect_v2_depth.h), feature by feature of +// the title's own depth view (kinect_depth_features.h). The two series are +// put on the host's steady clock (both recorders share it), their remaining +// lag found by cross-correlating the vertical centroid, and then each +// feature compared: correlation, the straight line taking the v2's value to +// the v1's (scale and offset), the RMS difference. +// +// None of this is Windows', so it is tested everywhere (kinect_depth_test). + +// One time series: times in microseconds (rising), values (NaN: none). +struct KinectSeries { + std::vector t; + std::vector v; + void add(int64_t time, double value) { t.push_back(time); v.push_back(value); } +}; + +// The value at a time by linear interpolation between the two samples +// around it, both present and at most max_gap_us apart; NaN otherwise. A +// time exactly on a sample takes it. +double kinect_series_at(const KinectSeries& series, int64_t time, int64_t max_gap_us); + +// The lag of b behind a, a(t) ~ b(t + lag): both resampled every step_us +// over their common span, then the Pearson correlation of the standardized +// samples for each lag in [-max_lag_us, max_lag_us] (step_us apart), the +// best refined by a parabola through it and its neighbours. found is false +// when too few samples overlap (fewer than min_pairs) or one is constant. +struct KinectLag { + bool found = false; + double lag_us = 0; + double r = 0; // the correlation at the best whole step + size_t pairs = 0; // samples compared at it +}; +KinectLag kinect_cross_correlate(const KinectSeries& a, const KinectSeries& b, int64_t max_lag_us, int64_t step_us, + size_t min_pairs = 20); + +// truth ~ slope * other + intercept, least squares, over the pairs where both +// are present. +struct KinectFit { + size_t n = 0; + double r = 0; // Pearson correlation (NaN when one is constant) + double slope = 0, intercept = 0; + double rmse = 0; // RMS of truth - other + double residual_rmse = 0; // RMS of truth - (slope * other + intercept) + double mean_truth = 0, mean_other = 0; +}; +KinectFit kinect_fit(const std::vector& truth, const std::vector& other); + +// A depth file's per-image summary for the comparison: the player is the +// slot with the most pixels in that image (a v1's skeleton slot and a v2's +// body index need not agree, so each file chooses its own). +struct KinectDepthSample { + int64_t steady_us = 0; + uint64_t number = 0; + float invalid = 0; // pixels without a depth, 0..1 + uint32_t player_pixels = 0; // pixels of any player + bool player = false; // some player in the image + KinectDepthFeatures features; // of the chosen slot +}; +KinectDepthSample kinect_depth_sample(const KinectDepthRecordFrame& frame); + +// The race's lean from the counts, as the title reads +640 against +644 +// (822C6200, docs/race-controls.md): ll > lr leans right ll/lr - 1, lr > ll +// left (negative) lr/ll - 1, at most 3.5 either way; a zero ll reads as 1.0. +double kinect_depth_lean(float lower_left, float lower_right); + +// The compared features, by name, and their value in a sample (NaN when +// the image does not give it: no player, or means of an image too sparse). +extern const std::vector kinect_depth_compare_features; +double kinect_depth_feature(const KinectDepthSample& sample, size_t feature); + +// The game's own crouches and jumps as events (sfr_kinect_depth_compare +// --events): the start of every run of 1 in a detector's original answer in +// the race CSV (kinect_race_record.h; per body record, rows where it was not +// asked skipped), its steady_us. +std::vector kinect_detector_event_starts(const std::vector& rows, size_t detector); + +// How far a feature moves at each event, on both sensors' aligned series +// (times rising; truth the v1's, other the v2's, NaN for none): the mean over +// [t + baseline_from, t + baseline_to] is the baseline, the response is the +// largest (upward) or smallest value over [t, t + response] less it. An +// event counts (measured) when both series give both. The means of the +// measured events, and how many moved the same way on both (signs alike). +struct KinectEventAmplitudes { + size_t events = 0, measured = 0, same_direction = 0; + double truth = 0, other = 0; // mean amplitudes (NaN when none measured) + double ratio() const { return other / truth; } +}; +KinectEventAmplitudes kinect_event_amplitudes(const std::vector& times, const std::vector& truth, + const std::vector& other, const std::vector& starts, + bool upward, int64_t baseline_from_us = -2000000, + int64_t baseline_to_us = -500000, int64_t response_us = 1200000); + +// Checking the offline features against what the game's depth view wrote +// for the same image (the race CSV's view columns, kinect_race_record.h): +// field by field, exactly (floats bit for bit). The centroid, extents and +// the four counts always; the means, near count and average only when the +// image had enough of the player for the game to write them (otherwise the +// game's memory still holds an older image's). +constexpr size_t kinect_depth_view_field_count = 16; +extern const std::array kinect_depth_view_fields; +struct KinectDepthViewCheck { + std::array compared{}, equal{}; + bool all_equal() const; + // The two values of a field, as text, for a report. + std::array game, tool; +}; +KinectDepthViewCheck kinect_depth_view_check(const KinectDepthViewRecord& game, const KinectDepthFeatures& tool); + +} diff --git a/src/kinect_depth_features.cpp b/src/kinect_depth_features.cpp new file mode 100644 index 0000000..7d687f6 --- /dev/null +++ b/src/kinect_depth_features.cpp @@ -0,0 +1,188 @@ +#include "kinect_depth_features.h" + +#include +#include +#include +#include + +namespace sfr { + +namespace { +constexpr uint32_t lanes = 8; +constexpr uint16_t depth_mask = 0xFFF8; +// An empty part's mean (the vector at 821AC140). +constexpr float empty_mean = 65528.0f; + +uint16_t add16(uint16_t a, uint32_t b) { return uint16_t(std::min(0xFFFF, uint32_t(a) + b)); } +// 32-bit saturating sums of non-negative values come out as the true sum +// clamped, whatever their order (each partial sum is clamped the same way). +uint32_t clamp32(uint64_t value) { return uint32_t(std::min(value, 0xFFFFFFFFull)); } +} + +KinectDepthCentroid kinect_depth_view_centroid(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot) { + KinectDepthCentroid out; + const uint16_t player = uint16_t(slot + 1); + const uint32_t vectors = width >> 3; + // The splats the code starts from: the smallest column is the width's + // low halfword, the rows its own counters (82439998..82439B7C). + std::array min_column, max_column{}, count{}; + min_column.fill(uint16_t(width)); + uint64_t column_sum = 0, row_sum = 0; + uint32_t min_row = height, max_row = 0; + uint16_t row_lane = 0; // v3: the row, a saturating 16-bit splat + const uint16_t* p = pixels; + for (uint32_t row = 0; row < height; ++row) { + std::array columns{}, rows{}, counts{}; + for (uint32_t v = 0; v < vectors; ++v, p += lanes) + for (uint32_t lane = 0; lane < lanes; ++lane) { + const uint16_t column = uint16_t(std::min(0xFFFF, v * lanes + lane)); + if ((p[lane] & ~depth_mask) != player) continue; + min_column[lane] = std::min(min_column[lane], column); + max_column[lane] = std::max(max_column[lane], column); + columns[lane] = add16(columns[lane], column); + rows[lane] = add16(rows[lane], row_lane); + counts[lane] = add16(counts[lane], 1); + } + row_lane = add16(row_lane, 1); + bool any = false; + for (uint32_t lane = 0; lane < lanes; ++lane) any = any || counts[lane]; + if (!any) continue; + for (uint32_t lane = 0; lane < lanes; ++lane) { + count[lane] = add16(count[lane], counts[lane]); + column_sum += columns[lane]; + row_sum += rows[lane]; + } + if (row < min_row) min_row = row; + if (row > max_row) max_row = row; + } + uint32_t total = 0; + for (uint16_t c : count) total += c; + out.pixels = total; + if (total) { + out.column = clamp32(column_sum) / total; + out.row = clamp32(row_sum) / total; + } else { + out.column = width >> 1; + out.row = height >> 1; + } + out.min_column = *std::min_element(min_column.begin(), min_column.end()); + // The largest column as the title takes it: lanes e and e + 4 merged by + // vminuw (82439D30), not vmaxuw, then the largest of the four. A player + // whose rightmost columns all fall in lanes whose partner lane saw less + // gets a smaller "largest column" -- kept, as it is what the game has. + uint16_t largest = 0; + for (uint32_t lane = 0; lane < 4; ++lane) + largest = std::max(largest, std::min(max_column[lane], max_column[lane + 4])); + out.max_column = largest; + out.min_row = min_row; + out.max_row = max_row; + return out; +} + +KinectDepthSplit kinect_depth_view_split(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot, + uint32_t centroid_column, uint32_t centroid_row) { + KinectDepthSplit out; + const uint16_t player = uint16_t(slot + 1); + const uint32_t vectors = width >> 3; + const uint16_t cc = uint16_t(centroid_column); + // Per part (KinectDepthPart order) the lanes' counts and the depth sum. + std::array, 4> counts{}; + std::array sums{}; + const uint16_t* p = pixels; + const auto rows = [&](uint32_t first, uint32_t last, uint32_t left, uint32_t right) { + for (uint32_t row = first; row < last; ++row) + for (uint32_t v = 0; v < vectors; ++v, p += lanes) + for (uint32_t lane = 0; lane < lanes; ++lane) { + const uint16_t column = uint16_t(std::min(0xFFFF, v * lanes + lane)); + if ((p[lane] & ~depth_mask) != player) continue; + const uint32_t part = cc > column ? left : right; + counts[part][lane] = add16(counts[part][lane], 1); + sums[part] += p[lane] & depth_mask; + } + }; + // The upper part (rows above the centroid's), then the lower; when the + // centroid row is 0 or the height, that part's loop does not run. + rows(0, std::min(centroid_row, height), upper_left, upper_right); + if (centroid_row != height) rows(std::min(centroid_row, height), height, lower_left, lower_right); + std::array count{}; + std::array c{}, s{}; + for (uint32_t part = 0; part < 4; ++part) { + for (uint16_t n : counts[part]) count[part] += n; + // fcfid and frsp: the integer through a double into a float. + c[part] = float(double(count[part])); + s[part] = float(double(clamp32(sums[part]))); + } + // ((upper right + upper left) + lower right) + lower left, each a float + // (fadds 8243A234, 8243A240, 8243A260), against (width * height) >> 9. + float total = c[upper_right] + c[upper_left]; + total = total + c[lower_right]; + total = total + c[lower_left]; + const uint32_t threshold = (uint32_t(width * height) >> 9) & 0x7FFFFF; + if (total < float(double(threshold))) return out; // only the counts, zeroed + out.written = true; + out.counts = c; + for (uint32_t part = 0; part < 4; ++part) { + // vrefp and two Newton-Raphson steps, as the recompiled code does + // them: separate multiplies and adds (vnmsubfp is c - a*b). + const float x = c[part]; + const float r0 = 1.0f / x; + float t = x * r0; + float e = 1.0f - t; + float r1 = r0 * e; + r1 = r1 + r0; + const bool refined = r1 == r1; // not NaN (vcmpeqfp) + t = x * r1; + e = 1.0f - t; + float r2 = r1 * e; + r2 = r2 + r1; + const float reciprocal = refined ? r2 : r0; + const float mean = s[part] * reciprocal; + out.means[part] = x > 0.0f ? mean : empty_mean; + } + // (((UR + UL) + LR) + LL) * 0.25 (the float at 8218FCE4). + float average = out.means[upper_right] + out.means[upper_left]; + average = average + out.means[lower_right]; + average = average + out.means[lower_left]; + average = average * 0.25f; + out.average = average; + // fctidz, the low word, its low halfword & 0xFFF8. + const double truncated = std::trunc(double(average)); + const int64_t whole = truncated >= 9.2e18 ? std::numeric_limits::max() + : truncated <= -9.2e18 ? std::numeric_limits::min() + : int64_t(truncated); + const uint16_t reference = uint16_t(uint32_t(uint64_t(whole))) & depth_mask; + std::array close{}; + const size_t vectors_all = (size_t(uint32_t(width * height)) >> 3); + for (size_t v = 0; v < vectors_all; ++v) + for (uint32_t lane = 0; lane < lanes; ++lane) { + const uint16_t value = pixels[v * lanes + lane]; + if ((value & ~depth_mask) != player) continue; + const uint16_t depth = value & depth_mask; + const uint16_t difference = reference > depth ? uint16_t(reference - depth) : uint16_t(depth - reference); + if (int16_t(600) > int16_t(difference)) close[lane] = add16(close[lane], 1); + } + for (uint16_t n : close) out.near_count += n; + return out; +} + +KinectDepthFeatures kinect_depth_view_features(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot) { + KinectDepthFeatures out; + out.slot = slot; + out.centroid = kinect_depth_view_centroid(pixels, width, height, slot); + out.split = kinect_depth_view_split(pixels, width, height, slot, out.centroid.column, out.centroid.row); + return out; +} + +KinectDepthImageStats kinect_depth_image_stats(const uint16_t* pixels, size_t count) { + KinectDepthImageStats out; + if (!count) return out; + size_t invalid = 0; + for (size_t i = 0; i < count; ++i) { + invalid += (pixels[i] & depth_mask) == 0; + ++out.players[pixels[i] & 7]; + } + out.invalid = float(double(invalid) / double(count)); + return out; +} + +} diff --git a/src/kinect_depth_features.h b/src/kinect_depth_features.h new file mode 100644 index 0000000..5eb6fa5 --- /dev/null +++ b/src/kinect_depth_features.h @@ -0,0 +1,122 @@ +#pragma once +#include +#include +#include +#include + +namespace sfr { + +// The title's own depth view, again, outside the game: what its two passes +// over the 320x240 depth-and-player-index image compute for one player, so +// that a recorded image (kinect_record_depth.h) gives the very numbers the +// race's detectors read (docs/kinect-sensor.md, "深度檢視的特徵"). +// +// The depth view (82439530, on the Kinect manager's worker thread) runs once +// per new depth image for each of the manager's two player slots (+7740, +// +7744; a skeleton slot 0..5, or negative for none): +// 82439998(slot, manager+7584, image, width, height) pass one +// 82439D80(slot, record+640, manager+7584+8*slot, image, width, height) +// where record = manager + 2784 + 800*slot is the player's body record (its +// +768 holds the skeleton, +784 the slot: 82438A98). Both are VMX code over +// eight pixels at a time; the image is read as one packed array of uint16, +// width/8 vectors a row (width a multiple of 8; 320 is). +// +// A pixel is the player's when (pixel & 7) == slot + 1: the mask 0xFFF8 is +// cleared (vandc with a splat of 0xFFF8: 82439BA4, 82439F80, 8243A028, +// 8243A430) and the rest compared with a splat of slot + 1 from a table of +// 1..6 on the stack (vcmpequh: 82439BC4, 82439FA0, 8243A048, 8243A434). Its +// depth is pixel & 0xFFF8 (millimetres << 3, never shifted down). +// +// Pass one, 82439998 (82439998..82439D7C): +// for every row r and column c of a player pixel: the sum of the columns, +// of the rows and the count, in eight 16-bit lanes (column c in lane c % 8) +// that saturate (vadduhs, 82439BDC..82439BE4; a row's lane sums are added +// into 32-bit words after the row, 82439BFC..82439C20), the lanes' +// smallest and largest column (vminuh/vmaxuh and vsel, +// 82439BB4..82439BD8), the first and last row with one (82439C08..82439C34). +// centroid = (column sum / count, row sum / count), unsigned division +// (divwu, 82439CB4, 82439CDC), or (width/2, height/2) with no pixel +// (82439CE8..82439CF4); stored as two words at manager+7584+8*slot +// (82439CB8, 82439CE0). The smallest column and row at manager+7632+8*slot +// (stwx at 82439D50, stw at 82439D28), the largest at manager+7680+8*slot +// (stwx at 82439D78, stw at 82439D64). The largest column is not quite +// that: the eight lanes' largest are merged pairwise (lane e with e + 4) +// by vminuw (82439D30) where the smallest column's code uses vminuw too +// (82439D1C), and only then the largest of the four taken (vmaxuw, +// 82439D60, 82439D6C); so it is the largest column whose partner lane +// reached at least as far. Without a pixel: smallest column = width, +// smallest row = height, largest = 0, 0. The count itself is not kept. +// Pass two, 82439D80 (82439D80..8243A4A4), with the centroid (cc, cr) of +// pass one: rows 0..cr-1 are the upper part (82439F68..82439FF0), rows +// cr..height-1 the lower (8243A010..8243A098); a pixel is on the left +// when its column < cc (vcmpgtuh cc > column, 82439F90, 8243A038), on the +// right otherwise. Per part and side: the player's pixel count (16-bit +// saturating lanes) and depth sum (32-bit words). Then (8243A09C..): +// total = ((upper right + upper left) + lower right) + lower left, float +// (fadds 8243A234, 8243A240, 8243A260) +// if total < float((width*height) >> 9) (150 for 320x240; mullw +// 8243A1D8, rlwinm 8243A1E4, fcmpu 8243A294): +// record+640..+655 = 0 (stvx v0, 8243A29C) and nothing else is +// written: the means, near count and average keep an older image's. +// else +// +640 lower left count, +644 lower right, +648 upper left, +652 +// upper right (floats; stvx at 8243A2D4) +// +656.. the same four parts' mean depth (sum * refined 1/count, +// vrefp and two Newton steps, 8243A300..8243A3A0; 65528.0, the +// constant at 821AC140, for a part with no pixel; stvx at 8243A3A4) +// +676 average = (((UR + UL) + LR) + LL) * 0.25 (8243A3A8..8243A3C4, +// stfs at 8243A3C8) +// +672 near: the player's pixels whose depth is less than 600 (75 mm) +// from the average truncated (fctidz 8243A3CC) and & 0xFFF8, compared +// as signed 16 bits (vcmpgtsh 8243A454; the loop 8243A42C..8243A468 +// goes over the whole image as width*height/8 vectors); stw 8243A4A0. +// The race's lean reads +640 (the image's lower left, the player's right +// as they face the sensor) against +644 (822C6200; nui_race.h). +// +// Every float operation is done one at a time as the recompiled code does +// it (no fused multiply-add), so the results are the game's bit for bit; +// kinect_depth_view_guest_test runs the generated code of both functions on +// the same images to check that. + +// Pass one's output for one slot. +struct KinectDepthCentroid { + uint32_t column = 0, row = 0; // manager+7584+8*slot, +4 + uint32_t min_column = 0, min_row = 0; // manager+7632+8*slot, +4 + uint32_t max_column = 0, max_row = 0; // manager+7680+8*slot, +4 + uint32_t pixels = 0; // the count (not kept by the game) +}; + +// Pass two's output (record +640..+679) for one slot. +struct KinectDepthSplit { + // Whether the image had enough of the player for more than the zeroed + // counts (total >= (width*height) >> 9); false leaves means, near and + // average as the previous image's, which the tool does not know. + bool written = false; + // lower left, lower right, upper left, upper right (+640, +644, +648, +652) + std::array counts{}; + // the same parts' mean depth, millimetres << 3 (+656, +660, +664, +668) + std::array means{}; + uint32_t near_count = 0; // +672 (not "near": a macro in ) + float average = 0; // +676 +}; +enum KinectDepthPart : uint32_t { lower_left = 0, lower_right = 1, upper_left = 2, upper_right = 3 }; + +KinectDepthCentroid kinect_depth_view_centroid(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot); +KinectDepthSplit kinect_depth_view_split(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot, + uint32_t centroid_column, uint32_t centroid_row); + +// Both passes for one slot, and the image's share of pixels without a depth +// (depth 0 after the mask) and its pixels of any player, for a summary. +struct KinectDepthFeatures { + uint32_t slot = 0; + KinectDepthCentroid centroid; + KinectDepthSplit split; +}; +KinectDepthFeatures kinect_depth_view_features(const uint16_t* pixels, uint32_t width, uint32_t height, uint32_t slot); +struct KinectDepthImageStats { + float invalid = 0; // pixels with no depth, 0..1 + std::array players{}; // pixels of each player index (0 none) +}; +KinectDepthImageStats kinect_depth_image_stats(const uint16_t* pixels, size_t count); + +} diff --git a/src/kinect_ideal_diagram.cpp b/src/kinect_ideal_diagram.cpp new file mode 100644 index 0000000..2f68620 --- /dev/null +++ b/src/kinect_ideal_diagram.cpp @@ -0,0 +1,93 @@ +#include "kinect_ideal_diagram.h" + +#include +#include + +namespace sfr { +namespace { +float half_tangent(float degrees) { return float(std::tan(double(degrees) * 3.14159265358979323846 / 360.0)); } +} + +const char* kinect_depth_choice_name(KinectDepthChoiceView view) { + switch (view) { + case KinectDepthChoiceView::fit: return "fit"; + case KinectDepthChoiceView::sensor: return "sensor"; + case KinectDepthChoiceView::ideal: break; + } + return "ideal"; +} + +KinectIdealDiagram kinect_ideal_diagram(const KinectIdealSnapshot& snapshot, float full_width, float full_height, + float ruler_left, float ruler_bottom) { + KinectIdealDiagram diagram; + ruler_left = std::max(0.0f, ruler_left); + ruler_bottom = std::max(0.0f, ruler_bottom); + const float width = full_width - ruler_left, height = full_height - ruler_bottom; + if (!(width > 0) || !(height > 0)) return diagram; + const float v1_tangent = half_tangent(kinect_v1_horizontal_fov_degrees); + const float v2_tangent = half_tangent(kinect_v2_horizontal_fov_degrees); + const float camera_x = snapshot.camera_x, camera_z = snapshot.camera_z; + const float players_z = camera_z + snapshot.distance; + // What must fit: both cameras, the players' line and every player, and + // at least 2.5 m to each side and 4 m deep. + float half_width = std::max(2.5f, std::fabs(camera_x) + 0.6f); + float z_low = std::min(0.0f, camera_z), z_high = std::max({4.5f, players_z + 0.5f, z_low + 2.5f}); + for (const KinectIdealPlayer& player : snapshot.players) { + if (!std::isfinite(player.x) || !std::isfinite(player.z)) continue; + half_width = std::max(half_width, std::fabs(player.x) + 0.4f); + z_low = std::min(z_low, player.z); + z_high = std::max(z_high, player.z + 0.5f); + } + half_width = std::min(half_width, 8.0f); + z_high = std::min(z_high, z_low + 10.0f); + // Room under the lower camera for its label. + z_low -= 0.45f; + diagram.scale = std::min(width / (2 * half_width), height / (z_high - z_low)); + diagram.z_low = z_low; + diagram.origin_x = ruler_left + width / 2; + // The slack goes above, so the cameras stay at the bottom. + diagram.origin_y = height; + diagram.v1_near_mark = diagram.place(camera_x, camera_z + kinect_v1_near_metres); + diagram.v1_far_mark = diagram.place(camera_x, camera_z + kinect_v1_far_metres); + // The rulers, every half metre over the whole drawing. + const float drawn_low = z_low, drawn_high = z_low + height / diagram.scale; + for (int half = int(std::ceil(drawn_low * 2 - 1e-4f)); half <= int(std::floor(drawn_high * 2 + 1e-4f)); ++half) + diagram.depth_ticks.push_back({diagram.place(0, half * 0.5f).y, half * 0.5f, half % 2 == 0}); + const float drawn_side = width / 2 / diagram.scale; + for (int half = -int(std::floor(drawn_side * 2 + 1e-4f)); half <= int(std::floor(drawn_side * 2 + 1e-4f)); ++half) + diagram.lateral_ticks.push_back({diagram.place(half * 0.5f, 0).x, half * 0.5f, half % 2 == 0}); + const auto place = [&](float x, float z) { return diagram.place(x, z); }; + const float far_z = z_low + height / diagram.scale; // the drawing's top edge + diagram.v2_view = {place(0, 0), place(-v2_tangent * far_z, far_z), place(v2_tangent * far_z, far_z)}; + const float v1_reach = std::max(0.0f, far_z - camera_z); + diagram.v1_view = {place(camera_x, camera_z), place(camera_x - v1_tangent * v1_reach, camera_z + v1_reach), + place(camera_x + v1_tangent * v1_reach, camera_z + v1_reach)}; + diagram.v1_near_y = place(0, camera_z + kinect_v1_near_metres).y; + diagram.v1_far_y = place(0, camera_z + kinect_v1_far_metres).y; + diagram.v1_near_x = {place(camera_x - v1_tangent * kinect_v1_near_metres, 0).x, + place(camera_x + v1_tangent * kinect_v1_near_metres, 0).x}; + diagram.v1_far_x = {place(camera_x - v1_tangent * kinect_v1_far_metres, 0).x, + place(camera_x + v1_tangent * kinect_v1_far_metres, 0).x}; + diagram.players_y = place(0, players_z).y; + for (const KinectIdealPlayer& player : snapshot.players) { + if (!std::isfinite(player.x) || !std::isfinite(player.z)) continue; + KinectDiagramPlayer& drawn = diagram.players.emplace_back(); + drawn.at = place(player.x, player.z); + drawn.slot = player.slot; + drawn.v1_depth = player.z - camera_z; + drawn.v2_depth = player.sensor_z; + drawn.in_v1_view = drawn.v1_depth > 0 && std::fabs(player.x - camera_x) <= v1_tangent * drawn.v1_depth; + drawn.in_v1_range = drawn.v1_depth >= kinect_v1_near_metres && drawn.v1_depth <= kinect_v1_far_metres; + drawn.in_v2_view = player.z > 0 && std::fabs(player.x) <= v2_tangent * player.z && + player.sensor_z >= kinect_v2_near_metres && player.sensor_z <= kinect_v2_far_metres; + // Both cameras' views at this depth. + const float v1_half = std::max(0.0f, v1_tangent * drawn.v1_depth), v2_half = std::max(0.0f, v2_tangent * player.z); + const float left = std::max(camera_x - v1_half, -v2_half), right = std::min(camera_x + v1_half, v2_half); + drawn.band_left = place(left, 0).x; + drawn.band_right = place(right, 0).x; + if (!(right > left) || drawn.v1_depth <= 0) drawn.band_left = drawn.band_right + 1; // seen by neither + } + return diagram; +} + +} diff --git a/src/kinect_ideal_diagram.h b/src/kinect_ideal_diagram.h new file mode 100644 index 0000000..ae691d0 --- /dev/null +++ b/src/kinect_ideal_diagram.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include + +namespace sfr { + +// The Kinect v2 depth view as the launcher chooses it, for the preview it +// opens (kinect_preview.h): which view, and Recompiled's tuning in metres and +// seconds (the launcher keeps centimetres and tenths: LauncherSettings). +// Plain data, so the launcher holds it on every platform; a Windows sensor +// turns it into its KinectDepthSettings (kinect_depth_settings, +// kinect_v2_depth.h), the same values the game reads from +// SFR_KINECT_V2_DEPTH_*. +enum class KinectDepthChoiceView : uint8_t { ideal, fit, sensor }; +struct KinectDepthChoice { + KinectDepthChoiceView view = KinectDepthChoiceView::ideal; + float distance = 2.5f, height = 1.0f, follow = 2.0f, deadband = 0.15f; + float skeleton_distance = 0; // 0: the depth's distance + bool operator==(const KinectDepthChoice&) const = default; +}; +// "ideal", "fit" or "sensor" (SFR_KINECT_V2_DEPTH_VIEW's names). +const char* kinect_depth_choice_name(KinectDepthChoiceView view); + +// Where the Recompiled (ideal) view's virtual v1 stands and where the +// players are, top-down, as the depth synthesis itself has them +// (kinect_ideal_snapshot, kinect_v2_depth.h): x and z in metres, in the +// levelled space the ideal view works in (the v2's own camera space turned so +// the floor is level; the v2's own space while no floor is known). The v2 +// stands at the origin of that space looking along +z. +struct KinectIdealPlayer { + uint32_t tracking_id = 0; + int slot = -1; // the title's player: 0 for 1P, 1 for 2P, -1 neither + float x = 0, z = 0; // SpineBase, levelled space + float sensor_z = 0; // the v2's own depth to it (its camera space z) +}; +struct KinectIdealSnapshot { + bool valid = false; // false: no sensor, or one that makes no depth + bool floor_known = false, placed = false; + float camera_x = 0, camera_z = 0; // the virtual v1, levelled space + float distance = 2.5f; // from the camera to the players' depth line + std::vector players; +}; + +// The two cameras' horizontal fields of view, and the distances each measures. +constexpr float kinect_v2_horizontal_fov_degrees = 70.6f, kinect_v1_horizontal_fov_degrees = 57.0f; +constexpr float kinect_v1_near_metres = 0.8f, kinect_v1_far_metres = 4.0f; +constexpr float kinect_v2_near_metres = 0.5f, kinect_v2_far_metres = 4.5f; + +// The top-down drawing of a snapshot in a width x height area (pixels, (0, 0) +// its top left; y down): the v2 at the bottom looking up, the virtual v1 +// behind or in front of it, the players. Pure arithmetic shared by the +// preview window (GDI) and the launcher (ImGui), and tested +// (kinect_ideal_diagram_test). x is drawn as the depth images draw it (the +// camera space's +x to the right), so the drawing agrees with the preview's +// pictures. +struct KinectDiagramPoint { + float x = 0, y = 0; +}; +struct KinectDiagramPlayer { + KinectDiagramPoint at; + int slot = -1; + float v1_depth = 0; // to the virtual v1, along its axis: the depth the game sees + float v2_depth = 0; // to the real v2 (its own z) + bool in_v1_view = false; // inside the virtual v1's horizontal field of view + bool in_v1_range = false; // and 0.8..4.0 m from it + bool in_v2_view = false; // inside the real v2's field of view and range + // The part of the players' row both cameras see at this player's depth + // (pixels; left > right when none). + float band_left = 0, band_right = 0; +}; +// A ruler mark: where it is (y for the depth ruler, x for the lateral one; +// pixels), its distance in metres, and whether it is a whole metre (a +// label and a faint grid line) or a half. +struct KinectDiagramTick { + float at = 0; + float metres = 0; + bool whole = false; +}; +struct KinectIdealDiagram { + float scale = 0; // pixels a metre; 0 when nothing fits + // The rulers: depth from the real v2 (its z) up the left edge, every + // 0.5 m over the drawing's whole height; across along the bottom (x, 0 + // on the v2's axis), every 0.5 m over its whole width. They sit in the + // strips the caller reserved (ruler_left, ruler_bottom) beside the drawing. + std::vector depth_ticks, lateral_ticks; + // The virtual v1's usable depths on its own axis: 0.8 m and 4.0 m from it. + KinectDiagramPoint v1_near_mark, v1_far_mark; + // Each camera's field of view: the apex (the camera) and the two far + // corners at the drawing's far edge (beyond the area at the sides; clip). + std::array v2_view{}, v1_view{}; + // The virtual v1's usable depths (0.8 and 4.0 m) across its view: y and + // each line's ends. + float v1_near_y = 0, v1_far_y = 0; + std::array v1_near_x{}, v1_far_x{}; + // The players' depth line (the camera stands distance behind it). + float players_y = 0; + std::vector players; + // A point of the levelled space (metres) in the drawing. + float origin_x = 0, origin_y = 0, z_low = 0; + KinectDiagramPoint place(float x, float z) const { return {origin_x + x * scale, origin_y - (z - z_low) * scale}; } +}; +// At least +-2.5 m wide and from the nearer camera to 4.5 m deep, grown so +// the cameras, the players' line and every player fit, with room under the +// lower camera for its label. ruler_left and ruler_bottom are pixels kept +// free at the left and bottom edges for the rulers' labels: the drawing is +// the rest. +KinectIdealDiagram kinect_ideal_diagram(const KinectIdealSnapshot& snapshot, float width, float height, + float ruler_left = 0, float ruler_bottom = 0); + +} diff --git a/src/kinect_nui_record.cpp b/src/kinect_nui_record.cpp new file mode 100644 index 0000000..64dbe9b --- /dev/null +++ b/src/kinect_nui_record.cpp @@ -0,0 +1,256 @@ +#include "kinect_nui_record.h" +#include "nui_skeleton.h" + +#include +#include +#include +#include +#include + +namespace sfr { + +namespace v = kinect_nui_value_kind; +// The race detectors of NUI_SKELETON_ANALYSIS.md (section 3.3 and appendix +// A), by their original functions, with the value each keeps besides its +// bits: brake and grab strength (entry +80, percent), the stance (entry +// +72), the trick's turn (entry +56, degrees), the lever and handle (entry +// +82/+83, signed percent), the front curve (the second entry's +81), the +// steam wipe's x (detector +76; its y is wipe_y) and the steering (entry +76, +// signed). +const std::array kinect_nui_detectors = {{ + {0x822C9050, "jump", v::none}, {0x822C8778, "crouch", v::none}, + {0x822CB840, "charge", v::none}, {0x822C8650, "arms_up", v::none}, + {0x822C9E90, "catch_lr", v::none}, {0x822CB340, "catch", v::none}, + {0x822C9180, "flying", v::none}, {0x822C9A80, "accel", v::none}, + {0x822CAF48, "accel2", v::none}, {0x822CA6B0, "side", v::none}, + {0x822C9BF0, "brake", v::entry_byte_80}, {0x822CB0B8, "grab", v::entry_byte_80}, + {0x822CA518, "kick", v::none}, {0x822CBD28, "kick2", v::none}, + {0x822CA810, "power", v::none}, {0x822CBF30, "power2", v::none}, + {0x822CAC90, "stance", v::entry_word_72}, {0x822C9938, "trick", v::entry_float_56}, + {0x822CC2D0, "hammer", v::none}, {0x822CC590, "over_throw", v::none}, + {0x822CCD98, "under_swing", v::none}, {0x822CD068, "under_throw", v::none}, + {0x822CDDB0, "inflator", v::none}, {0x822CC8F0, "shake", v::none}, + {0x822CDEE0, "wipe", v::detector_short_76}, {0x822CD638, "lever", v::entry_signed_82}, + {0x822CD960, "handle", v::entry_signed_83}, {0x822CD3F0, "front_curve", v::second_entry_signed_81}, + {0x822CE118, "swim", v::none}, {0x822C8D88, "hold_hands", v::none}, + {0x822C8958, "steer", v::entry_signed_76}, {0x822CB9B8, "steer_alt", v::entry_signed_76}, +}}; + +int kinect_nui_detector_index(uint32_t address) { + for (uint32_t i = 0; i < kinect_nui_detector_count; ++i) + if (kinect_nui_detectors[i].address == address) return int(i); + return -1; +} + +const std::array kinect_nui_joints = { + nui_joint::hand_left, nui_joint::hand_right, nui_joint::shoulder_left, nui_joint::shoulder_right, + nui_joint::shoulder_center, nui_joint::head, nui_joint::spine, nui_joint::hip_center}; +const std::array kinect_nui_joint_names = { + "hand_l", "hand_r", "shoulder_l", "shoulder_r", "shoulder_c", "head", "spine", "hip_c"}; + +namespace { +constexpr const char* hand_fields[] = {"flags", "theta", "r", "target_x", "target_y", "cursor_x", "cursor_y"}; +constexpr const char* hand_names[] = {"left", "right"}; +constexpr const char* detector_fields[] = {"result", "primary", "secondary", "value"}; +constexpr size_t fixed_columns = 6; +constexpr size_t column_count = fixed_columns + 2 * 7 + 3 * kinect_nui_joint_count + 2 + 4 * kinect_nui_detector_count + 1 + + 1 + kinect_nui_calibration_words; +// The calibration object's whole-number words (pixels). +bool calibration_integer(uint32_t index) { return index * 4 >= 112 && index * 4 <= 120; } + +void put_float(std::string& out, float value) { + out += ','; + if (!std::isfinite(value)) return; + char text[32]; + std::snprintf(text, sizeof text, "%.9g", double(value)); + out += text; +} +void put_int(std::string& out, long long value) { + out += ','; + out += std::to_string(value); +} +bool number(std::string_view text, double& value) { + if (text.empty()) return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtod(copy.c_str(), &end); + return end && *end == 0; +} +} + +std::string kinect_nui_record_header() { + std::string out = "steady_us,update,player,tracking_id,body_slot,racing"; + for (const char* hand : hand_names) + for (const char* field : hand_fields) out.append(",").append(hand).append("_").append(field); + for (const char* joint : kinect_nui_joint_names) + for (const char* axis : {"x", "y", "z"}) out.append(",").append(joint).append("_").append(axis); + out += ",lean_depth,lean_tilt"; + for (const KinectNuiDetector& detector : kinect_nui_detectors) + for (const char* field : detector_fields) out.append(",").append(detector.name).append("_").append(field); + out += ",wipe_y,calib_step"; + for (uint32_t i = 0; i < kinect_nui_calibration_words; ++i) out += ",calib_" + std::to_string(4 * i); + out += '\n'; + return out; +} + +void kinect_nui_record_row(const KinectNuiRecord& r, std::string& out) { + out += std::to_string(r.steady_us); + put_int(out, (long long)r.update); + put_int(out, r.player); + put_int(out, r.tracking_id); + put_int(out, r.body_slot); + put_int(out, r.racing ? 1 : 0); + for (const KinectNuiHand& hand : r.hands) { + if (!r.has_cursor) { out.append(7, ','); continue; } + put_int(out, hand.flags); + for (float value : {hand.theta, hand.r, hand.target_x, hand.target_y, hand.cursor_x, hand.cursor_y}) put_float(out, value); + } + for (const auto& joint : r.joints) { + if (!r.has_joints) { out.append(3, ','); continue; } + for (float value : joint) put_float(out, value); + } + if (r.has_lean) { put_float(out, r.lean_depth); put_float(out, r.lean_tilt); } + else out.append(2, ','); + for (const KinectNuiDetectorAnswer& answer : r.detectors) { + if (answer.result < 0) { out.append(4, ','); continue; } + put_int(out, answer.result); + put_int(out, answer.primary); + put_int(out, answer.secondary); + if (answer.has_value) put_float(out, answer.value); else out += ','; + } + if (r.has_wipe_y) put_float(out, r.wipe_y); else out += ','; + out += ','; + out += r.calibration_step; + for (uint32_t i = 0; i < kinect_nui_calibration_words; ++i) { + if (r.calibration_step.empty()) { out += ','; continue; } + if (calibration_integer(i)) put_int(out, int32_t(r.calibration[i])); + else put_float(out, std::bit_cast(r.calibration[i])); + } + out += '\n'; +} + +bool kinect_parse_nui_row(std::string_view line, KinectNuiRecord& r) { + const auto f = kinect_record_fields(line); + if (f.size() != column_count) return false; + double v = 0; + r = {}; + if (!number(f[0], v)) return false; // the header + r.steady_us = int64_t(v); + size_t at = 1; + const auto next = [&](double fallback = 0) { return number(f[at++], v) ? v : fallback; }; + const auto present = [&](size_t count) { + for (size_t i = 0; i < count; ++i) + if (!f[at + i].empty()) return true; + return false; + }; + r.update = uint64_t(next()); + r.player = uint32_t(next()); + r.tracking_id = uint32_t(next()); + r.body_slot = int32_t(next(-1)); + r.racing = next() != 0; + r.has_cursor = present(14); + for (KinectNuiHand& hand : r.hands) { + hand.flags = uint32_t(next()); + hand.theta = float(next()); + hand.r = float(next()); + hand.target_x = float(next()); + hand.target_y = float(next()); + hand.cursor_x = float(next()); + hand.cursor_y = float(next()); + } + r.has_joints = present(3 * kinect_nui_joint_count); + for (auto& joint : r.joints) + for (float& value : joint) value = float(next()); + r.has_lean = present(2); + r.lean_depth = float(next()); + r.lean_tilt = float(next()); + for (KinectNuiDetectorAnswer& answer : r.detectors) { + answer.result = int(next(-1)); + answer.primary = uint32_t(next()); + answer.secondary = uint32_t(next()); + answer.has_value = !f[at].empty(); + answer.value = float(next()); + } + r.has_wipe_y = !f[at].empty(); + r.wipe_y = float(next()); + r.calibration_step = std::string(f[at++]); + for (uint32_t i = 0; i < kinect_nui_calibration_words; ++i) { + const double value = next(); + r.calibration[i] = calibration_integer(i) ? uint32_t(int32_t(value)) : std::bit_cast(float(value)); + } + return true; +} + +KinectRecordFile* kinect_record_nui() { + static std::mutex lock; + static std::unique_ptr file; + std::lock_guard guard(lock); + const std::string& prefix = kinect_record_prefix(); + if (prefix.empty()) return nullptr; + if (!file) + file = std::make_unique(prefix + "-nui.csv", kinect_nui_record_header(), kinect_record_limit_bytes()); + return file.get(); +} + +void KinectNuiRecorder::detector(uint32_t player, uint32_t address, int result, uint32_t primary, uint32_t secondary, + bool has_value, float value) { + const int index = kinect_nui_detector_index(address); + if (player >= players || index < 0) return; + std::lock_guard guard(lock_); + // Asked more than once in an update (a player's race asks through two + // objects): the last answer and value, every bit any call added. + KinectNuiDetectorAnswer& answer = pending_[player].detectors[size_t(index)]; + answer.result = result; + answer.primary |= primary; + answer.secondary |= secondary; + if (has_value) { + answer.has_value = true; + answer.value = value; + } + ++answer.calls; +} + +void KinectNuiRecorder::wipe_y(uint32_t player, float value) { + if (player >= players) return; + std::lock_guard guard(lock_); + pending_[player].has_wipe_y = true; + pending_[player].wipe_y = value; +} + +void KinectNuiRecorder::lean(uint32_t player, float depth, float tilt) { + if (player >= players) return; + std::lock_guard guard(lock_); + pending_[player].has_lean = true; + pending_[player].lean_depth = depth; + pending_[player].lean_tilt = tilt; +} + +void KinectNuiRecorder::calibration(uint32_t player, const char* step, + const std::array& words) { + if (player >= players || !step || !*step) return; + std::lock_guard guard(lock_); + pending_[player].calibration_step = step; + pending_[player].calibration = words; +} + +void KinectNuiRecorder::take(uint32_t player, KinectNuiRecord& row) { + if (player >= players) return; + std::lock_guard guard(lock_); + KinectNuiRecord& pending = pending_[player]; + row.detectors = pending.detectors; + row.has_lean = pending.has_lean; + row.lean_depth = pending.lean_depth; + row.lean_tilt = pending.lean_tilt; + row.has_wipe_y = pending.has_wipe_y; + row.wipe_y = pending.wipe_y; + row.calibration_step = pending.calibration_step; + row.calibration = pending.calibration; + pending = {}; +} + +KinectNuiRecorder& kinect_nui_recorder() { + static KinectNuiRecorder recorder; + return recorder; +} + +} diff --git a/src/kinect_nui_record.h b/src/kinect_nui_record.h new file mode 100644 index 0000000..ffd4f5d --- /dev/null +++ b/src/kinect_nui_record.h @@ -0,0 +1,115 @@ +#pragma once +#include "kinect_record.h" + +#include +#include +#include +#include +#include + +namespace sfr { + +// The title's side of a recording (kinect_record.h), past what race.csv +// holds: with SFR_KINECT_RECORD set and a real Kinect playing, +// -nui.csv gets one row per input update (824920F8) per player with +// a body record: what the title's own consumers of the skeleton made of it +// on that update, for comparing a v1 session with a v2 one and for every +// later change (NUI_SKELETON_ANALYSIS.md, S2). +// +// - The menu hand cursor of each hand as 82492F30 left it (packet P = +// input + 32 + 2544 player + 16 + 400 hand): flags (P+0), theta (P+60), +// r (P+148), the target (P+96/+100) and the cursor (P+80/+84, pixels). +// - Joints of the player's body record (input block +12) as the title read +// them, after its smoothing and in its mirrored space (-x, y, -z): +// hands, shoulders, shoulder centre, head, spine and hip centre. +// - The race's lean pair from 822C6200 (the means of its two filters, +// source +44 from the depth, +88 from the body's tilt). +// - Every race detector the study lists (kinect_nui_detectors), asked since +// the player's last row: its answer, the bits it added to its result +// entry's +4 (primary) and +20 (secondary) words and its own value where +// it has one (see kinect_nui_detectors); empty when it was not asked. +// - When a race preparation measurement completed since the last row +// (822C65B0 arms forward, 822C6AD8 the body), its step and the +// calibration object's +0..+132 (+112..+120 whole numbers, the rest +// floats). +// Bounded by SFR_KINECT_RECORD_MB and written through the record queue +// like the other files, so the game's thread never waits for the disk. + +struct KinectNuiDetector { + uint32_t address; + const char* name; + // Where its value is: 0 none; else as kinect_nui_value_kind. + uint8_t value; +}; +namespace kinect_nui_value_kind { +constexpr uint8_t none = 0, entry_byte_80 = 1, entry_word_72 = 2, entry_float_56 = 3, entry_signed_82 = 4, + entry_signed_83 = 5, second_entry_signed_81 = 6, detector_short_76 = 7, entry_signed_76 = 8; +} +constexpr uint32_t kinect_nui_detector_count = 32; +extern const std::array kinect_nui_detectors; +int kinect_nui_detector_index(uint32_t address); + +struct KinectNuiDetectorAnswer { + int result = -1; // -1: not asked + uint32_t primary = 0, secondary = 0; + bool has_value = false; + float value = 0; + uint32_t calls = 0; +}; +struct KinectNuiHand { + uint32_t flags = 0; + float theta = 0, r = 0, target_x = 0, target_y = 0, cursor_x = 0, cursor_y = 0; +}; +constexpr uint32_t kinect_nui_joint_count = 8; +extern const std::array kinect_nui_joints; // nui_joint numbers +extern const std::array kinect_nui_joint_names; +constexpr uint32_t kinect_nui_calibration_words = 34; // +0..+132 + +struct KinectNuiRecord { + int64_t steady_us = 0; + uint64_t update = 0; // the input update's count + uint32_t player = 0; + uint32_t tracking_id = 0; + int32_t body_slot = -1; // the record's skeleton slot (+784) + bool racing = false; + bool has_cursor = false; + std::array hands{}; // left, right + bool has_joints = false; + std::array, kinect_nui_joint_count> joints{}; + bool has_lean = false; + float lean_depth = 0, lean_tilt = 0; + std::array detectors{}; + bool has_wipe_y = false; + float wipe_y = 0; // 822CDEE0's second coordinate (detector +78) + std::string calibration_step; // empty: none completed + std::array calibration{}; +}; + +std::string kinect_nui_record_header(); +void kinect_nui_record_row(const KinectNuiRecord& row, std::string& out); +bool kinect_parse_nui_row(std::string_view line, KinectNuiRecord& row); + +// The file, or null when recording is off. +KinectRecordFile* kinect_record_nui(); + +// What the race and its preparation saw of each player between two input +// updates, gathered from their hooks (any thread) and taken into the +// player's next row. +class KinectNuiRecorder { +public: + static constexpr uint32_t players = 2; + void detector(uint32_t player, uint32_t address, int result, uint32_t primary, uint32_t secondary, bool has_value, + float value); + void wipe_y(uint32_t player, float value); + void lean(uint32_t player, float depth, float tilt); + void calibration(uint32_t player, const char* step, const std::array& words); + // Moves what was gathered for the player into row (detectors, lean, + // calibration) and starts the player's next one empty. + void take(uint32_t player, KinectNuiRecord& row); +private: + std::mutex lock_; + std::array pending_{}; +}; +KinectNuiRecorder& kinect_nui_recorder(); + +} diff --git a/src/kinect_preview.h b/src/kinect_preview.h index 2343454..aae2525 100644 --- a/src/kinect_preview.h +++ b/src/kinect_preview.h @@ -1,6 +1,5 @@ #pragma once #include "kinect_sensor.h" -#include "kinect_calibration.h" #include #include @@ -19,7 +18,20 @@ namespace sfr { // elsewhere open() returns null. class KinectPreviewWindow { public: - static std::unique_ptr open(bool chinese); + // model is the launcher's choice: 1 the Xbox 360 Kinect (a v2 stands in + // when only that opens), 2 the Kinect v2 only (KinectSensor::open_selected). + // depth is the launcher's Kinect v2 depth view and Recompiled's tuning: + // the window's view list starts on it and its sensor makes that depth + // (KinectSensor::set_depth_settings); another view picked in the window + // stays the window's own and never changes the launcher's settings. + static std::unique_ptr open(bool chinese, int model = 1, const KinectDepthChoice& depth = {}); + // The launcher's choice again (every frame while it is open; any thread): + // new tuning is used at once, and a view changed in the launcher is shown. + void set_depth_choice(const KinectDepthChoice& depth); + // Where the Recompiled view's virtual v1 and the players are now + // (KinectSensor::ideal_snapshot); false while there is no Kinect v2 + // making depth. + bool ideal_snapshot(KinectIdealSnapshot& snapshot) const; // The same window over the game's own sensor, for the menu's Kinect // adjustment (kinect_tuner.h): it neither opens nor closes the sensor and // never takes its skeleton frames, which the game reads; the game hands @@ -39,9 +51,6 @@ class KinectPreviewWindow { // Why no sensor opened (KinectSensor::open's reason, "no-runtime", // "no-sensor", ...), or empty while it opens or once it has. std::string failure() const; - // The calibration the player finished in the window (its Calibrate - // button, launcher only), once: the launcher stores its crouch depth. - std::optional take_calibration(); struct Impl; private: explicit KinectPreviewWindow(std::unique_ptr); @@ -79,7 +88,8 @@ constexpr double kinect_frame_timeout_seconds = 1.5, kinect_first_frame_seconds // images: the depth camera's 320x240 exactly (the SDK's // NuiTransformSkeletonToDepthImage), the colour camera's 640x480 roughly // (its nominal focal length; the two cameras sit a few centimetres apart). -struct KinectImagePoint { float x = 0, y = 0; bool visible = false; }; +// KinectImagePoint is kinect_sensor.h's. A sensor that places points itself +// (KinectSensor::image_point, a Kinect v2) is asked first. KinectImagePoint kinect_depth_point(const std::array& p); KinectImagePoint kinect_colour_point(const std::array& p); diff --git a/src/kinect_preview_layout.cpp b/src/kinect_preview_layout.cpp new file mode 100644 index 0000000..fc675b7 --- /dev/null +++ b/src/kinect_preview_layout.cpp @@ -0,0 +1,73 @@ +#include "kinect_preview_layout.h" + +#include +#include + +namespace sfr { + +KinectPreviewLayout kinect_preview_layout(const KinectPreviewLayoutInput& input) { + KinectPreviewLayout layout; + const double colour = input.colour_aspect > 0 ? input.colour_aspect : 4.0 / 3.0; + const double depth = input.same_size ? colour : (input.depth_aspect > 0 ? input.depth_aspect : 4.0 / 3.0); + const int available_width = input.client_width - 2 * input.gap - input.gap; // both sides and between + const int available_height = input.client_height - input.top - input.bottom; + if (available_width <= 0 || available_height <= 0) return layout; + // One height for both; the widths follow from it. + const double height = std::min(double(available_height), double(available_width) / (colour + depth)); + const int panel_height = int(std::floor(height)); + int colour_width = int(std::floor(panel_height * colour + 1e-6)); + int depth_width = input.same_size ? colour_width : int(std::floor(panel_height * depth + 1e-6)); + // Rounding must never push the pair out of the area (the epsilon above + // could, at an exact fit, by a pixel). + if (colour_width + depth_width > available_width) { + if (input.same_size) colour_width = depth_width = available_width / 2; + else depth_width = available_width - colour_width; + } + if (panel_height <= 0 || colour_width <= 0 || depth_width <= 0) return layout; + layout.colour = {input.gap, input.top, input.gap + colour_width, input.top + panel_height}; + layout.depth = {2 * input.gap + colour_width, input.top, 2 * input.gap + colour_width + depth_width, + input.top + panel_height}; + return layout; +} + +KinectRecompiledLayout kinect_preview_recompiled_layout(const KinectRecompiledLayoutInput& input) { + KinectRecompiledLayout layout; + const auto aspect = [](double value, double fallback) { return value > 0 ? value : fallback; }; + const double v2_colour = aspect(input.v2_colour_aspect, 16.0 / 9.0), v2_depth = aspect(input.v2_depth_aspect, 512.0 / 424.0); + const double v1_colour = aspect(input.v1_colour_aspect, 4.0 / 3.0), v1_depth = aspect(input.v1_depth_aspect, 4.0 / 3.0); + // Two columns of one width; every picture is that wide (its height from + // its aspect), at the top of its cell, so colour sits exactly under + // colour and depth under depth, and a row is as tall as its taller one. + const double row_top = std::max(1.0 / v2_colour, 1.0 / v2_depth), row_bottom = std::max(1.0 / v1_colour, 1.0 / v1_depth); + const int available_width = input.client_width - 4 * input.gap; // both sides, between the columns, before the diagram + const int available_height = input.client_height - input.top - input.bottom; + if (available_width <= 0 || available_height <= input.caption) return layout; + // At most two thirds for the grid; as wide as the height allows. + const int grid_most = available_width - available_width / 3; + const double column = std::min(double(grid_most) / 2, double(available_height - input.caption) / (row_top + row_bottom)); + const int column_width = int(std::floor(column)); + if (column_width <= 0) return layout; + const int top_height = int(std::floor(column_width * row_top + 1e-6)), bottom_height = int(std::floor(column_width * row_bottom + 1e-6)); + const int left = input.gap, second = 2 * input.gap + column_width; + const int top = input.top, lower = input.top + top_height + input.caption; + layout.colour_cells[0] = {left, top, left + column_width, top + top_height}; + layout.colour_cells[1] = {left, lower, left + column_width, lower + bottom_height}; + layout.depth_cells[0] = {second, top, second + column_width, top + top_height}; + layout.depth_cells[1] = {second, lower, second + column_width, lower + bottom_height}; + const auto at_top = [&](const KinectPanelRect& cell, double aspect) { + const int height = std::min(cell.height(), int(std::floor(cell.width() / aspect + 1e-6))); + return KinectPanelRect{cell.left, cell.top, cell.right, cell.top + height}; + }; + layout.v2_colour = at_top(layout.colour_cells[0], v2_colour); + layout.v1_colour = at_top(layout.colour_cells[1], v1_colour); + layout.v2_depth = at_top(layout.depth_cells[0], v2_depth); + layout.v1_depth = at_top(layout.depth_cells[1], v1_depth); + // The diagram takes the rest of the width (a third or more), the + // grid's full height. + const int diagram_left = second + column_width + input.gap; + layout.diagram = {diagram_left, top, input.client_width - input.gap, lower + bottom_height}; + if (layout.diagram.width() <= 0) layout = {}; + return layout; +} + +} diff --git a/src/kinect_preview_layout.h b/src/kinect_preview_layout.h new file mode 100644 index 0000000..ee3b4e2 --- /dev/null +++ b/src/kinect_preview_layout.h @@ -0,0 +1,61 @@ +#pragma once + +namespace sfr { + +// Where the Kinect preview window puts its two picture panels (colour on the +// left, depth on the right). Pure arithmetic, so it is tested everywhere +// (kinect_preview_layout_test); the window paints and places skeletons from +// the same rectangles. +struct KinectPanelRect { + int left = 0, top = 0, right = 0, bottom = 0; + int width() const { return right - left; } + int height() const { return bottom - top; } +}; + +struct KinectPreviewLayoutInput { + int client_width = 0, client_height = 0; + // Each picture's width / height. + double colour_aspect = 4.0 / 3.0, depth_aspect = 4.0 / 3.0; + // Both panels exactly the same size, at the colour aspect (the common + // view of a Kinect v2, whose two pictures have the same shape). + bool same_size = false; + // Space around the panels: above them (the text), below (the hint line), + // and at the sides and between them. + int top = 110, bottom = 40, gap = 20; +}; + +struct KinectPreviewLayout { + KinectPanelRect colour, depth; +}; + +// The two panels side by side, aligned at their top, the same height, each +// as wide as its picture's aspect says, the pair as large as fits in the +// client area (scaled down uniformly, never stretched). Empty panels when +// nothing fits. +KinectPreviewLayout kinect_preview_layout(const KinectPreviewLayoutInput& input); + +// The Recompiled view's five areas: on the left a two by two grid, the real +// v2's whole pictures on top (colour, depth) and the virtual v1's below +// (colour reprojected into the v1's view, the depth the game gets), colour +// above colour and depth above depth; on the right, one tall panel for the +// top-down diagram, the rest of the width (a third or more) and the grid's +// full height. Both columns are one width and every picture is that wide, +// keeping its aspect, at the top of its cell (a row is as tall as its taller +// picture), as large as the window's height and two thirds of its width +// allow. Every row has a caption line above it (caption pixels, the gap +// between the rows: a line of text and some room; the first row's is inside +// top). Empty rectangles when nothing fits. +struct KinectRecompiledLayoutInput { + int client_width = 0, client_height = 0; + double v2_colour_aspect = 16.0 / 9.0, v2_depth_aspect = 512.0 / 424.0; + double v1_colour_aspect = 4.0 / 3.0, v1_depth_aspect = 4.0 / 3.0; + int top = 110, bottom = 40, gap = 20, caption = 32; +}; +struct KinectRecompiledLayout { + KinectPanelRect v2_colour, v2_depth, v1_colour, v1_depth; // the pictures + KinectPanelRect colour_cells[2], depth_cells[2]; // their cells (top row first) + KinectPanelRect diagram; +}; +KinectRecompiledLayout kinect_preview_recompiled_layout(const KinectRecompiledLayoutInput& input); + +} diff --git a/src/kinect_preview_window.cpp b/src/kinect_preview_window.cpp index bf55add..3057eb3 100644 --- a/src/kinect_preview_window.cpp +++ b/src/kinect_preview_window.cpp @@ -10,17 +10,23 @@ #include #include "camera_debug.h" +#include "kinect_preview_layout.h" +#include "kinect_v2_depth.h" +#include "kinect_v2_image.h" #include "window_placement.h" #include #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -33,9 +39,17 @@ constexpr COLORREF background = RGB(19, 24, 33), foreground = RGB(221, 228, 238) constexpr COLORREF body_colours[6] = {RGB(70, 207, 236), RGB(255, 176, 87), RGB(128, 225, 167), RGB(236, 112, 176), RGB(186, 150, 255), RGB(240, 230, 110)}; -void text(HDC dc, int x, int y, const std::wstring& words, COLORREF colour = foreground) { +// right: the x the text may not pass (0 for no limit), so a long line never +// runs under the buttons, the hint or the next caption. +void text(HDC dc, int x, int y, const std::wstring& words, COLORREF colour = foreground, int right = 0) { SetTextColor(dc, colour); - TextOutW(dc, x, y, words.c_str(), int(words.size())); + if (right > 0) { + if (right <= x) return; + const RECT clip{x, y, right, y + 24}; + ExtTextOutW(dc, x, y, ETO_CLIPPED, &clip, words.c_str(), UINT(words.size()), nullptr); + } else { + TextOutW(dc, x, y, words.c_str(), int(words.size())); + } } void draw_image(HDC dc, const RECT& area, const std::vector& pixels, int width, int height) { @@ -51,10 +65,42 @@ void draw_image(HDC dc, const RECT& area, const std::vector& pixels, i StretchDIBits(dc, area.left, area.top, area.right - area.left, area.bottom - area.top, 0, 0, width, height, pixels.data(), &info, DIB_RGB_COLORS, SRCCOPY); } + +// A depth and player index image in grey, near bright, each player tinted. +void shade_depth(const KinectImage& depth, std::vector& out) { + out.resize(size_t(depth.width) * depth.height); + for (size_t i = 0; i < out.size(); ++i) { + const uint16_t value = uint16_t(depth.pixels[2 * i] | depth.pixels[2 * i + 1] << 8); + const uint32_t millimetres = value >> 3, player = value & 7; + // Near is bright; nothing measured is black. + const uint32_t shade = millimetres == 0 ? 0 : + 255 - std::min(255, uint32_t(std::max(0, int(millimetres) - 500)) * 255 / 3500); + uint32_t r = shade, g = shade, b = shade; + if (player) { + const COLORREF tint = body_colours[(player - 1) % 6]; + r = GetRValue(tint) * (shade + 64) / 320; + g = GetGValue(tint) * (shade + 64) / 320; + b = GetBValue(tint) * (shade + 64) / 320; + } + out[i] = b | g << 8 | r << 16; + } +} + +// The diagram's colours: the real v2 blue, the virtual v1 orange, 1P purple, +// 2P green, anyone else grey; each fill a mix with the panel's background. +constexpr COLORREF diagram_background = RGB(26, 32, 44), v2_colour = RGB(86, 156, 255), v1_colour = RGB(255, 140, 80); +constexpr COLORREF player_colours[3] = {RGB(176, 128, 255), RGB(110, 214, 140), RGB(160, 168, 180)}; +constexpr COLORREF warning = RGB(255, 150, 130); +COLORREF mix(COLORREF colour, float amount) { + const auto part = [&](int a, int b) { return int(std::lround(a + (b - a) * amount)); }; + return RGB(part(GetRValue(diagram_background), GetRValue(colour)), part(GetGValue(diagram_background), GetGValue(colour)), + part(GetBValue(diagram_background), GetBValue(colour))); +} } struct KinectPreviewWindow::Impl { bool chinese = false; + int model = 1; // 1 Xbox 360 (a v2 stands in), 2 Kinect v2 only // Over the game's sensor (open_over): borrowed, never opened or closed // here, and its skeleton frames come from the game (handed). KinectSensor* borrowed = nullptr; @@ -78,15 +124,40 @@ struct KinectPreviewWindow::Impl { std::chrono::steady_clock::time_point opened_at{}, last_frame{}; // The tilt motor, turned from the window: a turn at a time, a second // or so each (the SDK asks for no more than a turn a second). - static constexpr int up_id = 101, down_id = 102, calibrate_id = 103, step = 5; - HWND up_button = nullptr, down_button = nullptr, calibrate_button = nullptr; - // The calibration (kinect_calibration.h), on this window's thread; its - // finished result waits under lock for the launcher. - KinectCalibration calibration; - bool calibration_saved = false; - std::optional finished; // under lock - bool new_frame = false; - HFONT button_font = nullptr; + static constexpr int up_id = 101, down_id = 102, view_id = 103, step = 5; + static constexpr int button_width = 96, view_width = 128; + HWND up_button = nullptr, down_button = nullptr, view_box = nullptr; + // The view list (a Kinect v2's): Recompiled, Simulated, Original, in + // KinectDepthChoiceView's order. It starts on the launcher's view; one + // picked here is this window's own, never written back. On the window's + // thread: choice (the view shown and the launcher's tuning), and the + // launcher's view last handed over, so that a view changed there shows. + // - Original: each v2 camera's whole picture (the old "Original" box), + // and the sensor's depth for the game made as sensor. + // - Simulated: the view both cameras share, and fit's depth. + // - Recompiled: ideal's depth, and the four pictures and the diagram. + KinectDepthChoice choice; + KinectDepthChoiceView launcher_view = KinectDepthChoiceView::ideal; + std::mutex choice_lock; + KinectDepthChoice handed_choice; // under choice_lock + bool choice_new = false; // under choice_lock + // The sensor the choice was last given to (set_depth_settings), so a new + // choice, or the sensor once it opens, is given it. A borrowed sensor + // (the game's) is never given one: the window shows the game's view. + const KinectSensor* settings_for = nullptr; + bool settings_sent = false; + bool game_ideal = false; // over the game: its own view is Recompiled + // Recompiled: the depth the game gets, and the diagram's snapshot. + KinectImage game_depth; + std::vector game_depth_pixels; + // The virtual v1's colour, pixel for pixel with that depth: each depth + // pixel's point (kinect_v1_depth_points) and where it lands in the v2's + // colour image, sampled again whenever either image is new. + std::vector v1_points, v1_colour_positions; + std::vector v1_colour_pixels; + KinectIdealSnapshot snapshot; + bool has_snapshot = false; + HFONT button_font = nullptr, small_font = nullptr, ruler_font = nullptr; std::atomic turning{false}, reread_angle{false}; std::jthread turner; // Last member: joins before everything above is destroyed. @@ -95,7 +166,7 @@ struct KinectPreviewWindow::Impl { void turn(int degrees) { std::shared_ptr source; { std::lock_guard guard(lock); source = sensor; } - if (!source || turning.exchange(true)) return; + if (!source || std::string_view(source->model()) == "Kinect v2" || turning.exchange(true)) return; EnableWindow(up_button, FALSE); EnableWindow(down_button, FALSE); turner = std::jthread([this, source, degrees] { @@ -110,124 +181,73 @@ struct KinectPreviewWindow::Impl { void place_buttons(HWND window) { RECT rect; GetClientRect(window, &rect); - const int width = 96, height = 32, gap = 10, top = 12; + const int width = button_width, height = 32, gap = 10, top = 12; MoveWindow(down_button, rect.right - gap - width, top, width, height, TRUE); MoveWindow(up_button, rect.right - 2 * (gap + width), top, width, height, TRUE); - if (calibrate_button) MoveWindow(calibrate_button, rect.right - 3 * (gap + width) - 40, top, width + 40, height, TRUE); + // A drop-down list's height is its open list's; its field is set to + // the buttons' height (CB_SETITEMHEIGHT). + MoveWindow(view_box, rect.right - 2 * (gap + width) - gap - view_width, top, view_width, 200, TRUE); } - void calibrate() { - if (calibration.step() == KinectCalibrationStep::idle || calibration.step() == KinectCalibrationStep::done) - calibration.start(); - else - calibration.cancel(); - calibration_saved = false; - update_calibrate_label(); + // The four pictures and the diagram (a Kinect v2 in Recompiled; over the + // game only when the game itself makes Recompiled). + bool recompiled_layout() const { + return choice.view == KinectDepthChoiceView::ideal && sensor_is_v2() && (!borrowed || game_ideal); } + // Each v2 camera's whole picture: Original, and the top row of Recompiled. + bool uncropped() const { return choice.view == KinectDepthChoiceView::sensor || recompiled_layout(); } - void update_calibrate_label() { - if (!calibrate_button) return; - const auto now = calibration.step(); - const std::wstring label = now == KinectCalibrationStep::idle ? say(L"Calibrate", L"校正") - : now == KinectCalibrationStep::done ? say(L"Calibrate again", L"重新校正") - : say(L"Stop calibrating", L"停止校正"); - SetWindowTextW(calibrate_button, label.c_str()); + // The launcher's newest choice: its tuning always, its view when it has + // changed there. Over the game the game's own view stays. + void take_choice() { + KinectDepthChoice given; + { + std::lock_guard guard(choice_lock); + if (!choice_new) return; + given = handed_choice; + choice_new = false; + } + if (borrowed) return; + KinectDepthChoice next = given; + next.view = given.view != launcher_view ? given.view : choice.view; + launcher_view = given.view; + if (next.view != choice.view && view_box) SendMessageW(view_box, CB_SETCURSEL, WPARAM(int(next.view)), 0); + if (!(next == choice)) { + choice = next; + settings_sent = false; + } } - // The player calibrated: the first tracked body (the nearest when there - // are several), as the race takes P1. - void feed_calibration() { - if (!new_frame) return; - new_frame = false; - const auto before = calibration.step(); - if (before == KinectCalibrationStep::idle || before == KinectCalibrationStep::done) return; - const KinectBody* player = nullptr; - for (const KinectBody& body : frame.bodies) - if (!player || body.position[2] < player->position[2]) player = &body; - const double seconds = std::chrono::duration(frame.captured_at.time_since_epoch()).count(); - calibration.observe(player, frame.floor_plane, seconds); - if (calibration.step() == KinectCalibrationStep::done && !calibration_saved) { - calibration_saved = true; - std::lock_guard guard(lock); - finished = calibration.result(); - } - if (calibration.step() != before) update_calibrate_label(); + // The window's title names the sensor: Kinect or Kinect v2. + std::wstring title_for(bool v2) const { + if (borrowed) return v2 ? (chinese ? L"Free Riders - Kinect v2 調整" : L"Free Riders - Kinect v2 adjustment") + : (chinese ? L"Free Riders - Kinect 調整" : L"Free Riders - Kinect adjustment"); + return v2 ? (chinese ? L"Free Riders - Kinect v2 預覽" : L"Free Riders - Kinect v2 preview") + : (chinese ? L"Free Riders - Kinect 預覽" : L"Free Riders - Kinect preview"); + } + bool title_is_v2 = false; + // Once a sensor is open, the title follows the one really in use (the + // Xbox 360 choice falls back to a v2). + void update_title(HWND window) { + std::shared_ptr source; + { std::lock_guard guard(lock); source = sensor; } + const KinectSensor* used = borrowed ? borrowed : source.get(); + if (!used) return; + const bool v2 = std::string_view(used->model()) == "Kinect v2"; + if (v2 == title_is_v2) return; + title_is_v2 = v2; + SetWindowTextW(window, title_for(v2).c_str()); } - // The current step's instruction, progress and live reading, over the top - // of the window. - void paint_calibration(HDC dc, int width) { - const auto now = calibration.step(); - if (now == KinectCalibrationStep::idle) return; - RECT panel{20, 8, width - 20, 100}; - HBRUSH brush = CreateSolidBrush(RGB(28, 38, 56)); - FillRect(dc, &panel, brush); - DeleteObject(brush); - HFONT big = CreateFontW(-24, 0, 0, 0, FW_SEMIBOLD, FALSE, FALSE, FALSE, DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, - CLIP_DEFAULT_PRECIS, CLEARTYPE_QUALITY, DEFAULT_PITCH, - chinese ? L"Microsoft JhengHei UI" : L"Segoe UI"); - const auto old = SelectObject(dc, big ? big : GetStockObject(DEFAULT_GUI_FONT)); - const KinectCalibrationResult& r = calibration.result(); - wchar_t line[320]; - std::wstring title, detail; - switch (now) { - case KinectCalibrationStep::stand: - title = say(L"1/5 Stand up straight, arms down, and keep still", L"1/5 站直、雙手放下，保持不動"); - break; - case KinectCalibrationStep::crouch: - title = say(L"2/5 Crouch as you would in a race, then stand up", L"2/5 像比賽時那樣蹲下，再站起來"); - std::swprintf(line, 320, chinese ? L"現在低了 %.0f 公分" : L"Now %.0f cm lower", std::max(0.0f, calibration.drop()) * 100); - detail = line; - break; - case KinectCalibrationStep::jump: - title = say(L"3/5 Jump (both feet off the floor)", L"3/5 跳一下（雙腳離地）"); - std::swprintf(line, 320, chinese ? L"蹲下 %.0f 公分　現在高了 %.0f 公分" : L"Crouch %.0f cm now %.0f cm higher", - r.crouch_drop * 100, std::max(0.0f, -calibration.drop()) * 100); - detail = line; - break; - case KinectCalibrationStep::lean_left: - case KinectCalibrationStep::lean_right: - title = now == KinectCalibrationStep::lean_left - ? say(L"4/5 Lean your body to your left as far as you steer, then stand straight", - L"4/5 身體往左傾（像轉彎那樣），再站直") - : say(L"5/5 Now lean to your right, then stand straight", L"5/5 再往右傾，然後站直"); - std::swprintf(line, 320, chinese ? L"現在傾斜 %.0f°（遊戲在 30° 時轉到底）" : L"Leaning %.0f° (the race steers fully at 30°)", - std::fabs(calibration.lean())); - detail = line; - break; - case KinectCalibrationStep::done: - std::swprintf(line, 320, chinese ? L"校正完成：蹲下深度設為 %u 公分" : L"Calibrated: crouch depth set to %u cm", r.crouch_cm); - title = line; - std::swprintf(line, 320, - chinese ? L"蹲下 %.0f 公分　跳躍 %ls　左傾 %.0f°、右傾 %.0f°（30° 轉到底）" - : L"Crouch %.0f cm jump %ls lean %.0f° left, %.0f° right (30° steers fully)", - r.crouch_drop * 100, - r.jump_rise > 0 ? (std::to_wstring(int(std::lround(r.jump_rise * 100))) + say(L" cm", L" 公分")).c_str() - : say(L"not seen", L"沒有偵測到").c_str(), - r.lean_left, r.lean_right); - detail = line; - break; - default: break; - } - text(dc, 32, 16, title, now == KinectCalibrationStep::done ? RGB(128, 225, 167) : foreground); - SelectObject(dc, old); - if (big) DeleteObject(big); - if (!calibration.tracked() && now != KinectCalibrationStep::done) - detail = say(L"Nobody in view: stand 1.5 to 3.5 m from the sensor, facing it", L"看不到你：請面向感應器，站在 1.5 到 3.5 公尺處"); - else if (calibration.waiting_long() && now != KinectCalibrationStep::done) - detail += say(L" (not seen yet: try it again, a little more)", L"　（還沒偵測到：請再做一次，動作大一點）"); - text(dc, 32, 52, detail, now == KinectCalibrationStep::done ? foreground : RGB(255, 205, 119)); - if (now != KinectCalibrationStep::done) { - RECT bar{32, 80, 32 + int((width - 104) * calibration.progress()), 90}; - HBRUSH fill = CreateSolidBrush(RGB(70, 207, 236)); - FillRect(dc, &bar, fill); - DeleteObject(fill); - } + bool sensor_is_v2() const { + std::lock_guard guard(lock); + return sensor != nullptr && std::string_view(sensor->model()) == "Kinect v2"; } + // A Kinect v2 has no tilt motor: the buttons stay greyed out for it. bool sensor_ready() const { std::lock_guard guard(lock); - return sensor != nullptr; + return sensor != nullptr && std::string_view(sensor->model()) != "Kinect v2"; } std::wstring say(const wchar_t* english, const wchar_t* chinese_words) const { @@ -238,6 +258,14 @@ struct KinectPreviewWindow::Impl { std::shared_ptr source; { std::lock_guard guard(lock); source = sensor; } if (!source) return; + const bool v2 = std::string_view(source->model()) == "Kinect v2"; + // The view's depth for the game, made by this window's own sensor. + if (v2 && !borrowed && (!settings_sent || settings_for != source.get())) { + source->set_depth_settings(kinect_depth_settings(choice)); + settings_sent = true; + settings_for = source.get(); + } + source->set_preview_common_view(!uncropped()); // a v2's; others ignore it const auto now = std::chrono::steady_clock::now(); if (borrowed) { std::lock_guard guard(handed_lock); @@ -248,7 +276,6 @@ struct KinectPreviewWindow::Impl { } } else if (source->next(frame)) { last_frame = now; - new_frame = true; } const bool new_colour = source->image(KinectImageKind::colour, colour); if (new_colour) last_frame = now; @@ -259,23 +286,25 @@ struct KinectPreviewWindow::Impl { } const bool new_depth = source->image(KinectImageKind::depth_and_player, depth); if (new_depth) last_frame = now; - if (new_depth && depth.bytes_per_pixel == 2) { - depth_pixels.resize(size_t(depth.width) * depth.height); - for (size_t i = 0; i < depth_pixels.size(); ++i) { - const uint16_t value = uint16_t(depth.pixels[2 * i] | depth.pixels[2 * i + 1] << 8); - const uint32_t millimetres = value >> 3, player = value & 7; - // Near is bright; nothing measured is black. - const uint32_t shade = millimetres == 0 ? 0 : - 255 - std::min(255, uint32_t(std::max(0, int(millimetres) - 500)) * 255 / 3500); - uint32_t r = shade, g = shade, b = shade; - if (player) { - const COLORREF tint = body_colours[(player - 1) % 6]; - r = GetRValue(tint) * (shade + 64) / 320; - g = GetGValue(tint) * (shade + 64) / 320; - b = GetBValue(tint) * (shade + 64) / 320; - } - depth_pixels[i] = b | g << 8 | r << 16; + if (new_depth && depth.bytes_per_pixel == 2) shade_depth(depth, depth_pixels); + if (v2 && recompiled_layout()) { + // The depth the game gets (game_image copies it; the game's own + // reads are unaffected), and where the ideal camera stands. + bool resample = new_colour; + if (source->game_image(KinectImageKind::depth_and_player, game_depth) && game_depth.bytes_per_pixel == 2) { + shade_depth(game_depth, game_depth_pixels); + // Back through the very view it was made with, into the + // sensor's space, and on into its colour image. + kinect_v1_depth_points(game_depth, kinect_image_view(game_depth), v1_points); + if (!source->colour_image_positions(v1_points, v1_colour_positions)) v1_colour_positions.clear(); + resample = true; } + const size_t pixels = size_t(game_depth.width) * game_depth.height; + if (resample && pixels && v1_colour_positions.size() == 2 * pixels && + colour_pixels.size() == size_t(colour.width) * colour.height) + kinect_colour_from_positions(v1_colour_positions.data(), pixels, colour_pixels.data(), colour.width, + colour.height, v1_colour_pixels); + has_snapshot = source->ideal_snapshot(snapshot); } if (reread_angle.exchange(false) || now - angle_read > std::chrono::seconds(1)) { has_angle = source->elevation(angle); @@ -283,12 +312,52 @@ struct KinectPreviewWindow::Impl { } } - void skeletons(HDC dc, const RECT& area, bool colour_view) { - const float sx = float(area.right - area.left) / (colour_view ? 640.0f : 320.0f); - const float sy = float(area.bottom - area.top) / (colour_view ? 480.0f : 240.0f); - const auto place = [&](const std::array& p) { - const KinectImagePoint point = colour_view ? kinect_colour_point(p) : kinect_depth_point(p); - return POINT{area.left + LONG(point.x * sx), area.top + LONG(point.y * sy)}; + void skeletons(HDC dc, const RECT& area, bool colour_view, KinectSensor* source) { + const KinectImageKind kind = colour_view ? KinectImageKind::colour : KinectImageKind::depth_and_player; + // The picture is the sensor's rectangle in the preview image's pixels + // (a Kinect v2's: the whole image, which follows the picture's shape + // and fills the panel), or a v1's whole fixed-size image. The panel + // shows that image whole, so its size is the scale. + KinectImageRect picture{0, 0, colour_view ? 640.0f : 320.0f, colour_view ? 480.0f : 240.0f}; + if (source) { + KinectImageRect given; + if (source->picture_rect(kind, given)) picture = given; + } + const float image_width = picture.x + picture.width, image_height = picture.y + picture.height; + if (!(image_width > 0) || !(image_height > 0)) return; + const float sx = float(area.right - area.left) / image_width; + const float sy = float(area.bottom - area.top) / image_height; + // A sensor that places points in its own images (a Kinect v2, whose + // images are fitted into the v1 sizes) is asked first, and a point it + // cannot place is left out; a v1's lens places every point. A point + // placed outside the picture still counts (its bone is clipped). + const auto place = [&](const std::array& p, POINT& at) { + KinectImagePoint point; + bool shown = true; + if (source && source->image_point(kind, p, point)) shown = point.visible; + else point = colour_view ? kinect_colour_point(p) : kinect_depth_point(p); + at = POINT{area.left + LONG(point.x * sx), area.top + LONG(point.y * sy)}; + return shown; + }; + // Everything is clipped to the picture, so that a bone leaving it is + // cut at its edge (never into the black bars around a fitted picture + // or the next panel); a joint is drawn only when its centre is inside + // the picture (all of the panel). + const RECT clip{std::max(area.left, area.left + LONG(std::floor(picture.x * sx))), + std::max(area.top, area.top + LONG(std::floor(picture.y * sy))), + std::min(area.right, area.left + LONG(std::ceil((picture.x + picture.width) * sx))), + std::min(area.bottom, area.top + LONG(std::ceil((picture.y + picture.height) * sy)))}; + bodies(dc, clip, place); + } + + // The skeletons with place putting each point in the window (false for a + // point it cannot place), clipped to clip. + using Placer = std::function&, POINT&)>; + void bodies(HDC dc, const RECT& clip, const Placer& place) { + const int saved = SaveDC(dc); + IntersectClipRect(dc, clip.left, clip.top, clip.right, clip.bottom); + const auto inside = [&](const POINT& p) { + return p.x >= clip.left && p.x < clip.right && p.y >= clip.top && p.y < clip.bottom; }; const auto old_brush = SelectObject(dc, GetStockObject(DC_BRUSH)); for (const KinectBody& body : frame.bodies) { @@ -298,13 +367,15 @@ struct KinectPreviewWindow::Impl { SetDCBrushColor(dc, colour); for (const auto bone : camera_debug_bones()) { if (!body.joint_states[bone.from] || !body.joint_states[bone.to]) continue; - const POINT a = place(body.joints[bone.from]), b = place(body.joints[bone.to]); + POINT a{}, b{}; + if (!place(body.joints[bone.from], a) || !place(body.joints[bone.to], b)) continue; MoveToEx(dc, a.x, a.y, nullptr); LineTo(dc, b.x, b.y); } for (uint32_t j = 0; j < nui_joint_count; ++j) { if (!body.joint_states[j]) continue; - const POINT p = place(body.joints[j]); + POINT p{}; + if (!place(body.joints[j], p) || !inside(p)) continue; const int r = body.joint_states[j] >= 2 ? 4 : 2; // inferred joints smaller Ellipse(dc, p.x - r, p.y - r, p.x + r + 1, p.y + r + 1); } @@ -312,6 +383,336 @@ struct KinectPreviewWindow::Impl { DeleteObject(pen); } SelectObject(dc, old_brush); + RestoreDC(dc, saved); + } + + // The frames' skeletons are in the skeleton's view, which Recompiled may + // put at its own distance (SFR_KINECT_V2_SKELETON_DISTANCE): a point of + // it is this much nearer along z than in the depth's view. + float skeleton_shift() const { + return choice.skeleton_distance > 0 ? choice.distance - choice.skeleton_distance : 0.0f; + } + + // Recompiled: the real v2's whole pictures on top, the virtual v1's + // below, the diagram on the right (kinect_preview_recompiled_layout). + void paint_recompiled(HDC dc, int width, int height, KinectSensor* source) { + KinectRecompiledLayoutInput input; + input.client_width = width; + input.client_height = height; + if (source) { + KinectImageRect rect; + if (source->picture_rect(KinectImageKind::colour, rect) && rect.height > 0) + input.v2_colour_aspect = rect.width / rect.height; + if (source->picture_rect(KinectImageKind::depth_and_player, rect) && rect.height > 0) + input.v2_depth_aspect = rect.width / rect.height; + } + const KinectRecompiledLayout layout = kinect_preview_recompiled_layout(input); + const auto win = [](const KinectPanelRect& r) { return RECT{r.left, r.top, r.right, r.bottom}; }; + // One short caption above each picture, stopping at the next cell. + const auto caption = [&](const KinectPanelRect& picture, int right, const wchar_t* english, const wchar_t* chinese_words) { + if (picture.width() > 0) text(dc, picture.left, picture.top - 22, say(english, chinese_words), dim, right - 8); + }; + caption(layout.v2_colour, layout.depth_cells[0].left, L"V2 colour", L"V2 彩色影像"); + caption(layout.v2_depth, layout.diagram.left, L"V2 depth", L"V2 深度影像"); + caption(layout.v1_colour, layout.depth_cells[1].left, L"Simulated colour", L"模擬彩色影像"); + caption(layout.v1_depth, layout.diagram.left, L"Simulated depth", L"模擬深度影像"); + caption(layout.diagram, width, L"Positions from above", L"俯視位置"); + const RECT v2_colour_area = win(layout.v2_colour), v2_depth_area = win(layout.v2_depth); + const RECT v1_colour_area = win(layout.v1_colour), v1_depth_area = win(layout.v1_depth); + draw_image(dc, v2_colour_area, colour_pixels, int(colour.width), int(colour.height)); + draw_image(dc, v2_depth_area, depth_pixels, int(depth.width), int(depth.height)); + if (layout.v2_colour.width() > 0) skeletons(dc, v2_colour_area, true, source); + if (layout.v2_depth.width() > 0) skeletons(dc, v2_depth_area, false, source); + // The v1's colour and depth, pixel for pixel the same (the colour is + // sampled for each depth pixel's own point), both with the skeleton + // through the v1's own lens (NuiTransformSkeletonToDepthImage) in the + // depth's view. + const auto v1_panel = [&](const RECT& area, const std::vector& pixels) { + if (area.right <= area.left) return; + if (pixels.size() != size_t(game_depth.width) * game_depth.height || pixels.empty()) + FillRect(dc, &area, static_cast(GetStockObject(BLACK_BRUSH))); + else draw_image(dc, area, pixels, int(game_depth.width), int(game_depth.height)); + const float sx = float(area.right - area.left) / float(kinect_v1_depth_width), + sy = float(area.bottom - area.top) / float(kinect_v1_depth_height); + const float shift = skeleton_shift(); + bodies(dc, area, [&](const std::array& p, POINT& at) { + const KinectImagePoint point = kinect_depth_point({p[0], p[1], p[2] + shift}); + if (!point.visible) return false; + at = POINT{area.left + LONG(point.x * sx), area.top + LONG(point.y * sy)}; + return true; + }); + }; + v1_panel(v1_colour_area, v1_colour_pixels); + v1_panel(v1_depth_area, game_depth_pixels); + if (layout.diagram.width() > 0) paint_diagram(dc, win(layout.diagram)); + } + + // A label's small rounded box, its top centre at (x, y), kept inside area. + static RECT tag_box(HDC dc, const RECT& area, int x, int y, const std::wstring& words) { + SIZE size{}; + GetTextExtentPoint32W(dc, words.c_str(), int(words.size()), &size); + const int w = size.cx + 12, h = size.cy + 4; + RECT box{x - w / 2, y, x - w / 2 + w, y + h}; + if (box.right > area.right - 2) OffsetRect(&box, area.right - 2 - box.right, 0); + if (box.left < area.left + 2) OffsetRect(&box, area.left + 2 - box.left, 0); + if (box.bottom > area.bottom - 2) OffsetRect(&box, 0, area.bottom - 2 - box.bottom); + return box; + } + void tag(HDC dc, const RECT& box, const std::wstring& words, COLORREF colour) { + HPEN pen = CreatePen(PS_SOLID, 1, colour); + HBRUSH brush = CreateSolidBrush(mix(colour, 0.25f)); + const auto old_pen = SelectObject(dc, pen); + const auto old_brush = SelectObject(dc, brush); + RoundRect(dc, box.left, box.top, box.right, box.bottom, 8, 8); + SelectObject(dc, old_brush); + SelectObject(dc, old_pen); + DeleteObject(brush); + DeleteObject(pen); + SetTextColor(dc, foreground); + RECT inner = box; + DrawTextW(dc, words.c_str(), int(words.size()), &inner, DT_CENTER | DT_VCENTER | DT_SINGLELINE | DT_NOPREFIX); + } + + // The top-down sketch (kinect_ideal_diagram): the real v2 at the bottom + // looking up, its field of view filled blue; the virtual v1, its view + // dashed orange, where the ideal view puts it; the players' depth line, + // each player with the part of the row both cameras see, its distance to + // each camera, and what it is outside of. + void paint_diagram(HDC dc, const RECT& area) { + HBRUSH back = CreateSolidBrush(diagram_background); + FillRect(dc, &area, back); + DeleteObject(back); + const int saved = SaveDC(dc); + IntersectClipRect(dc, area.left, area.top, area.right, area.bottom); + if (small_font) SelectObject(dc, small_font); + const int margin = 10; + const RECT inner{area.left + margin, area.top + margin, area.right - margin, area.bottom - margin}; + KinectIdealSnapshot shown; + if (has_snapshot) shown = snapshot; + else shown.distance = choice.distance; + // Strips for the rulers' labels: the depth ruler on the left, the + // lateral one along the bottom; the drawing is the rest. + SIZE label_size{}; + if (ruler_font) SelectObject(dc, ruler_font); + GetTextExtentPoint32W(dc, L"-8 m", 4, &label_size); + const LONG ruler_left = label_size.cx + 10, ruler_bottom = label_size.cy + 8; + if (small_font) SelectObject(dc, small_font); + const KinectIdealDiagram diagram = kinect_ideal_diagram(shown, float(inner.right - inner.left), float(inner.bottom - inner.top), + float(ruler_left), float(ruler_bottom)); + if (diagram.scale <= 0) { RestoreDC(dc, saved); return; } + const RECT drawing{inner.left + ruler_left, inner.top, inner.right, inner.bottom - ruler_bottom}; + const auto at = [&](const KinectDiagramPoint& p) { return POINT{inner.left + LONG(p.x), inner.top + LONG(p.y)}; }; + const auto x_of = [&](float x) { return inner.left + LONG(x); }; + const auto y_of = [&](float y) { return inner.top + LONG(y); }; + const auto triangle = [&](const std::array& corners, COLORREF fill, HPEN outline) { + const POINT points[3] = {at(corners[0]), at(corners[1]), at(corners[2])}; + HBRUSH brush = CreateSolidBrush(fill); + const auto old_brush = SelectObject(dc, brush); + const auto old_pen = SelectObject(dc, outline ? outline : GetStockObject(NULL_PEN)); + Polygon(dc, points, 3); + SelectObject(dc, old_pen); + SelectObject(dc, old_brush); + DeleteObject(brush); + }; + const auto outline = [&](const std::array& corners, HPEN pen) { + const auto old_pen = SelectObject(dc, pen); + const POINT apex = at(corners[0]), left = at(corners[1]), right = at(corners[2]); + MoveToEx(dc, left.x, left.y, nullptr); + LineTo(dc, apex.x, apex.y); + LineTo(dc, right.x, right.y); + SelectObject(dc, old_pen); + }; + const auto line = [&](LONG x0, LONG y0, LONG x1, LONG y1, HPEN pen) { + const auto old_pen = SelectObject(dc, pen); + MoveToEx(dc, x0, y0, nullptr); + LineTo(dc, x1, y1); + SelectObject(dc, old_pen); + }; + HPEN v2_pen = CreatePen(PS_SOLID, 2, v2_colour); + LOGBRUSH dash_brush{BS_SOLID, v1_colour, 0}; + HPEN v1_pen = ExtCreatePen(PS_GEOMETRIC | PS_DASH | PS_ENDCAP_FLAT, 2, &dash_brush, 0, nullptr); + HPEN v1_dots = CreatePen(PS_DOT, 1, mix(v1_colour, 0.7f)); + HPEN row_dots = CreatePen(PS_DOT, 1, dim); + SetBkMode(dc, TRANSPARENT); + // The rulers: depth from the real v2 up the left edge, across along + // the bottom (0 on the v2's axis); a mark every half metre, a label + // every metre where there is room for it. + HPEN ruler_pen = CreatePen(PS_SOLID, 1, dim); + HPEN grid_pen = CreatePen(PS_SOLID, 1, mix(foreground, 0.10f)); + if (ruler_font) SelectObject(dc, ruler_font); + SetTextColor(dc, dim); + const LONG axis_x = drawing.left - 2, axis_y = drawing.bottom + 2; + line(axis_x, drawing.top, axis_x, axis_y, ruler_pen); + line(axis_x, axis_y, drawing.right, axis_y, ruler_pen); + const auto metres_label = [](float metres) { + wchar_t words[16]; + if (metres == 0) std::swprintf(words, 16, L"0"); + else std::swprintf(words, 16, L"%d m", int(std::lround(metres))); + return std::wstring(words); + }; + const bool depth_labels = diagram.scale >= float(label_size.cy + 4); + for (const KinectDiagramTick& tick : diagram.depth_ticks) { + const LONG y = y_of(tick.at); + if (y < drawing.top || y > drawing.bottom) continue; + line(axis_x - (tick.whole ? 6 : 3), y, axis_x, y, ruler_pen); + if (!tick.whole || !depth_labels) continue; + const std::wstring words = metres_label(tick.metres); + RECT box{inner.left, y - label_size.cy / 2, axis_x - 8, y + label_size.cy / 2 + 1}; + DrawTextW(dc, words.c_str(), int(words.size()), &box, DT_RIGHT | DT_VCENTER | DT_SINGLELINE | DT_NOPREFIX); + } + const bool lateral_labels = diagram.scale >= float(label_size.cx + 6); + for (const KinectDiagramTick& tick : diagram.lateral_ticks) { + const LONG x = x_of(tick.at); + if (x < drawing.left || x > drawing.right) continue; + line(x, axis_y, x, axis_y + (tick.whole ? 5 : 3), ruler_pen); + if (!tick.whole || !lateral_labels) continue; + const std::wstring words = metres_label(tick.metres); + RECT box{x - label_size.cx, axis_y + 5, x + label_size.cx, inner.bottom + 2}; + DrawTextW(dc, words.c_str(), int(words.size()), &box, DT_CENTER | DT_TOP | DT_SINGLELINE | DT_NOPREFIX); + } + if (small_font) SelectObject(dc, small_font); + // The drawing itself stays inside its own part of the panel. + const int drawing_saved = SaveDC(dc); + IntersectClipRect(dc, drawing.left, drawing.top, drawing.right, drawing.bottom); + triangle(diagram.v2_view, mix(v2_colour, 0.22f), nullptr); + triangle(diagram.v1_view, mix(v1_colour, 0.22f), nullptr); + // A faint line across at every whole metre from the v2. + for (const KinectDiagramTick& tick : diagram.depth_ticks) + if (tick.whole) line(drawing.left, y_of(tick.at), drawing.right, y_of(tick.at), grid_pen); + outline(diagram.v2_view, v2_pen); + outline(diagram.v1_view, v1_pen ? v1_pen : v2_pen); + // The v1's usable depths, 0.8 and 4.0 m from it: dotted across its + // view, and marked and labelled on its own axis. + line(x_of(diagram.v1_near_x[0]), y_of(diagram.v1_near_y), x_of(diagram.v1_near_x[1]), y_of(diagram.v1_near_y), v1_dots); + line(x_of(diagram.v1_far_x[0]), y_of(diagram.v1_far_y), x_of(diagram.v1_far_x[1]), y_of(diagram.v1_far_y), v1_dots); + HPEN mark_pen = CreatePen(PS_SOLID, 2, v1_colour); + if (ruler_font) SelectObject(dc, ruler_font); + SetTextColor(dc, v1_colour); + for (const auto& [mark, words] : {std::pair{diagram.v1_near_mark, L"0.8 m"}, std::pair{diagram.v1_far_mark, L"4.0 m"}}) { + const POINT p = at(mark); + line(p.x - 6, p.y, p.x + 7, p.y, mark_pen); + TextOutW(dc, p.x + 9, p.y - label_size.cy, words, int(std::wcslen(words))); + } + if (small_font) SelectObject(dc, small_font); + // The players' depth line, distance in front of the virtual camera: drawn + // with nobody there too, so the launcher's distance slider shows at once. + line(drawing.left, y_of(diagram.players_y), drawing.right, y_of(diagram.players_y), row_dots); + const LONG radius = std::clamp(LONG(0.2f * diagram.scale), 9, 15); + for (const KinectDiagramPlayer& player : diagram.players) { + const COLORREF colour = player_colours[player.slot == 0 ? 0 : player.slot == 1 ? 1 : 2]; + const POINT centre = at(player.at); + if (player.band_right > player.band_left) { + const RECT band{x_of(player.band_left), centre.y - radius * 2 / 3, x_of(player.band_right), centre.y + radius * 2 / 3}; + HBRUSH brush = CreateSolidBrush(mix(colour, 0.2f)); + FillRect(dc, &band, brush); + DeleteObject(brush); + } + } + for (const KinectDiagramPlayer& player : diagram.players) { + const COLORREF colour = player_colours[player.slot == 0 ? 0 : player.slot == 1 ? 1 : 2]; + const POINT centre = at(player.at); + HBRUSH brush = CreateSolidBrush(colour); + HPEN pen = CreatePen(PS_SOLID, 1, RGB(255, 255, 255)); + const auto old_brush = SelectObject(dc, brush); + const auto old_pen = SelectObject(dc, pen); + Ellipse(dc, centre.x - radius, centre.y - radius, centre.x + radius + 1, centre.y + radius + 1); + SelectObject(dc, old_pen); + SelectObject(dc, old_brush); + DeleteObject(pen); + DeleteObject(brush); + const std::wstring name = player.slot == 0 ? L"1P" : player.slot == 1 ? L"2P" : L""; + if (!name.empty()) { + RECT label{centre.x - radius, centre.y - radius, centre.x + radius + 1, centre.y + radius + 1}; + SetTextColor(dc, RGB(255, 255, 255)); + DrawTextW(dc, name.c_str(), int(name.size()), &label, DT_CENTER | DT_VCENTER | DT_SINGLELINE | DT_NOPREFIX); + } + // Its distances, and what it is outside of, beside it. + wchar_t numbers[96]; + std::swprintf(numbers, 96, chinese ? L"遊戲 %.2f m v2 %.2f m" : L"game %.2f m v2 %.2f m", double(player.v1_depth), + double(player.v2_depth)); + std::vector> lines{{numbers, foreground}}; + if (!player.in_v1_view) lines.push_back({say(L"Outside the game's view", L"超出遊戲視野"), warning}); + else if (!player.in_v1_range) lines.push_back({say(L"Outside the game's 0.8-4.0 m", L"超出遊戲的 0.8–4.0 公尺"), warning}); + if (!player.in_v2_view) lines.push_back({say(L"Outside the v2's view", L"超出 v2 視野"), warning}); + LONG widest = 0; + for (const auto& [words, unused] : lines) { + SIZE size{}; + GetTextExtentPoint32W(dc, words.c_str(), int(words.size()), &size); + widest = std::max(widest, size.cx); + } + const LONG right_side = centre.x + radius + 6; + const LONG x = right_side + widest <= drawing.right ? right_side : std::max(drawing.left, centre.x - radius - 6 - widest); + LONG y = centre.y - LONG(lines.size()) * 8; + for (const auto& [words, colour_of_line] : lines) { + text(dc, x, y, words, colour_of_line); + y += 16; + } + } + // The cameras: a dot at each, and its name in a box under it (side + // by side when the two stand together). + const POINT v2_at = at(diagram.v2_view[0]), v1_at = at(diagram.v1_view[0]); + for (const auto& [where, colour] : {std::pair{v2_at, v2_colour}, std::pair{v1_at, v1_colour}}) { + HBRUSH brush = CreateSolidBrush(colour); + const auto old_brush = SelectObject(dc, brush); + const auto old_pen = SelectObject(dc, GetStockObject(NULL_PEN)); + Ellipse(dc, where.x - 4, where.y - 4, where.x + 5, where.y + 5); + SelectObject(dc, old_pen); + SelectObject(dc, old_brush); + DeleteObject(brush); + } + const std::wstring v2_name = say(L"Physical v2", L"實體 v2"), v1_name = say(L"Virtual Kinect", L"虛擬 Kinect"); + const RECT v2_box = tag_box(dc, drawing, v2_at.x, v2_at.y + 6, v2_name); + RECT v1_box = tag_box(dc, drawing, v1_at.x, v1_at.y + 6, v1_name); + RECT overlap{}; + if (IntersectRect(&overlap, &v2_box, &v1_box)) { + // Beside the v2's instead, on the side with more room. + const LONG w = v1_box.right - v1_box.left; + const bool left = v2_box.left - drawing.left > drawing.right - v2_box.right; + const LONG centre_x = left ? v2_box.left - 4 - w / 2 : v2_box.right + 4 + w / 2; + v1_box = tag_box(dc, drawing, centre_x, v2_box.top, v1_name); + } + tag(dc, v2_box, v2_name, v2_colour); + tag(dc, v1_box, v1_name, v1_colour); + // Nothing to place yet: say why. + const std::wstring hint = !has_snapshot ? say(L"Waiting for the depth image", L"等待深度影像") + : shown.players.empty() ? say(L"No players tracked", L"沒有追蹤到玩家") + : std::wstring(); + if (!hint.empty()) text(dc, drawing.left + 4, drawing.top, hint, dim, drawing.right); + // Recompiled's tuning as the launcher has it now, so a slider's move + // shows here at once. + if (recompiled_layout()) { + wchar_t first[96], second[96]; + std::swprintf(first, 96, chinese ? L"距離 %.2f m 高度 %.2f m" : L"Distance %.2f m Height %.2f m", + double(choice.distance), double(choice.height)); + std::swprintf(second, 96, chinese ? L"跟隨 %.1f s 死區 %.2f m" : L"Follow %.1f s Dead band %.2f m", + double(choice.follow), double(choice.deadband)); + const int step = int(label_size.cy) + 2; + int row_y = drawing.top + (hint.empty() ? 0 : step); + text(dc, drawing.left + 4, row_y, first, dim, drawing.right); + text(dc, drawing.left + 4, row_y + step, second, dim, drawing.right); + } + RestoreDC(dc, drawing_saved); + DeleteObject(v2_pen); + DeleteObject(ruler_pen); + DeleteObject(grid_pen); + DeleteObject(mark_pen); + if (v1_pen) DeleteObject(v1_pen); + DeleteObject(v1_dots); + DeleteObject(row_dots); + RestoreDC(dc, saved); + } + + // The one place the panels are placed: painting and the skeleton overlay + // both use these rectangles. + static KinectPreviewLayout panel_layout(int width, int height, double colour_aspect, double depth_aspect, + bool same_size) { + KinectPreviewLayoutInput input; + input.client_width = width; + input.client_height = height; + input.colour_aspect = colour_aspect; + input.depth_aspect = depth_aspect; + input.same_size = same_size; + return kinect_preview_layout(input); } void paint(HDC target, const RECT& rect) { @@ -346,10 +747,23 @@ struct KinectPreviewWindow::Impl { input.bodies = frame.bodies.size(); const KinectReadiness readiness = kinect_readiness(input); wchar_t line[256]; + // Where text may run to: the first row stops short of the buttons, + // the next rows (to y 76) of the tilt hint under them, the rest of + // the window's edge; so nothing overlaps at the smallest window. + const std::wstring keys = say(L"Tilt: buttons or Up / Down keys, 5 degrees a turn", L"調整角度：按鈕或鍵盤 ↑ ↓，每次 5 度"); + SIZE extent{}; + GetTextExtentPoint32W(dc, keys.c_str(), int(keys.size()), &extent); + const int buttons_left = width - 2 * (10 + button_width) - (10 + view_width) - 10; + const int hint_left = width - 20 - extent.cx; + const auto limit_at = [&](int y) { return y < 40 ? buttons_left : y < 76 ? hint_left - 12 : width - 20; }; if (!readiness.ready()) { // Not playable yet: each step with its state, then what to do // about the first one that failed. - const wchar_t* const names[] = {L"Runtime (SDK)", L"執行環境（SDK）", L"Sensor", L"感應器", + // The SDK of the sensor in use: 2.0 for a Kinect v2, 1.8 for a v1; + // before one opens, the one the launcher chose. + const bool v2_sdk = source ? std::string_view(source->model()) == "Kinect v2" : model == 2; + const wchar_t* const names[] = {v2_sdk ? L"Runtime (SDK 2.0)" : L"Runtime (SDK 1.8)", + v2_sdk ? L"執行環境（SDK 2.0）" : L"執行環境（SDK 1.8）", L"Sensor", L"感應器", L"Frames", L"資料傳送", L"Body", L"人體追蹤"}; int x = 20; for (size_t step = 0; step < size_t(KinectStep::count); ++step) { @@ -358,7 +772,7 @@ struct KinectPreviewWindow::Impl { state == KinectStepState::failed ? L"\u2717 " : L"\u2026 "; const std::wstring item = mark + say(names[2 * step], names[2 * step + 1]); text(dc, x, 16, item, state == KinectStepState::passed ? RGB(128, 225, 167) : - state == KinectStepState::failed ? RGB(255, 150, 130) : dim); + state == KinectStepState::failed ? RGB(255, 150, 130) : dim, limit_at(16)); SIZE size{}; GetTextExtentPoint32W(dc, item.c_str(), int(item.size()), &size); x += size.cx + 24; @@ -387,10 +801,9 @@ struct KinectPreviewWindow::Impl { : say(L"Opening the Kinect...", L"正在開啟 Kinect…"); break; } - text(dc, 20, 40, hint, readiness.first_failed == KinectStep::count ? foreground : RGB(255, 205, 119)); + text(dc, 20, 40, hint, readiness.first_failed == KinectStep::count ? foreground : RGB(255, 205, 119), limit_at(40)); } - const bool calibrating = calibration.step() != KinectCalibrationStep::idle; - if (source && readiness.ready() && !calibrating) { + if (source && readiness.ready()) { const std::string name = source->model(); const std::wstring model_name(name.begin(), name.end()); if (has_angle) @@ -400,7 +813,16 @@ struct KinectPreviewWindow::Impl { std::swprintf(line, 256, chinese ? L"%ls 追蹤到 %zu 人" : L"%ls %zu tracked", model_name.c_str(), frame.bodies.size()); std::wstring model(line); - text(dc, 20, 16, model); + text(dc, 20, 16, model, foreground, limit_at(16)); + // No depth stream for the game (it would get none, and its own + // detectors see nothing): say so, with why. + if (const std::string why = source->depth_failure(); !why.empty()) { + SIZE size{}; + GetTextExtentPoint32W(dc, model.c_str(), int(model.size()), &size); + const std::wstring reason(why.begin(), why.end()); + text(dc, 20 + size.cx + 24, 16, + say(L"\u2717 No depth for the game: ", L"\u2717 遊戲沒有深度影像：") + reason, RGB(255, 150, 130), limit_at(16)); + } // Per body: its id, distance, and whether its feet are really seen // (a foot the sensor only guesses is out of view). int y = 40; @@ -410,30 +832,58 @@ struct KinectPreviewWindow::Impl { body.tracking_id, body.sensor_index, body.position[2], feet ? say(L"feet in view", L"腳在畫面內").c_str() : say(L"feet out of view: tilt down or step back", L"腳不在畫面內：角度往下或往後站").c_str()); - text(dc, 20, y, line, feet ? body_colours[body.sensor_index % 6] : RGB(255, 205, 119)); + text(dc, 20, y, line, feet ? body_colours[body.sensor_index % 6] : RGB(255, 205, 119), limit_at(y)); + y += 20; + } + // A body cut by the picture's edge (a Kinect v2 says which; a v1 + // says nothing). The edges are the picture's, as drawn below, + // not mirrored: its left is the sensor's left, which is the + // PLAYER's right while facing the sensor. So a cut on the + // picture's left asks the player to step to their own left, + // and the reverse (kinect_edges_move). Display only. + for (const KinectBody& body : frame.bodies) { + if (!body.clipped_edges || y > 90) continue; + const uint8_t move = kinect_edges_move(body.clipped_edges); + std::wstring edges, what; + const auto add = [&](std::wstring& list, const wchar_t* en, const wchar_t* zh, const wchar_t* join) { + if (!list.empty()) list += join; + list += say(en, zh); + }; + if (body.clipped_edges & kinect_edge::top) add(edges, L"top", L"上方", L"、"); + if (body.clipped_edges & kinect_edge::bottom) add(edges, L"bottom", L"下方", L"、"); + if (body.clipped_edges & kinect_edge::left) add(edges, L"picture left", L"畫面左側", L"、"); + if (body.clipped_edges & kinect_edge::right) add(edges, L"picture right", L"畫面右側", L"、"); + if (move & kinect_edge_move::player_left) add(what, L"step to your left", L"請往你的左邊移動", L", "); + if (move & kinect_edge_move::player_right) add(what, L"step to your right", L"請往你的右邊移動", L", "); + if (move & kinect_edge_move::step_back) { + if (move & kinect_edge_move::tilt_up) add(what, L"step back or tilt the sensor up", L"請往後退，或把感應器角度往上", L", "); + else if (move & kinect_edge_move::tilt_down) add(what, L"step back or tilt the sensor down", L"請往後退，或把感應器角度往下", L", "); + else add(what, L"step back", L"請往後退", L", "); + } + std::wstring who; + if (frame.bodies.size() > 1) { + wchar_t slot[32]; + std::swprintf(slot, 32, chinese ? L"（位置 %u）" : L" (slot %u)", body.sensor_index); + who = slot; + } + text(dc, 20, y, say(L"✗ Body cut off", L"✗ 身體被畫面切到") + who + say(L": ", L"：") + edges + + say(L" — ", L"，") + what, RGB(255, 150, 130), limit_at(y)); y += 20; } } - const std::wstring keys = say(L"Tilt: buttons or Up / Down keys, 5 degrees a turn", L"調整角度：按鈕或鍵盤 ↑ ↓，每次 5 度"); - SIZE extent{}; - GetTextExtentPoint32W(dc, keys.c_str(), int(keys.size()), &extent); - text(dc, width - 20 - extent.cx, 52, turning.load() ? say(L"Turning...", L"轉動中…") : keys, dim); - // The two cameras side by side, 4:3 each, under the text. - const int top = 110, gap = 20; - const int panel_width = std::max(0, (width - 3 * gap) / 2); - const int panel_height = std::min(std::max(0, height - top - 40), panel_width * 3 / 4); - const RECT colour_area{gap, top, gap + panel_width, top + panel_height}; - const RECT depth_area{2 * gap + panel_width, top, 2 * gap + 2 * panel_width, top + panel_height}; - text(dc, colour_area.left, top - 22, say(L"COLOUR (skeleton roughly aligned)", L"彩色影像（骨架為近似對齊）"), dim); - text(dc, depth_area.left, top - 22, say(L"DEPTH (players tinted)", L"深度影像（玩家上色）"), dim); - draw_image(dc, colour_area, colour_pixels, int(colour.width), int(colour.height)); - draw_image(dc, depth_area, depth_pixels, int(depth.width), int(depth.height)); - skeletons(dc, colour_area, true); - skeletons(dc, depth_area, false); - paint_calibration(dc, width); + text(dc, hint_left, 52, turning.load() ? say(L"Turning...", L"轉動中…") : keys, dim); + const int gap = 20; + if (source && recompiled_layout()) paint_recompiled(dc, width, height, source.get()); + else paint_pair(dc, width, height, source.get()); + // Name the sensor in use: a v2 says Kinect v2. + const bool v2_sensor = source ? std::string_view(source->model()) == "Kinect v2" : model == 2; text(dc, gap, height - 28, borrowed ? say(L"Close this window to go back to the game.", L"關閉此視窗即可回到遊戲。") - : say(L"Close this window before starting the game: the Kinect opens for one program at a time.", - L"開始遊戲前會自動關閉此視窗：Kinect 一次只能給一個程式使用。"), dim); + : v2_sensor + ? say(L"Close this window before starting the game: the Kinect v2 opens for one program at a time.", + L"開始遊戲前會自動關閉此視窗：Kinect v2 一次只能給一個程式使用。") + : say(L"Close this window before starting the game: the Kinect opens for one program at a time.", + L"開始遊戲前會自動關閉此視窗：Kinect 一次只能給一個程式使用。"), + dim, width - gap); BitBlt(target, 0, 0, width, height, dc, 0, 0, SRCCOPY); SelectObject(dc, old_font); SelectObject(dc, old_bitmap); @@ -442,6 +892,41 @@ struct KinectPreviewWindow::Impl { DeleteDC(dc); } + // Simulated and Original (and a v1): colour and depth side by side. + void paint_pair(HDC dc, int width, int height, KinectSensor* source) { + // The two cameras side by side under the text, each panel following + // its picture's shape (kinect_preview_layout). A Kinect v2 shows the + // view both its cameras have unless Original is chosen; its sensor + // says so by giving the picture a rectangle, and both panels then + // have the same shape and size. + bool cropped = false; + double colour_aspect = 4.0 / 3.0, depth_aspect = 4.0 / 3.0; + if (source) { + KinectImageRect colour_rect, depth_rect; + const bool has_colour = source->picture_rect(KinectImageKind::colour, colour_rect); + const bool has_depth = source->picture_rect(KinectImageKind::depth_and_player, depth_rect); + cropped = has_colour && !uncropped(); + if (has_colour && colour_rect.height > 0) colour_aspect = colour_rect.width / colour_rect.height; + else if (colour.width && colour.height) colour_aspect = double(colour.width) / colour.height; + if (has_depth && depth_rect.height > 0) depth_aspect = depth_rect.width / depth_rect.height; + else if (depth.width && depth.height) depth_aspect = double(depth.width) / depth.height; + } + const KinectPreviewLayout layout = panel_layout(width, height, colour_aspect, depth_aspect, cropped); + const RECT colour_area{layout.colour.left, layout.colour.top, layout.colour.right, layout.colour.bottom}; + const RECT depth_area{layout.depth.left, layout.depth.top, layout.depth.right, layout.depth.bottom}; + const int gap = 20, top = 110; + // A caption stops short of the next one (the depth panel's, or the window's edge). + text(dc, gap, top - 22, + cropped ? say(L"COLOUR (cropped)", L"彩色影像（裁切）") : say(L"COLOUR", L"彩色影像"), dim, + depth_area.right > 0 ? depth_area.left - 8 : width - gap); + if (depth_area.right > 0) + text(dc, depth_area.left, top - 22, say(L"DEPTH", L"深度影像"), dim, width - gap); + draw_image(dc, colour_area, colour_pixels, int(colour.width), int(colour.height)); + draw_image(dc, depth_area, depth_pixels, int(depth.width), int(depth.height)); + if (colour_area.right > colour_area.left) skeletons(dc, colour_area, true, source); + if (depth_area.right > depth_area.left) skeletons(dc, depth_area, false, source); + } + static LRESULT CALLBACK procedure(HWND window, UINT message, WPARAM wp, LPARAM lp) { auto* self = reinterpret_cast(GetWindowLongPtrW(window, GWLP_USERDATA)); if (message == WM_NCCREATE) { @@ -453,19 +938,40 @@ struct KinectPreviewWindow::Impl { case WM_TIMER: if (self->stopping.stop_requested()) DestroyWindow(window); else { + self->take_choice(); self->refresh(); - self->feed_calibration(); + self->update_title(window); const bool ready = !self->turning.load() && self->sensor_ready(); - EnableWindow(self->up_button, ready); - EnableWindow(self->down_button, ready); - if (self->calibrate_button) EnableWindow(self->calibrate_button, self->sensor_ready()); + // Only on a change: enabling repaints a control. + const auto enable = [](HWND control, bool on) { + if (control && bool(IsWindowEnabled(control)) != on) EnableWindow(control, on); + }; + enable(self->up_button, ready); + enable(self->down_button, ready); + enable(self->view_box, self->sensor_is_v2()); // a v1 has one view only + // The window is WS_CLIPCHILDREN, so this repaint never draws + // over the list and buttons. InvalidateRect(window, nullptr, FALSE); } return 0; case WM_COMMAND: + if (LOWORD(wp) == view_id) { + // The list keeps the focus while it is open; the window takes + // the arrow keys back once it closes. + if (HIWORD(wp) == CBN_SELCHANGE) { + const LRESULT picked = SendMessageW(self->view_box, CB_GETCURSEL, 0, 0); + if (picked >= 0 && picked < 3 && KinectDepthChoiceView(picked) != self->choice.view) { + self->choice.view = KinectDepthChoiceView(picked); + self->settings_sent = false; + InvalidateRect(window, nullptr, FALSE); + } + } else if (HIWORD(wp) == CBN_CLOSEUP) { + SetFocus(window); + } + return 0; + } if (LOWORD(wp) == up_id) self->turn(step); else if (LOWORD(wp) == down_id) self->turn(-step); - else if (LOWORD(wp) == calibrate_id) self->calibrate(); SetFocus(window); // keep the arrow keys for the window return 0; case WM_KEYDOWN: @@ -509,11 +1015,13 @@ struct KinectPreviewWindow::Impl { cls.lpszClassName = L"SfrKinectPreview"; cls.hCursor = LoadCursorW(nullptr, MAKEINTRESOURCEW(32512)); if (!RegisterClassW(&cls) && GetLastError() != ERROR_CLASS_ALREADY_EXISTS) { is_closed.store(true); return; } + title_is_v2 = model == 2; HWND window = CreateWindowExW(0, cls.lpszClassName, - borrowed ? (chinese ? L"Free Riders - Kinect 調整" : L"Free Riders - Kinect adjustment") - : (chinese ? L"Free Riders - Kinect 預覽" : L"Free Riders - Kinect preview"), - WS_OVERLAPPEDWINDOW, CW_USEDEFAULT, CW_USEDEFAULT, 1360, 640, nullptr, nullptr, - instance, this); + title_for(model == 2).c_str(), + WS_OVERLAPPEDWINDOW | WS_CLIPCHILDREN, CW_USEDEFAULT, CW_USEDEFAULT, 1360, + // Recompiled's two rows of pictures want more height. + choice.view == KinectDepthChoiceView::ideal && (model == 2 || game_ideal) ? 820 : 640, + nullptr, nullptr, instance, this); if (!window) { is_closed.store(true); return; } shown.store(window); up_button = CreateWindowExW(0, L"BUTTON", chinese ? L"往上" : L"Up", WS_CHILD | WS_VISIBLE | WS_DISABLED | BS_PUSHBUTTON, @@ -523,15 +1031,23 @@ struct KinectPreviewWindow::Impl { button_font = CreateFontW(-16, 0, 0, 0, FW_NORMAL, FALSE, FALSE, FALSE, DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, CLIP_DEFAULT_PRECIS, CLEARTYPE_QUALITY, DEFAULT_PITCH, chinese ? L"Microsoft JhengHei UI" : L"Segoe UI"); - // Calibrating stores the launcher's crouch depth, so only the - // launcher's preview has it (not the one over the game's sensor). - if (!borrowed) { - calibrate_button = CreateWindowExW(0, L"BUTTON", say(L"Calibrate", L"校正").c_str(), - WS_CHILD | WS_VISIBLE | WS_DISABLED | BS_PUSHBUTTON, 0, 0, 0, 0, window, - reinterpret_cast(INT_PTR(calibrate_id)), instance, nullptr); - } - for (HWND button : {up_button, down_button, calibrate_button}) - if (button && button_font) SendMessageW(button, WM_SETFONT, WPARAM(button_font), TRUE); + for (HWND button : {up_button, down_button}) + if (button_font) SendMessageW(button, WM_SETFONT, WPARAM(button_font), TRUE); + small_font = CreateFontW(-13, 0, 0, 0, FW_NORMAL, FALSE, FALSE, FALSE, DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, + CLIP_DEFAULT_PRECIS, CLEARTYPE_QUALITY, DEFAULT_PITCH, + chinese ? L"Microsoft JhengHei UI" : L"Segoe UI"); + ruler_font = CreateFontW(-12, 0, 0, 0, FW_NORMAL, FALSE, FALSE, FALSE, DEFAULT_CHARSET, OUT_DEFAULT_PRECIS, + CLIP_DEFAULT_PRECIS, CLEARTYPE_QUALITY, DEFAULT_PITCH, + chinese ? L"Microsoft JhengHei UI" : L"Segoe UI"); + // The depth views, as the launcher names and orders them. + view_box = CreateWindowExW(0, L"COMBOBOX", nullptr, WS_CHILD | WS_VISIBLE | WS_DISABLED | WS_VSCROLL | CBS_DROPDOWNLIST, + 0, 0, 0, 0, window, reinterpret_cast(INT_PTR(view_id)), instance, nullptr); + if (button_font) SendMessageW(view_box, WM_SETFONT, WPARAM(button_font), TRUE); + for (const wchar_t* name : {L"Recompiled", chinese ? L"模擬" : L"Simulated", chinese ? L"原始" : L"Original"}) + SendMessageW(view_box, CB_ADDSTRING, 0, reinterpret_cast(name)); + SendMessageW(view_box, CB_SETCURSEL, WPARAM(int(choice.view)), 0); + // The field as tall as the buttons beside it (its border adds about 6). + SendMessageW(view_box, CB_SETITEMHEIGHT, WPARAM(-1), 26); place_buttons(window); if (borrowed) sfr::center_over_game(window); // over the game it adjusts ShowWindow(window, SW_SHOW); @@ -543,7 +1059,7 @@ struct KinectPreviewWindow::Impl { // A borrowed sensor is the game's: shared without an owner, so // closing the window leaves it open. std::shared_ptr opened = borrowed ? std::shared_ptr(borrowed, [](KinectSensor*) {}) - : KinectSensor::open(&why); + : KinectSensor::open_selected(model, &why); std::lock_guard guard(lock); sensor = std::move(opened); failure = sensor ? "" : why; @@ -562,6 +1078,8 @@ struct KinectPreviewWindow::Impl { } if (IsWindow(window)) DestroyWindow(window); if (button_font) DeleteObject(button_font); + if (small_font) DeleteObject(small_font); + if (ruler_font) DeleteObject(ruler_font); opener.join(); // The sensor closes now unless a tilt is still using it. { std::lock_guard guard(lock); sensor.reset(); } @@ -572,10 +1090,13 @@ struct KinectPreviewWindow::Impl { KinectPreviewWindow::KinectPreviewWindow(std::unique_ptr impl) : impl_(std::move(impl)) {} KinectPreviewWindow::~KinectPreviewWindow() = default; -std::unique_ptr KinectPreviewWindow::open(bool chinese) { +std::unique_ptr KinectPreviewWindow::open(bool chinese, int model, const KinectDepthChoice& depth) { try { auto impl = std::make_unique(); impl->chinese = chinese; + impl->model = model; + impl->choice = depth; + impl->launcher_view = depth.view; auto* raw = impl.get(); impl->worker = std::jthread([raw](std::stop_token stop) { raw->run(stop); }); return std::unique_ptr(new KinectPreviewWindow(std::move(impl))); @@ -590,6 +1111,19 @@ std::unique_ptr KinectPreviewWindow::open_over(KinectSensor auto impl = std::make_unique(); impl->chinese = PRIMARYLANGID(GetUserDefaultUILanguage()) == LANG_CHINESE; impl->borrowed = &sensor; + // The game's own view and tuning (its environment), which this + // window never changes: picking another view only changes the + // pictures' crop. + const KinectDepthSettings game = KinectDepthSettings::from_environment(); + impl->game_ideal = game.mode == KinectDepthViewMode::ideal; + impl->choice.view = impl->game_ideal ? KinectDepthChoiceView::ideal + : game.mode == KinectDepthViewMode::sensor ? KinectDepthChoiceView::sensor + : KinectDepthChoiceView::fit; + impl->choice.distance = game.distance; + impl->choice.height = game.height; + impl->choice.follow = game.follow; + impl->choice.deadband = game.deadband; + impl->choice.skeleton_distance = game.skeleton_distance; auto* raw = impl.get(); impl->worker = std::jthread([raw](std::stop_token stop) { raw->run(stop); }); return std::unique_ptr(new KinectPreviewWindow(std::move(impl))); @@ -605,6 +1139,18 @@ void KinectPreviewWindow::show_frame(const KinectFrame& frame) { impl_->handed_new = true; } +void KinectPreviewWindow::set_depth_choice(const KinectDepthChoice& depth) { + // Sent every frame; take_choice compares with what is shown. + std::lock_guard guard(impl_->choice_lock); + impl_->handed_choice = depth; + impl_->choice_new = true; +} + +bool KinectPreviewWindow::ideal_snapshot(KinectIdealSnapshot& snapshot) const { + const std::shared_ptr source = sensor(); + return source && source->ideal_snapshot(snapshot); +} + bool KinectPreviewWindow::closed() const { return impl_->is_closed.load(); } void KinectPreviewWindow::close() { if (const HWND window = impl_->shown.load(); window && !closed()) PostMessageW(window, WM_CLOSE, 0, 0); @@ -622,24 +1168,21 @@ std::string KinectPreviewWindow::failure() const { std::lock_guard guard(impl_->lock); return impl_->failure; } -std::optional KinectPreviewWindow::take_calibration() { - std::lock_guard guard(impl_->lock); - return std::exchange(impl_->finished, std::nullopt); -} } #else namespace sfr { struct KinectPreviewWindow::Impl {}; KinectPreviewWindow::KinectPreviewWindow(std::unique_ptr impl) : impl_(std::move(impl)) {} KinectPreviewWindow::~KinectPreviewWindow() = default; -std::unique_ptr KinectPreviewWindow::open(bool) { return nullptr; } +std::unique_ptr KinectPreviewWindow::open(bool, int, const KinectDepthChoice&) { return nullptr; } std::unique_ptr KinectPreviewWindow::open_over(KinectSensor&) { return nullptr; } void KinectPreviewWindow::show_frame(const KinectFrame&) {} +void KinectPreviewWindow::set_depth_choice(const KinectDepthChoice&) {} +bool KinectPreviewWindow::ideal_snapshot(KinectIdealSnapshot&) const { return false; } bool KinectPreviewWindow::closed() const { return true; } void KinectPreviewWindow::close() {} void KinectPreviewWindow::raise() {} std::shared_ptr KinectPreviewWindow::sensor() const { return nullptr; } std::string KinectPreviewWindow::failure() const { return {}; } -std::optional KinectPreviewWindow::take_calibration() { return std::nullopt; } } #endif diff --git a/src/kinect_race_record.cpp b/src/kinect_race_record.cpp new file mode 100644 index 0000000..cd5ec1e --- /dev/null +++ b/src/kinect_race_record.cpp @@ -0,0 +1,181 @@ +#include "kinect_race_record.h" + +#include +#include +#include +#include +#include + +namespace sfr { + +const std::array kinect_race_detectors = {0x822C8778, 0x822CB840, 0x822C9050, + 0x822CA6B0}; +const std::array kinect_race_detector_names = {"crouch", "charge", "jump", + "side"}; + +int kinect_race_detector_index(uint32_t address) { + for (uint32_t i = 0; i < kinect_race_detector_count; ++i) + if (kinect_race_detectors[i] == address) return int(i); + return -1; +} + +namespace { +constexpr size_t race_columns = 6 + 2 * kinect_race_detector_count; +constexpr const char* view_columns[] = { + "view_frame", "view_us", "slot", "centroid_col", "centroid_row", "min_col", "min_row", "max_col", "max_row", + "count_lower_left", "count_lower_right", "count_upper_left", "count_upper_right", + "mean_lower_left", "mean_lower_right", "mean_upper_left", "mean_upper_right", "near_count", "mean_depth", +}; +constexpr size_t view_column_count = sizeof view_columns / sizeof view_columns[0]; + +void put_float(std::string& out, float value) { + out += ','; + if (!std::isfinite(value)) return; + char text[32]; + std::snprintf(text, sizeof text, "%.9g", double(value)); + out += text; +} +void put_int(std::string& out, long long value) { + out += ','; + out += std::to_string(value); +} +bool number(std::string_view text, double& value) { + if (text.empty()) return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtod(copy.c_str(), &end); + return end && *end == 0; +} +bool whole(std::string_view text, long long& value) { + if (text.empty()) return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtoll(copy.c_str(), &end, 10); + return end && *end == 0; +} +} + +std::string kinect_race_record_header() { + std::string out = "steady_us,generation,record,tracking_id,depth_right,depth_left"; + for (const char* name : kinect_race_detector_names) { + out += ','; + out += name; + out += "_original,"; + out += name; + out += "_result"; + } + for (const char* name : view_columns) { + out += ','; + out += name; + } + out += '\n'; + return out; +} + +void kinect_race_record_row(const KinectRaceRecord& r, std::string& out) { + out += std::to_string(r.steady_us); + put_int(out, (long long)r.generation); + put_int(out, r.record); + put_int(out, r.tracking_id); + put_float(out, r.depth_right); + put_float(out, r.depth_left); + for (uint32_t i = 0; i < kinect_race_detector_count; ++i) { + if (r.original[i] >= 0) put_int(out, r.original[i]); else out += ','; + if (r.result[i] >= 0) put_int(out, r.result[i]); else out += ','; + } + const KinectDepthViewRecord& v = r.view; + if (!v.known) { + out.append(view_column_count, ','); + } else { + put_int(out, (long long)v.frame); + put_int(out, v.steady_us); + put_int(out, v.slot); + for (uint32_t value : {v.centroid.column, v.centroid.row, v.centroid.min_column, v.centroid.min_row, + v.centroid.max_column, v.centroid.max_row}) + put_int(out, value); + for (float value : v.counts) put_float(out, value); + for (float value : v.means) put_float(out, value); + put_int(out, v.near_count); + put_float(out, v.average); + } + out += '\n'; +} + +bool kinect_parse_race_row(std::string_view line, KinectRaceRecord& r) { + const auto f = kinect_record_fields(line); + // Files from before the depth view's columns have only the first ones. + if (f.size() != race_columns && f.size() != race_columns + view_column_count) return false; + double v = 0; + r = {}; + if (!number(f[0], v)) return false; // the header + r.steady_us = int64_t(v); + size_t at = 1; + const auto next = [&](double fallback = 0) { return number(f[at++], v) ? v : fallback; }; + r.generation = uint64_t(next()); + r.record = uint32_t(next()); + r.tracking_id = uint32_t(next()); + r.depth_right = float(next()); + r.depth_left = float(next()); + for (uint32_t i = 0; i < kinect_race_detector_count; ++i) { + r.original[i] = int(next(-1)); + r.result[i] = int(next(-1)); + } + if (f.size() == race_columns) return true; + KinectDepthViewRecord& view = r.view; + long long value = 0; + if (!whole(f[at], value)) return true; // no depth view yet for this record + view.known = true; + const auto integer = [&]() { return whole(f[at++], value) ? value : 0; }; + view.frame = uint64_t(integer()); + view.steady_us = int64_t(integer()); + view.slot = uint32_t(integer()); + view.centroid.column = uint32_t(integer()); + view.centroid.row = uint32_t(integer()); + view.centroid.min_column = uint32_t(integer()); + view.centroid.min_row = uint32_t(integer()); + view.centroid.max_column = uint32_t(integer()); + view.centroid.max_row = uint32_t(integer()); + for (float& count : view.counts) count = float(next(NAN)); + for (float& mean : view.means) mean = float(next(NAN)); + view.near_count = uint32_t(integer()); + view.average = float(next(NAN)); + return true; +} + +KinectRecordFile* kinect_record_race() { + static std::mutex lock; + static std::unique_ptr file; + std::lock_guard guard(lock); + const std::string& prefix = kinect_record_prefix(); + if (prefix.empty()) return nullptr; + if (!file) + file = std::make_unique(prefix + "-race.csv", kinect_race_record_header(), + kinect_record_limit_bytes()); + return file.get(); +} + +namespace { +std::mutex& view_lock() { + static std::mutex lock; + return lock; +} +std::unordered_map& views() { + static std::unordered_map table; + return table; +} +} + +void kinect_depth_view_publish(uint32_t record, const KinectDepthViewRecord& view) { + std::lock_guard guard(view_lock()); + views()[record] = view; +} + +bool kinect_depth_view_latest(uint32_t record, KinectDepthViewRecord& view) { + std::lock_guard guard(view_lock()); + const auto found = views().find(record); + if (found == views().end()) return false; + view = found->second; + return true; +} + +} diff --git a/src/kinect_race_record.h b/src/kinect_race_record.h new file mode 100644 index 0000000..8da5e51 --- /dev/null +++ b/src/kinect_race_record.h @@ -0,0 +1,77 @@ +#pragma once +#include "kinect_depth_features.h" +#include "kinect_record.h" + +#include +#include +#include +#include + +namespace sfr { + +// The race's side of a recording (kinect_record.h): with SFR_KINECT_RECORD +// set, -race.csv gets one row per game update per player record +// that a real sensor drives: the title's own depth view's lean pair (record +// +640, +644) as the title computed it, and what the title's own crouch, +// charge, jump and side detectors answered on that update (original) and +// what the race used (result: the same, nothing stands in for a real +// Kinect's depth). With a Kinect v1 these are the console's own depth-based +// gestures, with a Kinect v2 the same detectors over the depth the v2 makes +// (kinect_v2_depth.h), so a v1 session and a v2 session can be compared. +// After those columns, everything the depth view computed for the player +// from its newest image (KinectDepthViewRecord): the image's number, so the +// image itself (kinect_record_depth.h) can be matched and its features +// computed again offline (sfr_kinect_depth_features --validate). + +// The title's detectors a row records, in its column order. +constexpr uint32_t kinect_race_detector_count = 4; +extern const std::array kinect_race_detectors; // 822C8778 ... +extern const std::array kinect_race_detector_names; +// The column of a detector, or -1 when a row does not record it. +int kinect_race_detector_index(uint32_t address); + +// Everything the title's depth view wrote for one player from one depth +// image (kinect_depth_features.h has the layout and the code it comes +// from), read from guest memory right after the view ran on that image +// (nui_hooks.cpp, 82439530), so all of it is of the same image. +struct KinectDepthViewRecord { + bool known = false; // false: no depth view has run for this record yet + uint64_t frame = 0; // the depth image's number (NUI_IMAGE_FRAME +8, KinectImage::number) + int64_t steady_us = 0; // when the view finished with it + uint32_t slot = 0; // the skeleton slot (player index slot + 1) + KinectDepthCentroid centroid; // manager+7584/7632/7680 + 8*slot (pixels: not kept, 0) + // record +640..+679 as they are in memory: when the image had too few of + // the player's pixels the counts are zero and the rest an older image's. + std::array counts{}, means{}; + uint32_t near_count = 0; + float average = 0; +}; + +struct KinectRaceRecord { + int64_t steady_us = 0; + uint64_t generation = 0; // the sensor frame the update read + uint32_t record = 0; // the title's body record (guest address) + uint32_t tracking_id = 0; // its skeleton's tracking id + // The title's depth view's lean pair. + float depth_right = 0, depth_left = 0; + // Each detector's answer on this update: the title's own (original) and + // the one the race used (result); -1 when it was not asked. + std::array original{-1, -1, -1, -1}, result{-1, -1, -1, -1}; + // The depth view's newest features for this record (columns after the + // detectors; a file from before them still loads, with known false). + KinectDepthViewRecord view; +}; + +std::string kinect_race_record_header(); +void kinect_race_record_row(const KinectRaceRecord& row, std::string& out); +bool kinect_parse_race_row(std::string_view line, KinectRaceRecord& row); + +// The race file, or null when recording is off. +KinectRecordFile* kinect_record_race(); + +// The depth view's features of a body record, kept from the view's thread +// (kinect_depth_view_publish) for the race's rows (kinect_depth_view_latest). +void kinect_depth_view_publish(uint32_t record, const KinectDepthViewRecord& view); +bool kinect_depth_view_latest(uint32_t record, KinectDepthViewRecord& view); + +} diff --git a/src/kinect_record.cpp b/src/kinect_record.cpp new file mode 100644 index 0000000..4244524 --- /dev/null +++ b/src/kinect_record.cpp @@ -0,0 +1,556 @@ +#include "kinect_record.h" + +#include +#include +#include +#include +#include + +namespace sfr { + +int64_t kinect_record_now_us() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +const std::array kinect_record_joint_names = { + "spine_base", "spine_mid", "neck", "head", + "shoulder_left", "elbow_left", "wrist_left", "hand_left", + "shoulder_right", "elbow_right", "wrist_right", "hand_right", + "hip_left", "knee_left", "ankle_left", "foot_left", + "hip_right", "knee_right", "ankle_right", "foot_right", + "spine_shoulder", "hand_tip_left", "thumb_left", "hand_tip_right", "thumb_right", +}; + +// v1's 0..19 are v2's 0..19 except its shoulder centre (2), which is v2's +// SpineShoulder (20); v2's Neck (2) and 21..24 have no v1 joint. +const std::array kinect_record_v1_joint = { + 0, 1, -1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2, -1, -1, -1, -1, +}; + +void kinect_record_from_v1(const std::array, 20>& joints, const std::array& states, + KinectRecordBody& body) { + for (uint32_t j = 0; j < kinect_record_joint_count; ++j) { + const int from = kinect_record_v1_joint[j]; + body.joints[j] = from < 0 ? KinectRecordJoint{} : KinectRecordJoint{joints[from], states[from] < 0 ? 0 : states[from]}; + } +} + +float kinect_depth_invalid_ratio(const uint16_t* pixels, size_t count, int shift) { + if (!pixels || !count) return -1; + size_t invalid = 0; + // A v1 (SDK 1.8 NuiImageCamera.h) also has distinct "too far" (0x0FFF) + // and "unknown" (0x1FFF) values when asked for them; neither is a depth. + for (size_t i = 0; i < count; ++i) { + const uint32_t depth = pixels[i] >> shift; + invalid += depth == 0 || (shift == 3 && (depth == 0x0FFF || depth == 0x1FFF)); + } + return float(double(invalid) / double(count)); +} + +namespace { +void put(std::string& out, const char* text) { + out += ','; + out += text; +} +void put_float(std::string& out, float value) { + out += ','; + if (!std::isfinite(value)) return; // empty: not a number the reader can use + char text[32]; + std::snprintf(text, sizeof text, "%.6g", double(value)); + out += text; +} +void put_int(std::string& out, long long value) { + out += ','; + out += std::to_string(value); +} +void put_uint(std::string& out, unsigned long long value) { + out += ','; + out += std::to_string(value); +} +} + +std::string kinect_record_skeleton_header() { + std::string out = "steady_us,sensor,frame,sdk_time,floor_a,floor_b,floor_c,floor_d,gravity_x,gravity_y,gravity_z," + "depth_invalid,depth_frame,depth_steady_us,bodies,body_index,tracking_id," + "position_x,position_y,position_z,hand_left_state,hand_right_state"; + for (const char* name : kinect_record_joint_names) + for (const char* part : {"_x", "_y", "_z", "_state"}) { + out += ','; + out += name; + out += part; + } + out += ",clipped_edges\n"; + return out; +} + +void kinect_record_skeleton_rows(const KinectRecordFrame& frame, std::string& out) { + const auto row = [&](const KinectRecordBody* body) { + out += std::to_string(frame.steady_us); + put(out, frame.sensor.c_str()); + put_uint(out, frame.frame); + put_int(out, frame.sdk_time); + for (float v : frame.floor) put_float(out, v); + for (float v : frame.gravity) put_float(out, v); + if (frame.depth_invalid >= 0) { + put_float(out, frame.depth_invalid); + put_uint(out, frame.depth_frame); + put_int(out, frame.depth_steady_us); + } else { + out += ",,,"; + } + put_uint(out, frame.bodies.size()); + if (!body) { + // Nobody: the body's columns empty. + out.append(8 + 4 * kinect_record_joint_count, ','); + out += '\n'; + return; + } + put_uint(out, body->sensor_index); + put_uint(out, body->tracking_id); + if (body->has_position) + for (float v : body->position) put_float(out, v); + else + out += ",,,"; + for (int state : body->hand_states) + if (state >= 0) put_int(out, state); + else out += ','; + for (const KinectRecordJoint& joint : body->joints) { + if (joint.state < 0) { out += ",,,,"; continue; } + for (float v : joint.position) put_float(out, v); + put_int(out, joint.state); + } + if (body->clipped_edges >= 0) put_int(out, body->clipped_edges); + else out += ','; + out += '\n'; + }; + if (frame.bodies.empty()) row(nullptr); + for (const KinectRecordBody& body : frame.bodies) row(&body); +} + +std::vector kinect_record_fields(std::string_view line) { + while (!line.empty() && (line.back() == '\n' || line.back() == '\r')) line.remove_suffix(1); + std::vector fields; + size_t start = 0; + for (;;) { + const size_t comma = line.find(',', start); + fields.push_back(line.substr(start, comma == std::string_view::npos ? std::string_view::npos : comma - start)); + if (comma == std::string_view::npos) break; + start = comma + 1; + } + return fields; +} + +namespace { +bool number(std::string_view text, double& value) { + if (text.empty()) return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtod(copy.c_str(), &end); + return end && *end == 0; +} +// Whole numbers past a double's 53 bits (a v2's tracking ids, its +// RelativeTime) read exactly. +bool whole(std::string_view text, long long& value) { + if (text.empty()) return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtoll(copy.c_str(), &end, 10); + return end && *end == 0; +} +bool whole(std::string_view text, unsigned long long& value) { + if (text.empty() || text[0] == '-') return false; + const std::string copy(text); + char* end = nullptr; + value = std::strtoull(copy.c_str(), &end, 10); + return end && *end == 0; +} +} + +bool kinect_parse_skeleton_row(std::string_view line, KinectRecordRow& row) { + const auto fields = kinect_record_fields(line); + // 22 + the joints, and clipped_edges after them; recordings made before + // that column still load (clipped_edges stays -1 for them). + constexpr size_t columns = 22 + 4 * kinect_record_joint_count; + if (fields.size() != columns && fields.size() != columns + 1) return false; + double v = 0; + row = {}; + KinectRecordFrame& f = row.frame; + long long signed_value = 0; + unsigned long long unsigned_value = 0; + if (!whole(fields[0], signed_value)) return false; // the header, or a broken line + f.steady_us = signed_value; + f.sensor = std::string(fields[1]); + if (whole(fields[2], unsigned_value)) f.frame = unsigned_value; + if (whole(fields[3], signed_value)) f.sdk_time = signed_value; + for (int i = 0; i < 4; ++i) f.floor[i] = number(fields[4 + i], v) ? float(v) : 0.0f; + for (int i = 0; i < 3; ++i) f.gravity[i] = number(fields[8 + i], v) ? float(v) : 0.0f; + f.depth_invalid = number(fields[11], v) ? float(v) : -1.0f; + if (whole(fields[12], unsigned_value)) f.depth_frame = unsigned_value; + if (whole(fields[13], signed_value)) f.depth_steady_us = signed_value; + const size_t bodies = number(fields[14], v) ? size_t(v) : 0; + row.has_body = number(fields[15], v); + if (!row.has_body) return true; + (void)bodies; + KinectRecordBody& b = row.body; + b.sensor_index = uint32_t(v); + if (whole(fields[16], unsigned_value)) b.tracking_id = unsigned_value; + b.has_position = number(fields[17], v); + for (int i = 0; i < 3; ++i) b.position[i] = number(fields[17 + i], v) ? float(v) : 0.0f; + for (int i = 0; i < 2; ++i) b.hand_states[i] = number(fields[20 + i], v) ? int(v) : -1; + for (uint32_t j = 0; j < kinect_record_joint_count; ++j) { + const size_t at = 22 + 4 * j; + KinectRecordJoint& joint = b.joints[j]; + if (!number(fields[at + 3], v)) { joint = {}; continue; } + joint.state = int(v); + for (int i = 0; i < 3; ++i) joint.position[i] = number(fields[at + i], v) ? float(v) : NAN; + } + if (fields.size() == columns + 1 && number(fields[columns], v)) b.clipped_edges = int(v); + return true; +} + +const std::array kinect_record_jitter_joints = {0, 20, 7, 11}; + +KinectRecordStats kinect_record_stats(const std::vector& rows, std::string_view sensor) { + KinectRecordStats stats; + // Rows of one frame are together; the player is its nearest body. + struct Player { + bool present = false; + KinectRecordBody body; + int64_t steady_us = 0; + }; + std::vector players; + double depth_sum = 0; + uint64_t depth_count = 0; + for (size_t i = 0; i < rows.size();) { + if (rows[i].frame.sensor != sensor) { ++i; continue; } + const uint64_t number = rows[i].frame.frame; + Player player; + player.steady_us = rows[i].frame.steady_us; + if (rows[i].frame.depth_invalid >= 0) { + depth_sum += rows[i].frame.depth_invalid; + ++depth_count; + } + size_t j = i; + for (; j < rows.size() && rows[j].frame.sensor == sensor && rows[j].frame.frame == number && + rows[j].frame.steady_us == rows[i].frame.steady_us; + ++j) { + if (!rows[j].has_body) continue; + const float z = rows[j].body.joints[0].position[2]; + if (!player.present || z < player.body.joints[0].position[2]) { + player.present = true; + player.body = rows[j].body; + } + } + players.push_back(player); + i = j; + } + stats.frames = players.size(); + std::array squares{}; + std::array pairs{}; + double tracked = 0; + for (size_t f = 0; f < players.size(); ++f) { + const Player& now = players[f]; + if (f > 0) { + const Player& before = players[f - 1]; + if (before.present && !now.present) ++stats.lost; + if (before.present && now.present && before.body.tracking_id != now.body.tracking_id) ++stats.lost; + } + if (!now.present) continue; + ++stats.with_body; + if (now.body.clipped_edges > 0) ++stats.clipped; + uint32_t have = 0, seen = 0; + for (const KinectRecordJoint& joint : now.body.joints) { + if (joint.state < 0) continue; + ++have; + seen += joint.state == 2; + } + if (have) tracked += double(seen) / have; + if (f == 0) continue; + const Player& before = players[f - 1]; + // Consecutive frames of the same body, at most 0.1 s apart. + if (!before.present || before.body.tracking_id != now.body.tracking_id || + now.steady_us - before.steady_us > 100000) + continue; + for (size_t k = 0; k < kinect_record_jitter_joints.size(); ++k) { + const auto& a = before.body.joints[kinect_record_jitter_joints[k]]; + const auto& b = now.body.joints[kinect_record_jitter_joints[k]]; + if (a.state != 2 || b.state != 2) continue; + double d = 0; + for (int c = 0; c < 3; ++c) d += double(b.position[c] - a.position[c]) * (b.position[c] - a.position[c]); + squares[k] += d; + ++pairs[k]; + } + } + for (size_t k = 0; k < 4; ++k) + stats.jitter[k] = pairs[k] ? float(std::sqrt(squares[k] / double(pairs[k]) / 2.0)) : NAN; + stats.tracked = stats.with_body ? float(tracked / double(stats.with_body)) : 0.0f; + stats.depth_invalid = depth_count ? float(depth_sum / double(depth_count)) : -1.0f; + return stats; +} + +namespace { +std::atomic process_queue{nullptr}; +} + +uint64_t kinect_record_queue_budget_bytes() { + static const uint64_t budget = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_QUEUE_MB"); + const double mb = text && *text ? std::strtod(text, nullptr) : 256.0; + return uint64_t((mb >= 1 && mb <= 65536 ? mb : 256.0) * 1024 * 1024); + }(); + return budget; +} + +KinectRecordQueue& KinectRecordQueue::process() { + // Never destroyed: files kept in other statics drain into it while the + // process ends, whatever the order their destructors run in. + static KinectRecordQueue* queue = [] { + auto* made = new KinectRecordQueue(kinect_record_queue_budget_bytes()); + process_queue.store(made); + return made; + }(); + return *queue; +} + +std::string kinect_record_queue_status() { + const KinectRecordQueue* queue = process_queue.load(); + if (!queue) return {}; + char text[96]; + std::snprintf(text, sizeof text, " record_dropped=%llu record_queued_mb=%.1f", (unsigned long long)queue->dropped(), + double(queue->queued_bytes()) / (1024.0 * 1024.0)); + return text; +} + +KinectRecordQueue::KinectRecordQueue(uint64_t budget_bytes) : budget_(budget_bytes) {} + +KinectRecordQueue::~KinectRecordQueue() { + { + std::unique_lock guard(lock_); + // What is queued gets ten seconds to reach the disk. + done_.wait_for(guard, std::chrono::seconds(10), [&] { return items_.empty() && pending_.empty(); }); + stop_ = true; + dropped_ += items_.size(); + items_.clear(); + } + wake_.notify_all(); + if (writer_.joinable()) writer_.join(); +} + +void KinectRecordQueue::set_blocking(bool blocking) { + std::lock_guard guard(lock_); + blocking_ = blocking; +} + +uint64_t KinectRecordQueue::dropped() const { + std::lock_guard guard(lock_); + return dropped_; +} + +uint64_t KinectRecordQueue::queued_bytes() const { + std::lock_guard guard(lock_); + return queued_; +} + +bool KinectRecordQueue::push(KinectRecordSink* sink, std::string&& bytes) { + std::unique_lock guard(lock_); + if (stop_) return false; + if (!started_) { + started_ = true; + writer_ = std::thread([this] { run(); }); + } + const auto fits = [&] { return queued_ == 0 || queued_ + bytes.size() <= budget_; }; + if (!fits()) { + if (!blocking_) { + ++dropped_; + return false; + } + done_.wait(guard, [&] { return fits() || stop_; }); + if (stop_) return false; + } + queued_ += bytes.size(); + auto it = std::find_if(pending_.begin(), pending_.end(), [&](const auto& p) { return p.first == sink; }); + if (it == pending_.end()) pending_.emplace_back(sink, 1); + else ++it->second; + items_.push_back({sink, std::move(bytes)}); + guard.unlock(); + wake_.notify_one(); + return true; +} + +bool KinectRecordQueue::drain(KinectRecordSink* sink, std::chrono::milliseconds wait) { + std::unique_lock guard(lock_); + const auto none = [&] { + return std::none_of(pending_.begin(), pending_.end(), [&](const auto& p) { return p.first == sink; }); + }; + if (done_.wait_for(guard, wait, none)) return true; + // Out of time: what has not started is dropped; what is being written + // is waited for (its sink is about to go away). + uint64_t removed = 0; + for (auto it = items_.begin(); it != items_.end();) { + if (it->sink != sink) { ++it; continue; } + queued_ -= it->bytes.size(); + it = items_.erase(it); + ++removed; + } + dropped_ += removed; + for (auto it = pending_.begin(); it != pending_.end(); ++it) + if (it->first == sink) { + it->second -= removed; + if (!it->second) pending_.erase(it); + break; + } + done_.wait(guard, none); + return false; +} + +void KinectRecordQueue::run() { + auto logged_at = std::chrono::steady_clock::now() - std::chrono::seconds(2); + std::deque batch; + std::vector touched; + for (;;) { + std::unique_lock guard(lock_); + wake_.wait_for(guard, std::chrono::seconds(1), [&] { return !items_.empty() || stop_; }); + const auto now = std::chrono::steady_clock::now(); + if (dropped_ != logged_dropped_ && now - logged_at >= std::chrono::seconds(1)) { + // At most once a second, never from the recorders' own threads. + std::cerr << "NATIVE_KINECT_RECORD dropped=" << dropped_ << " (disk too slow)" << std::endl; + logged_dropped_ = dropped_; + logged_at = now; + } + if (items_.empty()) { + if (stop_) return; + continue; + } + batch.swap(items_); + guard.unlock(); + touched.clear(); + for (Item& item : batch) { + item.sink->record_write(item.bytes); + if (std::find(touched.begin(), touched.end(), item.sink) == touched.end()) touched.push_back(item.sink); + } + // Handed to the system after each batch: a game ending without its + // destructors, or killed, loses only what was still queued. + for (KinectRecordSink* sink : touched) sink->record_flush(); + guard.lock(); + for (const Item& item : batch) { + queued_ -= item.bytes.size(); + for (auto it = pending_.begin(); it != pending_.end(); ++it) + if (it->first == item.sink) { + if (!--it->second) pending_.erase(it); + break; + } + } + batch.clear(); + guard.unlock(); + done_.notify_all(); + } +} + +KinectRecordFile::KinectRecordFile(std::string path, std::string header, uint64_t limit_bytes, KinectRecordQueue* queue) + : queue_(queue ? *queue : KinectRecordQueue::process()), path_(std::move(path)), header_(std::move(header)), + limit_(limit_bytes) {} + +KinectRecordFile::~KinectRecordFile() { + queue_.drain(this, std::chrono::seconds(10)); + if (file_) std::fclose(file_); +} + +bool KinectRecordFile::write(const std::string& rows) { return write(std::string(rows)); } + +bool KinectRecordFile::write(std::string&& rows) { + std::lock_guard guard(lock_); + if (failed_ || full_) return false; + const uint64_t header = started_ ? 0 : header_.size(); + if (accounted_ + header + rows.size() > limit_) { + full_ = true; + std::cerr << "NATIVE_KINECT_RECORD full " << path_ << " after " << accounted_ << " bytes: stopped" << std::endl; + return false; + } + const uint64_t size = rows.size(); + // The file opens with the first row (on the writer thread), so turning + // recording on without a sensor leaves no empty file behind; the header + // goes with that row. + std::string bytes = header ? header_ + rows : std::move(rows); + if (!queue_.push(this, std::move(bytes))) return false; + started_ = true; + accounted_ += header + size; + return true; +} + +void KinectRecordFile::record_write(std::string& bytes) { + if (!opened_) { + opened_ = true; + file_ = std::fopen(path_.c_str(), "wb"); + if (!file_) { + failed_ = true; + std::cerr << "NATIVE_KINECT_RECORD failed to open " << path_ << std::endl; + return; + } + std::cerr << "NATIVE_KINECT_RECORD writing " << path_ << " (at most " << limit_ / (1024 * 1024) << " MB)" + << std::endl; + } + if (file_) std::fwrite(bytes.data(), 1, bytes.size(), file_); +} + +void KinectRecordFile::record_flush() { + if (file_) std::fflush(file_); +} + +namespace { +std::mutex& prefix_lock() { + static std::mutex lock; + return lock; +} +std::string& prefix_value() { + static std::string prefix = [] { + const char* text = std::getenv("SFR_KINECT_RECORD"); + return std::string(text && *text && std::string_view(text) != "0" ? text : ""); + }(); + return prefix; +} +std::unique_ptr& skeleton_file() { + static std::unique_ptr file; + return file; +} +} + +const std::string& kinect_record_prefix() { + std::lock_guard guard(prefix_lock()); + return prefix_value(); +} + +void kinect_record_start(const std::string& prefix) { + std::lock_guard guard(prefix_lock()); + if (skeleton_file()) return; + prefix_value() = prefix; +} + +uint64_t kinect_record_limit_bytes() { + static const uint64_t limit = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_MB"); + const double mb = text && *text ? std::strtod(text, nullptr) : 512.0; + return uint64_t((mb >= 1 && mb <= 16384 ? mb : 512.0) * 1024 * 1024); + }(); + return limit; +} + +KinectRecordFile* kinect_record_skeletons() { + std::lock_guard guard(prefix_lock()); + if (prefix_value().empty()) return nullptr; + auto& file = skeleton_file(); + if (!file) + file = std::make_unique(prefix_value() + "-skeleton.csv", kinect_record_skeleton_header(), + kinect_record_limit_bytes()); + return file.get(); +} + +void kinect_record_skeleton_frame(const KinectRecordFrame& frame) { + KinectRecordFile* file = kinect_record_skeletons(); + if (!file) return; + std::string rows; + kinect_record_skeleton_rows(frame, rows); + file->write(std::move(rows)); +} + +} diff --git a/src/kinect_record.h b/src/kinect_record.h new file mode 100644 index 0000000..a9b1fb4 --- /dev/null +++ b/src/kinect_record.h @@ -0,0 +1,237 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sfr { + +// Recording a real Kinect for offline study (docs/kinect-sensor.md, +// "錄製"): what each sensor tracked, frame by frame, in one CSV schema for a +// Kinect v1 and a Kinect v2, so that the same movement seen by both (two +// sensors opened by one sfr_kinect_probe, or the game and a probe side by +// side) can be lined up by the host's steady clock and compared. +// +// SFR_KINECT_RECORD= turns it on, in the game or the probe (the +// probe's --record sets it): -skeleton.csv gets one row per tracked +// body per sensor frame (one row with no body when nobody is tracked), as +// the sensor gave it (before any smoothing). The game adds +// -race.csv (kinect_race_record.h). Each file stops growing at +// SFR_KINECT_RECORD_MB megabytes (default 512) and says so once in the log. +// +// None of this is Windows', so it is tested everywhere (kinect_record_test). + +// The host's steady clock in microseconds: the same clock in every process +// of one machine on Windows (QueryPerformanceCounter), so a game and a +// probe recording side by side share it. +int64_t kinect_record_now_us(); + +// Kinect v2's JointType order and names, the joints of every row. A v1's +// twenty joints go into the v2 joints defined alike (its hip centre is +// SpineBase, its spine SpineMid, its shoulder centre SpineShoulder); its rows +// leave Neck and the fingertips and thumbs empty. +constexpr uint32_t kinect_record_joint_count = 25; +extern const std::array kinect_record_joint_names; +// v1's joint (NUI_SKELETON_POSITION_INDEX) in each of those, -1 for none. +extern const std::array kinect_record_v1_joint; + +struct KinectRecordJoint { + std::array position{}; // camera space, metres + int state = -1; // 0 not tracked, 1 inferred, 2 tracked; -1 the sensor has no such joint +}; + +struct KinectRecordBody { + uint32_t sensor_index = 0; // the sensor's skeleton slot (0..5), its body index + uint64_t tracking_id = 0; // the sensor's own, whole (a v2's is 64 bits) + std::array joints{}; + // A v1's skeleton position (its centre); a v2 has none. + bool has_position = false; + std::array position{}; + // A v2's hand states (0 unknown, 1 not tracked, 2 open, 3 closed, 4 + // lasso), -1 from a v1. + std::array hand_states{-1, -1}; + // A v2's clipped edges (kinect_edge bits of kinect_sensor.h: right 1, + // left 2, top 4, bottom 8; 0 none), -1 from a v1 (it does not say). + // The last column, so older recordings (without it) still load. + int clipped_edges = -1; +}; + +struct KinectRecordFrame { + int64_t steady_us = 0; // when the backend received it (kinect_record_now_us) + std::string sensor; // "v1" or "v2" + uint64_t frame = 0; // the sensor's own frame number (v1's dwFrameNumber), or the backend's count (v2) + // The SDK's own timestamp: v1's liTimeStamp (milliseconds), v2's + // RelativeTime (100 ns units); 0 when it gave none. + int64_t sdk_time = 0; + std::array floor{}; // floor clip plane, all zero when unknown + std::array gravity{}; // v1's normal to gravity; a v2 gives none (zero) + // The share of the newest depth image's pixels without a depth (0..1), + // that image's number and when it came; negative when no depth is open. + float depth_invalid = -1; + uint64_t depth_frame = 0; + int64_t depth_steady_us = 0; + std::vector bodies; +}; + +// A v1 skeleton's joints and states into a body's v2-ordered joints. +void kinect_record_from_v1(const std::array, 20>& joints, const std::array& states, + KinectRecordBody& body); + +// The share of pixels without a depth: a v1's depth-and-player pixels +// carry it above the three player bits (shift 3), a v2's are millimetres +// (shift 0); zero is "no depth" for both, and a v1's "too far" and +// "unknown" values too. +float kinect_depth_invalid_ratio(const uint16_t* pixels, size_t count, int shift); + +std::string kinect_record_skeleton_header(); +// The frame's rows (each ending in a newline), appended to out. +void kinect_record_skeleton_rows(const KinectRecordFrame& frame, std::string& out); + +// One parsed skeleton row (for kinect_record_summary and tests): the frame's +// fields and the body, has_body false for a frame with nobody. +struct KinectRecordRow { + KinectRecordFrame frame; // without bodies + bool has_body = false; + KinectRecordBody body; +}; +bool kinect_parse_skeleton_row(std::string_view line, KinectRecordRow& row); + +// A recording's steadiness, for one sensor (sfr_kinect_record_summary, +// docs/kinect-sensor.md): the A/B test of whether a second sensor's light +// makes this one worse, while the player stands still. Each frame's player +// is its nearest body (the smallest SpineBase z). +struct KinectRecordStats { + uint64_t frames = 0, with_body = 0; + // Frame-to-frame movement of a tracked joint, in metres RMS divided by + // the square root of two (the joint's own noise when it is still), for + // SpineBase, SpineShoulder, HandLeft and HandRight; NaN without pairs. + std::array jitter{}; + // Of the player's joints the sensor has, the share tracked (not + // inferred or lost), over frames with a player. + float tracked = 0; + // The mean share of depth pixels without a depth; negative when the + // recording has none. + float depth_invalid = -1; + // The player lost: a frame without a body after one with, or another + // tracking id taking the nearest place. + uint32_t lost = 0; + // Frames whose player was cut off by an edge of the picture (a v2's + // clipped_edges column; a v1's rows never say). + uint64_t clipped = 0; +}; +extern const std::array kinect_record_jitter_joints; // SpineBase, SpineShoulder, HandLeft, HandRight +KinectRecordStats kinect_record_stats(const std::vector& rows, std::string_view sensor); + +// Splits one CSV line at its commas (the recorder writes no quoted fields). +std::vector kinect_record_fields(std::string_view line); + +// Where a recorder's bytes go on the writer thread (KinectRecordQueue): the +// file classes below, or a test's fake. +class KinectRecordSink { +public: + virtual ~KinectRecordSink() = default; + // Writes one item's bytes (the writer thread only, in the order queued). + virtual void record_write(std::string& bytes) = 0; + // After a batch of items: hands what was written to the system. + virtual void record_flush() = 0; +}; + +// Recording never waits for the disk: every recorder (the CSV files, the +// depth images, a Kinect v2's raw frames; in the game and in the probe) +// hands an owned buffer to this queue and one writer thread per process +// writes and flushes it (after each batch, so a killed process loses only +// what was still queued). The queue holds at most budget bytes +// (SFR_KINECT_RECORD_QUEUE_MB, default 256): an item that does not fit is +// dropped and counted, never waited for ("NATIVE_KINECT_RECORD dropped=N +// (disk too slow)" at most once a second). A dropped depth or raw image +// leaves a gap in the image numbers and steady_us, which the readers and +// tools take as any gap. An offline tool can ask to wait instead +// (set_blocking). +class KinectRecordQueue { +public: + explicit KinectRecordQueue(uint64_t budget_bytes); + ~KinectRecordQueue(); // writes what is queued (at most 10 s), then stops + // The process's queue (SFR_KINECT_RECORD_QUEUE_MB). + static KinectRecordQueue& process(); + // Queues bytes for sink; false (dropped and counted) when the budget is + // full, unless blocking, when it waits for room. + bool push(KinectRecordSink* sink, std::string&& bytes); + // Waits (at most wait) until everything queued for sink is written; + // what is still queued then is dropped (counted). False on time out. + bool drain(KinectRecordSink* sink, std::chrono::milliseconds wait); + void set_blocking(bool blocking); + uint64_t dropped() const; + uint64_t queued_bytes() const; + uint64_t budget_bytes() const { return budget_; } +private: + struct Item { + KinectRecordSink* sink; + std::string bytes; + }; + void run(); + const uint64_t budget_; + mutable std::mutex lock_; + std::condition_variable wake_, done_; + std::deque items_; + std::vector> pending_; // per sink, queued or being written + uint64_t queued_ = 0, dropped_ = 0, logged_dropped_ = 0; + bool stop_ = false, blocking_ = false, started_ = false; + std::thread writer_; +}; +// SFR_KINECT_RECORD_QUEUE_MB, in bytes. +uint64_t kinect_record_queue_budget_bytes(); +// " record_dropped=N record_queued_mb=M" for a status line, empty when the +// process never recorded. +std::string kinect_record_queue_status(); + +// One CSV file the recorder writes, with its header first, bounded in size. +// Rows are taken whole from any thread and written by the record queue; the +// file opens (on the writer thread) with the first row. +class KinectRecordFile : public KinectRecordSink { +public: + KinectRecordFile(std::string path, std::string header, uint64_t limit_bytes, + KinectRecordQueue* queue = nullptr); // null: the process's + ~KinectRecordFile() override; // writes what is queued (at most 10 s), then closes + // False once the file could not be opened, is full, or the row was + // dropped (the queue full). + bool write(const std::string& rows); + bool write(std::string&& rows); + const std::string& path() const { return path_; } + void record_write(std::string& bytes) override; + void record_flush() override; +private: + KinectRecordQueue& queue_; + std::mutex lock_; + std::string path_, header_; + uint64_t limit_ = 0, accounted_ = 0; + bool started_ = false, full_ = false; + std::atomic failed_{false}; + // The writer thread's. + std::FILE* file_ = nullptr; + bool opened_ = false; +}; + +// The prefix SFR_KINECT_RECORD names, empty when recording is off. +const std::string& kinect_record_prefix(); +// Starts recording to this prefix from now on (the probe's --record); only +// before the first frame is recorded. +void kinect_record_start(const std::string& prefix); +// The skeleton file, or null when recording is off. +KinectRecordFile* kinect_record_skeletons(); +// A sensor frame into the skeleton file, when recording. +void kinect_record_skeleton_frame(const KinectRecordFrame& frame); +// SFR_KINECT_RECORD_MB, in bytes. +uint64_t kinect_record_limit_bytes(); +// The depth images (SFR_KINECT_RECORD_DEPTH) are kinect_record_depth.h's. + +} diff --git a/src/kinect_record_depth.cpp b/src/kinect_record_depth.cpp new file mode 100644 index 0000000..e89aa99 --- /dev/null +++ b/src/kinect_record_depth.cpp @@ -0,0 +1,343 @@ +#include "kinect_record_depth.h" + +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include + +namespace sfr { + +namespace { +constexpr char file_magic[8] = {'S', 'F', 'R', 'K', 'D', 'E', 'P', '2'}; +constexpr char frame_magic[4] = {'K', 'D', 'F', '2'}; + +void put32(std::string& out, uint32_t value) { + for (int i = 0; i < 4; ++i) out += char(uint8_t(value >> (8 * i))); +} +void put64(std::string& out, uint64_t value) { + for (int i = 0; i < 8; ++i) out += char(uint8_t(value >> (8 * i))); +} +void put_float(std::string& out, float value) { + uint32_t bits = 0; + std::memcpy(&bits, &value, 4); + put32(out, bits); +} +uint32_t get32(const uint8_t* p) { return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; } +uint64_t get64(const uint8_t* p) { return uint64_t(get32(p)) | uint64_t(get32(p + 4)) << 32; } +float get_float(const uint8_t* p) { + const uint32_t bits = get32(p); + float value = 0; + std::memcpy(&value, &bits, 4); + return value; +} +} + +const char* kinect_depth_source_name(KinectDepthSource source) { + switch (source) { + case KinectDepthSource::v1: return "v1"; + case KinectDepthSource::v2: return "v2"; + default: return "unknown"; + } +} + +void kinect_depth_frame_pixels(KinectDepthRecordFrame& frame, const uint8_t* bytes, uint32_t width, uint32_t height) { + frame.width = width; + frame.height = height; + frame.pixels.resize(size_t(width) * height); + for (size_t i = 0; i < frame.pixels.size(); ++i) frame.pixels[i] = uint16_t(bytes[2 * i] | bytes[2 * i + 1] << 8); +} + +std::string kinect_depth_file_header(uint32_t width, uint32_t height) { + std::string out(file_magic, 8); + put32(out, kinect_depth_file_version); + put32(out, kinect_depth_file_header_size); + put32(out, width); + put32(out, height); + put32(out, kinect_depth_pixel_format_player); + put32(out, kinect_depth_frame_header_size); + return out; +} + +void kinect_depth_zero_runs_encode(const uint16_t* pixels, size_t count, std::vector& tokens) { + tokens.clear(); + for (size_t i = 0; i < count;) { + if (pixels[i]) { + tokens.push_back(pixels[i++]); + continue; + } + size_t run = 0; + while (i < count && !pixels[i] && run < 0xFFFF) ++run, ++i; + tokens.push_back(0); + tokens.push_back(uint16_t(run)); + } +} + +bool kinect_depth_zero_runs_decode(const uint16_t* tokens, size_t token_count, size_t count, std::vector& pixels) { + pixels.clear(); + pixels.reserve(count); + for (size_t i = 0; i < token_count; ++i) { + if (tokens[i]) { + if (pixels.size() >= count) return false; + pixels.push_back(tokens[i]); + continue; + } + if (++i >= token_count || !tokens[i] || pixels.size() + tokens[i] > count) return false; + pixels.insert(pixels.end(), tokens[i], uint16_t(0)); + } + return pixels.size() == count; +} + +std::string kinect_depth_encode_frame(const KinectDepthRecordFrame& frame, bool allow_zero_runs) { + std::vector tokens; + bool runs = false; + if (allow_zero_runs) { + kinect_depth_zero_runs_encode(frame.pixels.data(), frame.pixels.size(), tokens); + runs = tokens.size() < frame.pixels.size(); + } + const std::vector& payload = runs ? tokens : frame.pixels; + std::string out(frame_magic, 4); + out.reserve(kinect_depth_frame_header_size + payload.size() * 2); + put32(out, uint32_t(payload.size() * 2)); + put64(out, uint64_t(frame.steady_us)); + put64(out, uint64_t(frame.handed_us)); + put64(out, uint64_t(frame.sdk_time)); + put64(out, frame.number); + put64(out, frame.sdk_frame); + out += char(uint8_t(frame.source)); + out += char(uint8_t(runs ? KinectDepthEncoding::zero_runs : KinectDepthEncoding::raw)); + out += '\0'; + out += '\0'; + for (float value : frame.origin) put_float(out, value); + put_float(out, frame.z_scale); + for (uint16_t value : payload) { + out += char(uint8_t(value)); + out += char(uint8_t(value >> 8)); + } + return out; +} + +bool kinect_depth_parse_file_header(const uint8_t* bytes, size_t size, uint32_t& width, uint32_t& height, + uint32_t& frame_header_size, std::string& why) { + if (size < kinect_depth_file_header_size) { why = "too short for a header"; return false; } + if (std::memcmp(bytes, file_magic, 8) != 0) { + why = std::memcmp(bytes, "SFRKDEP1", 8) == 0 ? "an old SFRKDEP1 file (no frame numbers): not read" + : "not a Kinect depth recording"; + return false; + } + const uint32_t version = get32(bytes + 8), header = get32(bytes + 12); + width = get32(bytes + 16); + height = get32(bytes + 20); + const uint32_t format = get32(bytes + 24); + frame_header_size = get32(bytes + 28); + if (version != kinect_depth_file_version || header != kinect_depth_file_header_size) { why = "unknown version"; return false; } + if (format != kinect_depth_pixel_format_player) { why = "unknown pixel format"; return false; } + if (!width || !height || width > 4096 || height > 4096) { why = "bad image size"; return false; } + if (frame_header_size < kinect_depth_frame_header_size || frame_header_size > 4096) { why = "bad frame header size"; return false; } + return true; +} + +KinectDepthReader::~KinectDepthReader() { + if (file_) std::fclose(file_); +} + +bool KinectDepthReader::open(const std::string& path) { + if (file_) std::fclose(file_); + file_ = std::fopen(path.c_str(), "rb"); + error_.clear(); + truncated_ = false; + if (!file_) { error_ = "cannot open " + path; return false; } + uint8_t header[kinect_depth_file_header_size]; + const size_t got = std::fread(header, 1, sizeof header, file_); + if (!kinect_depth_parse_file_header(header, got, width_, height_, frame_header_, error_)) { + std::fclose(file_); + file_ = nullptr; + return false; + } + return true; +} + +bool KinectDepthReader::next(KinectDepthRecordFrame& frame) { + if (!file_) return false; + std::vector header(frame_header_); + const size_t got = std::fread(header.data(), 1, header.size(), file_); + if (got == 0) return false; + if (got < header.size()) { truncated_ = true; return false; } + if (std::memcmp(header.data(), frame_magic, 4) != 0) { error_ = "a broken frame header"; return false; } + const uint32_t bytes = get32(&header[4]); + const size_t count = size_t(width_) * height_; + if (bytes % 2 || bytes > count * 4 + 16) { error_ = "a broken frame size"; return false; } + frame.steady_us = int64_t(get64(&header[8])); + frame.handed_us = int64_t(get64(&header[16])); + frame.sdk_time = int64_t(get64(&header[24])); + frame.number = get64(&header[32]); + frame.sdk_frame = get64(&header[40]); + frame.source = KinectDepthSource(header[48]); + const auto encoding = KinectDepthEncoding(header[49]); + for (int i = 0; i < 3; ++i) frame.origin[i] = get_float(&header[52 + 4 * i]); + frame.z_scale = get_float(&header[64]); + frame.width = width_; + frame.height = height_; + std::vector payload(bytes); + if (std::fread(payload.data(), 1, bytes, file_) < bytes) { truncated_ = true; return false; } + tokens_.resize(bytes / 2); + for (size_t i = 0; i < tokens_.size(); ++i) tokens_[i] = uint16_t(payload[2 * i] | payload[2 * i + 1] << 8); + if (encoding == KinectDepthEncoding::raw) { + if (tokens_.size() != count) { error_ = "a raw frame of the wrong size"; return false; } + frame.pixels = tokens_; + } else if (encoding == KinectDepthEncoding::zero_runs) { + if (!kinect_depth_zero_runs_decode(tokens_.data(), tokens_.size(), count, frame.pixels)) { + error_ = "broken zero runs"; + return false; + } + } else { + error_ = "unknown encoding"; + return false; + } + return true; +} + +std::string kinect_depth_space_warning(const std::string& path, uint64_t free_bytes, uint64_t limit_bytes) { + if (free_bytes >= 2 * limit_bytes) return {}; + return "NATIVE_KINECT_RECORD depth low disk: " + std::to_string(free_bytes / (1024 * 1024)) + " MB free for " + path + + ", less than twice the cap (" + std::to_string(limit_bytes / (1024 * 1024)) + " MB)"; +} + +KinectDepthRecordFile::KinectDepthRecordFile(std::string path, uint64_t limit_bytes, uint32_t every, + KinectRecordQueue* queue) + : queue_(queue ? *queue : KinectRecordQueue::process()), path_(std::move(path)), limit_(limit_bytes), + every_(every ? every : 1) {} + +KinectDepthRecordFile::~KinectDepthRecordFile() { + queue_.drain(this, std::chrono::seconds(10)); + if (file_) std::fclose(file_); +} + +uint64_t KinectDepthRecordFile::frames_written() const { + std::lock_guard guard(lock_); + return frames_; +} + +bool KinectDepthRecordFile::write(const KinectDepthRecordFrame& frame) { + std::lock_guard guard(lock_); + if (failed_ || full_) return false; + if (offered_++ % every_) return false; + if (!started_) { + width_ = frame.width; + height_ = frame.height; + } + if (frame.width != width_ || frame.height != height_ || frame.pixels.size() != size_t(width_) * height_) return false; + // The header goes with the first frame; the file opens (on the writer + // thread) when that arrives. + std::string bytes = started_ ? std::string() : kinect_depth_file_header(width_, height_); + bytes += kinect_depth_encode_frame(frame); + if (accounted_ + bytes.size() > limit_) { + full_ = true; + std::cerr << "NATIVE_KINECT_RECORD depth full " << path_ << " after " << accounted_ << " bytes (" << frames_ + << " images): stopped" << std::endl; + return false; + } + const uint64_t size = bytes.size(); + if (!queue_.push(this, std::move(bytes))) return false; + started_ = true; + accounted_ += size; + ++frames_; + return true; +} + +void KinectDepthRecordFile::record_write(std::string& bytes) { + if (!opened_) { + opened_ = true; + file_ = std::fopen(path_.c_str(), "wb"); + if (!file_) { + failed_ = true; + std::cerr << "NATIVE_KINECT_RECORD failed to open " << path_ << std::endl; + return; + } + uint64_t free_bytes = UINT64_MAX; + std::error_code error; + std::filesystem::path where = std::filesystem::absolute(std::filesystem::path(path_), error).parent_path(); + if (!error) { + const std::filesystem::space_info space = std::filesystem::space(where, error); + if (!error) free_bytes = space.available; + } + std::cerr << "NATIVE_KINECT_RECORD writing " << path_ << " (depth " << width_ << 'x' << height_ << ", at most " + << limit_ / (1024 * 1024) << " MB, " + << (every_ == 1 ? std::string("every image") : "1 image in " + std::to_string(every_)); + if (free_bytes != UINT64_MAX) std::cerr << ", " << free_bytes / (1024 * 1024) << " MB free"; + std::cerr << ')' << std::endl; + if (free_bytes != UINT64_MAX) { + const std::string warning = kinect_depth_space_warning(path_, free_bytes, limit_); + if (!warning.empty()) std::cerr << warning << std::endl; + } + } + if (file_) std::fwrite(bytes.data(), 1, bytes.size(), file_); +} + +void KinectDepthRecordFile::record_flush() { + if (file_) std::fflush(file_); +} + +namespace { +std::mutex& depth_lock() { + static std::mutex lock; + return lock; +} +bool& depth_switch() { + static bool on = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_DEPTH"); + return text && *text == '1'; + }(); + return on; +} +} + +bool kinect_record_depth_enabled() { + std::lock_guard guard(depth_lock()); + return depth_switch(); +} + +void kinect_record_depth_enable() { + std::lock_guard guard(depth_lock()); + depth_switch() = true; +} + +uint64_t kinect_record_depth_limit_bytes() { + static const uint64_t limit = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_DEPTH_MB"); + const double mb = text && *text ? std::strtod(text, nullptr) : 4096.0; + return uint64_t((mb >= 1 && mb <= 1048576 ? mb : 4096.0) * 1024 * 1024); + }(); + return limit; +} + +uint32_t kinect_record_depth_every() { + static const uint32_t every = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_DEPTH_EVERY"); + const long value = text && *text ? std::strtol(text, nullptr, 10) : 1; + return uint32_t(value >= 1 && value <= 1000 ? value : 1); + }(); + return every; +} + +std::string kinect_record_depth_path(const std::string& prefix, const std::string& tag) { + return tag.empty() ? prefix + "-depth.bin" : prefix + "-" + tag + "-depth.bin"; +} + +KinectDepthRecordFile* kinect_record_depth_file(const std::string& tag) { + if (!kinect_record_depth_enabled()) return nullptr; + const std::string& prefix = kinect_record_prefix(); + if (prefix.empty()) return nullptr; + std::lock_guard guard(depth_lock()); + static std::map> files; + auto& file = files[tag]; + if (!file) + file = std::make_unique(kinect_record_depth_path(prefix, tag), kinect_record_depth_limit_bytes(), + kinect_record_depth_every()); + return file.get(); +} + +} diff --git a/src/kinect_record_depth.h b/src/kinect_record_depth.h new file mode 100644 index 0000000..be5f21c --- /dev/null +++ b/src/kinect_record_depth.h @@ -0,0 +1,177 @@ +#pragma once +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sfr { + +// Recording the depth images a Kinect gives the game, for offline study +// (docs/kinect-sensor.md, "錄製"): the same 16-bit depth-and-player-index +// pixels the title's own depth view reads (millimetres << 3 | player index +// 1..6, 0 none), frame by frame, so its features can be computed again +// without the game (kinect_depth_features.h) and a real Kinect v1's images +// compared with the ones a Kinect v2 makes for the game (kinect_v2_depth.h). +// +// SFR_KINECT_RECORD_DEPTH=1 together with SFR_KINECT_RECORD= turns +// it on. The game writes every depth image it is handed through +// NuiImageStreamGetNextFrame to -depth.bin; sfr_kinect_probe (its +// --depth sets the switch) writes each sensor's own to -v1-depth.bin +// and -v2-depth.bin (a v1's raw depth, a v2's made one, as the game +// would get them). SFR_KINECT_RECORD_DEPTH_MB caps each file (default 4096; +// "NATIVE_KINECT_RECORD depth full" once in the log when reached) and +// SFR_KINECT_RECORD_DEPTH_EVERY=N keeps every Nth image only (default 1). +// +// The file, all little-endian: +// header, 32 bytes: +// 0 char[8] "SFRKDEP2" +// 8 uint32 version (2) +// 12 uint32 header size (32) +// 16 uint32 width (320) +// 20 uint32 height (240) +// 24 uint32 pixel format (1: uint16, millimetres << 3 | player index) +// 28 uint32 frame header size (68) +// then per frame, a 68-byte frame header: +// 0 char[4] "KDF2" +// 4 uint32 payload bytes +// 8 int64 steady_us: the host's steady clock (kinect_record_now_us) +// when the sensor's backend received the image (a v1) or +// made it (a v2); the clock all recordings share +// 16 int64 handed_us: the same clock when the game was handed it +// (NuiImageStreamGetNextFrame) or the probe read it +// 24 int64 sdk_time: a v1's liTimeStamp (ms), a v2's RelativeTime +// (100 ns) of its depth frame; 0 when unknown +// 32 uint64 number: the image's number, the one the game sees in +// NUI_IMAGE_FRAME +8 (counts from 1 per sensor and process) +// 40 uint64 sdk_frame: a v1's dwFrameNumber; 0 for a v2 (none) +// 48 uint8 source: 1 a real Kinect v1, 2 made by a Kinect v2 +// 49 uint8 encoding: 0 raw, 1 zero runs +// 50 uint16 0 +// 52 float[3] the virtual v1's origin (a v2's KinectDepthView; 0 for a v1) +// 64 float its depth stretch z_scale (1 for a v1) +// and the payload: width x height pixels, rows top-down, as uint16 (raw), +// or as uint16 tokens (zero runs): a non-zero token is a pixel, a zero +// token is followed by a count (1..65535) of zero pixels. +// A frame cut short at the end (the game ended mid-write) is left out by the +// reader. +// +// None of this is Windows', so it is tested everywhere (kinect_depth_test). + +constexpr uint32_t kinect_depth_file_version = 2; +constexpr uint32_t kinect_depth_file_header_size = 32; +constexpr uint32_t kinect_depth_frame_header_size = 68; +constexpr uint32_t kinect_depth_pixel_format_player = 1; + +enum class KinectDepthSource : uint8_t { unknown = 0, v1 = 1, v2 = 2 }; +enum class KinectDepthEncoding : uint8_t { raw = 0, zero_runs = 1 }; + +const char* kinect_depth_source_name(KinectDepthSource source); // "v1", "v2", "unknown" + +struct KinectDepthRecordFrame { + int64_t steady_us = 0; + int64_t handed_us = 0; + int64_t sdk_time = 0; + uint64_t number = 0; + uint64_t sdk_frame = 0; + KinectDepthSource source = KinectDepthSource::unknown; + std::array origin{}; + float z_scale = 1.0f; + uint32_t width = 0, height = 0; + std::vector pixels; // width x height +}; + +// The pixels of a KinectImage (2 bytes a pixel, host order: little-endian on +// every host the game runs on) into a frame. +void kinect_depth_frame_pixels(KinectDepthRecordFrame& frame, const uint8_t* bytes, uint32_t width, uint32_t height); + +std::string kinect_depth_file_header(uint32_t width, uint32_t height); +// The frame's header and payload; zero runs when they are smaller and allowed. +std::string kinect_depth_encode_frame(const KinectDepthRecordFrame& frame, bool allow_zero_runs = true); + +// Zero runs: tokens for pixels, and back (false when the tokens do not make +// exactly count pixels). +void kinect_depth_zero_runs_encode(const uint16_t* pixels, size_t count, std::vector& tokens); +bool kinect_depth_zero_runs_decode(const uint16_t* tokens, size_t token_count, size_t count, std::vector& pixels); + +// Reads a depth file frame by frame. +class KinectDepthReader { +public: + ~KinectDepthReader(); + bool open(const std::string& path); + // False at the end, at a frame cut short, or at a broken one (error()). + bool next(KinectDepthRecordFrame& frame); + uint32_t width() const { return width_; } + uint32_t height() const { return height_; } + // Why open or next failed; empty at a clean end. + const std::string& error() const { return error_; } + // Whether the last frame was cut short (and left out). + bool truncated() const { return truncated_; } +private: + std::FILE* file_ = nullptr; + uint32_t width_ = 0, height_ = 0, frame_header_ = 0; + std::string error_; + bool truncated_ = false; + std::vector tokens_; +}; + +// The parse of a header (for the reader and tests). +bool kinect_depth_parse_file_header(const uint8_t* bytes, size_t size, uint32_t& width, uint32_t& height, + uint32_t& frame_header_size, std::string& why); + +// The free space of the drive a file goes on: a warning line when it is less +// than twice the cap, else empty. +std::string kinect_depth_space_warning(const std::string& path, uint64_t free_bytes, uint64_t limit_bytes); + +// One depth file the recorder writes, opened on the first frame (a sensor +// that never gives one leaves no file), bounded in size, every Nth frame +// offered kept. Frames are taken whole from any thread and written by the +// record queue (kinect_record.h: the caller never waits for the disk; a frame +// the queue has no room for is dropped and counted). +class KinectDepthRecordFile : public KinectRecordSink { +public: + KinectDepthRecordFile(std::string path, uint64_t limit_bytes, uint32_t every, + KinectRecordQueue* queue = nullptr); // null: the process's + ~KinectDepthRecordFile() override; // writes what is queued (at most 10 s), then closes + // False when the frame was not taken (skipped by every, full, failed, + // dropped). + bool write(const KinectDepthRecordFrame& frame); + const std::string& path() const { return path_; } + // Frames taken for the file (queued or written). + uint64_t frames_written() const; + void record_write(std::string& bytes) override; + void record_flush() override; +private: + KinectRecordQueue& queue_; + mutable std::mutex lock_; + std::string path_; + uint64_t limit_ = 0, accounted_ = 0, offered_ = 0, frames_ = 0; + uint32_t every_ = 1, width_ = 0, height_ = 0; + bool started_ = false, full_ = false; + std::atomic failed_{false}; + // The writer thread's. + std::FILE* file_ = nullptr; + bool opened_ = false; +}; + +// SFR_KINECT_RECORD_DEPTH=1, or kinect_record_depth_enable() (the probe's +// --depth) before the first file is asked for. +bool kinect_record_depth_enabled(); +void kinect_record_depth_enable(); +// SFR_KINECT_RECORD_DEPTH_MB in bytes (default 4096 MB), and +// SFR_KINECT_RECORD_DEPTH_EVERY (default 1). +uint64_t kinect_record_depth_limit_bytes(); +uint32_t kinect_record_depth_every(); +// -depth.bin for an empty tag (the game), --depth.bin +// otherwise (the probe's "v1" and "v2"). +std::string kinect_record_depth_path(const std::string& prefix, const std::string& tag); +// The depth file for a tag when depth recording is on (with a prefix from +// SFR_KINECT_RECORD or kinect_record_start), null otherwise. +KinectDepthRecordFile* kinect_record_depth_file(const std::string& tag); + +} diff --git a/src/kinect_record_raw.cpp b/src/kinect_record_raw.cpp new file mode 100644 index 0000000..06b8fb4 --- /dev/null +++ b/src/kinect_record_raw.cpp @@ -0,0 +1,373 @@ +#include "kinect_record_raw.h" + +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include + +namespace sfr { + +namespace { +constexpr char file_magic[8] = {'S', 'F', 'R', 'K', 'R', 'A', 'W', '1'}; +constexpr char frame_magic[4] = {'K', 'R', 'W', '1'}; +constexpr uint32_t fixed_header_size = 64; +constexpr uint32_t body_size = 24; + +void put8(std::string& out, uint8_t value) { out += char(value); } +void put16(std::string& out, uint16_t value) { + out += char(uint8_t(value)); + out += char(uint8_t(value >> 8)); +} +void put32(std::string& out, uint32_t value) { + for (int i = 0; i < 4; ++i) out += char(uint8_t(value >> (8 * i))); +} +void put64(std::string& out, uint64_t value) { + for (int i = 0; i < 8; ++i) out += char(uint8_t(value >> (8 * i))); +} +void put_float(std::string& out, float value) { + uint32_t bits = 0; + std::memcpy(&bits, &value, 4); + put32(out, bits); +} +uint16_t get16(const uint8_t* p) { return uint16_t(p[0] | p[1] << 8); } +uint32_t get32(const uint8_t* p) { return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; } +uint64_t get64(const uint8_t* p) { return uint64_t(get32(p)) | uint64_t(get32(p + 4)) << 32; } +float get_float(const uint8_t* p) { + const uint32_t bits = get32(p); + float value = 0; + std::memcpy(&value, &bits, 4); + return value; +} +uint64_t header_bytes(uint32_t width, uint32_t height) { return fixed_header_size + 8ull * width * height; } +} + +const char* kinect_raw_rays_name(KinectRawRays rays) { + switch (rays) { + case KinectRawRays::sensor_table: return "sensor-table"; + case KinectRawRays::sensor_lens: return "sensor-lens"; + case KinectRawRays::typical_lens: return "typical-lens"; + default: return "unknown"; + } +} + +std::string kinect_raw_file_header(const KinectRawHeader& header) { + std::string out(file_magic, 8); + const size_t pixels = size_t(header.width) * header.height; + out.reserve(size_t(header_bytes(header.width, header.height))); + put32(out, kinect_raw_file_version); + put32(out, uint32_t(header_bytes(header.width, header.height))); + put32(out, header.width); + put32(out, header.height); + put32(out, uint32_t(header.rays_from)); + for (float value : header.lens) put_float(out, value); + put32(out, kinect_raw_frame_header_size); + put32(out, 0); + for (size_t i = 0; i < 2 * pixels; ++i) put_float(out, i < header.rays.size() ? header.rays[i] : 0.0f); + return out; +} + +void kinect_raw_runs_encode(const uint8_t* pixels, size_t count, std::vector>& runs) { + runs.clear(); + for (size_t i = 0; i < count;) { + const uint8_t value = pixels[i]; + uint32_t run = 0; + while (i < count && pixels[i] == value && run < 0xFFFF) ++run, ++i; + runs.emplace_back(value, uint16_t(run)); + } +} + +bool kinect_raw_runs_decode(const std::vector>& runs, size_t count, std::vector& pixels) { + pixels.clear(); + pixels.reserve(count); + for (const auto& [value, run] : runs) { + if (!run || pixels.size() + run > count) return false; + pixels.insert(pixels.end(), run, value); + } + return pixels.size() == count; +} + +std::string kinect_raw_encode_frame(const KinectRawFrame& frame) { + std::vector> runs; + kinect_raw_runs_encode(frame.body_index.data(), frame.body_index.size(), runs); + const uint32_t payload = uint32_t(frame.depth.size() * 2 + 4 + runs.size() * 4); + std::string out(frame_magic, 4); + out.reserve(kinect_raw_frame_header_size + payload); + put32(out, payload); + put64(out, uint64_t(frame.steady_us)); + put64(out, uint64_t(frame.depth_time)); + put64(out, uint64_t(frame.body_time)); + put64(out, frame.number); + put32(out, frame.flags); + const uint32_t bodies = uint32_t(std::min(frame.bodies.size(), kinect_raw_max_bodies)); + put32(out, bodies); + for (float value : frame.floor) put_float(out, value); + for (uint32_t i = 0; i < kinect_raw_max_bodies; ++i) { + const KinectRawBody body = i < bodies ? frame.bodies[i] : KinectRawBody{0, 0, 0, 0, {}}; + put64(out, body.tracking_id); + put8(out, body.body_index); + put8(out, body.tracked); + put8(out, body.spine_state); + put8(out, 0); + for (float value : body.spine_base) put_float(out, value); + } + // The depth in one go (it is most of the frame), little-endian. + const size_t at = out.size(); + out.resize(at + frame.depth.size() * 2); + char* p = out.data() + at; + for (uint16_t value : frame.depth) { + *p++ = char(uint8_t(value)); + *p++ = char(uint8_t(value >> 8)); + } + put32(out, uint32_t(runs.size())); + for (const auto& [value, run] : runs) { + put8(out, value); + put8(out, 0); + put16(out, run); + } + return out; +} + +bool kinect_raw_parse_header(const uint8_t* bytes, size_t size, KinectRawHeader& header, uint32_t& frame_header_size, + std::string& why) { + if (size < fixed_header_size) { why = "too short for a header"; return false; } + if (std::memcmp(bytes, file_magic, 8) != 0) { why = "not a Kinect v2 raw recording (SFRKRAW1)"; return false; } + const uint32_t version = get32(bytes + 8), total = get32(bytes + 12); + header.width = get32(bytes + 16); + header.height = get32(bytes + 20); + header.rays_from = KinectRawRays(get32(bytes + 24)); + for (int i = 0; i < 7; ++i) header.lens[i] = get_float(bytes + 28 + 4 * i); + frame_header_size = get32(bytes + 56); + if (version != kinect_raw_file_version) { why = "unknown version"; return false; } + if (!header.width || !header.height || header.width > 4096 || header.height > 4096) { why = "bad image size"; return false; } + if (total != header_bytes(header.width, header.height)) { why = "bad header size"; return false; } + if (frame_header_size < kinect_raw_frame_header_size || frame_header_size > 65536) { why = "bad frame header size"; return false; } + if (size < total) { why = "header cut short"; return false; } + const size_t pixels = size_t(header.width) * header.height; + header.rays.resize(2 * pixels); + for (size_t i = 0; i < 2 * pixels; ++i) header.rays[i] = get_float(bytes + fixed_header_size + 4 * i); + return true; +} + +KinectRawReader::~KinectRawReader() { close(); } + +void KinectRawReader::close() { + if (file_) std::fclose(file_); + file_ = nullptr; +} + +bool KinectRawReader::open(const std::string& path) { + if (file_) std::fclose(file_); + file_ = std::fopen(path.c_str(), "rb"); + error_.clear(); + truncated_ = false; + if (!file_) { error_ = "cannot open " + path; return false; } + std::vector bytes(fixed_header_size); + size_t got = std::fread(bytes.data(), 1, bytes.size(), file_); + if (got == fixed_header_size && std::memcmp(bytes.data(), file_magic, 8) == 0) { + const uint32_t total = get32(bytes.data() + 12); + if (total > fixed_header_size && total < (64u << 20)) { + bytes.resize(total); + got += std::fread(bytes.data() + fixed_header_size, 1, total - fixed_header_size, file_); + } + } + if (!kinect_raw_parse_header(bytes.data(), got, header_, frame_header_, error_)) { + std::fclose(file_); + file_ = nullptr; + return false; + } + return true; +} + +bool KinectRawReader::next(KinectRawFrame& frame) { + if (!file_) return false; + std::vector head(frame_header_); + const size_t got = std::fread(head.data(), 1, head.size(), file_); + if (got == 0) return false; + if (got < head.size()) { truncated_ = true; return false; } + if (std::memcmp(head.data(), frame_magic, 4) != 0) { error_ = "a broken frame header"; return false; } + const size_t pixels = size_t(header_.width) * header_.height; + const uint32_t payload = get32(&head[4]); + if (payload < pixels * 2 + 4 || payload > pixels * 2 + 4 + pixels * 4) { error_ = "a broken frame size"; return false; } + frame.steady_us = int64_t(get64(&head[8])); + frame.depth_time = int64_t(get64(&head[16])); + frame.body_time = int64_t(get64(&head[24])); + frame.number = get64(&head[32]); + frame.flags = get32(&head[40]); + const uint32_t bodies = get32(&head[44]); + if (bodies > kinect_raw_max_bodies) { error_ = "too many bodies"; return false; } + for (int i = 0; i < 4; ++i) frame.floor[i] = get_float(&head[48 + 4 * i]); + frame.bodies.resize(bodies); + for (uint32_t i = 0; i < bodies; ++i) { + const uint8_t* p = &head[64 + body_size * i]; + KinectRawBody& body = frame.bodies[i]; + body.tracking_id = get64(p); + body.body_index = p[8]; + body.tracked = p[9]; + body.spine_state = p[10]; + for (int k = 0; k < 3; ++k) body.spine_base[k] = get_float(p + 12 + 4 * k); + } + std::vector data(payload); + if (std::fread(data.data(), 1, payload, file_) < payload) { truncated_ = true; return false; } + frame.depth.resize(pixels); + for (size_t i = 0; i < pixels; ++i) frame.depth[i] = get16(&data[2 * i]); + const uint32_t count = get32(&data[2 * pixels]); + if (payload != pixels * 2 + 4 + size_t(count) * 4) { error_ = "a broken run count"; return false; } + std::vector> runs(count); + for (uint32_t i = 0; i < count; ++i) { + const uint8_t* p = &data[2 * pixels + 4 + 4 * size_t(i)]; + runs[i] = {p[0], get16(p + 2)}; + } + if (!kinect_raw_runs_decode(runs, pixels, frame.body_index)) { error_ = "broken body index runs"; return false; } + return true; +} + +KinectRawRecordFile::KinectRawRecordFile(std::string path, uint64_t limit_bytes, uint32_t every, KinectRecordQueue* queue) + : queue_(queue ? *queue : KinectRecordQueue::process()), path_(std::move(path)), limit_(limit_bytes), + every_(every ? every : 1) {} + +KinectRawRecordFile::~KinectRawRecordFile() { + queue_.drain(this, std::chrono::seconds(10)); + if (file_) std::fclose(file_); +} + +bool KinectRawRecordFile::is_open() const { + std::lock_guard guard(lock_); + return started_; +} + +uint64_t KinectRawRecordFile::frames_written() const { + std::lock_guard guard(lock_); + return frames_; +} + +bool KinectRawRecordFile::open(const KinectRawHeader& header) { + std::lock_guard guard(lock_); + if (started_) return !failed_; + started_ = true; + width_ = header.width; + height_ = header.height; + rays_from_ = header.rays_from; + // The header goes first; the file opens on the writer thread. + std::string bytes = kinect_raw_file_header(header); + accounted_ = bytes.size(); + if (!queue_.push(this, std::move(bytes))) { + failed_ = true; + std::cerr << "NATIVE_KINECT_RECORD raw header dropped (disk too slow): " << path_ << " not written" << std::endl; + return false; + } + return true; +} + +bool KinectRawRecordFile::write(const KinectRawFrame& frame) { + std::lock_guard guard(lock_); + if (!started_ || failed_ || full_) return false; + if (offered_++ % every_) return false; + const size_t pixels = size_t(width_) * height_; + if (frame.depth.size() != pixels || frame.body_index.size() != pixels) return false; + std::string bytes = kinect_raw_encode_frame(frame); + if (accounted_ + bytes.size() > limit_) { + full_ = true; + std::cerr << "NATIVE_KINECT_RECORD raw full " << path_ << " after " << accounted_ << " bytes (" << frames_ + << " frames): stopped" << std::endl; + return false; + } + const uint64_t size = bytes.size(); + if (!queue_.push(this, std::move(bytes))) return false; + accounted_ += size; + ++frames_; + return true; +} + +void KinectRawRecordFile::record_write(std::string& bytes) { + if (!opened_) { + opened_ = true; + file_ = std::fopen(path_.c_str(), "wb"); + if (!file_) { + failed_ = true; + std::cerr << "NATIVE_KINECT_RECORD failed to open " << path_ << std::endl; + return; + } + uint64_t free_bytes = UINT64_MAX; + std::error_code error; + const std::filesystem::path where = std::filesystem::absolute(std::filesystem::path(path_), error).parent_path(); + if (!error) { + const std::filesystem::space_info space = std::filesystem::space(where, error); + if (!error) free_bytes = space.available; + } + std::cerr << "NATIVE_KINECT_RECORD writing " << path_ << " (v2 raw depth " << width_ << 'x' << height_ << ", rays " + << kinect_raw_rays_name(rays_from_) << ", at most " << limit_ / (1024 * 1024) << " MB, " + << (every_ == 1 ? std::string("every frame") : "1 frame in " + std::to_string(every_)); + if (free_bytes != UINT64_MAX) std::cerr << ", " << free_bytes / (1024 * 1024) << " MB free"; + std::cerr << ')' << std::endl; + if (free_bytes != UINT64_MAX && free_bytes < 2 * limit_) + std::cerr << "NATIVE_KINECT_RECORD raw low disk: " << free_bytes / (1024 * 1024) << " MB free for " << path_ + << ", less than twice the cap (" << limit_ / (1024 * 1024) << " MB)" << std::endl; + } + if (file_) std::fwrite(bytes.data(), 1, bytes.size(), file_); +} + +void KinectRawRecordFile::record_flush() { + if (file_) std::fflush(file_); +} + +namespace { +std::mutex& raw_lock() { + static std::mutex lock; + return lock; +} +bool& raw_switch() { + static bool on = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_RAW"); + return text && *text == '1'; + }(); + return on; +} +} + +bool kinect_record_raw_enabled() { + std::lock_guard guard(raw_lock()); + return raw_switch(); +} + +void kinect_record_raw_enable() { + std::lock_guard guard(raw_lock()); + raw_switch() = true; +} + +uint64_t kinect_record_raw_limit_bytes() { + static const uint64_t limit = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_RAW_MB"); + const double mb = text && *text ? std::strtod(text, nullptr) : 16384.0; + return uint64_t((mb >= 1 && mb <= 4194304 ? mb : 16384.0) * 1024 * 1024); + }(); + return limit; +} + +uint32_t kinect_record_raw_every() { + static const uint32_t every = [] { + const char* text = std::getenv("SFR_KINECT_RECORD_RAW_EVERY"); + const long value = text && *text ? std::strtol(text, nullptr, 10) : 1; + return uint32_t(value >= 1 && value <= 1000 ? value : 1); + }(); + return every; +} + +std::string kinect_record_raw_path(const std::string& prefix) { return prefix + "-v2raw.bin"; } + +KinectRawRecordFile* kinect_record_raw_file() { + if (!kinect_record_raw_enabled()) return nullptr; + const std::string& prefix = kinect_record_prefix(); + if (prefix.empty()) return nullptr; + std::lock_guard guard(raw_lock()); + static std::unique_ptr file; + if (!file) + file = std::make_unique(kinect_record_raw_path(prefix), kinect_record_raw_limit_bytes(), + kinect_record_raw_every()); + return file.get(); +} + +} diff --git a/src/kinect_record_raw.h b/src/kinect_record_raw.h new file mode 100644 index 0000000..c8b7ce8 --- /dev/null +++ b/src/kinect_record_raw.h @@ -0,0 +1,187 @@ +#pragma once +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace sfr { + +// Recording what a Kinect v2 measured, before it is made into the game's +// v1-format depth (kinect_v2_depth.h), so the depth for the game can be made +// again offline through any view with any settings (sfr_kinect_depth_resynth, +// kinect_v2_resynth.h; docs/kinect-sensor.md). Each frame is exactly what the +// live synthesis used for one image: the depth and body index frames it +// paired (same RelativeTime), the newest body frame's bodies and floor as the +// view keeps them (KinectDepthViewer::bodies), and the times both views step +// on. +// +// SFR_KINECT_RECORD_RAW=1 together with SFR_KINECT_RECORD= (the +// probe's --raw sets the switch) writes -v2raw.bin, whenever a +// Kinect v2 makes depth for the game (the game, or sfr_kinect_probe). +// SFR_KINECT_RECORD_RAW_MB caps the file (default 16384; "NATIVE_KINECT_RECORD +// raw full" once in the log when reached), SFR_KINECT_RECORD_RAW_EVERY=N keeps +// every Nth frame only (default 1; the ideal and player views follow the +// frames they are given, so only every frame gives the live images back +// exactly). +// +// The file, all little-endian ("SFRKRAW1"): +// header (header size bytes): +// 0 char[8] "SFRKRAW1" +// 8 uint32 version (1) +// 12 uint32 header size (64 + 8 * width * height) +// 16 uint32 depth width (512) +// 20 uint32 depth height (424) +// 24 uint32 where the rays come from (KinectRawRays): 1 the sensor's +// own table (ICoordinateMapper::GetDepthFrameToCameraSpaceTable), +// 2 the sensor's lens (GetDepthCameraIntrinsics, without +// distortion), 3 a typical v2's lens +// 28 float[7] the sensor's depth lens: focal x, y, centre x, y (pixels), +// radial 2, 4, 6 (0 when unknown) +// 56 uint32 frame header size (208) +// 60 uint32 0 +// 64 float[2 * width * height] a ray per depth pixel, rows top-down: +// x / z, y / z of camera space (metres; +y up) +// then per frame, a 208-byte frame header: +// 0 char[4] "KRW1" +// 4 uint32 payload bytes +// 8 int64 steady_us: the host's steady clock when the image was made +// (as the depth recording's, kinect_record_depth.h) +// 16 int64 the depth frame's RelativeTime (100 ns; the body index +// frame's is within half a frame of it) +// 24 int64 the newest body frame's RelativeTime (100 ns; 0 none yet) +// 32 uint64 number: the made image's number (as in the depth recording) +// 40 uint32 flags: bit 0 the live synthesis had the sensor's mapper's +// points (else a typical lens's, kinect_v2_depth_points_from_lens) +// 44 uint32 bodies (0..6) +// 48 float[4] the body frame's floor plane (zero: none) +// 64 6 x 24 bytes, the first `bodies` used: uint64 tracking id (the 32 +// bits the game gets, kinect_v2_short_tracking_id), uint8 +// body index (0..5: the body index image's value), uint8 +// tracked (1), uint8 the SpineBase joint's state (0 not, 1 +// inferred, 2 tracked), uint8 0, float[3] SpineBase (metres) +// and the payload: the depth, width x height uint16 millimetres (0 none), +// then the body index as runs: uint32 run count, then per run uint8 value +// (0..5, 255 none), uint8 0, uint16 count (1..65535). +// A frame cut short at the end is left out by the reader. +// +// None of this is Windows', so it is tested everywhere (kinect_v2_depth_test). + +constexpr uint32_t kinect_raw_file_version = 1; +constexpr uint32_t kinect_raw_frame_header_size = 208; +constexpr uint32_t kinect_raw_max_bodies = 6; +constexpr uint32_t kinect_raw_flag_mapped = 1; + +enum class KinectRawRays : uint32_t { unknown = 0, sensor_table = 1, sensor_lens = 2, typical_lens = 3 }; +const char* kinect_raw_rays_name(KinectRawRays rays); // "sensor-table", "sensor-lens", "typical-lens" + +struct KinectRawHeader { + uint32_t width = 512, height = 424; + KinectRawRays rays_from = KinectRawRays::unknown; + std::array lens{}; // focal x, y, centre x, y, radial 2, 4, 6 + std::vector rays; // 2 per pixel +}; + +struct KinectRawBody { + uint64_t tracking_id = 0; + uint8_t body_index = 0; + uint8_t tracked = 1; + uint8_t spine_state = 0; + std::array spine_base{}; +}; + +struct KinectRawFrame { + int64_t steady_us = 0; + int64_t depth_time = 0; // 100 ns + int64_t body_time = 0; // 100 ns + uint64_t number = 0; + uint32_t flags = 0; + std::array floor{}; + std::vector bodies; + std::vector depth; // width x height millimetres + std::vector body_index; // width x height +}; + +std::string kinect_raw_file_header(const KinectRawHeader& header); +std::string kinect_raw_encode_frame(const KinectRawFrame& frame); + +// Body index runs: (value, count) pairs, and back (false when they do not +// make exactly count pixels). +void kinect_raw_runs_encode(const uint8_t* pixels, size_t count, std::vector>& runs); +bool kinect_raw_runs_decode(const std::vector>& runs, size_t count, std::vector& pixels); + +class KinectRawReader { +public: + ~KinectRawReader(); + bool open(const std::string& path); + void close(); + // False at the end, at a frame cut short, or at a broken one (error()). + bool next(KinectRawFrame& frame); + const KinectRawHeader& header() const { return header_; } + const std::string& error() const { return error_; } + bool truncated() const { return truncated_; } +private: + std::FILE* file_ = nullptr; + KinectRawHeader header_; + uint32_t frame_header_ = 0; + std::string error_; + bool truncated_ = false; +}; + +// The parse of a whole header held in memory (for the reader and tests). +bool kinect_raw_parse_header(const uint8_t* bytes, size_t size, KinectRawHeader& header, uint32_t& frame_header_size, + std::string& why); + +// The raw file a recorder writes: started by open() with its header (the +// rays are only known once the sensor has its calibration), bounded in size, +// every Nth frame offered kept. Frames are taken whole from any thread and +// written by the record queue (kinect_record.h: the caller never waits for +// the disk; a frame the queue has no room for is dropped and counted, a gap +// in the frame numbers). +class KinectRawRecordFile : public KinectRecordSink { +public: + KinectRawRecordFile(std::string path, uint64_t limit_bytes, uint32_t every, + KinectRecordQueue* queue = nullptr); // null: the process's + ~KinectRawRecordFile() override; // writes what is queued (at most 10 s), then closes + bool is_open() const; + bool open(const KinectRawHeader& header); + // False when the frame was not taken (not open, skipped by every, full, + // failed, dropped). + bool write(const KinectRawFrame& frame); + const std::string& path() const { return path_; } + // Frames taken for the file (queued or written). + uint64_t frames_written() const; + void record_write(std::string& bytes) override; + void record_flush() override; +private: + KinectRecordQueue& queue_; + mutable std::mutex lock_; + std::string path_; + uint64_t limit_ = 0, accounted_ = 0, offered_ = 0, frames_ = 0; + uint32_t every_ = 1, width_ = 0, height_ = 0; + KinectRawRays rays_from_ = KinectRawRays::unknown; + bool started_ = false, full_ = false; + std::atomic failed_{false}; + // The writer thread's. + std::FILE* file_ = nullptr; + bool opened_ = false; +}; + +// SFR_KINECT_RECORD_RAW=1, or kinect_record_raw_enable() (the probe's --raw). +bool kinect_record_raw_enabled(); +void kinect_record_raw_enable(); +// SFR_KINECT_RECORD_RAW_MB in bytes (default 16384 MB), and +// SFR_KINECT_RECORD_RAW_EVERY (default 1). +uint64_t kinect_record_raw_limit_bytes(); +uint32_t kinect_record_raw_every(); +// -v2raw.bin. +std::string kinect_record_raw_path(const std::string& prefix); +// The raw file when raw recording is on (with a prefix from +// SFR_KINECT_RECORD or kinect_record_start), null otherwise. +KinectRawRecordFile* kinect_record_raw_file(); + +} diff --git a/src/kinect_sensor.cpp b/src/kinect_sensor.cpp index aec9c12..6f0de77 100644 --- a/src/kinect_sensor.cpp +++ b/src/kinect_sensor.cpp @@ -2,44 +2,43 @@ #include #include +#include +#include +#include namespace sfr { KinectSensor::~KinectSensor() = default; -float kinect_level(KinectFrame& frame) { - const auto& g = frame.gravity; - const float length = std::sqrt(g[0] * g[0] + g[1] * g[1] + g[2] * g[2]); - if (!(length > 0.5f)) return 0.0f; - const std::array up = {g[0] / length, g[1] / length, g[2] / length}; - // cos 30 degrees: a sensor tilted further, or gravity the other way up, - // is not a tilt to take out. - if (up[1] < 0.8660254f) return 0.0f; - // The turn that takes up onto +y: about k = up x (0, 1, 0), by the angle - // between them (Rodrigues). - const std::array axis = {-up[2], 0.0f, up[0]}; - const float sine = std::sqrt(axis[0] * axis[0] + axis[2] * axis[2]), cosine = up[1]; - frame.gravity = {0.0f, 1.0f, 0.0f}; - if (sine < 1e-6f) return 0.0f; - const std::array k = {axis[0] / sine, 0.0f, axis[2] / sine}; - const auto turn = [&](const std::array& v) -> std::array { - const float dot = k[0] * v[0] + k[2] * v[2]; - const std::array cross = {k[1] * v[2] - k[2] * v[1], k[2] * v[0] - k[0] * v[2], - k[0] * v[1] - k[1] * v[0]}; - std::array out{}; - for (int i = 0; i < 3; ++i) out[i] = v[i] * cosine + cross[i] * sine + k[i] * dot * (1.0f - cosine); - return out; +std::string kinect_edges_text(uint8_t edges) { + std::string text; + const auto add = [&](uint8_t bit, const char* name) { + if (!(edges & bit)) return; + if (!text.empty()) text += '+'; + text += name; }; - for (KinectBody& body : frame.bodies) { - for (auto& joint : body.joints) joint = turn(joint); - body.position = turn(body.position); - } - // A plane through turned points keeps its distance; only its normal turns. - if (frame.floor_plane[0] != 0.0f || frame.floor_plane[1] != 0.0f || frame.floor_plane[2] != 0.0f) { - const auto normal = turn({frame.floor_plane[0], frame.floor_plane[1], frame.floor_plane[2]}); - frame.floor_plane = {normal[0], normal[1], normal[2], frame.floor_plane[3]}; + add(kinect_edge::top, "top"); + add(kinect_edge::bottom, "bottom"); + add(kinect_edge::left, "left"); + add(kinect_edge::right, "right"); + return text.empty() ? "none" : text; +} + +uint8_t kinect_edges_move(uint8_t edges) { + uint8_t move = 0; + const bool top = edges & kinect_edge::top, bottom = edges & kinect_edge::bottom; + const bool left = edges & kinect_edge::left, right = edges & kinect_edge::right; + // Cut at both ends of an axis: the picture is too small for the player. + if ((top && bottom) || (left && right)) move |= kinect_edge_move::step_back; + else { + if (top) move |= kinect_edge_move::tilt_up | kinect_edge_move::step_back; + if (bottom) move |= kinect_edge_move::tilt_down | kinect_edge_move::step_back; + // The picture's left is the player's right (the player faces the + // sensor, the picture is not mirrored): move to the player's own left. + if (left) move |= kinect_edge_move::player_left; + if (right) move |= kinect_edge_move::player_right; } - return std::atan2(sine, cosine) * 180.0f / 3.14159265358979f; + return move; } std::string kinect_open_failure(const std::string& v1, const std::string& v2) { @@ -50,6 +49,28 @@ std::string kinect_open_failure(const std::string& v1, const std::string& v2) { return "no-runtime"; } +int kinect_selected_model() { + const char* const value = std::getenv("SFR_KINECT_MODEL"); + return value && std::string_view(value) == "2" ? 2 : 1; +} + +bool kinect_host_smoothing() { + const char* const value = std::getenv("SFR_KINECT_SMOOTH"); + return value && *value == '1'; +} + +std::unique_ptr KinectSensor::open_selected(int model, std::string* why) { + if (model == 2) return open_model(2, why); + std::string v1_why, v2_why; + if (auto sensor = open_model(1, &v1_why)) return sensor; + if (auto sensor = open_model(2, &v2_why)) { + std::cerr << "NATIVE_KINECT no Kinect (Xbox 360) opened, using the Kinect v2 found instead\n"; + return sensor; + } + if (why) *why = kinect_open_failure(v1_why, v2_why); + return nullptr; +} + void kinect_v2_body(const std::array& joints, uint32_t tracking_id, uint32_t sensor_index, KinectBody& body) { body.tracking_id = tracking_id; @@ -116,8 +137,18 @@ std::array KinectPlayerSlots::ass void KinectPlayerSlots::retain(KinectFrame& frame) { assign(frame); + // Each player's body once: a sensor that repeats an id (a v2's 64-bit + // ids are shortened to the 32 the title keeps) must not make a third + // TRACKED skeleton, which the title's two-entry list cannot hold. + std::array kept{}; std::erase_if(frame.bodies, [&](const KinectBody& body) { - return !body.tracking_id || std::find(tracking_.begin(), tracking_.end(), body.tracking_id) == tracking_.end(); + if (!body.tracking_id) return true; + const auto slot = std::find(tracking_.begin(), tracking_.end(), body.tracking_id); + if (slot == tracking_.end()) return true; + bool& taken = kept[size_t(slot - tracking_.begin())]; + if (taken) return true; + taken = true; + return false; }); } diff --git a/src/kinect_sensor.h b/src/kinect_sensor.h index c65de94..95d6cd6 100644 --- a/src/kinect_sensor.h +++ b/src/kinect_sensor.h @@ -1,4 +1,5 @@ #pragma once +#include "kinect_ideal_diagram.h" #include "nui_skeleton.h" #include @@ -12,6 +13,8 @@ namespace sfr { +struct KinectDepthSettings; // kinect_v2_depth.h + // A real Kinect standing in as itself. The title was written against the // Xbox 360's NUI library, and the Kinect for Windows runtime (SDK 1.8, // Kinect10.dll) is that same library brought to the PC: its skeleton frame @@ -45,8 +48,31 @@ struct KinectBody { // A Kinect v2's fingertips and thumbs (left, right), which give which // way a palm faces; all zero from a v1 or when not seen. std::array, 2> hand_tips{}, thumbs{}; + // Which edges of the sensor's picture cut this body off (kinect_edge + // bits; 0 none). Only a Kinect v2 says (IBody::ClippedEdges); a v1 leaves + // it 0, which there means unknown, not "not cut". Display and log only: + // no detector or the game reads it. + uint8_t clipped_edges = 0; }; +// The bits of KinectBody::clipped_edges, the SDK's FrameEdges. Left and +// right are the picture's own, as the preview draws it (the sensor's raw +// image, not mirrored): the picture's left shows the sensor's left, which is +// the PLAYER's right while the player faces the sensor. +namespace kinect_edge { +constexpr uint8_t right = 0x1, left = 0x2, top = 0x4, bottom = 0x8, all = 0xF; +} +// "top+left" (picture edges, top, bottom, left, right order), "none" for 0. +std::string kinect_edges_text(uint8_t edges); +// What the player should do about clipped edges: bits of kinect_edge_move. +// A cut on the picture's left or right is cured by moving the other way +// round from the player's view: the picture's left edge is the player's +// right side (mirror), so the player steps to the player's own left. +namespace kinect_edge_move { +constexpr uint8_t step_back = 0x1, player_left = 0x2, player_right = 0x4, tilt_up = 0x8, tilt_down = 0x10; +} +uint8_t kinect_edges_move(uint8_t edges); + struct KinectFrame { uint64_t number = 0; // counts from one; 0 means nothing has arrived // Set by the acquisition worker, never by a consumer reading its cache. @@ -67,16 +93,100 @@ struct KinectImage { uint64_t number = 0; // counts from one; 0 means nothing has arrived uint32_t width = 0, height = 0, bytes_per_pixel = 0; std::vector pixels; + // For a recording (kinect_record_depth.h): when the backend had it (the + // host's steady clock, kinect_record_now_us), the SDK's own timestamp + // (a v1's liTimeStamp in ms, a v2's RelativeTime in 100 ns; 0 unknown) + // and frame number (a v1's dwFrameNumber; 0 for a v2), and for a depth + // image a Kinect v2 made for the game, the virtual v1 it was made + // through (KinectDepthView; a v1's own image leaves these as they are). + int64_t steady_us = 0, sdk_time = 0; + uint64_t sdk_frame = 0; + std::array view_origin{}; + float view_z_scale = 1.0f; + // And its turn (KinectDepthView::rotated, rotation), so the image can be + // taken back into the sensor's space (kinect_v1_depth_points). Not + // recorded. + bool view_rotated = false; + std::array view_rotation{1, 0, 0, 0, 1, 0, 0, 0, 1}; +}; + +// Where a point of skeleton space falls in one of the sensor's images, in +// that image's pixels; not visible when it cannot be placed. Visible means +// placed, not inside the image: a point beyond its edge is still visible +// (within a sane distance), and whoever draws it clips to the image. +struct KinectImagePoint { float x = 0, y = 0; bool visible = false; }; + +// Where the picture itself lies inside the preview's image of one kind +// (colour 640x480, depth 320x240), in those pixels. A picture of another +// shape (a Kinect v2's) is fitted with black bars around it, and nothing is +// drawn over the bars: skeletons are clipped at this rectangle. +struct KinectImageRect { + float x = 0, y = 0, width = 0, height = 0; + bool contains(float px, float py) const { + return px >= x && px < x + width && py >= y && py < y + height; + } }; class KinectSensor { public: virtual ~KinectSensor(); // The newest image of one camera, if there is one newer than - // image.number. A Kinect v1 opens its cameras unless SFR_KINECT_DEPTH=0; - // others have none. + // image.number. A Kinect v1 opens its cameras + // (and hands them to the game too: has_depth_stream). A Kinect v2 opens + // its cameras on the first call, for the preview window only, at the + // v1's sizes (kinect_v2_image.h), and closes them again when nobody has + // asked for a few seconds; the game reads game_image instead. Others + // have none. virtual bool image(KinectImageKind, KinectImage&) { return false; } - // Successfully opened streams, rather than requested configuration. + // The newest image of one camera for the game's own image streams + // (NuiImageStreamGetNextFrame), newer than image.number. A v1's are its + // image()s. A v2 makes a v1's depth and + // player index image of its own (kinect_v2_depth.h); its preview keeps + // image(). + virtual bool game_image(KinectImageKind kind, KinectImage& image) { return this->image(kind, image); } + // Why the game's depth stream is not there (a v1's depth stream or a v2's + // depth and body index readers would not open), or empty when it is. The + // race reads the title's own depth detectors and nothing stands in for a + // missing depth: the game just gets no depth, as on a console whose + // sensor has none. + virtual std::string depth_failure() const { return {}; } + // Where a skeleton point falls in this sensor's image of one kind (as + // image() gives it), when the sensor places points itself: true with the + // point, which may be not visible. False when it does not, and the + // preview uses a v1's lens (kinect_depth_point, kinect_colour_point). + virtual bool image_point(KinectImageKind, const std::array&, KinectImagePoint&) { return false; } + // Where the picture lies inside the preview's image of one kind. False + // (the default) means it fills the whole image, as a Kinect v1's does. + virtual bool picture_rect(KinectImageKind, KinectImageRect&) { return false; } + // What the preview shows of a sensor whose pictures differ in shape from + // a v1's (a Kinect v2): true, the default, the view both cameras have; + // false, each camera's whole picture (the window's "Original" box). + // Changes images, picture_rect and image_point from then on; never what + // the game or a recording gets. Others ignore it. + virtual void set_preview_common_view(bool common) { (void)common; } + // A Kinect v2's depth view and its tuning from now on, instead of the + // SFR_KINECT_V2_DEPTH_* environment it starts with (kinect_v2_depth.h): + // what the launcher's preview shows. Applied on the sensor's own thread + // at its next frame; the game never calls it. Others ignore it. + virtual void set_depth_settings(const KinectDepthSettings& settings) { (void)settings; } + // Where the Recompiled (ideal) view's virtual v1 stands and where the + // players are, as the newest depth image was made (a Kinect v2 keeps it + // in every view: KinectDepthViewer). False, leaving it alone, when the + // sensor makes no depth. + virtual bool ideal_snapshot(KinectIdealSnapshot& snapshot) { (void)snapshot; return false; } + // Where points of the sensor's own camera space (x, y, z metres, three + // floats each; a z that is not positive is no point) land in the + // preview's colour image (image(colour), its current view): x, y pairs + // in its pixels, NaN for a point it cannot place or none. A Kinect v2 + // through its coordinate mapper (the typical lens without it); false, + // leaving positions alone, for a sensor that cannot. + virtual bool colour_image_positions(const std::vector& points, std::vector& positions) { + (void)points; (void)positions; + return false; + } + // Successfully opened streams the game reads (NuiImageStreamOpen), + // rather than requested configuration. A v1's, and a v2's + // (its depth and body index readers opened). virtual bool has_depth_stream() const { return false; } virtual bool has_colour_stream() const { return false; } // The sensor's tilt motor, in degrees above level (a Kinect v1 turns @@ -90,6 +200,16 @@ class KinectSensor { // The first sensor the host has, started with skeleton tracking. Null // (with the reason in why) when there is no runtime or no sensor. static std::unique_ptr open(std::string* why = nullptr); + // One kind only: 1 a Kinect v1 (SDK 1.8), 2 a Kinect v2 (SDK 2.0). Both + // may be opened in one process (sfr_kinect_probe --both), each on its own + // runtime (whether the two get on has not been tried on hardware); a + // failure to open one leaves the other alone. + static std::unique_ptr open_model(int model, std::string* why = nullptr); + // What the launcher's camera choice asks for (SFR_KINECT_MODEL): 2 opens a + // Kinect v2 only; anything else is the Xbox 360 Kinect, and when no v1 + // opens but a v2 does the v2 is used all the same (one log line says so), + // so an old settings.ini with a v2 keeps working. + static std::unique_ptr open_selected(int model, std::string* why = nullptr); // Whether this build can talk to a sensor at all. static bool supported(); // "Kinect v1" or "Kinect v2", for the log. @@ -119,19 +239,20 @@ bool kinect_expire_frame(KinectFrame& frame, std::chrono::steady_clock::time_poi // Enrollment only lasts while its real body remains tracked. void kinect_prune_identities(const KinectFrame& frame, std::unordered_map& identities); -// Turns a frame so that its gravity points straight down. Skeleton space is -// the sensor's own: a sensor tilted up at the player (set low, as under a -// TV) sees them leaning back, and the title measures a stance leaning back. -// The sensor's accelerometer gives the frame's up (gravity); the bodies and -// the floor are turned about the sensor so that up is +y, and the frame's -// gravity becomes (0, 1, 0). A tilt beyond 30 degrees, or a frame without -// gravity, is left alone. Returns the tilt taken out, in degrees. -float kinect_level(KinectFrame& frame); - // Why no sensor opened, from why each kind failed ("no-runtime", // "no-sensor", ...): a runtime that is installed says more than one that is // not, so it is its reason that counts; with neither, "no-runtime". std::string kinect_open_failure(const std::string& v1, const std::string& v2); +// The model the launcher chose, from SFR_KINECT_MODEL: 2 for "kinect_v2", 1 +// otherwise (unset included: an old launcher only knew "kinect"). +int kinect_selected_model(); +// SFR_KINECT_SMOOTH=1: the backends smooth their skeletons themselves (a +// v1 through its SDK's NuiTransformSmooth, a v2 through KinectJointSmoother, +// the same filter), for sfr_kinect_probe (--smooth) and tests. Off by +// default: the game smooths each real skeleton once, as the title asks +// (its NuiTransformSmooth, nui_smooth_hooks.cpp), and leaves that out when +// this is on, so a skeleton is never smoothed twice. +bool kinect_host_smoothing(); // Which of the tracked bodies play as the title's two Kinect players. The // title follows a player by the slot and the tracking id it was identified diff --git a/src/kinect_sensor_none.cpp b/src/kinect_sensor_none.cpp index 61a2c83..41a7e21 100644 --- a/src/kinect_sensor_none.cpp +++ b/src/kinect_sensor_none.cpp @@ -9,4 +9,9 @@ std::unique_ptr KinectSensor::open(std::string* why) { return nullptr; } +std::unique_ptr KinectSensor::open_model(int, std::string* why) { + if (why) *why = "unsupported-platform"; + return nullptr; +} + } diff --git a/src/kinect_sensor_win32.cpp b/src/kinect_sensor_win32.cpp index a76ffb4..f84ae88 100644 --- a/src/kinect_sensor_win32.cpp +++ b/src/kinect_sensor_win32.cpp @@ -1,4 +1,5 @@ #include "kinect_sensor.h" +#include "kinect_record.h" #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -131,6 +132,7 @@ class WindowsKinect final : public KinectSensor { } const char* model() const override { return "Kinect v1"; } + std::string depth_failure() const override { return depth_failure_; } bool has_depth_stream() const override { return streams_[0].handle != nullptr; } bool has_colour_stream() const override { return streams_[1].handle != nullptr; } @@ -165,12 +167,11 @@ class WindowsKinect final : public KinectSensor { if (const char* text = std::getenv("SFR_KINECT_V1_FLAGS"); text && *text) flags = DWORD(std::strtoul(text, nullptr, 0)) | nui_initialize_flag_uses_skeleton; // The depth (with the player index) and colour cameras as well, for - // the title's own depth view and camera image; SFR_KINECT_DEPTH=0 - // opens the skeleton alone. - const char* images = std::getenv("SFR_KINECT_DEPTH"); - const bool with_images = !(images && *images == '0') && get(image_open_, "NuiImageStreamOpen") && - get(image_next_, "NuiImageStreamGetNextFrame") && + // the title's own depth view and camera image, always: the race reads + // the title's depth detectors, as on the console. + const bool with_images = get(image_open_, "NuiImageStreamOpen") && get(image_next_, "NuiImageStreamGetNextFrame") && get(image_release_, "NuiImageStreamReleaseFrame"); + if (!with_images) depth_failure_ = "no-image-api"; if (with_images) flags |= nui_initialize_flag_uses_depth_and_player_index | nui_initialize_flag_uses_color; const char* read = std::getenv("SFR_KINECT_V1_READ"); wait_in_sdk_ = !(read && std::string_view(read) == "event"); @@ -192,7 +193,10 @@ class WindowsKinect final : public KinectSensor { if (with_images) for (ImageStream& stream : streams_) { const HRESULT result = image_open_(stream.type, stream.resolution, 0, 2, nullptr, &stream.handle); - if (FAILED(result)) stream.handle = nullptr; + if (FAILED(result)) { + stream.handle = nullptr; + if (stream.type == 0) depth_failure_ = "stream-" + hresult(result); + } std::cerr << "NATIVE_KINECT_IMAGE_OPEN type=" << stream.type << " width=" << stream.width << " height=" << stream.height << " result=" << hresult(result) << std::endl; } @@ -241,6 +245,8 @@ class WindowsKinect final : public KinectSensor { uint64_t frames = 0, tracked = 0, waits = 0, failures = 0; HRESULT last_failure = S_OK; auto reported = std::chrono::steady_clock::now(); + // SFR_KINECT_RECORD (kinect_record.h): every frame, before smoothing. + const bool recording = kinect_record_skeletons() != nullptr; // Every five seconds, frames or not: a sensor that opens but sends // nothing (in use by another program, short of power, on a USB // controller it does not get on with) says so instead of staying @@ -253,6 +259,7 @@ class WindowsKinect final : public KinectSensor { if (failures) std::cerr << " last_error=" << hresult(last_failure); if (streams_[0].handle || streams_[1].handle) std::cerr << " depth_frames=" << image_frames_[0].exchange(0) << " colour_frames=" << image_frames_[1].exchange(0); + std::cerr << kinect_record_queue_status(); // recordings dropped for a slow disk, queued if (!frames && !failures) std::cerr << " (no skeleton frames: close any other program using the Kinect)"; std::cerr << std::endl; reported = now; @@ -274,6 +281,7 @@ class WindowsKinect final : public KinectSensor { continue; } frame.captured_at = std::chrono::steady_clock::now(); + if (recording) record(raw); if (smooth_ && smooth_frames_) smooth_(&raw, &smoothing); frame.floor_plane = {raw.floor_clip_plane.x, raw.floor_clip_plane.y, raw.floor_clip_plane.z, raw.floor_clip_plane.w}; @@ -302,6 +310,45 @@ class WindowsKinect final : public KinectSensor { } } + // One skeleton frame as the SDK gave it, into the recording: its own + // frame number and timestamp (liTimeStamp, milliseconds), the floor and + // gravity, every fully tracked skeleton, and the newest depth image's + // share of pixels without a depth. + void record(const NuiSkeletonFrame& raw) { + KinectRecordFrame out; + out.steady_us = kinect_record_now_us(); + out.sensor = "v1"; + out.frame = raw.frame_number; + out.sdk_time = raw.timestamp.QuadPart; + out.floor = {raw.floor_clip_plane.x, raw.floor_clip_plane.y, raw.floor_clip_plane.z, raw.floor_clip_plane.w}; + out.gravity = {raw.normal_to_gravity.x, raw.normal_to_gravity.y, raw.normal_to_gravity.z}; + { + std::lock_guard guard(lock_); + if (depth_quality_frame_ && out.steady_us - depth_quality_us_ <= 1000000) { + out.depth_invalid = depth_invalid_; + out.depth_frame = depth_quality_frame_; + out.depth_steady_us = depth_quality_us_; + } + } + for (uint32_t index = 0; index < uint32_t(nui_skeleton_count); ++index) { + const auto& skeleton = raw.skeletons[index]; + if (skeleton.tracking_state != nui_skeleton_tracked || !skeleton.tracking_id) continue; + KinectRecordBody& body = out.bodies.emplace_back(); + body.sensor_index = index; + body.tracking_id = skeleton.tracking_id; + body.has_position = true; + body.position = {skeleton.position.x, skeleton.position.y, skeleton.position.z}; + std::array, 20> joints{}; + std::array states{}; + for (int j = 0; j < nui_position_count; ++j) { + joints[j] = {skeleton.positions[j].x, skeleton.positions[j].y, skeleton.positions[j].z}; + states[j] = skeleton.position_states[j]; + } + kinect_record_from_v1(joints, states, body); + } + kinect_record_skeleton_frame(out); + } + // The newest frame of each open camera, if one has come since the last // look: the SDK keeps two, so reading at the skeleton's pace keeps up. void read_images() { @@ -317,12 +364,27 @@ class WindowsKinect final : public KinectSensor { scratch_.resize(size_t(row) * stream.height); for (uint32_t y = 0; y < stream.height; ++y) std::memcpy(scratch_.data() + size_t(y) * row, locked.bits + size_t(y) * locked.pitch, row); + // The depth image's share of pixels without a depth, for + // a recording (another sensor's light shows there first). + float invalid = -1; + if (i == 0 && stream.bytes_per_pixel == 2) + invalid = kinect_depth_invalid_ratio(reinterpret_cast(scratch_.data()), + size_t(stream.width) * stream.height, 3); std::lock_guard guard(lock_); + if (invalid >= 0) { + depth_invalid_ = invalid; + ++depth_quality_frame_; + depth_quality_us_ = kinect_record_now_us(); + } KinectImage& latest = images_[i]; latest.width = stream.width; latest.height = stream.height; latest.bytes_per_pixel = stream.bytes_per_pixel; latest.pixels.swap(scratch_); + // Its clocks, for a depth recording (kinect_record_depth.h). + latest.steady_us = kinect_record_now_us(); + latest.sdk_time = frame->timestamp.QuadPart; + latest.sdk_frame = frame->frame_number; ++latest.number; ++image_frames_[i]; } @@ -340,14 +402,13 @@ class WindowsKinect final : public KinectSensor { NuiSkeletonGetNextFrame next_frame_ = nullptr; NuiTransformSmooth smooth_ = nullptr; // The console's runtime hands the title its frames as tracked, and the - // title smooths them itself if it wants to (NuiTransformSmooth is part of - // the NUI library it links); smoothing here as well would add a frame of - // latency on top. SFR_KINECT_SMOOTH=1 smooths anyway, for sfr_kinect_probe - // or a jittery sensor. - bool smooth_frames_ = [] { - const char* value = std::getenv("SFR_KINECT_SMOOTH"); - return value && *value == '1'; - }(); + // title smooths them itself (NuiTransformSmooth, part of the NUI library + // it links, done natively in nui_smooth_hooks.cpp); smoothing here as + // well would add a frame of latency on top. SFR_KINECT_SMOOTH=1 + // (kinect_host_smoothing) smooths here instead, for sfr_kinect_probe + // --smooth; the game then leaves its own out. + bool smooth_frames_ = kinect_host_smoothing(); + std::string depth_failure_; // set while starting, before the worker runs NuiCameraElevationGetAngle elevation_get_ = nullptr; NuiCameraElevationSetAngle elevation_set_ = nullptr; NuiImageStreamOpen image_open_ = nullptr; @@ -357,6 +418,10 @@ class WindowsKinect final : public KinectSensor { std::array images_{}; std::array, 2> image_frames_{}; std::vector scratch_; + // The newest depth image's quality, under lock_ (record()). + float depth_invalid_ = -1; + uint64_t depth_quality_frame_ = 0; + int64_t depth_quality_us_ = 0; std::mutex lock_; KinectFrame latest_; uint64_t taken_ = 0; @@ -379,4 +444,14 @@ std::unique_ptr KinectSensor::open(std::string* why) { return nullptr; } +std::unique_ptr KinectSensor::open_model(int model, std::string* why) { + if (model == 1) { + if (auto sensor = std::make_unique(); sensor->start(why)) return sensor; + return nullptr; + } + if (model == 2) return open_kinect_v2(why); + if (why) *why = "unknown-model"; + return nullptr; +} + } diff --git a/src/kinect_smoothing.cpp b/src/kinect_smoothing.cpp index c6521bb..d8957c4 100644 --- a/src/kinect_smoothing.cpp +++ b/src/kinect_smoothing.cpp @@ -1,59 +1,14 @@ #include "kinect_smoothing.h" -#include #include namespace sfr { -namespace { -using Vec = std::array; -Vec add(const Vec& a, const Vec& b) { return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; } -Vec sub(const Vec& a, const Vec& b) { return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; } -Vec scale(const Vec& a, float s) { return {a[0] * s, a[1] * s, a[2] * s}; } -Vec mix(const Vec& a, const Vec& b, float t) { return add(scale(a, 1.0f - t), scale(b, t)); } // a towards b -float length(const Vec& a) { return std::sqrt(a[0] * a[0] + a[1] * a[1] + a[2] * a[2]); } -} void KinectJointSmoother::apply(KinectFrame& frame) { std::unordered_set seen; for (auto& body : frame.bodies) { seen.insert(body.tracking_id); - Body& history = bodies_[body.tracking_id]; - for (uint32_t j = 0; j < nui_joint_count; ++j) { - Joint& joint = history[j]; - const uint32_t state = body.joint_states[j]; - if (state == 0) { // not tracked: as measured, and start over - joint = {}; - continue; - } - // Inferred joints are guesses; the SDK smooths them twice as hard. - const float jitter = parameters_.jitter_radius * (state == 1 ? 2.0f : 1.0f); - const float deviation = parameters_.max_deviation * (state == 1 ? 2.0f : 1.0f); - const Vec measured = body.joints[j]; - Vec raw = measured, filtered, trend; - if (joint.frames == 0) { - filtered = raw; - trend = {}; - } else if (joint.frames == 1) { - filtered = scale(add(raw, joint.raw), 0.5f); - trend = mix(joint.trend, sub(filtered, joint.filtered), parameters_.correction); - } else { - // Inside the jitter radius, pull the measurement towards the - // last output in proportion to how small the move is. - const float moved = length(sub(raw, joint.filtered)); - if (moved <= jitter && jitter > 0) raw = mix(joint.filtered, raw, moved / jitter); - filtered = mix(raw, add(joint.filtered, joint.trend), parameters_.smoothing); - trend = mix(joint.trend, sub(filtered, joint.filtered), parameters_.correction); - } - Vec predicted = add(filtered, scale(trend, parameters_.prediction)); - // Never lead the measurement by more than the deviation radius. - const float lead = length(sub(predicted, measured)); - if (lead > deviation && lead > 0) predicted = mix(measured, predicted, deviation / lead); - joint.raw = measured; - joint.filtered = filtered; - joint.trend = trend; - joint.frames = joint.frames < 2 ? joint.frames + 1 : 2; - body.joints[j] = predicted; - } + nui_smooth_skeleton(bodies_[body.tracking_id], body.joints, body.joint_states, parameters_); } for (auto it = bodies_.begin(); it != bodies_.end();) it = seen.count(it->first) ? std::next(it) : bodies_.erase(it); diff --git a/src/kinect_smoothing.h b/src/kinect_smoothing.h index 80c1d56..739c7c8 100644 --- a/src/kinect_smoothing.h +++ b/src/kinect_smoothing.h @@ -1,5 +1,6 @@ #pragma once #include "kinect_sensor.h" +#include "nui_transform_smooth.h" #include #include @@ -7,36 +8,27 @@ namespace sfr { -// Kinect v1's SDK smooths a skeleton frame with NuiTransformSmooth; the v2 -// SDK has nothing of the kind, so its joints reach the game as measured and -// shake. This is the same filter -- Holt's double exponential smoothing with -// a jitter radius and a cap on how far a prediction may run -- applied to -// each tracked body's joints, keyed by its tracking id. -struct KinectSmoothingParameters { - float smoothing = 0.5f; // 0 follows the measurement, towards 1 the history - float correction = 0.5f; // how fast the trend follows the motion - float prediction = 0.5f; // frames of trend added ahead - float jitter_radius = 0.05f; // metres: closer moves are damped - float max_deviation = 0.04f; // metres: how far the output may lead the measurement -}; +// The NUI library's NuiTransformSmooth (nui_transform_smooth.h: Holt's +// double exponential smoothing with a jitter radius and a cap on how far a +// prediction may run, step for step as the title's library has it) over a +// host frame's bodies, keyed by tracking id. Only for SFR_KINECT_SMOOTH=1 +// (kinect_host_smoothing: sfr_kinect_probe --smooth, tests): the game +// smooths a real sensor's skeletons itself, once, in its own +// NuiTransformSmooth (nui_smooth_hooks.cpp). +using KinectSmoothingParameters = NuiSmoothParameters; class KinectJointSmoother { public: explicit KinectJointSmoother(KinectSmoothingParameters parameters = {}) : parameters_(parameters) {} - // Smooths every body in the frame in place. A joint that is not tracked - // passes through and starts over when it is seen again; inferred joints - // get twice the jitter and deviation radii, as in the SDK. Bodies that - // have left the frame are forgotten. + // Smooths every body in the frame in place, as the library smooths a + // TRACKED skeleton (inferred joints get twice the radii; a joint at the + // origin starts over). A body seen for the first time starts from its + // own measurement; bodies that have left the frame are forgotten. void apply(KinectFrame& frame); private: - struct Joint { - std::array raw{}, filtered{}, trend{}; - uint32_t frames = 0; - }; - using Body = std::array; KinectSmoothingParameters parameters_; - std::unordered_map bodies_; + std::unordered_map bodies_; }; } diff --git a/src/kinect_v2_depth.cpp b/src/kinect_v2_depth.cpp new file mode 100644 index 0000000..6f2ed1b --- /dev/null +++ b/src/kinect_v2_depth.cpp @@ -0,0 +1,546 @@ +#include "kinect_v2_depth.h" +#include "kinect_v2_image.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sfr { +namespace { +float number(const char* text, float fallback, float low, float high) { + if (!text || !*text) return fallback; + char* end = nullptr; + const float value = std::strtof(text, &end); + if (end == text || !std::isfinite(value)) return fallback; + return std::clamp(value, low, high); +} + +bool none(const std::array& p) { return p[0] == 0.0f && p[1] == 0.0f && p[2] == 0.0f; } + +bool known_floor(const std::array& floor) { return kinect_floor_known(floor); } + +std::array turn(const std::array& r, const std::array& p) { + return {r[0] * p[0] + r[1] * p[1] + r[2] * p[2], r[3] * p[0] + r[4] * p[1] + r[5] * p[2], + r[6] * p[0] + r[7] * p[1] + r[8] * p[2]}; +} +// The transpose: the turn undone. +std::array unturn(const std::array& r, const std::array& p) { + return {r[0] * p[0] + r[3] * p[1] + r[6] * p[2], r[1] * p[0] + r[4] * p[1] + r[7] * p[2], + r[2] * p[0] + r[5] * p[1] + r[8] * p[2]}; +} +} + +bool kinect_floor_known(const std::array& floor) { + if (!std::isfinite(floor[0]) || !std::isfinite(floor[1]) || !std::isfinite(floor[2]) || !std::isfinite(floor[3])) + return false; + const float length = std::sqrt(floor[0] * floor[0] + floor[1] * floor[1] + floor[2] * floor[2]); + return std::fabs(length - 1.0f) < 0.05f && floor[1] > 0.5f; +} + +bool kinect_floor_height_known(const std::array& floor) { + return kinect_floor_known(floor) && std::isfinite(floor[3]) && floor[3] >= kinect_floor_height_minimum; +} + +std::array kinect_level_rotation(const std::array& floor) { + if (!kinect_floor_known(floor)) return {1, 0, 0, 0, 1, 0, 0, 0, 1}; + const double length = std::sqrt(double(floor[0]) * floor[0] + double(floor[1]) * floor[1] + double(floor[2]) * floor[2]); + const double nx = floor[0] / length, ny = floor[1] / length, nz = floor[2] / length; + // Roll about z: (nx, ny) -> (0, h). + const double h = std::sqrt(nx * nx + ny * ny); + const double c = ny / h, s = nx / h; + // Then pitch about x: (h, nz) -> (1, 0). + const double c2 = h, s2 = -nz; + // R = Rx * Rz, Rz = [c -s 0; s c 0; 0 0 1], Rx = [1 0 0; 0 c2 -s2; 0 s2 c2]. + return {float(c), float(-s), 0.0f, + float(c2 * s), float(c2 * c), float(-s2), + float(s2 * s), float(s2 * c), float(c2)}; +} + +void kinect_floor_tilt(const std::array& floor, float& pitch_degrees, float& roll_degrees) { + pitch_degrees = roll_degrees = 0; + if (!kinect_floor_known(floor)) return; + constexpr double degrees = 180.0 / 3.14159265358979323846; + // Looking down, the up direction leans back towards the camera: -z. + pitch_degrees = float(std::atan2(-double(floor[2]), std::hypot(double(floor[0]), double(floor[1]))) * degrees); + roll_degrees = float(std::atan2(double(floor[0]), double(floor[1])) * degrees); +} + +uint16_t kinect_v1_depth_pixel(uint32_t millimetres, uint32_t player) { + if (millimetres < kinect_v1_depth_minimum_mm || millimetres > kinect_v1_depth_maximum_mm) + return kinect_v1_depth_no_value; + return uint16_t(millimetres << kinect_v1_player_index_shift | (player & 7u)); +} + +bool kinect_v1_depth_project(const std::array& point, float& x, float& y) { + // NuiTransformSkeletonToDepthImage's own test (z > FLT_EPSILON). + if (!(point[2] > std::numeric_limits::epsilon())) return false; + x = kinect_v1_depth_centre_x + point[0] * kinect_v1_depth_focal / point[2]; + y = kinect_v1_depth_centre_y - point[1] * kinect_v1_depth_focal / point[2]; + return std::isfinite(x) && std::isfinite(y); +} + +std::array kinect_depth_view_point(const KinectDepthView& view, const std::array& given) { + const std::array p = view.rotated ? turn(view.rotation, given) : given; + return {p[0] - view.origin[0], p[1] - view.origin[1], (p[2] - view.origin[2]) * view.z_scale}; +} + +std::array kinect_depth_view_unpoint(const KinectDepthView& view, const std::array& p) { + const std::array moved{p[0] + view.origin[0], p[1] + view.origin[1], p[2] / view.z_scale + view.origin[2]}; + return view.rotated ? unturn(view.rotation, moved) : moved; +} + +void kinect_depth_view_frame(const KinectDepthView& view, KinectFrame& frame) { + const auto move = [&](std::array& p) { + if (!none(p)) p = kinect_depth_view_point(view, p); + }; + for (KinectBody& body : frame.bodies) { + move(body.position); + for (auto& joint : body.joints) move(joint); + for (auto& tip : body.hand_tips) move(tip); + for (auto& thumb : body.thumbs) move(thumb); + } + // A direction turns with the camera (and is not moved). + if (view.rotated && !none(frame.gravity)) frame.gravity = turn(view.rotation, frame.gravity); + // n.p + d = 0 with p = R^T ((x', y', z' / k) + origin): + // (R n).((x', y', z' / k) + origin) + d = 0, so with m = R n, + // (mx, my, mz / k).p' + (d + m.origin) = 0, renormalised. + auto& floor = frame.floor_plane; + if (floor[0] == 0.0f && floor[1] == 0.0f && floor[2] == 0.0f) return; + if (view.rotated) { + const std::array normal = turn(view.rotation, {floor[0], floor[1], floor[2]}); + floor = {normal[0], normal[1], normal[2], floor[3]}; + } + floor[3] += floor[0] * view.origin[0] + floor[1] * view.origin[1] + floor[2] * view.origin[2]; + if (view.z_scale == 1.0f) return; + floor[2] /= view.z_scale; + const float length = std::sqrt(floor[0] * floor[0] + floor[1] * floor[1] + floor[2] * floor[2]); + if (!(length > 0.0f)) return; + for (float& value : floor) value /= length; + // A v2's gravity is its floor's normal (no accelerometer of its own). + frame.gravity = {floor[0], floor[1], floor[2]}; +} + +float kinect_v2_fit_scale(float v2_focal_length) { + if (!(v2_focal_length > 0.0f) || !std::isfinite(v2_focal_length)) v2_focal_length = kinect_v2_typical_depth_focal; + return kinect_v1_depth_focal / (v2_focal_length * float(kinect_v1_depth_height) / 424.0f); +} + +float kinect_depth_true_minimum(float z_scale) { return float(kinect_v1_depth_minimum_mm) / 1000.0f / z_scale; } +float kinect_depth_true_maximum(float z_scale) { return float(kinect_v1_depth_maximum_mm) / 1000.0f / z_scale; } + +KinectDepthSettings KinectDepthSettings::parse(const char* view, const char* distance, const char* height, + const char* fit_scale, const char* follow, const char* deadband, + const char* skeleton_stretch, const char* skeleton_distance) { + KinectDepthSettings settings; + const std::string_view name = view ? view : ""; + settings.mode = name == "sensor" ? KinectDepthViewMode::sensor + : name == "player" ? KinectDepthViewMode::player + : name == "ideal" ? KinectDepthViewMode::ideal + : KinectDepthViewMode::fit; + settings.player_view = settings.mode == KinectDepthViewMode::player; + settings.distance = number(distance, 2.5f, 1.0f, 4.0f); + settings.height = number(height, settings.mode == KinectDepthViewMode::ideal ? 1.0f : 0.8f, 0.2f, 2.0f); + settings.fit_scale = number(fit_scale, 0.0f, 1.0f, 2.0f); + settings.follow = number(follow, 2.0f, 0.2f, 20.0f); + settings.deadband = number(deadband, 0.15f, 0.0f, 1.0f); + // Real metres unless asked otherwise; "depth" follows the depth's k. + settings.skeleton_stretch = skeleton_stretch && std::string_view(skeleton_stretch) == "depth" + ? 0.0f + : number(skeleton_stretch, 1.0f, 1.0f, 2.0f); + settings.skeleton_distance = number(skeleton_distance, 0.0f, 1.0f, 4.0f); + return settings; +} + +const char* KinectDepthSettings::view_name() const { + switch (mode) { + case KinectDepthViewMode::sensor: return "sensor"; + case KinectDepthViewMode::player: return "player"; + case KinectDepthViewMode::ideal: return "ideal"; + case KinectDepthViewMode::fit: break; + } + return "fit"; +} + +KinectDepthSettings kinect_depth_settings(const KinectDepthChoice& choice) { + const auto text = [](float value) { + char out[32]; + std::snprintf(out, sizeof out, "%.2f", double(value)); + return std::string(out); + }; + const std::string distance = text(choice.distance), height = text(choice.height), follow = text(choice.follow), + deadband = text(choice.deadband), skeleton = text(choice.skeleton_distance); + // The ideal view's height default is 1.0 and fit's 0.8; the tuning is + // Recompiled's, so it is read as ideal's and the view set afterwards. + KinectDepthSettings settings = KinectDepthSettings::parse("ideal", distance.c_str(), height.c_str(), nullptr, + follow.c_str(), deadband.c_str(), nullptr, + choice.skeleton_distance > 0 ? skeleton.c_str() : nullptr); + const KinectDepthSettings view = KinectDepthSettings::parse(kinect_depth_choice_name(choice.view), nullptr, nullptr); + settings.mode = view.mode; + settings.player_view = view.player_view; + return settings; +} + +KinectDepthSettings KinectDepthSettings::from_environment() { + return parse(std::getenv("SFR_KINECT_V2_DEPTH_VIEW"), std::getenv("SFR_KINECT_V2_DEPTH_DISTANCE"), + std::getenv("SFR_KINECT_V2_DEPTH_HEIGHT"), std::getenv("SFR_KINECT_V2_DEPTH_FIT_SCALE"), + std::getenv("SFR_KINECT_V2_DEPTH_FOLLOW"), std::getenv("SFR_KINECT_V2_DEPTH_DEADBAND"), + std::getenv("SFR_KINECT_V2_SKELETON_STRETCH"), std::getenv("SFR_KINECT_V2_SKELETON_DISTANCE")); +} +KinectDepthView KinectDepthViewLatch::place(const std::array& reference, + const std::array& floor) const { + KinectDepthView view; + if (!settings_.player_view) return view; + // Along the sensor's z, so the reference is distance away. + view.origin[2] = reference[2] - settings_.distance; + // And along its y so that the camera is height above the floor as seen + // from the reference point: the reference's height over the floor less + // the camera's is where it sits in the image, as for a level v1. + if (known_floor(floor)) { + const float above = floor[0] * reference[0] + floor[1] * reference[1] + floor[2] * reference[2] + floor[3]; + view.origin[1] = reference[1] - (above - settings_.height); + } + return view; +} + +const KinectDepthView& KinectDepthViewLatch::update(const KinectFrame& frame, double seconds) { + if (!settings_.player_view) { + view_ = {}; + return view_; + } + const bool anyone = !frame.bodies.empty(); + if (anyone) last_seen_ = seconds; + if (state_ != State::empty && !anyone && seconds - last_seen_ >= release_seconds) { + // Nobody for a while: the next person places it again. The view + // stays meanwhile (there is nobody to place). + state_ = State::empty; + count_ = 0; + } + if (!anyone || state_ == State::latched) return view_; + const KinectBody* body = nullptr; + if (state_ == State::settling) + for (const KinectBody& candidate : frame.bodies) + if (candidate.tracking_id == tracking_id_) body = &candidate; + if (!body) { + // Someone new (or the one settling left): the most central body. + body = &*std::min_element(frame.bodies.begin(), frame.bodies.end(), [](const KinectBody& a, const KinectBody& b) { + return std::fabs(a.position[0]) < std::fabs(b.position[0]); + }); + state_ = State::settling; + tracking_id_ = body->tracking_id; + count_ = 0; + sum_ = {}; + view_ = place(body->position, frame.floor_plane); + } + for (int i = 0; i < 3; ++i) sum_[i] += body->position[i]; + if (++count_ >= settle_frames) { + const std::array mean{float(sum_[0] / count_), float(sum_[1] / count_), float(sum_[2] / count_)}; + view_ = place(mean, frame.floor_plane); + state_ = State::latched; + } + return view_; +} + +KinectIdealState kinect_ideal_step(const KinectIdealState& previous, const KinectDepthSettings& settings, double seconds, + const KinectFrame& bodies) { + KinectIdealState state = previous; + const double dt = previous.started ? std::max(0.0, seconds - previous.seconds) : 0.0; + state.started = true; + state.seconds = seconds; + // The floor: the first known one as it is, then smoothed; an unknown + // one (a v2 sends zeros while it has none) leaves it. + // The normal and the distance apart: a v2 knows the normal (gravity) + // long before the distance, which it sends as 0 meanwhile; smoothing that + // 0 in would put the camera height metres above the floor. + bool newly_level = false; + if (kinect_floor_known(bodies.floor_plane)) { + const bool has_height = kinect_floor_height_known(bodies.floor_plane); + if (!state.floor_known) { + state.floor = bodies.floor_plane; + state.floor_known = newly_level = true; + state.height_known = has_height; + if (!has_height) state.floor[3] = 0; + } else { + const float a = float(1.0 - std::exp(-dt / kinect_ideal_floor_seconds)); + for (int i = 0; i < 3; ++i) state.floor[i] += a * (bodies.floor_plane[i] - state.floor[i]); + const float length = + std::sqrt(state.floor[0] * state.floor[0] + state.floor[1] * state.floor[1] + state.floor[2] * state.floor[2]); + if (length > 0.0f) + for (int i = 0; i < 3; ++i) state.floor[i] /= length; + if (has_height && !state.height_known) { + state.floor[3] = bodies.floor_plane[3]; + state.height_known = true; + } else if (has_height) { + state.floor[3] += a * (bodies.floor_plane[3] - state.floor[3]); + } + } + } + // The players' mean SpineBase in the levelled space. + const std::array level = kinect_level_rotation(state.floor_known ? state.floor : std::array{}); + double sum_x = 0, sum_z = 0; + uint32_t count = 0; + for (const KinectBody& body : bodies.bodies) { + const auto& p = body.position; + if (none(p) || !std::isfinite(p[0]) || !std::isfinite(p[1]) || !std::isfinite(p[2])) continue; + const std::array q = state.floor_known ? turn(level, p) : p; + sum_x += q[0]; + sum_z += q[2]; + ++count; + } + state.bodies = count; + if (!count) { + // Nobody: the camera holds; the next body is taken at once. + state.had_bodies = false; + return state; + } + const std::array measured{float(sum_x / count), float(sum_z / count)}; + if (!state.placed || !previous.had_bodies || newly_level) { + state.reference = measured; + state.placed = true; + } else { + const float a = float(1.0 - std::exp(-dt / double(settings.follow))); + for (int i = 0; i < 2; ++i) { + const float error = measured[i] - state.reference[i]; + if (std::fabs(error) <= settings.deadband) continue; + state.reference[i] += a * (error - std::copysign(settings.deadband, error)); + } + } + state.had_bodies = true; + return state; +} + +KinectDepthView kinect_ideal_view(const KinectIdealState& state, const KinectDepthSettings& settings) { + KinectDepthView view; + // Nobody yet: straight in front of the v2's own place. + const std::array reference = state.placed ? state.reference : std::array{0.0f, settings.distance}; + view.origin[0] = reference[0]; + view.origin[2] = reference[1] - settings.distance; + if (state.floor_known) { + view.rotated = true; + view.rotation = kinect_level_rotation(state.floor); + // Levelled, the floor is y = -d: the camera height above it. Without + // a distance yet, the sensor's own height (y = 0). + if (state.height_known) view.origin[1] = -state.floor[3] + settings.height; + } + return view; +} + +KinectIdealSnapshot kinect_ideal_snapshot(const KinectFrame& bodies, const KinectIdealState& state, + const KinectDepthSettings& settings) { + KinectIdealSnapshot snapshot; + snapshot.valid = true; + snapshot.floor_known = state.floor_known; + snapshot.placed = state.placed; + const KinectDepthView view = kinect_ideal_view(state, settings); + snapshot.camera_x = view.origin[0]; + snapshot.camera_z = view.origin[2]; + snapshot.distance = settings.distance; + const std::array level = kinect_level_rotation(state.floor_known ? state.floor : std::array{}); + for (const KinectBody& body : bodies.bodies) { + const auto& p = body.position; + if (none(p) || !std::isfinite(p[0]) || !std::isfinite(p[1]) || !std::isfinite(p[2])) continue; + const std::array q = state.floor_known ? turn(level, p) : p; + KinectIdealPlayer& player = snapshot.players.emplace_back(); + player.tracking_id = body.tracking_id; + player.x = q[0]; + player.z = q[2]; + player.sensor_z = p[2]; + } + return snapshot; +} + +KinectDepthViewer::KinectDepthViewer(const KinectDepthSettings& settings) : settings_(settings), latch_(settings) { + fit_scale_ = settings.fit_scale > 0 ? settings.fit_scale : kinect_v2_fit_scale(kinect_v2_typical_depth_focal); + if (settings_.mode == KinectDepthViewMode::fit) view_.z_scale = fit_scale_; + if (settings_.mode == KinectDepthViewMode::ideal) view_ = kinect_ideal_view(ideal_, settings_); +} + +void KinectDepthViewer::set_settings(const KinectDepthSettings& settings) { + const bool same_latch = settings.mode == settings_.mode && settings.distance == settings_.distance && + settings.height == settings_.height; + settings_ = settings; + if (!same_latch) latch_ = KinectDepthViewLatch(settings_); + if (settings_.fit_scale > 0) fit_scale_ = settings_.fit_scale; + switch (settings_.mode) { + case KinectDepthViewMode::sensor: view_ = {}; break; + case KinectDepthViewMode::fit: view_ = {}; view_.z_scale = fit_scale_; break; + case KinectDepthViewMode::player: view_ = latch_.view(); break; + case KinectDepthViewMode::ideal: view_ = kinect_ideal_view(ideal_, settings_); break; + } +} + +void KinectDepthViewer::set_fit_scale(float z_scale) { + fit_scale_ = z_scale; + if (settings_.mode == KinectDepthViewMode::fit) view_.z_scale = fit_scale_; +} + +const KinectDepthView& KinectDepthViewer::body_frame(const KinectFrame& frame, double seconds) { + bodies_.floor_plane = frame.floor_plane; + bodies_.gravity = frame.gravity; + bodies_.bodies.clear(); + for (const KinectBody& body : frame.bodies) { + KinectBody& kept = bodies_.bodies.emplace_back(); + kept.tracking_id = body.tracking_id; + kept.sensor_index = body.sensor_index; + kept.position = body.position; + kept.joint_states[0] = body.joint_states[0]; + } + bodies_seconds_ = seconds; + switch (settings_.mode) { + case KinectDepthViewMode::player: view_ = latch_.update(frame, seconds); break; + case KinectDepthViewMode::sensor: view_ = {}; break; + case KinectDepthViewMode::fit: view_ = {}; view_.z_scale = fit_scale_; break; + case KinectDepthViewMode::ideal: break; // stepped with the depth + } + return view_; +} + +KinectDepthView KinectDepthViewer::skeleton_view() const { + KinectDepthView view = view_; + if (settings_.mode == KinectDepthViewMode::fit && settings_.skeleton_stretch > 0) + view.z_scale = settings_.skeleton_stretch; + if (settings_.mode == KinectDepthViewMode::ideal && settings_.skeleton_distance > 0) + view.origin[2] += settings_.distance - settings_.skeleton_distance; + return view; +} + +const KinectDepthView& KinectDepthViewer::depth_frame(double seconds) { + // Stepped in every view (the preview's diagram); only ideal is made through it. + ideal_ = kinect_ideal_step(ideal_, settings_, seconds, bodies_); + if (settings_.mode == KinectDepthViewMode::ideal) view_ = kinect_ideal_view(ideal_, settings_); + return view_; +} + +KinectDepthStats kinect_v2_synthesize_depth(const float* points, const uint8_t* body_index, size_t count, + const KinectDepthView& view, KinectImage& out) { + constexpr uint32_t width = kinect_v1_depth_width, height = kinect_v1_depth_height; + constexpr size_t pixels = size_t(width) * height; + constexpr float empty = std::numeric_limits::infinity(); + std::vector nearest(pixels, empty); + std::vector player(pixels, 0); + for (size_t i = 0; i < count; ++i) { + const float* p = points + 3 * i; + if (!(p[2] > 0.0f) || !std::isfinite(p[0]) || !std::isfinite(p[1]) || !std::isfinite(p[2])) continue; + const std::array seen = kinect_depth_view_point(view, {p[0], p[1], p[2]}); + float x = 0, y = 0; + if (!kinect_v1_depth_project(seen, x, y)) continue; + // Pixel n is at n (NuiTransformDepthImageToSkeleton's (x - 160)). + const float column = std::floor(x + 0.5f), row = std::floor(y + 0.5f); + if (column < 0 || row < 0 || column >= float(width) || row >= float(height)) continue; + const size_t at = size_t(row) * width + size_t(column); + if (seen[2] < nearest[at]) { + nearest[at] = seen[2]; + const uint32_t body = body_index ? body_index[i] : kinect_v2_no_body; + player[at] = uint8_t(body < 6 ? body + 1 : 0); + } + } + KinectDepthStats stats; + // Holes one pixel wide (the two cameras' pixels do not line up one to + // one), from the image before any is filled, only where both sides agree. + std::vector depth = nearest; + for (uint32_t row = 0; row < height; ++row) + for (uint32_t column = 0; column < width; ++column) { + const size_t at = size_t(row) * width + column; + if (nearest[at] != empty) continue; + const auto pair = [&](size_t a, size_t b) { + if (nearest[a] == empty || nearest[b] == empty || player[a] != player[b]) return false; + if (std::fabs(nearest[a] - nearest[b]) > 0.05f) return false; + depth[at] = (nearest[a] + nearest[b]) / 2.0f; + player[at] = player[a]; + return true; + }; + if ((column > 0 && column + 1 < width && pair(at - 1, at + 1)) || + (row > 0 && row + 1 < height && pair(at - width, at + width))) + ++stats.filled; + } + out.width = width; + out.height = height; + out.bytes_per_pixel = 2; + out.pixels.assign(pixels * 2, 0); + for (size_t at = 0; at < pixels; ++at) { + if (depth[at] == empty) continue; + const uint32_t millimetres = uint32_t(std::lround(double(depth[at]) * 1000.0)); + const uint16_t value = kinect_v1_depth_pixel(millimetres, player[at]); + out.pixels[2 * at] = uint8_t(value); + out.pixels[2 * at + 1] = uint8_t(value >> 8); + if (value != kinect_v1_depth_no_value) { + ++stats.valid; + if (value & 7u) ++stats.players; + } + } + return stats; +} + +KinectDepthView kinect_image_view(const KinectImage& image) { + KinectDepthView view; + view.origin = image.view_origin; + view.z_scale = image.view_z_scale > 0 ? image.view_z_scale : 1.0f; + view.rotated = image.view_rotated; + view.rotation = image.view_rotation; + return view; +} + +void kinect_v1_depth_points(const KinectImage& depth, const KinectDepthView& view, std::vector& points) { + const size_t pixels = size_t(depth.width) * depth.height; + points.assign(pixels * 3, 0.0f); + if (depth.bytes_per_pixel != 2 || depth.pixels.size() < pixels * 2) return; + for (uint32_t row = 0; row < depth.height; ++row) + for (uint32_t column = 0; column < depth.width; ++column) { + const size_t at = size_t(row) * depth.width + column; + const uint32_t millimetres = uint32_t(depth.pixels[2 * at] | depth.pixels[2 * at + 1] << 8) >> kinect_v1_player_index_shift; + if (!millimetres) continue; + const float z = float(millimetres) / 1000.0f; + const std::array seen{(float(column) - kinect_v1_depth_centre_x) * z / kinect_v1_depth_focal, + -(float(row) - kinect_v1_depth_centre_y) * z / kinect_v1_depth_focal, z}; + const std::array p = kinect_depth_view_unpoint(view, seen); + points[3 * at] = p[0]; + points[3 * at + 1] = p[1]; + points[3 * at + 2] = p[2]; + } +} + +void kinect_v2_depth_points_from_lens(const uint16_t* millimetres, uint32_t width, uint32_t height, + std::vector& points) { + points.assign(size_t(width) * height * 3, 0.0f); + constexpr float focal = 366.0f, centre_x = 256.0f, centre_y = 212.0f; + for (uint32_t y = 0; y < height; ++y) + for (uint32_t x = 0; x < width; ++x) { + const size_t at = size_t(y) * width + x; + const float z = float(millimetres[at]) / 1000.0f; + if (!(z > 0.0f)) continue; + points[3 * at] = (float(x) - centre_x) * z / focal; + points[3 * at + 1] = -(float(y) - centre_y) * z / focal; + points[3 * at + 2] = z; + } +} + +void kinect_v2_depth_points_from_rays(const uint16_t* millimetres, const float* rays, size_t count, + std::vector& points) { + points.assign(count * 3, 0.0f); + for (size_t at = 0; at < count; ++at) { + const float z = float(millimetres[at]) / 1000.0f; + if (!(z > 0.0f)) continue; + points[3 * at] = rays[2 * at] * z; + points[3 * at + 1] = rays[2 * at + 1] * z; + points[3 * at + 2] = z; + } +} + +void kinect_v2_depth_rays_from_lens(float focal_x, float focal_y, float centre_x, float centre_y, uint32_t width, + uint32_t height, std::vector& rays) { + rays.assign(size_t(width) * height * 2, 0.0f); + for (uint32_t y = 0; y < height; ++y) + for (uint32_t x = 0; x < width; ++x) { + const size_t at = size_t(y) * width + x; + rays[2 * at] = (float(x) - centre_x) / focal_x; + rays[2 * at + 1] = -(float(y) - centre_y) / focal_y; + } +} + +uint32_t kinect_v2_short_tracking_id(uint64_t id) { return uint32_t(id) ? uint32_t(id) : uint32_t(id >> 32) | 1u; } + +} diff --git a/src/kinect_v2_depth.h b/src/kinect_v2_depth.h new file mode 100644 index 0000000..709c458 --- /dev/null +++ b/src/kinect_v2_depth.h @@ -0,0 +1,344 @@ +#pragma once +#include "kinect_ideal_diagram.h" +#include "kinect_sensor.h" + +#include +#include +#include +#include + +namespace sfr { + +// A Kinect v2 standing in for a Kinect v1's depth camera, for the game's own +// depth view (docs/kinect-sensor.md, "Kinect v2 深度"). +// On the console the race's lean, crouch, jump and side step are decided by +// the title's code reading the 320x240 depth-and-player-index image, not the +// skeleton. A v1 on the PC hands its image over as it is; a v2's depth camera +// is another camera (512x424, a wider lens, another position for the +// player), so its depth is not resampled but reprojected: every v2 depth +// pixel becomes a point in camera space (the sensor's own coordinate mapper), +// is moved into a virtual v1 depth camera (KinectDepthView) and projected +// through the v1's nominal lens, the nearest point winning each pixel. +// +// None of this is Windows', so it is tested everywhere (kinect_v2_depth_test). + +// The virtual v1 depth camera: NuiImageCamera.h's nominal focal length at +// 320x240 (NUI_CAMERA_DEPTH_NOMINAL_FOCAL_LENGTH_IN_PIXELS), centred, as +// NuiTransformSkeletonToDepthImage projects (NuiSkeleton.h): +// x = 160 + X * f / Z, y = 120 - Y * f / Z, pixel x at x (rounded). +constexpr uint32_t kinect_v1_depth_width = 320, kinect_v1_depth_height = 240; +constexpr float kinect_v1_depth_focal = 285.63f; +constexpr float kinect_v1_depth_centre_x = 160.0f, kinect_v1_depth_centre_y = 120.0f; +// NuiImageCamera.h: the default range a v1 measures (NUI_IMAGE_DEPTH_MINIMUM +// and _MAXIMUM, in millimetres before the shift), and the player index's +// three bits (NUI_IMAGE_PLAYER_INDEX_SHIFT). +constexpr uint32_t kinect_v1_depth_minimum_mm = 800, kinect_v1_depth_maximum_mm = 4000; +constexpr uint32_t kinect_v1_player_index_shift = 3; +// A pixel without a depth. A v1 stream opened without +// NUI_IMAGE_STREAM_FLAG_DISTINCT_OVERFLOW_DEPTH_VALUES (as ours is, flags 0) +// reports too near, too far and unknown all as NUI_IMAGE_DEPTH_NO_VALUE (0), +// player 0; the distinct values (0x0FFF << 3 too far, 0x1FFF << 3 unknown, +// 0 too near) are only for that flag. +constexpr uint16_t kinect_v1_depth_no_value = 0; + +// One v1 depth-and-player-index pixel: the millimetres shifted up three, the +// player (1..6, the skeleton slot + 1; 0 none) in the low three bits; outside +// the v1's range, no value. +uint16_t kinect_v1_depth_pixel(uint32_t millimetres, uint32_t player); + +// A camera-space point (metres) through the v1's nominal depth lens: false +// for a point at or behind the camera. +bool kinect_v1_depth_project(const std::array& point, float& x, float& y); + +// Where the virtual v1 stands, in the v2's camera space: it is the v2 moved +// by origin (no turn), and its depth is stretched by z_scale, so a point p +// is (p - origin) with z times z_scale to it. The fit view (origin zero, +// z_scale k) is the v2's whole field of view through the v1's lens: a point +// at angle a is drawn at the v1 pixel for angle a / k, and a person at d +// looks like one of the same size at k d, with that depth. +// The ideal view also turns: when rotated, a point is first turned by +// rotation (row-major 3x3, kinect_level_rotation: the floor made level) and +// then moved, R p - origin; origin is then in that levelled space. +struct KinectDepthView { + std::array origin{}; + float z_scale = 1.0f; + bool rotated = false; + std::array rotation{1, 0, 0, 0, 1, 0, 0, 0, 1}; +}; +std::array kinect_depth_view_point(const KinectDepthView& view, const std::array& p); +// The other way: a point the virtual camera sees, in the v2's space. +std::array kinect_depth_view_unpoint(const KinectDepthView& view, const std::array& p); +// A frame's skeletons (centres, joints, a v2's fingertips and thumbs; a +// point left at exactly zero means none and stays so) and floor plane into +// the virtual camera, so that skeleton and depth image agree. The floor plane +// goes through the same space (its normal turned with a rotated view and +// renormalised under a stretch), and a stretched frame's gravity is the new +// floor normal, as a v2's is the floor's; without a stretch gravity is a +// direction: turned with a rotated view, otherwise as it was. +void kinect_depth_view_frame(const KinectDepthView& view, KinectFrame& frame); + +// A floor plane the sensor actually gave (n.p + d = 0): a unit normal +// pointing mostly up (y > 0.5); a v2 sends zeros until it has found one. +bool kinect_floor_known(const std::array& floor); +// The turn that makes a floor level: R n = (0, 1, 0), first a roll about +// the camera's z (taking out the normal's x), then a pitch about its x +// (taking out its z), so the optical axis keeps its heading (no yaw). +// Row-major. The identity for a floor that is not known. +std::array kinect_level_rotation(const std::array& floor); +// The camera's tilt from a floor normal, in degrees, for the log: pitch +// positive when it looks down, roll positive when the floor's normal leans +// to the camera's +x. +void kinect_floor_tilt(const std::array& floor, float& pitch_degrees, float& roll_degrees); + +// The fit view's stretch: the v1's focal length over the v2's scaled so its +// 424 rows fit the v1's 240, f1 / (f2 * 240 / 424); about 1.378 for a +// typical v2 (focal length 366). +float kinect_v2_fit_scale(float v2_focal_length); +constexpr float kinect_v2_typical_depth_focal = 366.0f; +// The true distances (metres) that land inside the v1's 800..4000 mm under a +// stretch. +float kinect_depth_true_minimum(float z_scale); +float kinect_depth_true_maximum(float z_scale); + +// A Kinect v2 always makes the depth (no switch). SFR_KINECT_V2_DEPTH_VIEW +// (developer tuning) chooses where the virtual v1 is: +// - fit (the default): at the v2, with depth stretched so the v2's whole +// view fits the v1's image (KinectDepthView); no state, so one or two +// players and relays are the game's own business. The stretch is +// kinect_v2_fit_scale of the sensor's own focal length (its typical one +// until the sensor gives it), or SFR_KINECT_V2_DEPTH_FIT_SCALE (1.0..2.0; +// 1.0 is the sensor view). +// - sensor: at the v2, unstretched (the v1's narrower view of it). +// - player: where a v1 would be for the player: the player's reference point +// (latched, KinectDepthViewLatch) SFR_KINECT_V2_DEPTH_DISTANCE metres away +// (default 2.5, 1.0..4.0) and, when the floor is known, the camera +// SFR_KINECT_V2_DEPTH_HEIGHT metres above it (default 0.8, 0.2..2.0). +// - ideal: a real v1 standing where it should: level (the floor plane made +// horizontal), SFR_KINECT_V2_DEPTH_DISTANCE in front of the players +// (default 2.5), SFR_KINECT_V2_DEPTH_HEIGHT above the floor (default 1.0), +// following the players' mean SpineBase slowly (KinectIdealState: +// SFR_KINECT_V2_DEPTH_FOLLOW seconds, default 2.0, 0.2..20, outside a dead +// band of SFR_KINECT_V2_DEPTH_DEADBAND metres, default 0.15, 0..1), with +// no stretch: sizes, depths and movements are a v1's at that distance. +enum class KinectDepthViewMode : uint8_t { fit, sensor, player, ideal }; +struct KinectDepthSettings { + KinectDepthViewMode mode = KinectDepthViewMode::fit; + bool player_view = false; // mode == player + float distance = 2.5f, height = 0.8f; // height 1.0 by default for ideal + float fit_scale = 0; // set by SFR_KINECT_V2_DEPTH_FIT_SCALE; 0 from the sensor's lens + float follow = 2.0f, deadband = 0.15f; // ideal + // The skeleton handed to the game, apart from the depth: fit's z stretch + // for the skeleton, SFR_KINECT_V2_SKELETON_STRETCH (1.0..2.0; unset 1.0, + // the sensor's real metres; "depth", stored as 0, the depth's k), and + // the ideal camera's distance for the skeleton, + // SFR_KINECT_V2_SKELETON_DISTANCE (1.0..4.0; 0, unset, the depth's). + // The title never projects a skeleton into the depth image (it never + // calls NuiTransformSkeletonToDepthImage), so the two need not agree; + // its menu cursor is the 3D angle of shoulder to hand, which a z stretch + // of k makes about 1/k as sensitive for a hand reaching forward. + float skeleton_stretch = 1.0f, skeleton_distance = 0; + static KinectDepthSettings from_environment(); + // The values from text as the environment gives them (null for unset). + static KinectDepthSettings parse(const char* view, const char* distance, const char* height, + const char* fit_scale = nullptr, const char* follow = nullptr, + const char* deadband = nullptr, const char* skeleton_stretch = nullptr, + const char* skeleton_distance = nullptr); + const char* view_name() const; +}; +// The launcher's choice (kinect_ideal_diagram.h) as the settings the game +// would read for it: parse() of the same text SFR_KINECT_V2_DEPTH_* would +// carry, so the clamps and defaults are the game's. The tuning is kept for +// every view (fit and sensor do not use it), so the ideal state a viewer keeps +// for the preview's diagram follows the launcher's sliders whatever is shown. +KinectDepthSettings kinect_depth_settings(const KinectDepthChoice& choice); + +// The player view's placement, latched rather than following: the player's +// reference point (their skeleton centre, a v2's SpineBase) is taken when +// someone first becomes tracked -- the most central body, averaged over its +// first settle_frames frames (0.5 s), with the first frame's placement +// meanwhile -- and then kept while anyone is tracked, so that side steps and +// the hips going down and up stay in the image as movement. Once nobody has +// been tracked for release_seconds it is let go, and the next person latches +// it again. The sensor view is the v2's own place throughout. +class KinectDepthViewLatch { +public: + static constexpr uint32_t settle_frames = 15; + static constexpr double release_seconds = 1.0; + explicit KinectDepthViewLatch(const KinectDepthSettings& settings) : settings_(settings) {} + // The view for a frame of the sensor's own space, at seconds (any clock + // that goes forward). + const KinectDepthView& update(const KinectFrame& frame, double seconds); + const KinectDepthView& view() const { return view_; } + bool latched() const { return state_ == State::latched; } + // Where the camera goes for this reference point and floor. + KinectDepthView place(const std::array& reference, const std::array& floor) const; +private: + enum class State { empty, settling, latched }; + KinectDepthSettings settings_; + State state_ = State::empty; + KinectDepthView view_; + uint32_t tracking_id_ = 0, count_ = 0; + std::array sum_{}; + double last_seen_ = 0; +}; + +// The ideal view's camera, stepped once per depth frame by a pure function +// of the previous state, the frame's time and the newest body frame's +// bodies and floor (kinect_ideal_step), so that a raw recording +// (kinect_record_raw.h) gives the same images again offline. +// - The floor plane is smoothed (kinect_ideal_floor_seconds; the first known +// one taken as it is, an unknown one leaves it) and makes the turn that +// levels the camera (kinect_level_rotation). No floor yet: no turn, and the +// camera at the sensor's own height. +// - The reference is the mean SpineBase of all bodies given (one or two +// players alike), in the levelled space; its x and z are followed by an +// exponential of time constant follow seconds, only by what lies beyond the +// dead band (per axis), so leaning, side steps and hops stay movements in +// the image while a slow drift, or a player joining or leaving, glides in. +// The first body after nobody (and after the floor first becomes known) +// is taken at once; nobody holds the camera where it was. Height never +// follows: the camera is height above the floor. +// - The camera: x on the reference, distance in front of it (z), height above +// the floor, level, the v1's lens, no stretch. +constexpr double kinect_ideal_floor_seconds = 1.0; +// A Kinect v2 sends a floor plane whose normal is known (from its gravity) +// but whose distance is 0 until it has really found the floor, and for as +// long as it does not see it (FloorClipPlane's W): the normal then levels +// the camera, but its height above the floor is unknown, so the camera stays +// at the sensor's own height. A distance of at least +// kinect_floor_height_minimum (metres) is a height; the first one is taken at +// once, later ones are smoothed, and a plane without one leaves it. +constexpr float kinect_floor_height_minimum = 0.05f; +bool kinect_floor_height_known(const std::array& floor); +struct KinectIdealState { + bool started = false; // stepped at least once + double seconds = 0; // the last step's time + bool floor_known = false; // the normal: levelled + bool height_known = false; // and its distance: the camera's height above the floor + std::array floor{}; // smoothed, the sensor's own space + bool placed = false; // a reference was taken + bool had_bodies = false; // the last step had someone + std::array reference{}; // smoothed x, z, levelled space + uint32_t bodies = 0; // bodies in the last step +}; +KinectIdealState kinect_ideal_step(const KinectIdealState& previous, const KinectDepthSettings& settings, double seconds, + const KinectFrame& bodies); +KinectDepthView kinect_ideal_view(const KinectIdealState& state, const KinectDepthSettings& settings); +// The ideal view top-down (kinect_ideal_diagram.h): the virtual camera's x +// and z from kinect_ideal_view, and each body's SpineBase turned into the +// same levelled space (bodies as KinectDepthViewer::bodies keeps them; slot +// left -1 for the caller to fill). +KinectIdealSnapshot kinect_ideal_snapshot(const KinectFrame& bodies, const KinectIdealState& state, + const KinectDepthSettings& settings); + +// The virtual camera for every view, the one place the live sensor +// (kinect_v2_win32.cpp) and the offline re-synthesis (kinect_v2_resynth.h) +// both keep it, so the two cannot drift apart: +// - body_frame for every body frame, in the sensor's own space and before +// any smoothing, at seconds on the body frames' clock (a v2's RelativeTime +// in seconds): the player view's latch follows it, the ideal view keeps its +// bodies (tracking id, body index, SpineBase) and floor for the next depth +// frame. Returns the depth's view; the skeleton handed to the game goes +// through skeleton_view(). +// - depth_frame for every depth image about to be made, at seconds on the +// depth frames' clock: the ideal view steps. Returns the view the image is +// made through. +// fit, sensor and player are as they were; ideal is new. The ideal state is +// stepped in every view (only ideal makes its images through it), so the +// preview's diagram shows where Recompiled's camera would be whatever view +// is made. +class KinectDepthViewer { +public: + explicit KinectDepthViewer(const KinectDepthSettings& settings); + const KinectDepthSettings& settings() const { return settings_; } + // Other settings from now on (the preview's view and the launcher's + // tuning, KinectSensor::set_depth_settings). The ideal state is kept (a + // new tuning applies from the next step), the player view's latch starts + // again when its view or placement changes, and the fit stretch is the + // setting's when it has one, otherwise the one already known. + void set_settings(const KinectDepthSettings& settings); + // The fit view's stretch (from the sensor's lens, or the setting). + void set_fit_scale(float z_scale); + float fit_scale() const { return fit_scale_; } + const KinectDepthView& body_frame(const KinectFrame& frame, double seconds); + const KinectDepthView& depth_frame(double seconds); + const KinectDepthView& view() const { return view_; } + // The view the skeleton handed to the game goes through: the depth's, + // except for fit with its own skeleton stretch (the depth's origin, z + // scaled by skeleton_stretch) and ideal with its own skeleton distance + // (the same level camera moved along z so the reference is that far). + KinectDepthView skeleton_view() const; + // The newest body frame as the ideal view keeps it (bodies with their + // tracking id, sensor index, SpineBase position and its state in + // joint_states[0]; the floor), and its time. + const KinectFrame& bodies() const { return bodies_; } + double bodies_seconds() const { return bodies_seconds_; } + bool latched() const { return latch_.latched(); } + const KinectIdealState& ideal() const { return ideal_; } + // Where the ideal view's camera is now (the view itself in ideal). + KinectDepthView ideal_view() const { return kinect_ideal_view(ideal_, settings_); } +private: + KinectDepthSettings settings_; + float fit_scale_ = 1.0f; + KinectDepthViewLatch latch_; + KinectIdealState ideal_; + KinectFrame bodies_; + double bodies_seconds_ = 0; + KinectDepthView view_; +}; + +// How a synthesized image came out. +struct KinectDepthStats { + uint32_t valid = 0; // pixels with a depth + uint32_t players = 0; // of those, a player's + uint32_t filled = 0; // single-pixel holes filled +}; + +// The v1 image from a v2 frame. points are the v2 depth pixels in camera +// space (x, y, z metres, three floats a pixel; a z that is not positive and +// finite is no point), body_index each pixel's body (0..5, +// kinect_v2_no_body for none; null for none at all), count the pixels. +// Each point is moved into the view and projected; the nearest wins its +// pixel. A pixel left empty between two that agree (both horizontal or both +// vertical neighbours, within 50 mm and the same player) takes their mean; +// anything else empty, or outside the v1's range, is no value. out is +// 320x240, 2 bytes a pixel in host order (KinectImage); its number is left +// alone. +KinectDepthStats kinect_v2_synthesize_depth(const float* points, const uint8_t* body_index, size_t count, + const KinectDepthView& view, KinectImage& out); + +// The view a made image went through, from its own fields (view_origin, +// view_z_scale, view_rotated, view_rotation). +KinectDepthView kinect_image_view(const KinectImage& image); +// The other way from kinect_v2_synthesize_depth: each pixel of a v1 depth +// image (2 bytes a pixel, the millimetres shifted up three) back to the +// point of the sensor's own camera space it shows. The pixel's centre +// through the v1's nominal lens at its depth, (x - 160, -(y - 120)) z / f, +// then out of the view (kinect_depth_view_unpoint). Three floats a pixel, +// zeros for a pixel without a value. A point is exact in depth and within +// half a pixel across (the synthesis rounds to the nearest pixel), so a +// colour sampled for it falls on the same pixel as its depth. +void kinect_v1_depth_points(const KinectImage& depth, const KinectDepthView& view, std::vector& points); + +// Camera-space points from v2 depth millimetres through a typical v2's lens +// (focal length 366 at 512x424, centre (256, 212); kinect_v2_depth_estimate +// the other way), for when the sensor's mapper gives none. +void kinect_v2_depth_points_from_lens(const uint16_t* millimetres, uint32_t width, uint32_t height, + std::vector& points); +// Camera-space points from v2 depth millimetres and a ray per pixel (x / z, +// y / z, camera y up; ICoordinateMapper::GetDepthFrameToCameraSpaceTable's +// table): (rx z, ry z, z). For the offline re-synthesis of a raw recording. +void kinect_v2_depth_points_from_rays(const uint16_t* millimetres, const float* rays, size_t count, + std::vector& points); +// The rays of a lens without distortion (focal lengths and centre in depth +// pixels): ((x - cx) / fx, -(y - cy) / fy). +void kinect_v2_depth_rays_from_lens(float focal_x, float focal_y, float centre_x, float centre_y, uint32_t width, + uint32_t height, std::vector& rays); +// A v2's tracking id as the player slots key on it (32 bits: its low half, +// or the high half with bit 0 set when the low half is 0). +uint32_t kinect_v2_short_tracking_id(uint64_t id); + +// Recording the images made (SFR_KINECT_RECORD_DEPTH): kinect_record_depth.h, +// with the view each was made through (KinectImage::view_origin, view_z_scale). + +} diff --git a/src/kinect_v2_image.cpp b/src/kinect_v2_image.cpp new file mode 100644 index 0000000..52452e9 --- /dev/null +++ b/src/kinect_v2_image.cpp @@ -0,0 +1,244 @@ +#include "kinect_v2_image.h" + +#include +#include +#include + +namespace sfr { +namespace { +// The span of source pixels one target pixel covers along an axis: whole +// pixels from first up to (not including) last, at least one. A hair of +// slack, so that an exact third (1920 to 640) lands on exact pixels. +void span(double from, double to, uint32_t size, uint32_t& first, uint32_t& last) { + const double low = std::floor(from + 1e-3), high = std::ceil(to - 1e-3); + first = uint32_t(std::clamp(low, 0.0, double(size - 1))); + last = uint32_t(std::clamp(high, double(first + 1), double(size))); +} + +// The common view's fit for a frame of any size (the crop stays the same part +// of the image): source pixel to preview pixel. +KinectFit frame_fit(KinectImageKind kind, uint32_t width, uint32_t height, KinectV2View view) { + const bool colour = kind == KinectImageKind::colour; + const KinectV2Crop whole = kinect_v2_crop(kind, view); + const float fx = float(width) / float(colour ? kinect_v2_colour_width : kinect_v2_depth_width); + const float fy = float(height) / float(colour ? kinect_v2_colour_height : kinect_v2_depth_height); + const KinectV2Crop crop{whole.x * fx, whole.y * fy, whole.width * fx, whole.height * fy}; + const KinectImageSize size = kinect_v2_preview_size(kind, view); + return kinect_fit_crop(crop, size.width, size.height); +} + +// Colour fitted into the preview's image: each target pixel the average +// of the source pixels under it; sample(x, y) gives one source pixel as +// B, G, R, 0. +template +void fit_colour(uint32_t width, uint32_t height, KinectV2View view, KinectImage& out, Sample sample) { + const KinectImageSize size = kinect_v2_preview_size(KinectImageKind::colour, view); + const uint32_t tw = size.width, th = size.height; + out.width = tw; + out.height = th; + out.bytes_per_pixel = 4; + out.pixels.assign(size_t(tw) * th * 4, 0); + if (!width || !height) return; + const KinectFit fit = frame_fit(KinectImageKind::colour, width, height, view); + for (uint32_t ty = 0; ty < th; ++ty) { + const double sy = (ty + 0.5 - fit.offset_y) / fit.scale; + if (sy < 0 || sy >= height) continue; // a bar + uint32_t y0, y1; + span((ty - fit.offset_y) / fit.scale, (ty + 1 - fit.offset_y) / fit.scale, height, y0, y1); + for (uint32_t tx = 0; tx < tw; ++tx) { + const double sx = (tx + 0.5 - fit.offset_x) / fit.scale; + if (sx < 0 || sx >= width) continue; + uint32_t x0, x1; + span((tx - fit.offset_x) / fit.scale, (tx + 1 - fit.offset_x) / fit.scale, width, x0, x1); + uint32_t b = 0, g = 0, r = 0, count = 0; + for (uint32_t y = y0; y < y1; ++y) + for (uint32_t x = x0; x < x1; ++x) { + const uint32_t pixel = sample(x, y); + b += pixel & 0xFF; + g += pixel >> 8 & 0xFF; + r += pixel >> 16 & 0xFF; + ++count; + } + uint8_t* to = &out.pixels[(size_t(ty) * tw + tx) * 4]; + to[0] = uint8_t((b + count / 2) / count); + to[1] = uint8_t((g + count / 2) / count); + to[2] = uint8_t((r + count / 2) / count); + to[3] = 0; + } + } +} + +bool sane(const KinectImagePoint& point, uint32_t width, uint32_t height) { + // Only a guard against absurd coordinates (they would overflow the + // window's integer maths). A point outside the image is still placed, so + // that a bone from one inside still reaches the edge: the preview window + // clips its drawing to the picture, and draws a joint only when its + // centre is inside. + return point.x > -float(width) && point.x < 2.0f * float(width) && point.y > -float(height) && + point.y < 2.0f * float(height); +} +} + +KinectFit kinect_fit(uint32_t source_width, uint32_t source_height, uint32_t target_width, uint32_t target_height) { + KinectFit fit; + if (!source_width || !source_height) return fit; + fit.scale = std::min(float(target_width) / float(source_width), float(target_height) / float(source_height)); + fit.offset_x = (float(target_width) - float(source_width) * fit.scale) / 2.0f; + fit.offset_y = (float(target_height) - float(source_height) * fit.scale) / 2.0f; + return fit; +} + +KinectImagePoint kinect_fit_point(const KinectFit& fit, float x, float y, uint32_t target_width, + uint32_t target_height) { + if (!std::isfinite(x) || !std::isfinite(y) || !(fit.scale > 0)) return {}; + KinectImagePoint point{fit.offset_x + x * fit.scale, fit.offset_y + y * fit.scale, true}; + point.visible = sane(point, target_width, target_height); + return point; +} + +KinectV2Crop kinect_v2_crop(KinectImageKind kind, KinectV2View view) { + const bool colour = kind == KinectImageKind::colour; + if (view == KinectV2View::original) + return {0, 0, float(colour ? kinect_v2_colour_width : kinect_v2_depth_width), + float(colour ? kinect_v2_colour_height : kinect_v2_depth_height)}; + const float colour_half_x = float(kinect_v2_colour_width) / 2.0f / kinect_v2_colour_focal; + const float colour_half_y = float(kinect_v2_colour_height) / 2.0f / kinect_v2_colour_focal; + const float depth_half_x = float(kinect_v2_depth_width) / 2.0f / kinect_v2_depth_focal; + const float depth_half_y = float(kinect_v2_depth_height) / 2.0f / kinect_v2_depth_focal; + // The intersection of the two fields of view, whatever its shape. + const float half_x = std::min(colour_half_x, depth_half_x); + const float half_y = std::min(colour_half_y, depth_half_y); + const float focal = colour ? kinect_v2_colour_focal : kinect_v2_depth_focal; + const float centre_x = colour ? kinect_v2_colour_centre_x : kinect_v2_depth_centre_x; + const float centre_y = colour ? kinect_v2_colour_centre_y : kinect_v2_depth_centre_y; + return {centre_x - half_x * focal, centre_y - half_y * focal, 2.0f * half_x * focal, 2.0f * half_y * focal}; +} + +KinectFit kinect_fit_crop(const KinectV2Crop& crop, uint32_t target_width, uint32_t target_height) { + KinectFit fit; + if (!(crop.width > 0) || !(crop.height > 0)) return fit; + fit.scale = std::min(float(target_width) / crop.width, float(target_height) / crop.height); + fit.offset_x = (float(target_width) - crop.width * fit.scale) / 2.0f - crop.x * fit.scale; + fit.offset_y = (float(target_height) - crop.height * fit.scale) / 2.0f - crop.y * fit.scale; + return fit; +} + +KinectImageSize kinect_v2_preview_size(KinectImageKind kind, KinectV2View view) { + const bool colour = kind == KinectImageKind::colour; + const KinectV2Crop crop = kinect_v2_crop(kind, view); + const uint32_t cap = colour ? kinect_v2_colour_preview_long_side : kinect_v2_depth_preview_long_side; + // The long side is the cap, the short one follows the crop's shape. + if (crop.width >= crop.height) + return {cap, std::max(1u, uint32_t(std::lround(double(cap) * crop.height / crop.width)))}; + return {std::max(1u, uint32_t(std::lround(double(cap) * crop.width / crop.height))), cap}; +} + +KinectImageRect kinect_v2_picture_rect(KinectImageKind kind, KinectV2View view) { + // The image is the crop (no bars), so the picture is all of it. + const KinectImageSize size = kinect_v2_preview_size(kind, view); + return {0, 0, float(size.width), float(size.height)}; +} + +uint32_t kinect_yuy2_bgrx(uint8_t y, uint8_t u, uint8_t v) { + const int c = int(y) - 16, d = int(u) - 128, e = int(v) - 128; + const auto clamp = [](int value) { return uint32_t(std::clamp(value, 0, 255)); }; + const uint32_t r = clamp((298 * c + 409 * e + 128) >> 8); + const uint32_t g = clamp((298 * c - 100 * d - 208 * e + 128) >> 8); + const uint32_t b = clamp((298 * c + 516 * d + 128) >> 8); + return b | g << 8 | r << 16; +} + +void kinect_v2_colour_from_yuy2(const uint8_t* yuy2, uint32_t width, uint32_t height, KinectV2View view, + KinectImage& out) { + const size_t stride = size_t(width) * 2; + fit_colour(width, height, view, out, [yuy2, stride](uint32_t x, uint32_t y) { + const uint8_t* pair = yuy2 + size_t(y) * stride + size_t(x & ~1u) * 2; + return kinect_yuy2_bgrx(pair[(x & 1) ? 2 : 0], pair[1], pair[3]); + }); +} + +void kinect_v2_colour_from_bgra(const uint8_t* bgra, uint32_t width, uint32_t height, KinectV2View view, + KinectImage& out) { + const size_t stride = size_t(width) * 4; + fit_colour(width, height, view, out, [bgra, stride](uint32_t x, uint32_t y) { + const uint8_t* pixel = bgra + size_t(y) * stride + size_t(x) * 4; + return uint32_t(pixel[0]) | uint32_t(pixel[1]) << 8 | uint32_t(pixel[2]) << 16; + }); +} + +void kinect_v2_depth_image(const uint16_t* millimetres, const uint8_t* body_index, uint32_t width, + uint32_t height, KinectV2View view, KinectImage& out) { + const KinectImageSize size = kinect_v2_preview_size(KinectImageKind::depth_and_player, view); + const uint32_t tw = size.width, th = size.height; + out.width = tw; + out.height = th; + out.bytes_per_pixel = 2; + out.pixels.assign(size_t(tw) * th * 2, 0); + if (!width || !height) return; + const KinectFit fit = frame_fit(KinectImageKind::depth_and_player, width, height, view); + for (uint32_t ty = 0; ty < th; ++ty) { + const double sy = std::floor((ty + 0.5 - fit.offset_y) / fit.scale); + if (sy < 0 || sy >= height) continue; + for (uint32_t tx = 0; tx < tw; ++tx) { + const double sx = std::floor((tx + 0.5 - fit.offset_x) / fit.scale); + if (sx < 0 || sx >= width) continue; + const size_t at = size_t(sy) * width + size_t(sx); + const uint32_t depth = std::min(millimetres[at], 0x1FFF); + const uint32_t body = body_index ? body_index[at] : kinect_v2_no_body; + const uint16_t value = uint16_t(depth << 3 | (body < 6 ? body + 1 : 0)); + uint8_t* to = &out.pixels[(size_t(ty) * tw + tx) * 2]; + to[0] = uint8_t(value); + to[1] = uint8_t(value >> 8); + } + } +} + +KinectImagePoint kinect_v2_depth_to_preview(float x, float y, KinectV2View view) { + return kinect_fit_point(frame_fit(KinectImageKind::depth_and_player, kinect_v2_depth_width, kinect_v2_depth_height, view), + x, y, kinect_v2_preview_size(KinectImageKind::depth_and_player, view).width, + kinect_v2_preview_size(KinectImageKind::depth_and_player, view).height); +} + +KinectImagePoint kinect_v2_colour_to_preview(float x, float y, KinectV2View view) { + return kinect_fit_point(frame_fit(KinectImageKind::colour, kinect_v2_colour_width, kinect_v2_colour_height, view), x, y, + kinect_v2_preview_size(KinectImageKind::colour, view).width, + kinect_v2_preview_size(KinectImageKind::colour, view).height); +} + +// Typical intrinsics of a v2 (kinect_v2_depth_focal, kinect_v2_colour_focal). +KinectImagePoint kinect_v2_depth_estimate(const std::array& p, KinectV2View view) { + if (!(p[2] > 0.1f)) return {}; + return kinect_v2_depth_to_preview(kinect_v2_depth_centre_x + p[0] * kinect_v2_depth_focal / p[2], + kinect_v2_depth_centre_y - p[1] * kinect_v2_depth_focal / p[2], view); +} + +KinectImagePoint kinect_v2_colour_estimate(const std::array& p, KinectV2View view) { + if (!(p[2] > 0.1f)) return {}; + return kinect_v2_colour_to_preview(kinect_v2_colour_centre_x + p[0] * kinect_v2_colour_focal / p[2], + kinect_v2_colour_centre_y - p[1] * kinect_v2_colour_focal / p[2], view); +} + +KinectImagePoint kinect_v2_place(KinectImageKind kind, const std::array& p, const float* mapped, + KinectV2View view) { + const bool colour = kind == KinectImageKind::colour; + if (mapped && std::isfinite(mapped[0]) && std::isfinite(mapped[1])) + return colour ? kinect_v2_colour_to_preview(mapped[0], mapped[1], view) + : kinect_v2_depth_to_preview(mapped[0], mapped[1], view); + return colour ? kinect_v2_colour_estimate(p, view) : kinect_v2_depth_estimate(p, view); +} + +void kinect_colour_from_positions(const float* positions, size_t count, const uint32_t* colour, uint32_t width, + uint32_t height, std::vector& out) { + out.assign(count, 0); + if (!colour || !width || !height) return; + for (size_t i = 0; i < count; ++i) { + const float x = positions[2 * i], y = positions[2 * i + 1]; + if (!std::isfinite(x) || !std::isfinite(y)) continue; + // Pixel n covers n .. n + 1. + const float column = std::floor(x), row = std::floor(y); + if (column < 0 || row < 0 || column >= float(width) || row >= float(height)) continue; + out[i] = colour[size_t(row) * width + size_t(column)]; + } +} + +} \ No newline at end of file diff --git a/src/kinect_v2_image.h b/src/kinect_v2_image.h new file mode 100644 index 0000000..4968749 --- /dev/null +++ b/src/kinect_v2_image.h @@ -0,0 +1,148 @@ +#pragma once +#include "kinect_sensor.h" + +#include +#include +#include + +namespace sfr { + +// A Kinect v2's cameras as the preview window shows them. The preview was +// made for a Kinect v1's images (KinectImage): colour 640x480 and depth and +// player index 320x240, both 4:3. A v2's colour camera is 1920x1080 (16:9, +// about 84x54 degrees) and its depth camera 512x424 (about 71x60 degrees), so +// the two would show different parts of the room. The preview therefore shows +// only the view the cameras have in common (kinect_v2_crop), in images of +// that shape (kinect_v2_preview_size) that its panels follow, with no bars. This is the preview's own picture: +// the depth the game gets (kinect_v2_depth.h) is made from the whole image. +// Skeleton points are placed by the sensor's own coordinate mapper +// (ICoordinateMapper) and moved the same way (kinect_v2_place), with the +// typical lens (kinect_v2_*_estimate) when the mapper is not there or cannot +// place a point, so the preview places them as it does a v1's. +// +// None of this is Windows', so it is tested everywhere (kinect_v2_image_test). + +constexpr uint32_t kinect_v2_colour_width = 1920, kinect_v2_colour_height = 1080; +constexpr uint32_t kinect_v2_depth_width = 512, kinect_v2_depth_height = 424; +// The v1 sizes the preview draws (a v2's follow its pictures' shape: +// kinect_v2_preview_size). +constexpr uint32_t kinect_preview_colour_width = 640, kinect_preview_colour_height = 480; +constexpr uint32_t kinect_preview_depth_width = 320, kinect_preview_depth_height = 240; +// A v2 body index pixel that is no body (BodyIndexFrame). +constexpr uint8_t kinect_v2_no_body = 255; + +// Where a source image of one size lands in a target of another: scaled +// uniformly to fit, centred. +struct KinectImageSize { + uint32_t width = 0, height = 0; +}; +struct KinectFit { + float scale = 0; // target pixels per source pixel + float offset_x = 0, offset_y = 0; // where the source's (0, 0) lands +}; +KinectFit kinect_fit(uint32_t source_width, uint32_t source_height, uint32_t target_width, uint32_t target_height); + +// What the preview shows of each camera: the view both cameras have +// (common, the default) or each camera's whole picture (original: in its own +// shape, colour 640x360 and depth 512x424, no bars). A runtime choice +// of the preview window, passed to everything below; never the game's. +enum class KinectV2View { common, original }; + +// The part of a sensor image the preview shows, in that image's own pixels +// (colour 1920x1080, depth 512x424). +struct KinectV2Crop { + float x = 0, y = 0, width = 0, height = 0; +}; + +// The typical lenses (pixels, and the optical centre) the crop is worked out +// from. The depth ones are a v2's usual (the sensor's own differ a few +// pixels). The colour ones are NOMINAL: there is no intrinsics API for the +// colour camera, and the cameras sit a few centimetres apart (about a degree +// at two metres), so the alignment of the two pictures is approximate. +constexpr float kinect_v2_depth_focal = 366.0f, kinect_v2_depth_centre_x = 256.0f, kinect_v2_depth_centre_y = 212.0f; +constexpr float kinect_v2_colour_focal = 1060.0f, kinect_v2_colour_centre_x = 960.0f, kinect_v2_colour_centre_y = 540.0f; + +// The view both v2 cameras have (the intersection of their fields of view, +// about 1.373:1), in tangent space (x / z, y / z) about each camera's optical +// axis: half-height the lower of the two cameras' (colour 0.509, depth +// 0.579), half-width likewise (depth 0.699, colour 0.906). Colour is x +// 219..1701, y 0..1080; depth x 0..512, y 25.7..398.3; in the original view +// the whole image. The one place the crop is worked out: the pixel +// conversion, the point mapping and the picture rectangle all come from it. +KinectV2Crop kinect_v2_crop(KinectImageKind kind, KinectV2View view); + +// The long side of a v2's preview image: colour 640, depth 512 (the depth +// camera's own width). The short side follows the crop's shape, so the image +// IS the crop with no bars: common colour 640x466, depth 512x373; original +// colour 640x360, depth 512x424. +constexpr uint32_t kinect_v2_colour_preview_long_side = 640, kinect_v2_depth_preview_long_side = 512; +KinectImageSize kinect_v2_preview_size(KinectImageKind kind, KinectV2View view); + +// Where a crop of a source image lands in a target: scaled uniformly to fit, +// centred. The fit is for the source's own pixels (its (0, 0) lands where +// the source's corner would), so kinect_fit_point places a source position. +KinectFit kinect_fit_crop(const KinectV2Crop& crop, uint32_t target_width, uint32_t target_height); + +// Where a v2's picture lies in the preview's image of one kind: all of it +// (the image is the crop, at kinect_v2_preview_size). +KinectImageRect kinect_v2_picture_rect(KinectImageKind kind, KinectV2View view); + +// A source pixel position (the sensor's own image space) in the target. +KinectImagePoint kinect_fit_point(const KinectFit& fit, float x, float y, uint32_t target_width, + uint32_t target_height); + +// One YUY2 (Y0 U Y1 V) pixel pair's colour for one of its pixels, as B, G, +// R, 0 (ITU-R BT.601, video range: the format a v2's colour camera sends). +uint32_t kinect_yuy2_bgrx(uint8_t y, uint8_t u, uint8_t v); + +// A colour image, its common view (kinect_v2_crop, scaled to the frame's +// size) fitted into kinect_v2_preview_size (KinectImage colour: +// B, G, R, unused) by averaging the source pixels each target pixel covers. +// The source is either YUY2 (2 bytes a pixel, pairs sharing U and V) or +// BGRA (4 bytes a pixel). The image's number is left alone. +void kinect_v2_colour_from_yuy2(const uint8_t* yuy2, uint32_t width, uint32_t height, KinectV2View view, + KinectImage& out); +void kinect_v2_colour_from_bgra(const uint8_t* bgra, uint32_t width, uint32_t height, KinectV2View view, + KinectImage& out); + +// A depth image (millimetres, 0 for nothing measured) with each pixel's body +// (0..5, kinect_v2_no_body for none; null when unknown), fitted into the +// preview size as a v1's depth and player index: the millimetres +// shifted up three (clamped to the 13 bits there are), the body plus one in +// the low three bits. Nearest pixel, so that no depth is averaged across an +// edge. The image's number is left alone. +void kinect_v2_depth_image(const uint16_t* millimetres, const uint8_t* body_index, uint32_t width, + uint32_t height, KinectV2View view, KinectImage& out); + +// Image positions in the sensor's own images (512x424 depth, 1920x1080 +// colour) moved into the preview's images, as the coordinate mapper gives +// them. A point the mapper could not place (infinite) is not visible. +KinectImagePoint kinect_v2_depth_to_preview(float x, float y, KinectV2View view); +KinectImagePoint kinect_v2_colour_to_preview(float x, float y, KinectV2View view); + +// A skeleton point (camera space, metres) in the preview's image of one +// kind. mapped is the coordinate mapper's answer for it in the sensor's own +// image (x, y; depth pixels for depth_and_player, colour pixels for colour), +// or null without one. The mapper answers -infinity for a point it cannot +// place; then, and without an answer, the typical lens places it instead +// (kinect_v2_*_estimate). +KinectImagePoint kinect_v2_place(KinectImageKind kind, const std::array& p, const float* mapped, + KinectV2View view); + +// A colour picture made pixel by pixel from another one: positions holds, +// for each of count output pixels, where it lands in the colour image +// (x, y pairs in its pixels, as KinectSensor::colour_image_positions gives +// them; NaN for none), which is width x height pixels of B, G, R, unused. +// The nearest pixel's colour, or black for none or outside the image. +void kinect_colour_from_positions(const float* positions, size_t count, const uint32_t* colour, uint32_t width, + uint32_t height, std::vector& out); + +// A skeleton point (camera space, metres) through a v2 +// camera's typical lens, in the preview's images. Close (the cameras' own +// calibration differs a few pixels from sensor to sensor), and the colour +// one ignores that the colour camera sits about 5 cm beside the depth +// camera, so it is only a fallback. +KinectImagePoint kinect_v2_depth_estimate(const std::array& p, KinectV2View view); +KinectImagePoint kinect_v2_colour_estimate(const std::array& p, KinectV2View view); + +} diff --git a/src/kinect_v2_resynth.cpp b/src/kinect_v2_resynth.cpp new file mode 100644 index 0000000..8b9477a --- /dev/null +++ b/src/kinect_v2_resynth.cpp @@ -0,0 +1,97 @@ +#include "kinect_v2_resynth.h" +#include "kinect_v2_image.h" + +#include +#include + +namespace sfr { + +KinectFrame kinect_raw_body_frame(const KinectRawFrame& frame) { + KinectFrame out; + out.floor_plane = frame.floor; + // A v2's gravity is its floor's normal. + out.gravity = {frame.floor[0], frame.floor[1], frame.floor[2]}; + for (const KinectRawBody& raw : frame.bodies) { + if (!raw.tracked) continue; + KinectBody& body = out.bodies.emplace_back(); + body.tracking_id = uint32_t(raw.tracking_id); + body.sensor_index = raw.body_index; + body.position = raw.spine_base; + body.joint_states[0] = raw.spine_state; + } + return out; +} + +void kinect_raw_set_bodies(const KinectFrame& bodies, KinectRawFrame& frame) { + frame.floor = bodies.floor_plane; + frame.bodies.clear(); + for (const KinectBody& body : bodies.bodies) { + if (frame.bodies.size() >= kinect_raw_max_bodies) break; + KinectRawBody& raw = frame.bodies.emplace_back(); + raw.tracking_id = body.tracking_id; + raw.body_index = uint8_t(body.sensor_index); + raw.tracked = 1; + raw.spine_state = uint8_t(body.joint_states[0]); + raw.spine_base = body.position; + } +} + +float kinect_raw_fit_scale(const KinectRawHeader& header) { + const float focal = header.lens[1]; + return kinect_v2_fit_scale(focal > 100.0f && focal < 1000.0f ? focal : kinect_v2_typical_depth_focal); +} + +bool kinect_raw_rays_orientation(const KinectRawHeader& header, bool& flip_x, bool& flip_y) { + flip_x = flip_y = false; + const size_t pixels = size_t(header.width) * header.height; + if (header.rays.size() != 2 * pixels || !header.width || !header.height) return false; + const size_t left = size_t(header.height / 2) * header.width, top = header.width / 2; + const float x = header.rays[2 * left], y = header.rays[2 * top + 1]; + if (!std::isfinite(x) || !std::isfinite(y) || x == 0.0f || y == 0.0f) return false; + flip_x = x > 0.0f; + flip_y = y < 0.0f; + return true; +} + +KinectV2Resynth::KinectV2Resynth(const KinectDepthSettings& settings, const KinectRawHeader& header) + : viewer_(settings), rays_(header.rays) { + if (!(settings.fit_scale > 0)) viewer_.set_fit_scale(kinect_raw_fit_scale(header)); + bool flip_x = false, flip_y = false; + rays_usable_ = kinect_raw_rays_orientation(header, flip_x, flip_y); + if (!rays_usable_) { + warning_ = "the recording's rays are unusable: every frame through a typical v2's lens"; + } else if (flip_x || flip_y) { + // Camera space has +x to the image's right and +y up; a table the + // other way round is turned so the points are the mapper's. + warning_ = std::string("the recording's rays point ") + (flip_x ? "x" : "") + (flip_x && flip_y ? " and " : "") + + (flip_y ? "y" : "") + " the other way: turned to camera space"; + for (size_t i = 0; i < rays_.size(); i += 2) { + if (flip_x) rays_[i] = -rays_[i]; + if (flip_y) rays_[i + 1] = -rays_[i + 1]; + } + } +} + +KinectDepthStats KinectV2Resynth::make(const KinectRawFrame& frame, KinectImage& out) { + // A body frame the live path had not had before this image. + if (frame.body_time != 0 && frame.body_time != last_body_time_) { + viewer_.body_frame(kinect_raw_body_frame(frame), double(frame.body_time) * 1e-7); + last_body_time_ = frame.body_time; + } + const KinectDepthView view = viewer_.depth_frame(double(frame.depth_time) * 1e-7); + const size_t pixels = frame.depth.size(); + if ((frame.flags & kinect_raw_flag_mapped) && rays_usable_ && rays_.size() == 2 * pixels) + kinect_v2_depth_points_from_rays(frame.depth.data(), rays_.data(), pixels, points_); + else + kinect_v2_depth_points_from_lens(frame.depth.data(), kinect_v2_depth_width, kinect_v2_depth_height, points_); + const KinectDepthStats stats = kinect_v2_synthesize_depth( + points_.data(), frame.body_index.size() == pixels ? frame.body_index.data() : nullptr, pixels, view, out); + out.number = frame.number; + out.steady_us = frame.steady_us; + out.sdk_time = frame.depth_time; + out.view_origin = view.origin; + out.view_z_scale = view.z_scale; + return stats; +} + +} diff --git a/src/kinect_v2_resynth.h b/src/kinect_v2_resynth.h new file mode 100644 index 0000000..f8a3e18 --- /dev/null +++ b/src/kinect_v2_resynth.h @@ -0,0 +1,52 @@ +#pragma once +#include "kinect_record_raw.h" +#include "kinect_v2_depth.h" + +#include +#include + +namespace sfr { + +// Making the game's depth again from a raw Kinect v2 recording +// (kinect_record_raw.h; sfr_kinect_depth_resynth, docs/kinect-sensor.md), +// through the same view state (KinectDepthViewer) and the same synthesis +// (kinect_v2_synthesize_depth) as the live sensor: frame by frame, the body +// frame it carries goes to body_frame when it is a new one, then the depth +// frame steps the view and is made. Points come from the recording's rays +// for frames the live path had the sensor's mapper for, from the typical +// lens (as live) for the others. +// +// None of this is Windows', so it is tested everywhere (kinect_v2_depth_test). + +// A raw frame's bodies and floor as the view keeps a body frame +// (KinectDepthViewer::bodies), and the other way for the recorder. +KinectFrame kinect_raw_body_frame(const KinectRawFrame& frame); +void kinect_raw_set_bodies(const KinectFrame& bodies, KinectRawFrame& frame); + +// The fit view's stretch a raw recording's lens gives (the sensor's focal +// length when the header has it, the typical v2's otherwise). +float kinect_raw_fit_scale(const KinectRawHeader& header); + +// Whether a header's rays look like camera space (+x to the image's right, +// +y up): the middle of the left column points left, of the top row up. +// flip_x / flip_y say which axis points the other way; false when the rays +// are unusable (zero or not finite). +bool kinect_raw_rays_orientation(const KinectRawHeader& header, bool& flip_x, bool& flip_y); + +class KinectV2Resynth { +public: + // settings.fit_scale 0 takes kinect_raw_fit_scale(header). + KinectV2Resynth(const KinectDepthSettings& settings, const KinectRawHeader& header); + KinectDepthStats make(const KinectRawFrame& frame, KinectImage& out); + const KinectDepthViewer& viewer() const { return viewer_; } + // A note about the rays (flipped, unusable), empty when all is well. + const std::string& warning() const { return warning_; } +private: + KinectDepthViewer viewer_; + std::vector rays_, points_; + bool rays_usable_ = false; + int64_t last_body_time_ = 0; + std::string warning_; +}; + +} diff --git a/src/kinect_v2_win32.cpp b/src/kinect_v2_win32.cpp index fac02d4..062cb49 100644 --- a/src/kinect_v2_win32.cpp +++ b/src/kinect_v2_win32.cpp @@ -1,5 +1,10 @@ #include "kinect_sensor.h" #include "kinect_smoothing.h" +#include "kinect_v2_image.h" +#include "kinect_v2_depth.h" +#include "kinect_v2_resynth.h" +#include "kinect_record.h" +#include "kinect_record_raw.h" #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -8,15 +13,26 @@ #define NOMINMAX #endif #include +#include #include +// CoTaskMemFree, for the coordinate mapper's table (a raw recording's rays). +#pragma comment(lib, "ole32.lib") +#include +#include #include +#include #include #include #include #include +#include +#include #include +#include +#include #include +#include // Kinect v2 through the Kinect for Windows SDK 2.0's runtime (Kinect20.dll), // loaded when asked for, like v1's. Its API is COM: the interfaces below are @@ -42,18 +58,194 @@ struct IBody; struct IBodyFrame; struct IBodyFrameReader; struct IBodyFrameSource; +struct IColorFrameSource; +struct IDepthFrameSource; +struct IBodyIndexFrameSource; +struct ICoordinateMapper; +// The cameras, for the preview window only (image()): Kinect.h's +// ColorImageFormat. +enum ColorImageFormat : int32_t { + ColorImageFormat_None = 0, ColorImageFormat_Rgba = 1, ColorImageFormat_Yuv = 2, + ColorImageFormat_Bgra = 3, ColorImageFormat_Bayer = 4, ColorImageFormat_Yuy2 = 5, +}; + +// Kinect.h's IKinectSensor, in its order: the three IsAvailableChanged event +// methods, Open, Close, get_IsOpen, get_IsAvailable, then the sources +// (colour, depth, body, body index, infrared, long-exposure infrared, +// audio), OpenMultiSourceFrameReader and get_CoordinateMapper (then +// get_UniqueKinectId and get_KinectCapabilities, unused). The slot numbers +// (IUnknown's QueryInterface, AddRef and Release are 0..2) were checked +// against IKinectSensorVtbl in the SDK 2.0 (v2.0_1409) Kinect.h. struct IKinectSensor : IUnknown { - virtual HRESULT STDMETHODCALLTYPE SubscribeIsAvailableChanged(WaitableHandle*) = 0; - virtual HRESULT STDMETHODCALLTYPE UnsubscribeIsAvailableChanged(WaitableHandle) = 0; - virtual HRESULT STDMETHODCALLTYPE GetIsAvailableChangedEventData(WaitableHandle, IUnknown**) = 0; - virtual HRESULT STDMETHODCALLTYPE Open() = 0; - virtual HRESULT STDMETHODCALLTYPE Close() = 0; - virtual HRESULT STDMETHODCALLTYPE get_IsOpen(BOOLEAN*) = 0; - virtual HRESULT STDMETHODCALLTYPE get_IsAvailable(BOOLEAN*) = 0; - virtual HRESULT STDMETHODCALLTYPE get_ColorFrameSource(IUnknown**) = 0; - virtual HRESULT STDMETHODCALLTYPE get_DepthFrameSource(IUnknown**) = 0; - virtual HRESULT STDMETHODCALLTYPE get_BodyFrameSource(IBodyFrameSource**) = 0; + virtual HRESULT STDMETHODCALLTYPE SubscribeIsAvailableChanged(WaitableHandle*) = 0; // 3 + virtual HRESULT STDMETHODCALLTYPE UnsubscribeIsAvailableChanged(WaitableHandle) = 0; // 4 + virtual HRESULT STDMETHODCALLTYPE GetIsAvailableChangedEventData(WaitableHandle, IUnknown**) = 0; // 5 + virtual HRESULT STDMETHODCALLTYPE Open() = 0; // 6 + virtual HRESULT STDMETHODCALLTYPE Close() = 0; // 7 + virtual HRESULT STDMETHODCALLTYPE get_IsOpen(BOOLEAN*) = 0; // 8 + virtual HRESULT STDMETHODCALLTYPE get_IsAvailable(BOOLEAN*) = 0; // 9 + virtual HRESULT STDMETHODCALLTYPE get_ColorFrameSource(IColorFrameSource**) = 0; // 10 + virtual HRESULT STDMETHODCALLTYPE get_DepthFrameSource(IDepthFrameSource**) = 0; // 11 + virtual HRESULT STDMETHODCALLTYPE get_BodyFrameSource(IBodyFrameSource**) = 0; // 12 + virtual HRESULT STDMETHODCALLTYPE get_BodyIndexFrameSource(IBodyIndexFrameSource**) = 0; // 13 + virtual HRESULT STDMETHODCALLTYPE get_InfraredFrameSource(IUnknown**) = 0; // 14 + virtual HRESULT STDMETHODCALLTYPE get_LongExposureInfraredFrameSource(IUnknown**) = 0; // 15 + virtual HRESULT STDMETHODCALLTYPE get_AudioSource(IUnknown**) = 0; // 16 + virtual HRESULT STDMETHODCALLTYPE OpenMultiSourceFrameReader(DWORD, IUnknown**) = 0; // 17 + virtual HRESULT STDMETHODCALLTYPE get_CoordinateMapper(ICoordinateMapper**) = 0; // 18 +}; + +// Kinect.h's DepthSpacePoint and ColorSpacePoint: a pixel position in the +// depth (512x424) or colour (1920x1080) image, -infinity for a point the +// mapper cannot place. +struct DepthSpacePoint { float x, y; }; +// Kinect.h's CameraIntrinsics. +struct CameraIntrinsics { + float focal_x, focal_y, centre_x, centre_y, radial2, radial4, radial6; +}; +static_assert(sizeof(CameraIntrinsics) == 28, "CameraIntrinsics layout"); +struct ColorSpacePoint { float x, y; }; +static_assert(sizeof(CameraSpacePoint) == 12 && sizeof(DepthSpacePoint) == 8 && sizeof(ColorSpacePoint) == 8, + "space point layouts"); + +// Kinect.h's ICoordinateMapper (IID 8784DF2D-16B0-481C-A11E-55E70BF25018), +// in ICoordinateMapperVtbl's order up to the last method called, +// GetDepthCameraIntrinsics (19). Slots 12..19 were checked against +// ICoordinateMapperVtbl in the SDK 2.0 (v2.0_1409) Kinect.h like the rest: +// MapDepthPointsToCameraSpace, MapDepthPointsToColorSpace, +// MapDepthFrameToCameraSpace, MapDepthFrameToColorSpace, +// MapColorFrameToDepthSpace, MapColorFrameToCameraSpace, +// GetDepthFrameToCameraSpaceTable, GetDepthCameraIntrinsics. +// Only the array forms are called, with one point: the single-point forms +// take CameraSpacePoint by value, and an array of one is the same answer +// without depending on how a 12-byte struct is passed. The preview's +// skeleton overlay uses it (image_point); without it, or for a point it +// cannot place, the typical lens of a v2 (kinect_v2_*_estimate). +struct ICoordinateMapper : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeCoordinateMappingChanged(WaitableHandle*) = 0; // 3 + virtual HRESULT STDMETHODCALLTYPE UnsubscribeCoordinateMappingChanged(WaitableHandle) = 0; // 4 + virtual HRESULT STDMETHODCALLTYPE GetCoordinateMappingChangedEventData(WaitableHandle, IUnknown**) = 0; // 5 + virtual HRESULT STDMETHODCALLTYPE MapCameraPointToDepthSpace(CameraSpacePoint, DepthSpacePoint*) = 0; // 6 + virtual HRESULT STDMETHODCALLTYPE MapCameraPointToColorSpace(CameraSpacePoint, ColorSpacePoint*) = 0; // 7 + virtual HRESULT STDMETHODCALLTYPE MapDepthPointToCameraSpace(DepthSpacePoint, UINT16, CameraSpacePoint*) = 0; // 8 + virtual HRESULT STDMETHODCALLTYPE MapDepthPointToColorSpace(DepthSpacePoint, UINT16, ColorSpacePoint*) = 0; // 9 + virtual HRESULT STDMETHODCALLTYPE MapCameraPointsToDepthSpace(UINT, const CameraSpacePoint*, UINT, + DepthSpacePoint*) = 0; // 10 + virtual HRESULT STDMETHODCALLTYPE MapCameraPointsToColorSpace(UINT, const CameraSpacePoint*, UINT, + ColorSpacePoint*) = 0; // 11 + virtual HRESULT STDMETHODCALLTYPE MapDepthPointsToCameraSpace(UINT, const DepthSpacePoint*, UINT, const UINT16*, + UINT, CameraSpacePoint*) = 0; // 12 + virtual HRESULT STDMETHODCALLTYPE MapDepthPointsToColorSpace(UINT, const DepthSpacePoint*, UINT, const UINT16*, + UINT, ColorSpacePoint*) = 0; // 13 + // Every pixel of a depth frame (512x424 millimetres) into camera space; + // -infinity for a pixel without a depth. the depth made for the game's + // reprojection (kinect_v2_depth.h). + virtual HRESULT STDMETHODCALLTYPE MapDepthFrameToCameraSpace(UINT, const UINT16*, UINT, CameraSpacePoint*) = 0; // 14 + virtual HRESULT STDMETHODCALLTYPE MapDepthFrameToColorSpace(UINT, const UINT16*, UINT, ColorSpacePoint*) = 0; // 15 + virtual HRESULT STDMETHODCALLTYPE MapColorFrameToDepthSpace(UINT, const UINT16*, UINT, DepthSpacePoint*) = 0; // 16 + virtual HRESULT STDMETHODCALLTYPE MapColorFrameToCameraSpace(UINT, const UINT16*, UINT, CameraSpacePoint*) = 0; // 17 + // A ray (PointF: x / z, y / z) per depth pixel, the caller's to free + // with CoTaskMemFree; for a raw recording's header (kinect_record_raw.h). + virtual HRESULT STDMETHODCALLTYPE GetDepthFrameToCameraSpaceTable(UINT32*, void**) = 0; // 18 + // The depth camera's lens (focal lengths and centre in 512x424 pixels, + // radial distortion), for the fit view's stretch; zero until the + // sensor has sent its calibration. + virtual HRESULT STDMETHODCALLTYPE GetDepthCameraIntrinsics(CameraIntrinsics*) = 0; // 19 +}; + +struct IColorFrame; +struct IColorFrameReader; +struct IDepthFrame; +struct IDepthFrameReader; +struct IBodyIndexFrame; +struct IBodyIndexFrameReader; + +// IColorFrameSource: FrameCaptured events, get_IsActive, OpenReader (then +// CreateFrameDescription, get_FrameDescription, get_KinectSensor, unused). +struct IColorFrameSource : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameCaptured(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameCaptured(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameCapturedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE get_IsActive(BOOLEAN*) = 0; + virtual HRESULT STDMETHODCALLTYPE OpenReader(IColorFrameReader**) = 0; +}; + +// Every frame reader, as the body's: FrameArrived events, then +// AcquireLatestFrame (then get_IsPaused, put_IsPaused and the source, unused). +struct IColorFrameReader : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameArrived(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameArrived(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameArrivedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE AcquireLatestFrame(IColorFrame**) = 0; +}; + +// IColorFrame: get_RawColorImageFormat, get_FrameDescription, +// CopyRawFrameDataToArray, AccessRawUnderlyingBuffer, +// CopyConvertedFrameDataToArray (then CreateFrameDescription, +// get_ColorCameraSettings, get_RelativeTime, get_ColorFrameSource, unused). +struct IColorFrame : IUnknown { + virtual HRESULT STDMETHODCALLTYPE get_RawColorImageFormat(ColorImageFormat*) = 0; + virtual HRESULT STDMETHODCALLTYPE get_FrameDescription(IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE CopyRawFrameDataToArray(UINT capacity, BYTE* data) = 0; + virtual HRESULT STDMETHODCALLTYPE AccessRawUnderlyingBuffer(UINT* capacity, BYTE** buffer) = 0; + virtual HRESULT STDMETHODCALLTYPE CopyConvertedFrameDataToArray(UINT capacity, BYTE* data, ColorImageFormat) = 0; +}; + +// IDepthFrameSource: FrameCaptured events, get_IsActive, OpenReader (then +// the reliable distances, get_FrameDescription, get_KinectSensor, unused). +struct IDepthFrameSource : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameCaptured(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameCaptured(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameCapturedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE get_IsActive(BOOLEAN*) = 0; + virtual HRESULT STDMETHODCALLTYPE OpenReader(IDepthFrameReader**) = 0; +}; + +struct IDepthFrameReader : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameArrived(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameArrived(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameArrivedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE AcquireLatestFrame(IDepthFrame**) = 0; +}; + +// IDepthFrame: CopyFrameDataToArray (3), AccessUnderlyingBuffer (4), +// get_FrameDescription (5), get_RelativeTime (6; then the source and the +// reliable distances, unused), as IDepthFrameVtbl in Kinect.h. Millimetres, +// 0 for nothing measured. +struct IDepthFrame : IUnknown { + virtual HRESULT STDMETHODCALLTYPE CopyFrameDataToArray(UINT capacity, UINT16* data) = 0; + virtual HRESULT STDMETHODCALLTYPE AccessUnderlyingBuffer(UINT* capacity, UINT16** buffer) = 0; + virtual HRESULT STDMETHODCALLTYPE get_FrameDescription(IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE get_RelativeTime(INT64*) = 0; // TIMESPAN, 100 ns units +}; + +// IBodyIndexFrameSource: FrameCaptured events, get_IsActive, OpenReader +// (then get_FrameDescription, get_KinectSensor, unused). +struct IBodyIndexFrameSource : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameCaptured(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameCaptured(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameCapturedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE get_IsActive(BOOLEAN*) = 0; + virtual HRESULT STDMETHODCALLTYPE OpenReader(IBodyIndexFrameReader**) = 0; +}; + +struct IBodyIndexFrameReader : IUnknown { + virtual HRESULT STDMETHODCALLTYPE SubscribeFrameArrived(WaitableHandle*) = 0; + virtual HRESULT STDMETHODCALLTYPE UnsubscribeFrameArrived(WaitableHandle) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFrameArrivedEventData(WaitableHandle, IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE AcquireLatestFrame(IBodyIndexFrame**) = 0; +}; + +// IBodyIndexFrame: CopyFrameDataToArray (3), AccessUnderlyingBuffer (4), +// get_FrameDescription (5), get_RelativeTime (6; then the source, unused), +// as IBodyIndexFrameVtbl in Kinect.h. Each depth pixel's body (0..5, its +// index in GetAndRefreshBodyData's array), 255 for none. +struct IBodyIndexFrame : IUnknown { + virtual HRESULT STDMETHODCALLTYPE CopyFrameDataToArray(UINT capacity, BYTE* data) = 0; + virtual HRESULT STDMETHODCALLTYPE AccessUnderlyingBuffer(UINT* capacity, BYTE** buffer) = 0; + virtual HRESULT STDMETHODCALLTYPE get_FrameDescription(IUnknown**) = 0; + virtual HRESULT STDMETHODCALLTYPE get_RelativeTime(INT64*) = 0; }; struct IBodyFrameSource : IUnknown { @@ -72,9 +264,12 @@ struct IBodyFrameReader : IUnknown { virtual HRESULT STDMETHODCALLTYPE AcquireLatestFrame(IBodyFrame**) = 0; }; +// IBodyFrame: GetAndRefreshBodyData (3), get_FloorClipPlane (4), +// get_RelativeTime (5; then get_BodyFrameSource, unused). struct IBodyFrame : IUnknown { virtual HRESULT STDMETHODCALLTYPE GetAndRefreshBodyData(UINT capacity, IBody** bodies) = 0; virtual HRESULT STDMETHODCALLTYPE get_FloorClipPlane(Vector4*) = 0; + virtual HRESULT STDMETHODCALLTYPE get_RelativeTime(INT64*) = 0; // TIMESPAN, 100 ns units }; struct IBody : IUnknown { @@ -111,6 +306,16 @@ class WindowsKinectV2 final : public KinectSensor { ~WindowsKinectV2() override { worker_.request_stop(); if (worker_.joinable()) worker_.join(); + { + std::lock_guard guard(image_lock_); + close_images(); + } + release(game_depth_reader_); + release(game_index_reader_); + { + std::lock_guard guard(mapper_lock_); + release(mapper_); + } release(reader_); if (sensor_) sensor_->Close(); release(sensor_); @@ -150,10 +355,23 @@ class WindowsKinectV2 final : public KinectSensor { return false; } std::cerr << "NATIVE_KINECT model=v2 smooth=" << (smooth_ ? 1 : 0) << '\n'; + open_game_depth(); // failing to open leaves the skeleton path (game_depth_ false) worker_ = std::jthread([this](std::stop_token stop) { run(stop); }); return true; } + // the Kinect v2 depth (kinect_v2_depth.h): a v1's depth stream for the + // game, made on the sensor's own thread. + bool has_depth_stream() const override { return game_depth_; } + std::string depth_failure() const override { return game_depth_ ? std::string() : depth_failure_; } + bool game_image(KinectImageKind kind, KinectImage& out) override { + if (kind != KinectImageKind::depth_and_player || !game_depth_) return false; + std::lock_guard guard(game_image_lock_); + if (game_image_.number == 0 || game_image_.number == out.number) return false; + out = game_image_; + return true; + } + bool next(KinectFrame& frame) override { std::lock_guard guard(lock_); if (latest_.number == taken_) return false; @@ -162,13 +380,605 @@ class WindowsKinectV2 final : public KinectSensor { return true; } + // The cameras, for the preview window (kinect_v2_image.h). Opened on + // the first call rather than with the sensor, so the game, which never + // asks (has_depth_stream stays false), reads skeletons alone; read on + // the caller's thread, so their cost is the preview's. + bool image(KinectImageKind kind, KinectImage& out) override { + std::lock_guard guard(image_lock_); + const auto now = std::chrono::steady_clock::now(); + image_asked_ = now; + if (!images_open_ && now >= images_retry_at_) open_images(now); + if (kind == KinectImageKind::colour) read_colour(); + else read_depth(); + const KinectImage& latest = images_[size_t(kind)]; + if (latest.number == 0 || latest.number == out.number) return false; + out = latest; + return true; + } + + void set_preview_common_view(bool common) override { + preview_common_.store(common, std::memory_order_relaxed); + } + + // The preview's view and the launcher's tuning: taken up on the sensor's + // thread (apply_depth_settings) before its next depth image. + void set_depth_settings(const KinectDepthSettings& settings) override { + std::lock_guard guard(depth_settings_lock_); + pending_depth_settings_ = settings; + } + + bool ideal_snapshot(KinectIdealSnapshot& snapshot) override { + if (!game_depth_) return false; + std::lock_guard guard(snapshot_lock_); + snapshot = snapshot_; + return true; + } + + bool picture_rect(KinectImageKind kind, KinectImageRect& rect) override { + rect = kinect_v2_picture_rect(kind, preview_view()); + return true; + } + + // The preview's sizes, through the sensor's own coordinate mapper (its + // factory calibration of both cameras), or a typical v2's lens without it + // or for a point it cannot place (kinect_v2_place). + bool image_point(KinectImageKind kind, const std::array& given, KinectImagePoint& point) override { + float mapped[2] = {}; + bool answered = false; + // The frames handed out are in the virtual v1's place in + // made-depth mode; the sensor's own images are not. + std::array p = given; + if (game_depth_) { + std::lock_guard guard(lock_); + p = kinect_depth_view_unpoint(published_view_, given); + } + if (ICoordinateMapper* mapper = coordinate_mapper()) { + std::lock_guard mapping(map_lock_); + const CameraSpacePoint camera{p[0], p[1], p[2]}; + HRESULT result = E_FAIL; + if (kind == KinectImageKind::colour) { + ColorSpacePoint out{-INFINITY, -INFINITY}; + result = mapper->MapCameraPointsToColorSpace(1, &camera, 1, &out); + mapped[0] = out.x; + mapped[1] = out.y; + } else { + DepthSpacePoint out{-INFINITY, -INFINITY}; + result = mapper->MapCameraPointsToDepthSpace(1, &camera, 1, &out); + mapped[0] = out.x; + mapped[1] = out.y; + } + answered = SUCCEEDED(result); + } + const bool placed = answered && std::isfinite(mapped[0]) && std::isfinite(mapped[1]); + mapped_points_[placed ? 0 : 1].fetch_add(1, std::memory_order_relaxed); + point = kinect_v2_place(kind, p, answered ? mapped : nullptr, preview_view()); + return true; + } + + // Sensor-space points into the preview's colour image, all in one call + // to the mapper (the points that are points only), the typical lens for + // what it cannot place (kinect_v2_place), as image_point places one. + bool colour_image_positions(const std::vector& points, std::vector& positions) override { + const size_t count = points.size() / 3; + positions.assign(count * 2, NAN); + std::vector given; + std::vector index; + given.reserve(count); + index.reserve(count); + for (size_t i = 0; i < count; ++i) { + const float x = points[3 * i], y = points[3 * i + 1], z = points[3 * i + 2]; + if (!(z > 0.0f) || !std::isfinite(x) || !std::isfinite(y) || !std::isfinite(z)) continue; + given.push_back({x, y, z}); + index.push_back(i); + } + if (given.empty()) return true; + std::vector mapped(given.size(), ColorSpacePoint{-INFINITY, -INFINITY}); + bool answered = false; + if (ICoordinateMapper* mapper = coordinate_mapper()) { + std::lock_guard mapping(map_lock_); + answered = SUCCEEDED(mapper->MapCameraPointsToColorSpace(UINT(given.size()), given.data(), UINT(mapped.size()), + mapped.data())); + } + const KinectV2View view = preview_view(); + for (size_t j = 0; j < given.size(); ++j) { + const float at[2] = {mapped[j].x, mapped[j].y}; + const KinectImagePoint point = + kinect_v2_place(KinectImageKind::colour, {given[j].x, given[j].y, given[j].z}, answered ? at : nullptr, view); + if (!point.visible) continue; + positions[2 * index[j]] = point.x; + positions[2 * index[j] + 1] = point.y; + } + return true; + } + private: + KinectV2View preview_view() const { + return preview_common_.load(std::memory_order_relaxed) ? KinectV2View::common : KinectV2View::original; + } + + // The sensor's coordinate mapper, asked for on the preview's first point + // (the game never asks): null when the runtime would not give one. + ICoordinateMapper* coordinate_mapper() { + std::lock_guard guard(mapper_lock_); + if (!mapper_asked_) { + mapper_asked_ = true; + const HRESULT result = sensor_->get_CoordinateMapper(&mapper_); + if (FAILED(result)) mapper_ = nullptr; + std::cerr << "NATIVE_KINECT_MAPPER model=v2 result=" << hresult(result) << (mapper_ ? "" : " (lens estimate)") + << std::endl; + } + return mapper_; + } + + // Under image_lock_ from here to read_depth. + void open_images(std::chrono::steady_clock::time_point now) { + IColorFrameSource* colour = nullptr; + HRESULT colour_result = sensor_->get_ColorFrameSource(&colour); + if (SUCCEEDED(colour_result) && colour) colour_result = colour->OpenReader(&colour_reader_); + release(colour); + IDepthFrameSource* depth = nullptr; + HRESULT depth_result = sensor_->get_DepthFrameSource(&depth); + if (SUCCEEDED(depth_result) && depth) depth_result = depth->OpenReader(&depth_reader_); + release(depth); + IBodyIndexFrameSource* index = nullptr; + HRESULT index_result = sensor_->get_BodyIndexFrameSource(&index); + if (SUCCEEDED(index_result) && index) index_result = index->OpenReader(&index_reader_); + release(index); + images_open_ = colour_reader_ || depth_reader_; + // Nothing opened: try again in a while rather than every call. + if (!images_open_) images_retry_at_ = now + std::chrono::seconds(5); + std::cerr << "NATIVE_KINECT_IMAGE model=v2 open colour=" << hresult(colour_result) << " depth=" + << hresult(depth_result) << " body_index=" << hresult(index_result) << std::endl; + } + + void close_images() { + if (!images_open_ && !index_reader_) return; + release(colour_reader_); + release(depth_reader_); + release(index_reader_); + images_open_ = false; + has_body_index_ = false; + std::cerr << "NATIVE_KINECT_IMAGE model=v2 closed" << std::endl; + } + + void read_colour() { + if (!colour_reader_) return; + IColorFrame* frame = nullptr; + // E_PENDING (or no frame) until a new one arrives: nothing new. + if (FAILED(colour_reader_->AcquireLatestFrame(&frame)) || !frame) { release(frame); return; } + constexpr UINT pixels = kinect_v2_colour_width * kinect_v2_colour_height; + ColorImageFormat format = ColorImageFormat_None; + if (FAILED(frame->get_RawColorImageFormat(&format))) format = ColorImageFormat_None; + HRESULT result = E_FAIL; + KinectImage& made = images_[size_t(KinectImageKind::colour)]; + const KinectV2View view = preview_view(); + if (format == ColorImageFormat_Yuy2) { + // The camera's own format: copied as it is and converted only + // where the preview's smaller image samples it. + colour_scratch_.resize(size_t(pixels) * 2); + result = frame->CopyRawFrameDataToArray(UINT(colour_scratch_.size()), colour_scratch_.data()); + if (SUCCEEDED(result)) + kinect_v2_colour_from_yuy2(colour_scratch_.data(), kinect_v2_colour_width, kinect_v2_colour_height, view, made); + } + if (FAILED(result)) { + colour_scratch_.resize(size_t(pixels) * 4); + result = frame->CopyConvertedFrameDataToArray(UINT(colour_scratch_.size()), colour_scratch_.data(), + ColorImageFormat_Bgra); + if (SUCCEEDED(result)) + kinect_v2_colour_from_bgra(colour_scratch_.data(), kinect_v2_colour_width, kinect_v2_colour_height, view, made); + } + release(frame); + if (SUCCEEDED(result)) { + ++made.number; + image_frames_[size_t(KinectImageKind::colour)].fetch_add(1, std::memory_order_relaxed); + } else if (!colour_failure_logged_) { + colour_failure_logged_ = true; + std::cerr << "NATIVE_KINECT_IMAGE model=v2 colour_copy=" << hresult(result) << " format=" << int(format) + << std::endl; + } + } + + void read_depth() { + constexpr UINT pixels = kinect_v2_depth_width * kinect_v2_depth_height; + if (index_reader_) { + // Which body each pixel is, for the preview's tint; its own + // reader, so it may be a frame apart from the depth. + IBodyIndexFrame* frame = nullptr; + if (SUCCEEDED(index_reader_->AcquireLatestFrame(&frame)) && frame) { + body_index_.resize(pixels); + has_body_index_ = SUCCEEDED(frame->CopyFrameDataToArray(pixels, body_index_.data())); + } + release(frame); + } + if (!depth_reader_) return; + IDepthFrame* frame = nullptr; + if (FAILED(depth_reader_->AcquireLatestFrame(&frame)) || !frame) { release(frame); return; } + depth_scratch_.resize(pixels); + const HRESULT result = frame->CopyFrameDataToArray(pixels, depth_scratch_.data()); + release(frame); + if (FAILED(result)) { + if (!depth_failure_logged_) { + depth_failure_logged_ = true; + std::cerr << "NATIVE_KINECT_IMAGE model=v2 depth_copy=" << hresult(result) << std::endl; + } + return; + } + { + // How much of the image has no depth, for a recording + // (kinect_record.h): another sensor's light shows there first. + std::lock_guard quality(depth_quality_lock_); + depth_invalid_ = kinect_depth_invalid_ratio(depth_scratch_.data(), pixels, 0); + ++depth_quality_frame_; + depth_quality_us_ = kinect_record_now_us(); + } + KinectImage& made = images_[size_t(KinectImageKind::depth_and_player)]; + kinect_v2_depth_image(depth_scratch_.data(), has_body_index_ ? body_index_.data() : nullptr, + kinect_v2_depth_width, kinect_v2_depth_height, preview_view(), made); + ++made.number; + image_frames_[size_t(KinectImageKind::depth_and_player)].fetch_add(1, std::memory_order_relaxed); + } + + // The newest depth image's share of pixels without a depth, into a + // recorded frame; none while the depth reader is closed (the game never + // opens it; the probe does while recording). + void depth_quality(KinectRecordFrame& record) { + std::lock_guard quality(depth_quality_lock_); + if (!depth_quality_frame_ || kinect_record_now_us() - depth_quality_us_ > 1000000) return; + record.depth_invalid = depth_invalid_; + record.depth_frame = depth_quality_frame_; + record.depth_steady_us = depth_quality_us_; + } + + // The cameras close again once the preview has not asked for a while + // (its window closed), from the sensor's own thread; never waits for a + // preview busy converting a frame. + void close_idle_images() { + std::unique_lock guard(image_lock_, std::try_to_lock); + if (!guard.owns_lock() || !images_open_) return; + if (std::chrono::steady_clock::now() - image_asked_ > std::chrono::seconds(3)) close_images(); + } + + // The game's own depth and body index readers, apart from the preview's + // (every reader gets every frame), so neither takes the other's frames. + void open_game_depth() { + IDepthFrameSource* depth = nullptr; + HRESULT depth_result = sensor_->get_DepthFrameSource(&depth); + if (SUCCEEDED(depth_result) && depth) depth_result = depth->OpenReader(&game_depth_reader_); + release(depth); + IBodyIndexFrameSource* index = nullptr; + HRESULT index_result = sensor_->get_BodyIndexFrameSource(&index); + if (SUCCEEDED(index_result) && index) index_result = index->OpenReader(&game_index_reader_); + release(index); + // Without the body index there are no players in the image, which is + // what the game's depth view looks for: both or neither. + game_depth_ = game_depth_reader_ && game_index_reader_; + if (!game_depth_) { + release(game_depth_reader_); + release(game_index_reader_); + depth_failure_ = FAILED(depth_result) ? "depth-" + hresult(depth_result) + : "body-index-" + hresult(index_result); + } + std::ostringstream line; + const KinectDepthSettings& settings = viewer_.settings(); + line << "NATIVE_KINECT_V2_DEPTH " << (game_depth_ ? "on" : "off") << " view=" << settings.view_name(); + if (settings.mode == KinectDepthViewMode::fit) { + if (settings.fit_scale > 0) { + fit_scale_ = settings.fit_scale; + fit_source_ = "setting"; + } else { + fit_lens_pending_ = !fit_from_sensor(); + } + viewer_.set_fit_scale(fit_scale_); + line << fit_description(); + } else if (settings.mode == KinectDepthViewMode::player) { + line << " distance=" << settings.distance << " height=" << settings.height; + } else if (settings.mode == KinectDepthViewMode::ideal) { + // No stretch: the v1's 0.8..4 m is measured from the virtual + // camera, which stands distance in front of the players. + char text[200]; + std::snprintf(text, sizeof text, + " distance=%.2f height=%.2f follow=%.2f deadband=%.2f true_range=0.80-4.00m from the virtual camera" + " (the players at %.2fm)", + double(settings.distance), double(settings.height), double(settings.follow), + double(settings.deadband), double(settings.distance)); + line << text; + } + // The skeleton's own transform: in real metres (1.0) unless + // SFR_KINECT_V2_SKELETON_STRETCH says otherwise ("depth": the + // depth's k, which may still change once the sensor gives its lens). + if (settings.mode == KinectDepthViewMode::fit) { + char text[96]; + if (settings.skeleton_stretch > 0) + std::snprintf(text, sizeof text, " skeleton_stretch=%.3f%s", double(settings.skeleton_stretch), + settings.skeleton_stretch == 1.0f ? " (real metres)" : ""); + else + std::snprintf(text, sizeof text, " skeleton_stretch=depth's (%.3f)", double(fit_scale_)); + line << text; + } + if (settings.mode == KinectDepthViewMode::ideal) + line << " skeleton_distance=" + << (settings.skeleton_distance > 0 ? std::to_string(settings.skeleton_distance) : std::string("depth's")); + line << " depth=" << hresult(depth_result) << " body_index=" << hresult(index_result); + std::cerr << line.str() << std::endl; + } + + // The fit view's stretch from the sensor's own depth lens; false while + // it has not sent its calibration (the typical lens meanwhile). + bool fit_from_sensor() { + ICoordinateMapper* mapper = coordinate_mapper(); + if (!mapper) return false; + CameraIntrinsics lens{}; + std::lock_guard mapping(map_lock_); + if (FAILED(mapper->GetDepthCameraIntrinsics(&lens)) || !(lens.focal_y > 100.0f && lens.focal_y < 1000.0f)) + return false; + fit_focal_ = lens.focal_y; + fit_scale_ = kinect_v2_fit_scale(lens.focal_y); + fit_source_ = "sensor"; + return true; + } + + std::string fit_description() const { + char text[160]; + std::snprintf(text, sizeof text, " scale=%.3f focal=%.1f (%s) true_range=%.2f-%.2fm", double(fit_scale_), + double(fit_focal_), fit_source_, double(kinect_depth_true_minimum(fit_scale_)), + double(kinect_depth_true_maximum(fit_scale_))); + return text; + } + + // Where the virtual v1 is for this body frame of the sensor's own space + // (KinectDepthViewer: the player view's latch follows it, the ideal view + // keeps its bodies for the next depth image). + KinectDepthView game_view(const KinectFrame& frame, double seconds) { + // The sensor sends its lens a moment after it starts: asked again + // about once a second until then. + if (viewer_.settings().mode == KinectDepthViewMode::fit && fit_lens_pending_ && ++fit_lens_tries_ % 30 == 1 && + fit_from_sensor()) { + fit_lens_pending_ = false; + viewer_.set_fit_scale(fit_scale_); + std::cerr << "NATIVE_KINECT_V2_DEPTH fit" << fit_description() << std::endl; + } + return viewer_.body_frame(frame, seconds); + } + + // New settings from set_depth_settings, on the sensor's thread. The fit + // view's stretch is the setting's, or the sensor's lens (asked for now + // if it has not been). One log line when the view changes, none for + // tuning alone (a slider being dragged sends one every frame). + void apply_depth_settings() { + std::optional settings; + { + std::lock_guard guard(depth_settings_lock_); + settings.swap(pending_depth_settings_); + } + if (!settings) return; + const bool new_view = settings->mode != viewer_.settings().mode; + viewer_.set_settings(*settings); + if (settings->mode == KinectDepthViewMode::fit) { + if (settings->fit_scale > 0) { + fit_scale_ = settings->fit_scale; + fit_source_ = "setting"; + fit_lens_pending_ = false; + } else if (std::string_view(fit_source_) != "sensor") { + fit_lens_pending_ = !fit_from_sensor(); + } + viewer_.set_fit_scale(fit_scale_); + } + if (new_view) { + char text[160]; + std::snprintf(text, sizeof text, " distance=%.2f height=%.2f follow=%.2f deadband=%.2f", + double(settings->distance), double(settings->height), double(settings->follow), + double(settings->deadband)); + std::cerr << "NATIVE_KINECT_V2_DEPTH set view=" << settings->view_name() << text + << (settings->mode == KinectDepthViewMode::fit ? fit_description() : std::string()) << std::endl; + } + } + + // The ideal camera and the players for the preview's diagram, after a + // depth image is made; 1P and 2P as the title's slots would take them. + void publish_snapshot() { + KinectIdealSnapshot made = kinect_ideal_snapshot(viewer_.bodies(), viewer_.ideal(), viewer_.settings()); + const auto slots = snapshot_slots_.assign(viewer_.bodies()); + for (KinectIdealPlayer& player : made.players) + for (size_t slot = 0; slot < slots.size(); ++slot) + if (slots[slot] && slots[slot]->tracking_id == player.tracking_id) player.slot = int(slot); + std::lock_guard guard(snapshot_lock_); + snapshot_ = std::move(made); + } + + // The ideal view's camera for the log: where it stands (levelled space), + // its tilt taken out, the reference it follows, and how many bodies. + std::string ideal_description(const KinectDepthView& view) const { + const KinectIdealState& state = viewer_.ideal(); + float pitch = 0, roll = 0; + kinect_floor_tilt(state.floor, pitch, roll); + // The floor as the sensor gives it now and as smoothed, the sensor's + // height above it (its distance), and the camera's: what decides + // where a player lands up and down in the made image. + const auto& given = viewer_.bodies().floor_plane; + char text[480]; + std::snprintf(text, sizeof text, + " depth_camera=%.3f,%.3f,%.3f level=%d pitch=%.1f roll=%.1f floor_d=%.3f reference=%.3f,%.3f%s" + " reference_bodies=%u floor_given=%.3f,%.3f,%.3f,%.3f sensor_height=%s camera_height=%s height=%.2f", + double(view.origin[0]), double(view.origin[1]), double(view.origin[2]), state.floor_known ? 1 : 0, + double(pitch), double(roll), double(state.floor[3]), double(state.reference[0]), + double(state.reference[1]), state.placed ? "" : " (none yet)", state.bodies, double(given[0]), + double(given[1]), double(given[2]), double(given[3]), + state.height_known ? std::to_string(state.floor[3]).substr(0, 5).c_str() : "unknown", + state.height_known ? std::to_string(state.floor[3] + view.origin[1]).substr(0, 5).c_str() + : "the sensor's", + double(viewer_.settings().height)); + return text; + } + + // A raw recording (kinect_record_raw.h) opens once the sensor's mapper + // gives points (its calibration is then in, and with it the table), or + // after about five seconds without (the lens, or a typical one). + void record_raw(KinectRawRecordFile& file, bool mapped, const KinectImage& made) { + constexpr UINT pixels = kinect_v2_depth_width * kinect_v2_depth_height; + if (!file.is_open()) { + if (!mapped && ++raw_waited_ < 150) return; + KinectRawHeader header; + header.width = kinect_v2_depth_width; + header.height = kinect_v2_depth_height; + if (ICoordinateMapper* mapper = coordinate_mapper()) { + std::lock_guard mapping(map_lock_); + CameraIntrinsics lens{}; + if (SUCCEEDED(mapper->GetDepthCameraIntrinsics(&lens)) && lens.focal_y > 100.0f && lens.focal_y < 1000.0f) + header.lens = {lens.focal_x, lens.focal_y, lens.centre_x, lens.centre_y, lens.radial2, lens.radial4, + lens.radial6}; + UINT32 count = 0; + void* table = nullptr; + if (SUCCEEDED(mapper->GetDepthFrameToCameraSpaceTable(&count, &table)) && table && count == pixels) { + const float* rays = static_cast(table); + header.rays.assign(rays, rays + 2 * size_t(pixels)); + header.rays_from = KinectRawRays::sensor_table; + } + if (table) CoTaskMemFree(table); + } + if (header.rays_from == KinectRawRays::sensor_table) { + bool flip_x = false, flip_y = false; + if (!kinect_raw_rays_orientation(header, flip_x, flip_y)) header.rays_from = KinectRawRays::unknown; + } + if (header.rays_from != KinectRawRays::sensor_table) { + const bool lens = header.lens[1] > 0; + kinect_v2_depth_rays_from_lens(lens ? header.lens[0] : 366.0f, lens ? header.lens[1] : 366.0f, + lens ? header.lens[2] : 256.0f, lens ? header.lens[3] : 212.0f, + kinect_v2_depth_width, kinect_v2_depth_height, header.rays); + header.rays_from = lens ? KinectRawRays::sensor_lens : KinectRawRays::typical_lens; + } + if (!file.open(header)) return; + } + KinectRawFrame frame; + frame.steady_us = made.steady_us; + frame.depth_time = game_depth_time_; + frame.body_time = depth_body_time_; + frame.number = made.number; + frame.flags = mapped ? kinect_raw_flag_mapped : 0; + kinect_raw_set_bodies(viewer_.bodies(), frame); + frame.depth = game_millimetres_; + if (game_body_index_.size() == pixels) frame.body_index = game_body_index_; + else frame.body_index.assign(pixels, kinect_v2_no_body); + file.write(frame); + } + + // The newest depth and body index frames, and once both are of the same + // moment, the game's image from them through the view of the newest body + // frame. On the sensor's thread. + void poll_game_depth() { + constexpr UINT pixels = kinect_v2_depth_width * kinect_v2_depth_height; + IDepthFrame* depth = nullptr; + if (SUCCEEDED(game_depth_reader_->AcquireLatestFrame(&depth)) && depth) { + game_millimetres_.resize(pixels); + INT64 time = 0; + if (SUCCEEDED(depth->CopyFrameDataToArray(pixels, game_millimetres_.data())) && + SUCCEEDED(depth->get_RelativeTime(&time))) + game_depth_time_ = time; + } + release(depth); + IBodyIndexFrame* index = nullptr; + if (SUCCEEDED(game_index_reader_->AcquireLatestFrame(&index)) && index) { + game_body_index_.resize(pixels); + INT64 time = 0; + if (SUCCEEDED(index->CopyFrameDataToArray(pixels, game_body_index_.data())) && + SUCCEEDED(index->get_RelativeTime(&time))) + game_index_time_ = time; + } + release(index); + // Both readers' frames are the same capture (one camera); a frame + // apart they would put the players' outline a frame off its depth. + if (!game_depth_time_ || game_depth_time_ == game_made_time_ || + std::llabs(game_depth_time_ - game_index_time_) > 166666) // half a frame, 100 ns units + return; + game_made_time_ = game_depth_time_; + bool mapped = false; + if (ICoordinateMapper* mapper = coordinate_mapper()) { + game_points_.resize(size_t(pixels) * 3); + std::lock_guard mapping(map_lock_); + mapped = SUCCEEDED(mapper->MapDepthFrameToCameraSpace( + pixels, game_millimetres_.data(), pixels, reinterpret_cast(game_points_.data()))); + // A mapper without its calibration yet answers no point at all. + if (mapped) { + mapped = false; + for (size_t i = 0; i < pixels && !mapped; ++i) + mapped = std::isfinite(game_points_[3 * i + 2]) && game_points_[3 * i + 2] > 0; + } + } + if (!mapped) kinect_v2_depth_points_from_lens(game_millimetres_.data(), kinect_v2_depth_width, + kinect_v2_depth_height, game_points_); + (mapped ? game_mapped_frames_ : game_lens_frames_).fetch_add(1, std::memory_order_relaxed); + // The view of the newest body frame (fit, sensor, player), or the + // ideal view stepped now on the depth frame's own clock. + const KinectDepthView view = viewer_.depth_frame(double(game_depth_time_) * 1e-7); + publish_snapshot(); + if (viewer_.settings().mode == KinectDepthViewMode::ideal && + viewer_.ideal().floor_known != ideal_floor_logged_known_) { + ideal_floor_logged_known_ = viewer_.ideal().floor_known; + if (ideal_floor_logged_known_) + std::cerr << "NATIVE_KINECT_V2_DEPTH ideal floor found: levelled" << ideal_description(view) << std::endl; + } + // The floor's distance (the sensor's height) comes and goes apart + // from its normal: one line each time that changes, at most one a + // second, so a log shows why the camera stands where it does. + if (viewer_.settings().mode == KinectDepthViewMode::ideal && viewer_.ideal().floor_known && + viewer_.ideal().height_known != ideal_height_logged_known_) { + const auto now = std::chrono::steady_clock::now(); + if (now - ideal_height_logged_at_ >= std::chrono::seconds(1)) { + ideal_height_logged_known_ = viewer_.ideal().height_known; + ideal_height_logged_at_ = now; + std::cerr << "NATIVE_KINECT_V2_DEPTH ideal " + << (ideal_height_logged_known_ ? "floor height found" : "floor without a height (W=0): the camera at the sensor's own height") + << ideal_description(view) << std::endl; + } + } + if (viewer_.settings().mode == KinectDepthViewMode::ideal && !viewer_.ideal().floor_known && !ideal_no_floor_logged_) { + ideal_no_floor_logged_ = true; + std::cerr << "NATIVE_KINECT_V2_DEPTH ideal no floor plane yet: not levelled, the camera at the sensor's own height" + << std::endl; + } + KinectImage made; + const KinectDepthStats stats = + kinect_v2_synthesize_depth(game_points_.data(), game_body_index_.size() == pixels ? game_body_index_.data() : nullptr, + pixels, view, made); + // Its clocks and view, for a depth recording (kinect_record_depth.h: + // the game records what it is handed, the probe what it reads). + made.steady_us = kinect_record_now_us(); + made.sdk_time = game_depth_time_; + made.view_origin = view.origin; + made.view_z_scale = view.z_scale; + made.view_rotated = view.rotated; + made.view_rotation = view.rotation; + { + std::lock_guard guard(game_image_lock_); + made.number = game_image_.number + 1; + game_image_ = made; + } + game_frames_.fetch_add(1, std::memory_order_relaxed); + game_valid_.store(stats.valid, std::memory_order_relaxed); + game_players_.store(stats.players, std::memory_order_relaxed); + // SFR_KINECT_RECORD_RAW: what this image was made from. + if (KinectRawRecordFile* raw = kinect_record_raw_file()) record_raw(*raw, mapped, made); + } + void run(std::stop_token stop) { KinectFrame frame; std::array mapped; uint64_t frames = 0, tracked = 0; auto reported = std::chrono::steady_clock::now(); + // Per body slot: the clipped edges last written to the log and when, + // for NATIVE_KINECT_CLIPPED (on a change, once a second at most). + std::array logged_edges{}; + std::array logged_edges_at{}; + // SFR_KINECT_RECORD (kinect_record.h): every frame, raw. + const bool recording = kinect_record_skeletons() != nullptr; + uint64_t recorded_frames = 0; + const auto began = std::chrono::steady_clock::now(); while (!stop.stop_requested()) { + close_idle_images(); + if (game_depth_) { + apply_depth_settings(); + poll_game_depth(); + } IBodyFrame* body_frame = nullptr; // E_PENDING until the next frame (30 a second). if (FAILED(reader_->AcquireLatestFrame(&body_frame)) || !body_frame) { @@ -181,8 +991,24 @@ class WindowsKinectV2 final : public KinectSensor { const bool read = SUCCEEDED(body_frame->GetAndRefreshBodyData(body_count, bodies)); Vector4 floor{}; if (FAILED(body_frame->get_FloorClipPlane(&floor))) floor = {}; + // The body frames' clock: the recordings', and the depth view's + // (KinectDepthViewer), so a raw recording replays it. + INT64 relative_time = 0; + if (FAILED(body_frame->get_RelativeTime(&relative_time))) relative_time = 0; release(body_frame); - if (!read) continue; + if (!read) { + for (IBody*& body : bodies) release(body); + continue; + } + KinectRecordFrame record; + if (recording) { + record.steady_us = kinect_record_now_us(); + record.sensor = "v2"; + record.frame = ++recorded_frames; + record.sdk_time = relative_time; + record.floor = {floor.x, floor.y, floor.z, floor.w}; + depth_quality(record); + } frame.captured_at = captured_at; frame.floor_plane = {floor.x, floor.y, floor.z, floor.w}; // v2 gives no gravity of its own; the floor's normal is up. @@ -201,7 +1027,7 @@ class WindowsKinectV2 final : public KinectSensor { uint32_t(joints[j].state < 0 ? 0 : joints[j].state)}; // The player slots key on 32 bits; v2's ids count up // from a large base, so the low half stays distinct. - const uint32_t short_id = uint32_t(id) ? uint32_t(id) : uint32_t(id >> 32) | 1u; + const uint32_t short_id = kinect_v2_short_tracking_id(id); KinectBody& made = frame.bodies.emplace_back(); kinect_v2_body(mapped, short_id, index, made); // The hands' state (open, a fist, lasso), which a v1 does not see. @@ -209,21 +1035,111 @@ class WindowsKinectV2 final : public KinectSensor { if (SUCCEEDED(body->get_HandLeftState(&state))) made.hand_states[0] = uint8_t(state < 0 ? 0 : state); state = 0; if (SUCCEEDED(body->get_HandRightState(&state))) made.hand_states[1] = uint8_t(state < 0 ? 0 : state); + // Which edges of the picture cut it off: shown and logged only. + DWORD edges = 0; + if (SUCCEEDED(body->get_ClippedEdges(&edges))) made.clipped_edges = uint8_t(edges & kinect_edge::all); + if (recording) { + // All 25 joints as the sensor gave them, before smoothing. + KinectRecordBody& kept = record.bodies.emplace_back(); + kept.sensor_index = index; + kept.tracking_id = id; + for (uint32_t j = 0; j < kinect_v2_joint_count; ++j) + kept.joints[j] = {{joints[j].position.x, joints[j].position.y, joints[j].position.z}, + joints[j].state < 0 ? 0 : int(joints[j].state)}; + kept.hand_states = {int(made.hand_states[0]), int(made.hand_states[1])}; + kept.clipped_edges = made.clipped_edges; + } } release(body); } + { + const auto clip_now = std::chrono::steady_clock::now(); + std::array seen{}; + for (const KinectBody& tracked_body : frame.bodies) { + const size_t slot = tracked_body.sensor_index % 6; + seen[slot] = true; + if (tracked_body.clipped_edges == logged_edges[slot] || + clip_now - logged_edges_at[slot] < std::chrono::seconds(1)) continue; + std::cerr << "NATIVE_KINECT_CLIPPED model=v2 body=" << tracked_body.sensor_index + << " edges=" << kinect_edges_text(tracked_body.clipped_edges) + << " mask=" << unsigned(tracked_body.clipped_edges) << '\n'; + logged_edges[slot] = tracked_body.clipped_edges; + logged_edges_at[slot] = clip_now; + } + // A body that is gone starts again from "none" when it returns. + for (size_t slot = 0; slot < seen.size(); ++slot) + if (!seen[slot]) logged_edges[slot] = 0; + } + if (recording) kinect_record_skeleton_frame(record); + // Made depth: where the virtual v1 stands, placed by + // the sensor's own view of the player (latched, or followed). + KinectDepthView view; + if (game_depth_) { + const double seconds = relative_time ? double(relative_time) * 1e-7 + : std::chrono::duration(captured_at - began).count(); + depth_body_time_ = relative_time; + game_view(frame, seconds); + // The skeleton's own view: fit's skeleton is in real metres + // (SFR_KINECT_V2_SKELETON_STRETCH, default 1.0) while its + // depth is stretched by k; ideal's is the depth's unless + // SFR_KINECT_V2_SKELETON_DISTANCE sets it apart (the title + // never projects a skeleton into its depth; image_point + // undoes this for the preview). + view = viewer_.skeleton_view(); + } if (smooth_) smoother_.apply(frame); + // The skeletons the game reads in the virtual camera's place too, + // so that they fall on their own outlines in its depth image. + if (game_depth_) kinect_depth_view_frame(view, frame); ++frames; if (!frame.bodies.empty()) ++tracked; { std::lock_guard guard(lock_); frame.number = latest_.number + 1; latest_ = frame; + published_view_ = view; } const auto now = std::chrono::steady_clock::now(); if (now - reported >= std::chrono::seconds(5)) { std::cerr << "NATIVE_KINECT model=v2 frames=" << frames << " with_body=" << tracked - << " bodies=" << frame.bodies.size() << '\n'; + << " bodies=" << frame.bodies.size(); + for (const KinectBody& cut : frame.bodies) + if (cut.clipped_edges) + std::cerr << " clipped" << cut.sensor_index << '=' << kinect_edges_text(cut.clipped_edges); + // The preview's cameras, while it has them open. + const uint64_t colour_frames = image_frames_[size_t(KinectImageKind::colour)].exchange(0); + const uint64_t depth_frames = image_frames_[size_t(KinectImageKind::depth_and_player)].exchange(0); + if (colour_frames || depth_frames) + std::cerr << " preview_colour_frames=" << colour_frames << " preview_depth_frames=" << depth_frames; + // The overlay's points: placed by the mapper, by the lens. + const uint64_t mapped = mapped_points_[0].exchange(0), estimated = mapped_points_[1].exchange(0); + if (mapped || estimated) + std::cerr << " preview_mapped_points=" << mapped << " preview_estimated_points=" << estimated; + if (game_depth_) { + // The game's depth images: how many were made, how much of + // the newest has a depth and how much of it is players, + // and where the virtual v1 stands. + constexpr double pixels = double(kinect_v1_depth_width * kinect_v1_depth_height); + const uint64_t made = game_frames_.exchange(0), lens = game_lens_frames_.exchange(0); + game_mapped_frames_.exchange(0); + std::cerr << " depth_synth_frames=" << made << " depth_valid=" + << game_valid_.load(std::memory_order_relaxed) / pixels + << " depth_player_pixels=" << game_players_.load(std::memory_order_relaxed) + << " depth_view=" << viewer_.view().origin[0] << ',' << viewer_.view().origin[1] << ',' + << viewer_.view().origin[2] + << " depth_latched=" << (viewer_.latched() ? 1 : 0); + if (viewer_.settings().mode == KinectDepthViewMode::ideal) std::cerr << ideal_description(viewer_.view()); + // fit: the skeleton's own z scale (1 by default, the + // depth's k with "depth"); ideal with its own distance. + if (viewer_.settings().mode == KinectDepthViewMode::fit || viewer_.settings().skeleton_distance > 0) + std::cerr << " skeleton_view=" << view.origin[0] << ',' << view.origin[1] << ',' << view.origin[2] + << " skeleton_z_scale=" << view.z_scale; + if (lens) std::cerr << " depth_lens_frames=" << lens; + if (KinectRawRecordFile* raw = kinect_record_raw_file()) + std::cerr << " raw_frames=" << raw->frames_written(); + } + // Recordings dropped for a slow disk, and what is queued. + std::cerr << kinect_record_queue_status() << '\n'; reported = now; frames = tracked = 0; } @@ -233,17 +1149,84 @@ class WindowsKinectV2 final : public KinectSensor { HMODULE library_ = nullptr; IKinectSensor* sensor_ = nullptr; IBodyFrameReader* reader_ = nullptr; - // v2's SDK smooths nothing, so its joints shake from frame to frame; the - // filter Kinect v1's SDK offers is applied here (kinect_smoothing.h). - // On by default; SFR_KINECT_SMOOTH=0 hands the joints over as measured. - bool smooth_ = [] { - const char* value = std::getenv("SFR_KINECT_SMOOTH"); - return !value || *value != '0'; - }(); + // v2's SDK smooths nothing; the title does (its NuiTransformSmooth, done + // natively for a real sensor's skeletons in nui_smooth_hooks.cpp), so the + // joints go to the game as measured and are smoothed once, there. + // SFR_KINECT_SMOOTH=1 (kinect_host_smoothing) smooths them here with the + // same filter instead (kinect_smoothing.h), for sfr_kinect_probe --smooth; + // the game then leaves its own out. Recordings take the joints before. + bool smooth_ = kinect_host_smoothing(); KinectJointSmoother smoother_; std::mutex lock_; KinectFrame latest_; uint64_t taken_ = 0; + // The preview's cameras, under image_lock_. + std::mutex image_lock_; + IColorFrameReader* colour_reader_ = nullptr; + IDepthFrameReader* depth_reader_ = nullptr; + IBodyIndexFrameReader* index_reader_ = nullptr; + bool images_open_ = false, has_body_index_ = false; + bool colour_failure_logged_ = false, depth_failure_logged_ = false; + std::chrono::steady_clock::time_point image_asked_{}, images_retry_at_{}; + std::array images_{}; // by KinectImageKind + std::vector colour_scratch_, body_index_; + std::vector depth_scratch_; + std::array, 2> image_frames_{}; + // The preview's coordinate mapper, under mapper_lock_, and how many of + // its points it placed and how many the lens did, for the log. + std::mutex mapper_lock_; + ICoordinateMapper* mapper_ = nullptr; + bool mapper_asked_ = false; + std::array, 2> mapped_points_{}; + std::atomic preview_common_{true}; // the preview's view (set_preview_common_view) + // One call into the mapper at a time (the preview's thread and the + // sensor's both use it). + std::mutex map_lock_; + // The Kinect v2 depth (kinect_v2_depth.h): the game's readers (open + // before the worker starts, so game_depth_ never changes after), their + // newest frames and times, the camera-space points, and the image made, + // under game_image_lock_. The view the newest frame was moved by is + // published_view_, under lock_. + // Where the virtual v1 stands (every view; shared with the offline + // re-synthesis, kinect_v2_resynth.h), on the sensor's thread. + KinectDepthViewer viewer_{KinectDepthSettings::from_environment()}; + // Settings handed over by set_depth_settings, waiting for the sensor's + // thread; and what ideal_snapshot answers, with the slots it is labelled by. + std::mutex depth_settings_lock_; + std::optional pending_depth_settings_; + std::mutex snapshot_lock_; + KinectIdealSnapshot snapshot_; + KinectPlayerSlots snapshot_slots_; // the sensor's thread + INT64 depth_body_time_ = 0; // the newest body frame's RelativeTime, for a raw recording + uint32_t raw_waited_ = 0; // depth images made before a raw recording opened + bool ideal_floor_logged_known_ = false, ideal_no_floor_logged_ = false; + bool ideal_height_logged_known_ = false; + std::chrono::steady_clock::time_point ideal_height_logged_at_{}; + // The fit view's stretch and where it came from (setting, sensor, or a + // typical v2 until the sensor says), on the sensor's thread. + float fit_focal_ = kinect_v2_typical_depth_focal; + float fit_scale_ = kinect_v2_fit_scale(kinect_v2_typical_depth_focal); + const char* fit_source_ = "typical"; + bool fit_lens_pending_ = false; + uint32_t fit_lens_tries_ = 0; + bool game_depth_ = false; + std::string depth_failure_; // why game_depth_ is false, set before the worker runs + IDepthFrameReader* game_depth_reader_ = nullptr; + IBodyIndexFrameReader* game_index_reader_ = nullptr; + std::vector game_millimetres_; + std::vector game_body_index_; + std::vector game_points_; + INT64 game_depth_time_ = 0, game_index_time_ = 0, game_made_time_ = 0; + std::mutex game_image_lock_; + KinectImage game_image_; + KinectDepthView published_view_; + std::atomic game_frames_{0}, game_mapped_frames_{0}, game_lens_frames_{0}; + std::atomic game_valid_{0}, game_players_{0}; + // The newest depth image's quality, under depth_quality_lock_. + std::mutex depth_quality_lock_; + float depth_invalid_ = -1; + uint64_t depth_quality_frame_ = 0; + int64_t depth_quality_us_ = 0; std::jthread worker_; }; diff --git a/src/launcher_main.cpp b/src/launcher_main.cpp index 7a02df9..8c95684 100644 --- a/src/launcher_main.cpp +++ b/src/launcher_main.cpp @@ -88,12 +88,16 @@ enum Text { Sound, SoundHint, Volume, SkipMovies, SkipMoviesHint, SkipLogos, SkipLogosHint, VertexCache, VertexCacheHint, GpuPipeline, GpuPipelineHint, RaceEvery, RaceEveryHint, TitleStats, TitleStatsHint, - CameraLabel, CameraHint, CameraHintOff, CameraHintPicture, CameraHintKinect, CameraOff, CameraPicture, CameraMotion, CameraDevice, CameraDeviceHint, CameraNone, + CameraLabel, CameraHint, CameraHintOff, CameraHintKinect, CameraHintKinectV2, CameraOff, CameraMotion, CameraDevice, CameraDeviceHint, CameraNone, CameraTest, CameraTesting, CameraWorks, CameraSilent, CameraClosed, CameraMirror, CameraMirrorHint, - CameraKinect, KinectRow, KinectRowHint, KinectMissing, + CameraKinect, CameraKinectV2, + V2View, V2ViewSensor, V2ViewFit, V2ViewIdeal, V2ViewHintSensor, V2ViewHintFit, V2ViewHintIdeal, + V2Distance, V2DistanceHint, V2Height, V2HeightHint, V2Follow, V2FollowHint, V2Deadband, V2DeadbandHint, + V2Advanced, + KinectMissing, KinectNoRuntime, KinectNoSensor, KinectFailed, KinectDownload, KinectDownloadV2, KinectInstallHint, VoiceLabel, VoiceHint, - KinectPreviewOpen, KinectPreviewClose, CameraPreviewOpen, CameraPreviewClose, KinectCalibrated, + KinectPreviewOpen, KinectPreviewClose, CameraPreviewOpen, CameraPreviewClose, CameraDebug, CameraDebugHint, MenuSteady, MenuSteadyHint, MenuSteadyOff, ImageDirectory, ImageDirectoryHint, AssetDirectory, AssetDirectoryHint, Browse, Found, Missing, FilesHint, SaveDirectory, SaveDirectoryHint, SaveFound, SaveNone, SaveDefault, SaveOpen, @@ -129,7 +133,6 @@ enum Text { OnlineReachNotFound, OnlineReachRefused, OnlineReachNoAnswer, OnlineReachUnexpected, OnlineFriendsEnter, OnlineNoAddress, OnlineFirewall, OnlineChecking, OnlineFailedTitle, OnlineFailedRetry, OnlineStartAnyway, - KinectSmoothLabel, KinectSmoothHint, KinectLeanLabel, KinectLeanHint, KinectCrouchLabel, KinectCrouchHint, CustomWindowSize, CustomWindowSizeRow, CustomWindowSizeHint, AspectModeLabel, AspectModeHint, AspectAuto, AspectOriginal, UiAlignLabel, UiAlignHint, UiAlignEdge, UiAlignCenter, @@ -184,17 +187,16 @@ constexpr std::array, TextCount> texts{{ {"The game window's title shows, every half second, the frame rate, the average and longest frame, the graphics, and how often the Kinect or webcam body arrives. Applies on the next game launch.", "遊戲視窗的標題每半秒顯示 FPS、平均與最長畫格時間、繪圖後端與倍率，以及 Kinect 或 webcam 骨架每秒幾格。下次啟動遊戲時套用。"}, {"Camera", "攝影機"}, - {"Motion tracks your body for Player 1. Controller input takes priority; after 1.5 seconds idle, fresh camera tracking resumes. Lost tracking falls back to the controller. Player 2 keeps its assigned controller.", - "「體感」追蹤你的身體來控制 1P。操作手把時優先接手，停止操作 1.5 秒後恢復攝影機追蹤；失去追蹤時退回手把。2P 保留原本指定的手把。"}, + {"Webcam tracks your body for Player 1. Controller input takes priority; after 1.5 seconds idle, fresh camera tracking resumes. Lost tracking falls back to the controller. Player 2 keeps its assigned controller.", + "「Webcam」追蹤你的身體來控制 1P。操作手把時優先接手，停止操作 1.5 秒後恢復攝影機追蹤；失去追蹤時退回手把。2P 保留原本指定的手把。"}, {"No camera: the game is played with controllers and the keyboard.", "不使用攝影機：用手把或鍵盤遊玩。"}, - {"Opens a webcam only for the pictures the game shows of you; you still play with controllers and the keyboard.", - "開啟 webcam，只把影像交給遊戲顯示；操作仍然用手把或鍵盤。"}, - {"A real Kinect tracks the players' bodies, as on the console: its skeletons and its depth and colour cameras go to the game, which reads your moves itself. Controllers still work in the menus.", - "由實體 Kinect 追蹤玩家的身體，跟主機上一樣：骨架、深度與彩色影像直接交給遊戲，由遊戲自己判斷動作。選單中仍可使用手把。"}, + {"An Xbox 360 Kinect tracks the players' bodies, as on the console. Controllers still work in the menus.", + "由 Xbox 360 版 Kinect追蹤玩家的身體，跟主機上一樣，選單中仍可使用手把。"}, + {"An Xbox One Kinect v2 tracks the players' bodies. Controllers still work in the menus.", + "由 Xbox One 版 Kinect v2 追蹤玩家的身體，選單中仍可使用手把。"}, {"Off", "關閉"}, - {"Picture", "畫面"}, - {"Motion", "體感"}, + {"Webcam", "Webcam"}, {"Which camera", "使用哪一台"}, {"The camera to read, when the host has more than one.", "當電腦上有多台攝影機時，要讀取哪一台。"}, {"No camera found", "找不到攝影機"}, @@ -207,9 +209,29 @@ constexpr std::array, TextCount> texts{{ {"Turn this on when the camera shows you as a mirror does. With it wrong, raising one hand moves the other, and the cursor runs off the side of the screen.", "若攝影機畫面像照鏡子一樣左右相反，請開啟此項。設定相反時，舉起一隻手會動到另一隻，游標也會跑到畫面邊緣。"}, {"Kinect", "Kinect"}, - {"Kinect sensor", "Kinect 感測器"}, - {"A real Kinect tracks you as the console did: an Xbox 360 Kinect (Kinect for Windows SDK 1.8) or a Kinect v2 for Xbox One (SDK 2.0); the first one found is used. The preview shows whether it works, what it sees and the skeletons it tracks, and tilts it. It closes when the game starts.", - "由實體 Kinect 像主機一樣追蹤你的身體：Xbox 360 版（需 Kinect for Windows SDK 1.8）或 Xbox One 版 Kinect v2（需 SDK 2.0），會使用第一台找到的。預覽會顯示它是否正常、拍到的畫面與追蹤到的骨架，也能調整角度；開始遊戲時會自動關閉。"}, + {"Kinect v2", "Kinect v2"}, + {"Depth view", "深度視角"}, + {"Original", "原始"}, + {"Simulated", "模擬"}, + {"Recompiled", "Recompiled"}, + {"Hands the game the Kinect v2's view as it is.", + "把 Kinect v2 的視角直接交給遊戲。"}, + {"Fits the Kinect v2's whole view into the game's own picture size.", + "把 Kinect v2 的整個視野縮放成遊戲原本的畫面大小。"}, + {"A virtual Kinect standing level, at a set distance and height, that follows the players slowly.", + "虛擬一台水平、固定距離與高度的 Kinect，並緩慢跟著玩家。"}, + {"Distance", "距離"}, + {"How far in front of the players the virtual camera stands. About 2.5 m suits the game best.", + "虛擬攝影機在玩家前方多遠。約 2.5 公尺最適合遊戲。"}, + {"Height", "高度"}, + {"How high above the floor the virtual camera is.", "虛擬攝影機離地多高。"}, + {"Follow", "跟隨速度"}, + {"How many seconds the virtual camera takes to follow the players when they move. Larger is slower and steadier.", + "玩家移動後，虛擬攝影機花幾秒跟上。數字越大越慢、越穩。"}, + {"Dead band", "死區"}, + {"The virtual camera ignores a player's movement inside this distance. 0 follows every movement.", + "玩家在這個距離內移動時，虛擬攝影機不跟隨。設為 0 則每次移動都跟。"}, + {"Advanced settings", "進階設定"}, {"No Kinect found", "找不到 Kinect"}, {"No Kinect SDK is installed", "尚未安裝 Kinect SDK"}, {"No Kinect connected", "沒有連接 Kinect"}, @@ -223,15 +245,14 @@ constexpr std::array, TextCount> texts{{ "恢復 Kinect 的語音指令：start、OK、back、next、pause……英文或中文都可以（開始、確定、返回、下一步、暫停…）。使用 Windows 語音辨識與預設麥克風：要用 Kinect 的麥克風陣列，請把它設為預設錄音裝置。"}, {"Open preview", "開啟預覽"}, {"Close preview", "關閉預覽"}, - {"Show picture", "顯示畫面"}, - {"Close picture", "關閉畫面"}, - {"Kinect calibrated: crouch depth saved", "Kinect 校正完成：已儲存蹲下深度"}, + {"Open preview", "開啟預覽"}, + {"Close preview", "關閉預覽"}, {"Skeleton debug window", "骨架 Debug 視窗"}, {"Shows only the skeleton received by the game, in front and side views, with no camera image. Closing this window does not stop the game. Changes apply on the next game launch.", "僅顯示遊戲收到的骨架，提供正面與側面視圖，不顯示攝影機影像。關閉此視窗不會停止遊戲。設定會在下次啟動遊戲時套用。"}, {"Menu cursor steadying", "選單游標穩定"}, - {"A hand held still on a button still wanders a few millimetres, and the cursor with it. Movement inside this circle is ignored in the menus (never while racing); a larger circle is steadier but a little stickier. 0 turns it off.", - "手停在按鈕上時仍會晃動幾公釐，游標也跟著抖。在選單中（比賽時不作用），手在這個圓圈內的移動會被忽略；圈越大越穩，但會稍微黏手。設為 0 關閉。"}, + {"A hand held still on a button still wanders a few millimetres, and the cursor with it. In the menus, movement inside this circle is ignored, and it has no effect while racing; a larger number is steadier but a little stickier. 0 turns it off.", + "手停在按鈕上時仍會晃動幾公釐，游標也跟著抖。在選單時，手在這個圓圈內的移動會被忽略，比賽中無效；數字越大越穩定，但會稍微黏手，設為0關閉。"}, {"Off", "關閉"}, {"Game code image", "遊戲程式映像"}, {"The game's decoded code and data (complete.txt, image.bin).", "解碼後的遊戲程式與資料（complete.txt、image.bin）。"}, @@ -453,15 +474,6 @@ constexpr std::array, TextCount> texts{{ {"If you start anyway, the game keeps trying every few seconds, and Xbox LIVE opens once the server answers.", "若仍要啟動，遊戲會每隔幾秒再試著連線，連上伺服器後就能進入 Xbox LIVE。"}, {"Start anyway", "仍要啟動"}, - {"Smooth Kinect v2 tracking", "平滑 Kinect v2 追蹤"}, - {"Steadies a Kinect v2's joints with the same filter Kinect v1's SDK uses, for about a frame of delay. Turn it off to feel the sensor as measured.", - "用與 Kinect v1 SDK 相同的濾波器讓 Kinect v2 的關節更穩定，約多一格延遲。關閉則使用感應器原始量測。"}, - {"Lean sensitivity", "傾身靈敏度"}, - {"How much steering a lean gives in races. Lower it if small movements turn too much.", - "比賽中身體傾斜帶來的轉向幅度。若小動作就轉太多，請調低。"}, - {"Crouch depth", "蹲下深度"}, - {"How far your hips must drop to count as a crouch. Raise it if you crouch by accident; lower it if crouches are missed.", - "臀部要下降多少才算蹲下。若常誤判為蹲下就調高；若蹲下沒反應就調低。"}, {"Custom", "自訂"}, {"Custom window size", "自訂視窗大小"}, {"Width and height in pixels (160 to 16384). Press Enter or leave the box to apply.", @@ -1427,6 +1439,8 @@ struct Launcher { // power adapter not in), or starting it. std::string kinect_failure; // The camera choice under the pointer while its list is open, or -1. + int v2_view_hovered = -1; + bool advanced_open = false, voice_open = false; int camera_hovered = -1; // The preview window, when open, holds the Kinect (one session a // process); it has the tilt buttons. @@ -1987,6 +2001,35 @@ struct Launcher { ImGui::Dummy(ImVec2(0, 4 * scale)); } + // A section heading that folds: the look of settings_section, with a + // triangle instead of the dash (right when folded, down when open). + void collapsible_section(Text label, bool& open) { + ImGui::Dummy(ImVec2(0, 4 * scale)); + const char* text = tr(label); + const float line_height = ImGui::GetTextLineHeight(), height = line_height + 6 * scale; + const ImVec2 start = ImGui::GetCursorScreenPos(); + const float width = ImGui::GetContentRegionAvail().x; + ImGui::PushID(text); + if (ImGui::InvisibleButton("##section", ImVec2(width, height))) open = !open; + const bool hot = ImGui::IsItemHovered() || ImGui::IsItemFocused(); + ImGui::PopID(); + ImDrawList* draw = ImGui::GetWindowDrawList(); + const ImU32 colour = color(cyan, hot ? 1.0f : 0.85f); + const float half = line_height * 0.28f; + const ImVec2 centre(start.x + half + 2 * scale, start.y + height * 0.5f); + if (open) + draw->AddTriangleFilled(ImVec2(centre.x - half, centre.y - half * 0.6f), ImVec2(centre.x + half, centre.y - half * 0.6f), + ImVec2(centre.x, centre.y + half * 0.8f), colour); + else + draw->AddTriangleFilled(ImVec2(centre.x - half * 0.6f, centre.y - half), ImVec2(centre.x - half * 0.6f, centre.y + half), + ImVec2(centre.x + half * 0.8f, centre.y), colour); + const float text_x = start.x + 2 * half + 10 * scale; + draw->AddText(ImVec2(text_x, start.y + 3 * scale), colour, text); + const float line_x = text_x + ImGui::CalcTextSize(text).x + 8 * scale; + draw->AddLine(ImVec2(line_x, centre.y), ImVec2(start.x + width, centre.y), ImGui::GetColorU32(ImGuiCol_Separator), 1.0f); + ImGui::Dummy(ImVec2(0, 4 * scale)); + } + void display_settings() { std::vector sizes = reported_sizes; const bool original = settings.aspect_ratio == "16:9"; @@ -2677,22 +2720,104 @@ struct Launcher { ImGui::PopID(); } + // A Kinect v2's depth view, and Recompiled's tuning when that is chosen. + void kinect_v2_view_rows(float width, float scale) { + // Recompiled first (the default), then Simulated, then Original. + static constexpr const char* values[] = {"ideal", "fit", "sensor"}; + static constexpr Text labels[] = {V2ViewIdeal, V2ViewFit, V2ViewSensor}; + static constexpr Text hints[] = {V2ViewHintIdeal, V2ViewHintFit, V2ViewHintSensor}; + int chosen = 0; + for (int i = 0; i < 3; ++i) if (settings.kinect_v2_view == values[i]) chosen = i; + const int described = v2_view_hovered >= 0 ? v2_view_hovered : chosen; + setting_row(tr(V2View), tr(hints[described]), width, scale, [&] { + v2_view_hovered = -1; + ImGui::SetNextItemWidth(width); + if (ImGui::BeginCombo("##v2_view", tr(labels[chosen]))) { + for (int i = 0; i < 3; ++i) { + if (ImGui::Selectable(tr(labels[i]), chosen == i)) settings.kinect_v2_view = values[i]; + if (ImGui::IsItemHovered() || ImGui::IsItemFocused()) v2_view_hovered = i; + } + ImGui::EndCombo(); + } + }); + } + + // Recompiled's fine tuning, one slider a row (in the Advanced settings). + void kinect_v2_tuning_rows(float width, float scale) { + // One slider a row, in whole units, as wide as the camera row. + const auto slider = [&](Text label, Text hint, const char* id, uint32_t& value, int low, int high, int step, + const std::function& show) { + setting_row(tr(label), tr(hint), width, scale, [&] { + int shown_value = int(value); + char shown[48]; + show(shown, sizeof shown, shown_value); + ImGui::SetNextItemWidth(width); + bool moved = ImGui::SliderInt(id, &shown_value, low, high, shown); + moved |= slider_steps(&shown_value, low, high, step); + if (moved) value = uint32_t(std::clamp(shown_value, low, high)); + }); + }; + const auto centimetres = [](char* out, size_t size, int value) { std::snprintf(out, size, "%.2f m", value / 100.0); }; + slider(V2Distance, V2DistanceHint, "##v2_distance", settings.kinect_v2_distance, 100, 400, 5, centimetres); + slider(V2Height, V2HeightHint, "##v2_height", settings.kinect_v2_height, 20, 200, 5, centimetres); + slider(V2Follow, V2FollowHint, "##v2_follow", settings.kinect_v2_follow, 2, 200, 1, + [](char* out, size_t size, int value) { std::snprintf(out, size, "%.1f s", value / 10.0); }); + slider(V2Deadband, V2DeadbandHint, "##v2_deadband", settings.kinect_v2_deadband, 0, 100, 1, centimetres); + } + void motion_settings() { const float switch_width = ImGui::GetFrameHeight() * 1.9f; const float camera_width = 150 * scale; // The hint says what the chosen setting does, or while the list is // open the one under the pointer (from the frame before). - static constexpr const char* camera_values[] = {"off", "picture", "motion", "kinect"}; - static constexpr Text camera_hints[] = {CameraHintOff, CameraHintPicture, CameraHint, CameraHintKinect}; + static constexpr const char* camera_values[] = {"off", "motion", "kinect", "kinect_v2"}; + static constexpr Text camera_hints[] = {CameraHintOff, CameraHint, CameraHintKinect, CameraHintKinectV2}; + // The picture-only choice is gone: an old setting means off. + if (settings.camera == "picture") settings.camera = "off"; int camera_chosen = 0; for (int i = 0; i < 4; ++i) if (settings.camera == camera_values[i]) camera_chosen = i; const int camera_described = camera_hovered >= 0 ? camera_hovered : camera_chosen; - setting_row(tr(CameraLabel), tr(camera_hints[camera_described]), camera_width, scale, [&] { - const Text labels[] = {CameraOff, CameraPicture, CameraMotion, CameraKinect}; + const bool kinect_v2_chosen = settings.camera == "kinect_v2"; + const bool kinect_chosen = settings.camera == "kinect" || kinect_v2_chosen; + // What the sensor sees, in a window of its own: whether it works, + // its cameras and skeletons, and its tilt. Its button sits in the + // camera's row. + if (kinect_chosen && kinect_preview) { + if (const std::string why = kinect_preview->failure(); !why.empty()) kinect_failure = why; + if (kinect_preview->closed()) kinect_preview.reset(); + // The view and Recompiled's tuning as they are now, so the open + // preview follows the sliders; the preview never writes back. + else kinect_preview->set_depth_choice(sfr::launcher_kinect_depth_choice(settings)); + } + const float preview_button = (std::max)(ImGui::CalcTextSize(tr(KinectPreviewOpen)).x, + ImGui::CalcTextSize(tr(KinectPreviewClose)).x) + ImGui::GetStyle().FramePadding.x * 4; + // A webcam's picture window sits in the same place (Windows only). + bool webcam_preview = false; +#ifdef _WIN32 + webcam_preview = settings.camera == "motion"; +#endif + // Which camera: the host's list, read once and again whenever the + // camera is turned on, since one may have been plugged in since. + // A Kinect is the host's only one: no list, only whether it answers. + if (settings.camera == "motion" && cameras_listed_for != settings.camera) { + cameras = sfr::CameraCapture::devices(); + cameras_listed_for = settings.camera; + } + if (camera_preview && camera_preview->closed()) camera_preview.reset(); + // The camera is remembered by name, so the row shows the saved + // name even when that camera is not here today. + const char* current = settings.camera_device.empty() + ? (cameras.empty() ? tr(CameraNone) : cameras.front().c_str()) + : settings.camera_device.c_str(); + const float row_width = kinect_chosen || webcam_preview ? camera_width + ImGui::GetStyle().ItemSpacing.x + preview_button + : camera_width; + setting_row(tr(CameraLabel), tr(camera_hints[camera_described]), row_width, scale, [&] { + const Text labels[] = {CameraOff, CameraMotion, CameraKinect, CameraKinectV2}; // A real sensor only where there is a runtime to reach it. - const int choices = sfr::KinectSensor::supported() ? 4 : 3; + const int choices = sfr::KinectSensor::supported() ? 4 : 2; const int chosen = (std::min)(camera_chosen, choices - 1); camera_hovered = -1; + const float control_x = ImGui::GetCursorPosX(); ImGui::SetNextItemWidth(camera_width); if (ImGui::BeginCombo("##camera", tr(labels[chosen]))) { for (int i = 0; i < choices; ++i) { @@ -2701,131 +2826,69 @@ struct Launcher { } ImGui::EndCombo(); } - }); - // Which camera: the hosts's list, read once and again whenever the - // camera is turned on, since one may have been plugged in since. - // A Kinect is the host's only one: no list, only whether it answers. - if (settings.camera == "kinect") { - // What the sensor sees, in a window of its own: whether it works, - // its cameras and skeletons, and its tilt. - if (kinect_preview) { - if (const std::string why = kinect_preview->failure(); !why.empty()) kinect_failure = why; - // A finished calibration (the preview's Calibrate) sets the - // crouch depth from the player's own crouch. - if (const auto calibrated = kinect_preview->take_calibration()) { - settings.kinect_crouch_cm = calibrated->crouch_cm; - sfr::save_launcher_settings(settings_file, settings); - status = tr(KinectCalibrated); - status_error = false; - } - if (kinect_preview->closed()) kinect_preview.reset(); - } - const Text label = kinect_preview ? KinectPreviewClose : KinectPreviewOpen; - const float button = (std::max)(ImGui::CalcTextSize(tr(KinectPreviewOpen)).x, - ImGui::CalcTextSize(tr(KinectPreviewClose)).x) + ImGui::GetStyle().FramePadding.x * 4; - setting_row(tr(KinectRow), tr(KinectRowHint), button, scale, [&] { - if (ImGui::Button(tr(label), ImVec2(button, 0))) { + if (kinect_chosen) { + ImGui::SameLine(); + if (ImGui::Button(tr(kinect_preview ? KinectPreviewClose : KinectPreviewOpen), ImVec2(preview_button, 0))) { if (kinect_preview) kinect_preview.reset(); else { kinect_failure.clear(); - kinect_preview = sfr::KinectPreviewWindow::open(language == 1); + kinect_preview = sfr::KinectPreviewWindow::open(language == 1, kinect_v2_chosen ? 2 : 1, + sfr::launcher_kinect_depth_choice(settings)); } } - }); + } else if (webcam_preview) { + ImGui::SameLine(); + ImGui::BeginDisabled(cameras.empty()); + if (ImGui::Button(tr(camera_preview ? KinectPreviewClose : KinectPreviewOpen), ImVec2(preview_button, 0))) { + if (camera_preview) camera_preview.reset(); + else camera_preview = sfr::CameraPreviewWindow::open(current, language == 1); + } + ImGui::EndDisabled(); + } + // A webcam's list sits under the row's controls, beside the hint, + // as wide as the row; the hint is drawn after this from the same + // line, on the left. + if (settings.camera == "motion") { + const float line_x = ImGui::GetCursorPosX(), line_y = ImGui::GetCursorPosY(); + ImGui::SetCursorPos(ImVec2(control_x, line_y)); + ImGui::SetNextItemWidth(row_width); + if (ImGui::BeginCombo("##camera_device", current)) { + for (const auto& camera : cameras) + if (ImGui::Selectable(camera.c_str(), camera == settings.camera_device)) { + settings.camera_device = camera; + camera_trial.store(CameraTrial::none); + } + ImGui::EndCombo(); + } + ImGui::SetCursorPos(ImVec2(line_x, line_y)); + } + }); + if (kinect_chosen) { // Anything short of a working sensor: what to install and check. if (!kinect_failure.empty()) { const Text why = kinect_failure == "no-runtime" || kinect_failure == "incomplete-runtime" ? KinectNoRuntime : kinect_failure == "no-sensor" ? KinectNoSensor : KinectFailed; const float padding = ImGui::GetStyle().FramePadding.x * 4; - const float v1_width = ImGui::CalcTextSize(tr(KinectDownload)).x + padding; + // The Xbox 360 choice falls back to a v2, so it offers both; + // the v2 choice never opens a v1. + const float v1_width = kinect_v2_chosen ? 0.0f : ImGui::CalcTextSize(tr(KinectDownload)).x + padding; const float v2_width = ImGui::CalcTextSize(tr(KinectDownloadV2)).x + padding; setting_row(tr(why), tr(KinectInstallHint), - v1_width + v2_width + ImGui::GetStyle().ItemSpacing.x, scale, [&] { - if (ImGui::Button(tr(KinectDownload), ImVec2(v1_width, 0))) - sfr::launcher::open_url("https://www.microsoft.com/download/details.aspx?id=40278"); - ImGui::SameLine(); + v1_width + v2_width + (kinect_v2_chosen ? 0.0f : ImGui::GetStyle().ItemSpacing.x), scale, [&] { + if (!kinect_v2_chosen) { + if (ImGui::Button(tr(KinectDownload), ImVec2(v1_width, 0))) + sfr::launcher::open_url("https://www.microsoft.com/download/details.aspx?id=40278"); + ImGui::SameLine(); + } if (ImGui::Button(tr(KinectDownloadV2), ImVec2(v2_width, 0))) sfr::launcher::open_url("https://www.microsoft.com/download/details.aspx?id=44561"); }); } - // Tuning (Issue #97); applied the next time the game starts. - const float kinect_switch = ImGui::GetFrameHeight() * 1.9f; - const float kinect_slider = 260 * scale; - setting_row(tr(KinectSmoothLabel), tr(KinectSmoothHint), kinect_switch, scale, - [&] { toggle("##kinect_smooth", &settings.kinect_smooth); }); - setting_row(tr(KinectLeanLabel), tr(KinectLeanHint), kinect_slider, scale, [&] { - int lean = int(settings.kinect_lean); - ImGui::SetNextItemWidth(kinect_slider); - bool moved = ImGui::SliderInt("##kinect_lean", &lean, 10, 100, "%d%%"); - moved |= slider_steps(&lean, 10, 100, 5); - if (moved) settings.kinect_lean = uint32_t(lean); - }); - setting_row(tr(KinectCrouchLabel), tr(KinectCrouchHint), kinect_slider, scale, [&] { - int crouch = int(settings.kinect_crouch_cm); - ImGui::SetNextItemWidth(kinect_slider); - bool moved = ImGui::SliderInt("##kinect_crouch", &crouch, 5, 35, "%d cm"); - moved |= slider_steps(&crouch, 5, 35, 1); - if (moved) settings.kinect_crouch_cm = uint32_t(crouch); - }); + // No smoothing setting: the game smooths a real Kinect's + // skeletons itself, once, as the title asks (nui_smooth_hooks.cpp). + if (kinect_v2_chosen) kinect_v2_view_rows(row_width, scale); } else if (settings.camera != "off") { - if (cameras_listed_for != settings.camera) { - cameras = sfr::CameraCapture::devices(); - cameras_listed_for = settings.camera; - } - const float test_width = ImGui::CalcTextSize(tr(CameraTesting)).x + ImGui::GetStyle().FramePadding.x * 4; - if (camera_preview && camera_preview->closed()) camera_preview.reset(); -#ifdef _WIN32 - const float preview_width = (std::max)(ImGui::CalcTextSize(tr(CameraPreviewOpen)).x, - ImGui::CalcTextSize(tr(CameraPreviewClose)).x) + - ImGui::GetStyle().FramePadding.x * 4; - const float buttons_width = test_width + ImGui::GetStyle().ItemSpacing.x + preview_width; -#else - const float buttons_width = test_width; -#endif - const float device_width = 230 * scale + buttons_width; - // The camera is remembered by name, so the row shows the saved - // name even when that camera is not here today. - const char* current = settings.camera_device.empty() - ? (cameras.empty() ? tr(CameraNone) : cameras.front().c_str()) - : settings.camera_device.c_str(); - const Text trial[] = {CameraTest, CameraTesting, CameraWorks, CameraSilent, CameraClosed}; - setting_row(tr(CameraDevice), tr(CameraDeviceHint), device_width, scale, [&] { - ImGui::SetNextItemWidth(device_width - buttons_width - ImGui::GetStyle().ItemSpacing.x); - if (ImGui::BeginCombo("##camera_device", current)) { - for (const auto& camera : cameras) - if (ImGui::Selectable(camera.c_str(), camera == settings.camera_device)) { - settings.camera_device = camera; - camera_trial.store(CameraTrial::none); - } - ImGui::EndCombo(); - } - ImGui::SameLine(); - const auto state = camera_trial.load(); - // One of them holds the camera at a time. - ImGui::BeginDisabled(cameras.empty() || state == CameraTrial::looking || camera_preview); - if (ImGui::Button(tr(trial[int(state)]), ImVec2(test_width, 0))) { - camera_trial.store(CameraTrial::looking); - camera_trial_worker = std::jthread([this, wanted = std::string(current)] { - auto camera = sfr::CameraCapture::open(640, 480, wanted); - if (!camera) { camera_trial.store(CameraTrial::closed); return; } - sfr::CameraFrame frame; - for (int attempt = 0; attempt < 300 && !frame.number; ++attempt) - if (!camera->next(frame)) std::this_thread::sleep_for(std::chrono::milliseconds(10)); - camera_trial.store(frame.number ? CameraTrial::pictures : CameraTrial::silent); - }); - } - ImGui::EndDisabled(); -#ifdef _WIN32 - ImGui::SameLine(); - ImGui::BeginDisabled(cameras.empty() || state == CameraTrial::looking); - if (ImGui::Button(tr(camera_preview ? CameraPreviewClose : CameraPreviewOpen), ImVec2(preview_width, 0))) { - if (camera_preview) camera_preview.reset(); - else camera_preview = sfr::CameraPreviewWindow::open(current, language == 1); - } - ImGui::EndDisabled(); -#endif - }); if (settings.camera == "motion") { setting_row(tr(CameraMirror), tr(CameraMirrorHint), switch_width, scale, [&] { toggle("##camera_mirror", &settings.camera_mirror); }); @@ -2835,25 +2898,33 @@ struct Launcher { #endif } } - if (settings.camera == "motion" || settings.camera == "kinect") { - const float slider_width = 220 * scale; - setting_row(tr(MenuSteady), tr(MenuSteadyHint), slider_width, scale, [&] { - int radius = int((std::min)(settings.menu_steady_radius, 30u)); - char shown[32]; - if (radius) std::snprintf(shown, sizeof shown, "%d mm", radius); - else std::snprintf(shown, sizeof shown, "%s", tr(MenuSteadyOff)); - ImGui::SetNextItemWidth(slider_width); - bool moved = ImGui::SliderInt("##menu_steady", &radius, 0, 30, shown); - moved |= slider_steps(&radius, 0, 30, 1); - if (moved) settings.menu_steady_radius = uint32_t(std::clamp(radius, 0, 30)); - }); - // For a bug report: numbers in game.log (motion_trace.h). - setting_row(tr(MotionTrace), tr(MotionTraceHint), ImGui::GetFrameHeight() * 1.9f, scale, - [&] { toggle("##motion_trace", &settings.motion_trace); }); + if (settings.camera == "motion" || settings.camera == "kinect" || kinect_v2_chosen) { + // Fine tuning, folded away until asked for: Recompiled's and the + // menu cursor's. With a Kinect the sliders are as wide as the + // camera row above them. + collapsible_section(V2Advanced, advanced_open); + if (advanced_open) { + if (kinect_v2_chosen && settings.kinect_v2_view == "ideal") kinect_v2_tuning_rows(row_width, scale); + const float slider_width = kinect_chosen ? row_width : 220 * scale; + setting_row(tr(MenuSteady), tr(MenuSteadyHint), slider_width, scale, [&] { + int radius = int((std::min)(settings.menu_steady_radius, 30u)); + char shown[32]; + if (radius) std::snprintf(shown, sizeof shown, "%d mm", radius); + else std::snprintf(shown, sizeof shown, "%s", tr(MenuSteadyOff)); + ImGui::SetNextItemWidth(slider_width); + bool moved = ImGui::SliderInt("##menu_steady", &radius, 0, 30, shown); + moved |= slider_steps(&radius, 0, 30, 1); + if (moved) settings.menu_steady_radius = uint32_t(std::clamp(radius, 0, 30)); + }); + // For a bug report: numbers in game.log (motion_trace.h). + setting_row(tr(MotionTrace), tr(MotionTraceHint), ImGui::GetFrameHeight() * 1.9f, scale, + [&] { toggle("##motion_trace", &settings.motion_trace); }); + } } if (sfr::VoiceRecognizer::supported()) { - settings_section(SectionVoice); - setting_row(tr(VoiceLabel), tr(VoiceHint), switch_width, scale, [&] { toggle("##voice", &settings.voice); }); + collapsible_section(SectionVoice, voice_open); + if (voice_open) + setting_row(tr(VoiceLabel), tr(VoiceHint), switch_width, scale, [&] { toggle("##voice", &settings.voice); }); } } diff --git a/src/launcher_settings.cpp b/src/launcher_settings.cpp index 9d05950..19a9091 100644 --- a/src/launcher_settings.cpp +++ b/src/launcher_settings.cpp @@ -135,13 +135,19 @@ LauncherSettings parse_launcher_settings(const std::string& text) { else if (key == "vulkan") read_flag(value, settings.vulkan); else if (key == "touch_controls") read_flag(value, settings.touch_controls); else if (key == "tilt") read_flag(value, settings.tilt); - else if (key == "camera" && (value == "off" || value == "picture" || value == "motion" || value == "kinect")) settings.camera = value; + else if (key == "camera" && (value == "off" || value == "picture" || value == "motion" || value == "kinect" || value == "kinect_v2")) settings.camera = value; else if (key == "camera_device" && value.size() <= 128) settings.camera_device = value; else if (key == "camera_mirror") read_flag(value, settings.camera_mirror); - else if (key == "kinect_smooth") read_flag(value, settings.kinect_smooth); - else if (key == "kinect_lean") read_number(value, 10, 100, settings.kinect_lean); - else if (key == "kinect_crouch_cm") read_number(value, 5, 35, settings.kinect_crouch_cm); + // kinect_smooth (the v2's smoothing switch) is gone: the game smooths + // a real Kinect's skeletons once, as the title asks. An old file's + // key is ignored like any unknown one and not written again. else if (key == "menu_steady_radius") read_number(value, 0, 30, settings.menu_steady_radius); + else if (key == "kinect_v2_view" && (value == "fit" || value == "sensor" || value == "ideal")) + settings.kinect_v2_view = value; + else if (key == "kinect_v2_distance") read_number(value, 100, 400, settings.kinect_v2_distance); + else if (key == "kinect_v2_height") read_number(value, 20, 200, settings.kinect_v2_height); + else if (key == "kinect_v2_follow") read_number(value, 2, 200, settings.kinect_v2_follow); + else if (key == "kinect_v2_deadband") read_number(value, 0, 100, settings.kinect_v2_deadband); else if (key == "voice") read_flag(value, settings.voice); else if (key == "camera_debug") read_flag(value, settings.camera_debug); else if (key == "motion_trace") read_flag(value, settings.motion_trace); @@ -221,10 +227,12 @@ std::string format_launcher_settings(const LauncherSettings& s) { << "camera=" << s.camera << '\n' << "camera_device=" << s.camera_device << '\n' << "camera_mirror=" << s.camera_mirror << '\n' - << "kinect_smooth=" << s.kinect_smooth << '\n' - << "kinect_lean=" << s.kinect_lean << '\n' - << "kinect_crouch_cm=" << s.kinect_crouch_cm << '\n' << "menu_steady_radius=" << (std::min)(s.menu_steady_radius, 30u) << '\n' + << "kinect_v2_view=" << s.kinect_v2_view << '\n' + << "kinect_v2_distance=" << s.kinect_v2_distance << '\n' + << "kinect_v2_height=" << s.kinect_v2_height << '\n' + << "kinect_v2_follow=" << s.kinect_v2_follow << '\n' + << "kinect_v2_deadband=" << s.kinect_v2_deadband << '\n' << "voice=" << s.voice << '\n' << "camera_debug=" << s.camera_debug << '\n' << "motion_trace=" << s.motion_trace << '\n' @@ -304,6 +312,29 @@ bool has_game_save(const std::filesystem::path& saves) { return false; } +namespace { +bool v2_recompiled(const LauncherSettings& s) { return s.camera == "kinect_v2" && s.kinect_v2_view == "ideal"; } +// A hundredths value as the decimal text the game reads (250 -> "2.50"). +std::string centi(uint32_t value) { + char text[32]; + std::snprintf(text, sizeof text, "%u.%02u", value / 100, value % 100); + return text; +} +} // namespace + +KinectDepthChoice launcher_kinect_depth_choice(const LauncherSettings& s) { + KinectDepthChoice choice; + choice.view = s.kinect_v2_view == "fit" ? KinectDepthChoiceView::fit + : s.kinect_v2_view == "sensor" ? KinectDepthChoiceView::sensor + : KinectDepthChoiceView::ideal; + // As game_environment writes them: centimetres, follow in tenths. + choice.distance = float(s.kinect_v2_distance) / 100.0f; + choice.height = float(s.kinect_v2_height) / 100.0f; + choice.follow = float(s.kinect_v2_follow) / 10.0f; + choice.deadband = float(s.kinect_v2_deadband) / 100.0f; + return choice; +} + std::vector> game_environment(const LauncherSettings& s, const std::filesystem::path& directory) { const auto [window_width, window_height] = window_size(s); @@ -360,14 +391,24 @@ std::vector> game_environment(const Launcher {"SFR_GRAPHICS", s.vulkan ? "vulkan" : "d3d12"}, // "off" leaves the camera alone; "picture" opens it for the image the // title shows; "motion" also drives the Kinect player's body with it; - // "kinect" is a real sensor tracking the players (kinect_sensor.h). - {"SFR_CAMERA", s.camera == "off" ? "" : s.camera}, + // "kinect" is a real sensor tracking the players (kinect_sensor.h): + // the Xbox 360 one ("kinect", a v2 stands in when no v1 opens) or the + // Kinect v2 ("kinect_v2"). Both reach the game as SFR_CAMERA=kinect, + // the model as SFR_KINECT_MODEL (2 opens a v2 only; unset or 1 is + // the Xbox 360 Kinect). + {"SFR_CAMERA", s.camera == "off" ? "" : s.camera == "kinect_v2" ? "kinect" : s.camera}, + {"SFR_KINECT_MODEL", s.camera == "kinect_v2" ? "2" : s.camera == "kinect" ? "1" : ""}, + // A Kinect v2's depth view and Recompiled's tuning (kinect_v2_depth.h); + // only for the v2 choice, so an inherited value is cleared otherwise. + {"SFR_KINECT_V2_DEPTH_VIEW", s.camera == "kinect_v2" ? s.kinect_v2_view : ""}, + {"SFR_KINECT_V2_DEPTH_DISTANCE", v2_recompiled(s) ? centi(s.kinect_v2_distance) : ""}, + {"SFR_KINECT_V2_DEPTH_HEIGHT", v2_recompiled(s) ? centi(s.kinect_v2_height) : ""}, + {"SFR_KINECT_V2_DEPTH_FOLLOW", v2_recompiled(s) ? centi(s.kinect_v2_follow * 10) : ""}, + {"SFR_KINECT_V2_DEPTH_DEADBAND", v2_recompiled(s) ? centi(s.kinect_v2_deadband) : ""}, + // No skeleton distance of its own any more: clear an inherited one. + {"SFR_KINECT_V2_SKELETON_DISTANCE", ""}, {"SFR_CAMERA_DEVICE", s.camera_device}, {"SFR_CAMERA_MIRROR", s.camera_mirror ? "1" : "0"}, - // Left unset at the defaults, so the game's own choices stand. - {"SFR_KINECT_SMOOTH", s.kinect_smooth ? "" : "0"}, - {"SFR_KINECT_LEAN_SCALE", s.kinect_lean == 100 ? "" : fixed_decimal(3.5 * s.kinect_lean / 100.0)}, - {"SFR_KINECT_CROUCH_DEPTH", s.kinect_crouch_cm == 15 ? "" : fixed_decimal(s.kinect_crouch_cm / 100.0)}, {"SFR_MENU_HAND_STEADY", s.menu_steady_radius ? "1" : "0"}, {"SFR_MENU_STEADY_RADIUS", s.menu_steady_radius ? std::to_string((std::min)(s.menu_steady_radius, 30u)) : ""}, {"SFR_VOICE", s.voice ? "1" : ""}, diff --git a/src/launcher_settings.h b/src/launcher_settings.h index f4ef76d..9c1ab7d 100644 --- a/src/launcher_settings.h +++ b/src/launcher_settings.h @@ -1,4 +1,6 @@ #pragma once +#include "kinect_ideal_diagram.h" + #include #include #include @@ -60,8 +62,10 @@ struct LauncherSettings { // gamepad drives; "camera" tracks a real body with a webcam, which cannot // share the pad's sticks, so the two are one choice. "picture" keeps the // pad in charge and only gives the title a camera image. - // "kinect" is a real Kinect tracking the players, as on the console. - std::string camera = "off"; // "off", "picture", "motion" or "kinect" + // "kinect" is a real Xbox 360 Kinect tracking the players, as on the + // console (a Kinect v2 stands in when no v1 opens); "kinect_v2" is a + // Kinect v2 for Xbox One only. + std::string camera = "off"; // "off", "picture", "motion", "kinect" or "kinect_v2" // Which of the host's cameras, by the name it lists (SFR_CAMERA_DEVICE). // A name and not a number, because the list changes: a phone camera or a // capture card comes and goes, and the number then means another camera. @@ -71,18 +75,28 @@ struct LauncherSettings { // it is a setting: with it wrong, left and right are the wrong way // round and the cursor runs off the side of the screen. bool camera_mirror = false; // SFR_CAMERA_MIRROR - // A real Kinect's tuning (Issue #97). Smoothing steadies a v2's joints - // (v1's SDK smooths its own when asked); the lean is the steering a body - // lean gives, as a percent of the most; the crouch is how far the hips - // drop for a crouch, in centimetres. The defaults are the game's own. - bool kinect_smooth = true; // SFR_KINECT_SMOOTH=0 when off - uint32_t kinect_lean = 100; // SFR_KINECT_LEAN_SCALE = 3.5 x percent - uint32_t kinect_crouch_cm = 15; // SFR_KINECT_CROUCH_DEPTH in metres + // A real Kinect has no tuning here any more: the game smooths its + // skeletons once as the title asks (an old kinect_smooth key is ignored), + // and its own depth detectors decide lean and crouch. // The menu cursor's steadying (pose_stability.h), webcam and Kinect: the // circle a hand held still may wander in without moving the cursor, in // millimetres; 0 turns it off. SFR_MENU_STEADY_RADIUS and // SFR_MENU_HAND_STEADY. - uint32_t menu_steady_radius = 8; + uint32_t menu_steady_radius = 0; + // How a Kinect v2's depth becomes the v1's (SFR_KINECT_V2_DEPTH_VIEW): + // "ideal" (Recompiled, the launcher's default: a virtual level v1 that + // follows the players), "fit" (Simulated, the v2's whole view fitted into + // the v1's) or "sensor" (Original, the v2's own view). The game itself, + // without the variable, still makes fit (KinectDepthSettings); only the + // launcher's default is Recompiled. Recompiled's own tuning, in centimetres + // (follow in tenths of a second): how far in front of the players the + // virtual camera stands, how high above the floor, how slowly it follows, + // the dead band it ignores. + std::string kinect_v2_view = "ideal"; + uint32_t kinect_v2_distance = 250; // 100..400 + uint32_t kinect_v2_height = 100; // 20..200 + uint32_t kinect_v2_follow = 20; // 2..200 + uint32_t kinect_v2_deadband = 15; // 0..100 // Voice commands through the host's speech recognizer and the default // microphone (a Kinect's array, made the default, is heard), in English // or Traditional Chinese; the pad's buttons still say them too. @@ -154,6 +168,11 @@ std::filesystem::path resolved_save_directory(const LauncherSettings& settings, // Whether a saves' folder holds a game save (/00000001/SfrAllData.sav/). bool has_game_save(const std::filesystem::path& save_directory); +// A Kinect v2's depth view and Recompiled's tuning in metres and seconds, +// for the launcher's preview (kinect_ideal_diagram.h): the same values +// game_environment hands the game for Recompiled. +KinectDepthChoice launcher_kinect_depth_choice(const LauncherSettings& settings); + // The game's environment for these settings: the play defaults (no // diagnostic limits or traces, 60 fps cap) plus the player's choices. An // empty value removes the variable. diff --git a/src/native_presentation.cpp b/src/native_presentation.cpp index f34afab..5ea5d9f 100644 --- a/src/native_presentation.cpp +++ b/src/native_presentation.cpp @@ -1,6 +1,7 @@ #include "avatar_transform.h" #include "native_presentation.h" #include "render_scale.h" +#include "consecutive_failures.h" #include #include "native_graphics.h" @@ -454,6 +455,8 @@ struct NativePresentation::Impl { // VK_SUBOPTIMAL_KHR: the layer is no longer the swap chain's size): the // swap chain is made again next frame, a few times in a row before giving up. uint32_t present_failures = 0; + // A resize refused while the window is dragged: tried again next frame. + uint32_t resize_failures = 0; // MoltenVK still shows an image it calls suboptimal (scaled into the // layer). When a swap chain made again does not help, the game goes on // with it instead of closing (Issue #191); a normal present ends this. @@ -1977,12 +1980,21 @@ void NativePresentation::present(uint32_t area_width, uint32_t area_height, floa std::this_thread::sleep_for(std::chrono::milliseconds(16)); // no frame to pace by return; } + // Dragging the window edge can leave a size the surface refuses + // for a moment (it changes again before the swap chain is made): + // try again next frame, and give up only if it keeps failing. + if (sfr::failure_within_limit(impl_->resize_failures)) { + std::cerr << "NATIVE_PRESENTATION resize_refused attempt=" << impl_->resize_failures << '\n'; + impl_->resize_swap_chain = true; + return; + } #ifdef SFR_IOS impl_->rebuild_swap_chain = true; // a new surface on the same layer, next frame return; #endif throw std::runtime_error("failed to resize the native swap chain"); } + impl_->resize_failures = 0; impl_->rebuild_swap_chain_sync(); if (impl_->blit_pipeline) impl_->build_blit_targets(); std::cerr << "NATIVE_PRESENTATION resized images=" << impl_->swap_chain->getTextureCount() @@ -2166,6 +2178,9 @@ void NativePresentation::present(uint32_t area_width, uint32_t area_height, floa #ifdef _WIN32 ShowWindow(impl_->window, SW_SHOWNORMAL); UpdateWindow(impl_->window); + // Come to the front: the launcher allowed this process to take the + // foreground, and a game that opens behind other windows is easy to miss. + SetForegroundWindow(impl_->window); #else SDL_ShowWindow(impl_->window); #endif diff --git a/src/nui_analysis_hooks.cpp b/src/nui_analysis_hooks.cpp new file mode 100644 index 0000000..4c14a53 --- /dev/null +++ b/src/nui_analysis_hooks.cpp @@ -0,0 +1,145 @@ +#include "ppc_recomp_shared.h" +#include "diagnostic_hooks.h" +#include "guest_memory.h" +#include "kinect_nui_record.h" +#include "nui_analysis_trace.h" + +#include +#include +#include +#include +#include +#include +#include + +// Observation of the title's own use of its skeletons (docs/kinect-sensor.md, +// "觀察用 trace"), with SFR_NUI_ANALYSIS_TRACE=1 only. Each hook runs the +// original as it is and afterwards only reads guest memory. +// +// - NUI_TRACE_CALIB: the race preparation's ("On your Gear!") measuring +// detectors write a player's base values into the calibration object at +// [source + 92] and answer 1 once they have: 822C65B0 (arms forward: +132 +// the hip centre's z, +112/+116 the depth centroid's column and row, +120 +// the outline's height) and 822C6AD8 (the body: +0..+76 lengths and +// directions, +124/+128 head and hips above the floor, +132 the hips' z). +// Each time one of them answers 1 for a source after not doing so, the +// whole object +0..+132 is written once. With SFR_KINECT_RECORD and a +// real Kinect it also goes into the player's next row of -nui.csv +// (kinect_nui_record.h), trace or not. +// - NUI_TRACE_BIND: the binding of players to body records (82222B80, every +// frame from 82222908): its mode at box +176, the two players' records at +// box +120/+124 and their enrollments at box +136/+140 (box = [83E52F88]), +// with the candidates it counted (+180/+184) and the team mode (83E515E7: 3 +// is a relay); a line whenever one of them changes. + +namespace { +constexpr uint32_t nui_box = 0x83E52F88, team_mode = 0x83E515E7, body_reader = 0x82918418; + +sfr::GuestMemory& memory() { return *sfr::active_memory; } +uint32_t word(uint64_t address) { return memory().readable(address, 4) ? memory().load(address) : 0u; } +float number(uint64_t address) { return std::bit_cast(word(address)); } + +// The body record a race source reads (its object's vtable[1]), when that +// is the title's plain reader (lwz r3,4(r3)): no guest call is made. +uint32_t record_of_source(uint32_t source) { + const uint32_t object = source ? word(source) : 0; + const uint32_t vtable = object ? word(object) : 0; + return vtable && word(uint64_t(vtable) + 4) == body_reader ? word(uint64_t(object) + 4) : 0; +} +uint32_t tracking_id_of_record(uint32_t record) { + const uint32_t data = record ? word(uint64_t(record) + 768) : 0; + return data ? word(uint64_t(data) + 4) : 0; +} + +// The player (0, 1) whose body record this is: the binding's (box +// +120/+124), or the input's (its block +12); -1 for none. +int player_of_record(uint32_t record) { + if (!record) return -1; + if (const uint32_t box = word(nui_box)) + for (uint32_t p = 0; p < 2; ++p) + if (word(uint64_t(box) + 120 + 4 * p) == record) return int(p); + if (const uint32_t input = word(0x83E52FB8)) + for (uint32_t p = 0; p < 2; ++p) + if (word(uint64_t(input) + 32 + 2544 * p + 12) == record) return int(p); + return -1; +} + +void calibration_completed(const char* step, uint32_t address, uint32_t source, uint32_t result) { + static std::map, uint32_t> last; + uint32_t& before = last[{address, source}]; + const bool completed = result == 1 && before != 1; + before = result; + if (!completed) return; + const uint32_t calib = word(uint64_t(source) + 92); + const uint32_t record = record_of_source(source); + // SFR_KINECT_RECORD: into the player's next nui.csv row + // (kinect_nui_record.h), for a real Kinect's player. + if (calib && memory().readable(calib, 136) && sfr::nui_body_from_sensor() && sfr::kinect_record_nui()) { + std::array words{}; + for (uint32_t i = 0; i < words.size(); ++i) words[i] = word(uint64_t(calib) + 4 * i); + if (const int player = player_of_record(record); player >= 0) + sfr::kinect_nui_recorder().calibration(uint32_t(player), step, words); + } + static sfr::NuiTraceLimiter limiter("CALIB", 20.0, 2000); + if (!limiter.take()) return; + std::ostringstream line; + line << "NUI_TRACE_CALIB step=" << step << " function=0x" << std::hex << address << " source=0x" << source + << " calib=0x" << calib << " record=0x" << record << std::dec << " tracking_id=" << tracking_id_of_record(record) + << " slot=" << (record ? int(word(uint64_t(record) + 784)) : -1); + if (calib && memory().readable(calib, 136)) { + for (uint32_t offset = 0; offset <= 132; offset += 4) { + line << " +" << offset << '='; + // +112/+116/+120 are whole numbers (the depth centroid and the + // outline's height, in pixels); the rest floats. + if (offset >= 112 && offset <= 120) line << int32_t(word(uint64_t(calib) + offset)); + else line << number(uint64_t(calib) + offset); + } + } + std::cerr << line.str() << '\n'; +} +} + +PPC_FUNC_IMPL(__imp__sub_822C65B0); +SFR_INPUT_HOOK(sub_822C65B0) { + sfr::enter_function(ctx, "sub_822C65B0", 0x822C65B0); + const uint32_t source = ctx.r4.u32; + __imp__sub_822C65B0(ctx, base); + calibration_completed("arms_forward", 0x822C65B0, source, ctx.r3.u32); +} + +PPC_FUNC_IMPL(__imp__sub_822C6AD8); +SFR_INPUT_HOOK(sub_822C6AD8) { + sfr::enter_function(ctx, "sub_822C6AD8", 0x822C6AD8); + const uint32_t source = ctx.r4.u32; + __imp__sub_822C6AD8(ctx, base); + calibration_completed("body", 0x822C6AD8, source, ctx.r3.u32); +} + +PPC_FUNC_IMPL(__imp__sub_82222B80); +SFR_INPUT_HOOK(sub_82222B80) { + sfr::enter_function(ctx, "sub_82222B80", 0x82222B80); + __imp__sub_82222B80(ctx, base); + if (!sfr::nui_analysis_trace()) return; + const uint32_t box = word(nui_box); + if (!box || !memory().readable(box, 188)) return; + const std::array now{word(uint64_t(box) + 176), word(uint64_t(box) + 120), word(uint64_t(box) + 124), + word(uint64_t(box) + 136), word(uint64_t(box) + 140), word(uint64_t(box) + 180), + uint32_t(memory().readable(team_mode, 1) ? memory().load(team_mode) : 0xFF)}; + static std::array before{}; + static bool seen = false; + // The candidate count (+180) alone changes whenever someone steps in or + // out; only the binding itself makes a line. + const bool changed = !seen || now[0] != before[0] || now[1] != before[1] || now[2] != before[2] || + now[3] != before[3] || now[4] != before[4] || now[6] != before[6]; + if (!changed) return; + static sfr::NuiTraceLimiter limiter("BIND", 10.0, 2000); + if (!limiter.take()) return; // tried again next frame: the change is still there + seen = true; + before = now; + std::ostringstream line; + line << "NUI_TRACE_BIND mode=" << int32_t(now[0]) << " team_mode=" << now[6] << " box=0x" << std::hex << box + << " record0=0x" << now[1] << " record1=0x" << now[2] << std::dec << " tracking_id0=" << tracking_id_of_record(now[1]) + << " tracking_id1=" << tracking_id_of_record(now[2]) << " enrollment0=" << int32_t(now[3]) + << " enrollment1=" << int32_t(now[4]) << " candidates=" << now[5] << " candidates_new=" << word(uint64_t(box) + 184); + std::cerr << line.str() << '\n'; +} diff --git a/src/nui_analysis_trace.h b/src/nui_analysis_trace.h new file mode 100644 index 0000000..19540d1 --- /dev/null +++ b/src/nui_analysis_trace.h @@ -0,0 +1,56 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace sfr { + +// SFR_NUI_ANALYSIS_TRACE=1: observation lines for the study of how the title +// uses its skeletons (docs/kinect-sensor.md, "觀察用 trace"). They change +// nothing the title does; each kind writes lines starting NUI_TRACE_, +// spaced out and capped (NuiTraceLimiter), so game.log stays small however +// long a session runs. +inline bool nui_analysis_trace() { + static const bool enabled = [] { + const char* text = std::getenv("SFR_NUI_ANALYSIS_TRACE"); + return text && *text == '1'; + }(); + return enabled; +} + +// One kind's budget: at most per_second lines a second, spaced evenly (a +// call asking for n lines is answered at most every n / per_second +// seconds), and at most total lines in all, after which one line says the +// kind has stopped. Thread safe. +class NuiTraceLimiter { +public: + NuiTraceLimiter(const char* kind, double per_second, uint64_t total) + : kind_(kind), interval_(per_second > 0 ? 1.0 / per_second : 1.0), total_(total) {} + bool take(uint32_t lines = 1) { + if (!nui_analysis_trace()) return false; + std::lock_guard guard(lock_); + if (stopped_) return false; + const auto now = std::chrono::steady_clock::now(); + if (now < next_) return false; + if (written_ + lines > total_) { + stopped_ = true; + std::cerr << "NUI_TRACE_" << kind_ << " stopped after " << written_ << " lines (cap)\n"; + return false; + } + written_ += lines; + next_ = now + std::chrono::duration_cast( + std::chrono::duration(interval_ * lines)); + return true; + } +private: + const char* kind_; + double interval_; + uint64_t total_, written_ = 0; + bool stopped_ = false; + std::chrono::steady_clock::time_point next_{}; + std::mutex lock_; +}; + +} diff --git a/src/nui_hooks.cpp b/src/nui_hooks.cpp index 1123ae6..d34d143 100644 --- a/src/nui_hooks.cpp +++ b/src/nui_hooks.cpp @@ -3,6 +3,9 @@ #include "guest_memory.h" #include "camera_player.h" #include "kinect_sensor.h" +#include "kinect_race_record.h" +#include "kinect_nui_record.h" +#include "kinect_record_depth.h" #include "kinect_preview.h" #include "kinect_tuner.h" #include "camera_tuner.h" @@ -16,6 +19,7 @@ #include "nui_player_routing.h" #include "nui_menu_progress.h" #include "nui_speech.h" +#include "nui_analysis_trace.h" #include "voice_commands.h" #include "local_profile.h" #include "touch_controls.h" @@ -60,7 +64,7 @@ sfr::SkeletonJoints camera_joints{}; bool camera_has_joints = false; // SFR_CAMERA=kinect: a real sensor tracks the players, as on the console. // Its skeletons go to the title untouched, and the race reads them through -// the title's own detectors instead of the pad (nui_body_from_sensor). +// the title's own depth detectors instead of the pad (nui_body_from_sensor). std::unique_ptr kinect; // The menu's Kinect adjustment window over that sensor (kinect_tuner.h). std::mutex tuner_lock; @@ -70,7 +74,14 @@ std::unique_ptr tuner; std::unique_ptr camera_tuner; std::once_flag kinect_start_once; std::atomic sensor_body{false}; -std::atomic sensor_depth{false}; +// When the last depth frame was handed to the game (steady clock, ms): a gap +// of more than half a second is logged once the frames come again, and +// nothing else changes -- the game's own detectors read what it gets. +std::atomic depth_given_ms{INT64_MIN/2}; +// How many depth images the game has been handed, and the newest one's +// number: the depth view (82439530) is then known to have run on that image +// (SFR_KINECT_RECORD, record_depth_view). +std::atomic depth_given_count{0}, depth_given_number{0}; std::atomic kinect_generation{0}; // kinect_frame_generation sfr::KinectFrame kinect_frame; sfr::KinectPlayerSlots kinect_players; @@ -210,13 +221,18 @@ const auto started = std::chrono::steady_clock::now(); void start_kinect() { std::call_once(kinect_start_once,[] { std::string why; - kinect=sfr::KinectSensor::open(&why); - sensor_depth.store(kinect && kinect->has_depth_stream(),std::memory_order_relaxed); + kinect=sfr::KinectSensor::open_selected(sfr::kinect_selected_model(),&why); sensor_body.store(kinect!=nullptr,std::memory_order_relaxed); std::cerr << "NATIVE_KINECT started=" << (kinect?1:0); if(kinect) std::cerr << " model=" << kinect->model() << " depth=" << kinect->has_depth_stream(); else std::cerr << " reason=" << why << " fallback=pad"; std::cerr << '\n'; + // No substitute: without a depth stream the game gets none either + // (NuiImageStreamOpen is left to the library, which fails without a + // sensor) and its own detectors see nothing. + if(kinect && !kinect->has_depth_stream()) + std::cerr << "NATIVE_KINECT_DEPTH unavailable model=" << kinect->model() << " reason=" + << (kinect->depth_failure().empty() ? "unknown" : kinect->depth_failure()) << '\n'; }); } } @@ -350,7 +366,6 @@ bool pause_gesture_active(bool racing) { bool sfr::nui_body_from_sensor() { return sensor_body.load(std::memory_order_relaxed); } -bool sfr::nui_sensor_has_depth() { return sensor_depth.load(std::memory_order_relaxed); } uint64_t sfr::kinect_frame_generation() { return kinect_generation.load(std::memory_order_relaxed); } // NuiInitialize(flags, ?) @@ -393,7 +408,7 @@ SFR_INPUT_HOOK(sub_8276FEE0) { // the raw depth and player index into its buffer (+72) for the depth view's // silhouettes (82439530), which steer the race and more -- and releases it. // The library is not initialized here, so the originals fail and the object -// is thrown away. With a real Kinect (unless SFR_KINECT_DEPTH=0) these give it the +// is thrown away. With a real Kinect these give it the // sensor's frames instead: type 0 depth and player index at resolution 1 // (320x240), type 1 colour at resolution 2 (640x480), in a texture made the // way the object makes its own, so everything after is the title's code. @@ -409,8 +424,7 @@ ImageStream image_streams[2]; bool kinect_images_wanted() { static const bool wanted=[] { const char* camera=std::getenv("SFR_CAMERA"); - const char* depth=std::getenv("SFR_KINECT_DEPTH"); - return camera && std::string_view(camera)=="kinect" && !(depth && *depth=='0'); + return camera && std::string_view(camera)=="kinect"; }(); return wanted; } @@ -499,12 +513,37 @@ SFR_INPUT_HOOK(sub_82767148) { asked[0]=asked[1]=given[0]=given[1]=0; reported=now; } - if(!kinect || !kinect->image(kind,stream->image) || stream->image.width!=stream->width || + // game_image: a v1's camera as it is, or a v2's made depth image + // (a Kinect v2's), never the preview's. + if(!kinect || !kinect->game_image(kind,stream->image) || stream->image.width!=stream->width || stream->image.height!=stream->height || stream->image.bytes_per_pixel!=stream->bytes_per_pixel) { ctx.r3.u64=0x83010001u; // E_NUI_FRAME_NO_DATA: the title tries again later return; } ++given[stream->type&1]; + if(stream->type==0) { + const int64_t now=now_ms(); + const int64_t before=depth_given_ms.exchange(now,std::memory_order_relaxed); + if(before>INT64_MIN/4 && now-before>500) + std::cerr<<"NUI_KINECT_DEPTH gap ms="<image.number,std::memory_order_relaxed); + depth_given_count.fetch_add(1,std::memory_order_release); + // SFR_KINECT_RECORD_DEPTH (kinect_record_depth.h): every image the + // game is handed, as it is handed (a v1's own, or the one a v2 made). + if(sfr::KinectDepthRecordFile* file=sfr::kinect_record_depth_file("")) { + sfr::KinectDepthRecordFrame record; + record.steady_us=stream->image.steady_us; + record.handed_us=sfr::kinect_record_now_us(); + record.sdk_time=stream->image.sdk_time; + record.number=stream->image.number; + record.sdk_frame=stream->image.sdk_frame; + record.source=std::string_view(kinect->model())=="Kinect v2"?sfr::KinectDepthSource::v2:sfr::KinectDepthSource::v1; + record.origin=stream->image.view_origin; + record.z_scale=stream->image.view_z_scale; + sfr::kinect_depth_frame_pixels(record,stream->image.pixels.data(),stream->width,stream->height); + file->write(record); + } + } auto& memory=*sfr::active_memory; const uint32_t rect=stream->frame+32; // D3DLOCKED_RECT: pitch, bits call_guest8(ctx,base,sub_824F3CF0,{stream->texture,0,rect,0,0}); @@ -706,23 +745,14 @@ SFR_INPUT_HOOK(sub_827707B0) { // until they leave, preserving their source slot numbers. kinect_players.retain(kinect_frame); ++frame_number; - // Levelling turns the bodies, which the console never did: it is - // for the skeleton-only fallback, and - // SFR_KINECT_LEVEL=1 or 0 chooses either way. - static const bool level=[]{ - const char* t=std::getenv("SFR_KINECT_LEVEL"); - if(t && *t) return *t!='0'; - return !sfr::nui_sensor_has_depth(); - }(); const float gravity_y=kinect_frame.gravity[1]; - const float tilt=received && level?sfr::kinect_level(kinect_frame):0.0f; const uint64_t generation=kinect_generation.fetch_add(1,std::memory_order_relaxed)+1; // Every three seconds, what reaches the title: the tilt taken // out and the first body's centre line and hands, with the hands' // tracking states (2 tracked, 1 inferred, 0 not). if(generation%90==1) { std::ostringstream line; - line<<"NATIVE_KINECT_BODY frame="<-nui.csv per player with a body record (or with anything the race +// gathered for it) on each input update: the cursor as 82492F30 left it, +// the record's joints (after the title's smoothing, mirrored), and what the +// race detectors and the race preparation's measurements gave since the +// player's last row. Queued: the game never waits for the disk. +void record_nui_rows(uint32_t input) { + if(!input || !sfr::nui_body_from_sensor()) return; + sfr::KinectRecordFile* file=sfr::kinect_record_nui(); + if(!file) return; + auto& memory=*sfr::active_memory; + const auto word=[&](uint64_t at) {return memory.readable(at,4)?memory.load(at):0u;}; + const auto number=[&](uint64_t at) {return std::bit_cast(word(at));}; + const bool racing=word(0x83E52F8C)!=0; + const int64_t now=sfr::kinect_record_now_us(); + std::string text; + for(uint32_t player=0;player<2;++player) { + sfr::KinectNuiRecord row; + sfr::kinect_nui_recorder().take(player,row); + const uint64_t block=uint64_t(input)+32+2544*player; + const uint32_t body=word(block+12); + const bool gathered=row.has_lean || !row.calibration_step.empty() || + std::any_of(row.detectors.begin(),row.detectors.end(),[](const auto& answer){return answer.result>=0;}); + if(!body && !gathered) continue; + row.steady_us=now; + row.update=input_frames; + row.player=player; + row.racing=racing; + if(body) { + const uint32_t data=word(uint64_t(body)+768); + row.tracking_id=data?word(uint64_t(data)+4):0; + row.body_slot=int32_t(word(uint64_t(body)+784)); + row.has_cursor=true; + for(uint32_t hand=0;hand<2;++hand) { + const uint64_t packet=block+16+400*hand; + sfr::KinectNuiHand& out=row.hands[hand]; + out.flags=word(packet); + out.theta=number(packet+60); + out.r=number(packet+148); + out.target_x=number(packet+96); + out.target_y=number(packet+100); + out.cursor_x=number(packet+80); + out.cursor_y=number(packet+84); + } + row.has_joints=true; + for(uint32_t i=0;iwrite(std::move(text)); // queued: never waits for the disk +} +} // The title's START screen does not use every later menu-manager path. // Observe the completed common input update as well as the menu cursor below. SFR_MENU_HOOK(sub_824920F8) { sfr::enter_function(ctx,"sub_824920F8",0x824920F8); const uint32_t input=ctx.r3.u32; __imp__sub_824920F8(ctx,base); + try { + trace_cursor(input); + record_nui_rows(input); + } catch(const std::exception& error) { + std::cerr<<"NUI_TRACE_CURSOR unreadable="<(uint64_t(view)+72)<(m.load(address)); }; + for(uint32_t k=0;k<2;++k) { + const int32_t slot=int32_t(m.load(uint64_t(manager)+7740+4*k)); + if(slot<0 || slot>5) continue; + const uint64_t at=uint64_t(manager)+8*uint32_t(slot); + const uint64_t record=uint64_t(manager)+2784+800*uint32_t(slot); + sfr::KinectDepthViewRecord view; + view.known=true; + view.frame=frame; + view.steady_us=now; + view.slot=uint32_t(slot); + view.centroid.column=m.load(at+7584); + view.centroid.row=m.load(at+7588); + view.centroid.min_column=m.load(at+7632); + view.centroid.min_row=m.load(at+7636); + view.centroid.max_column=m.load(at+7680); + view.centroid.max_row=m.load(at+7684); + for(uint32_t i=0;i<4;++i) { + view.counts[i]=f32(record+640+4*i); + view.means[i]=f32(record+656+4*i); + } + view.near_count=m.load(record+672); + view.average=f32(record+676); + // By the record, and by the slot (a record's +784, 82438A98) for a + // race whose detectors are asked about another copy of it. + sfr::kinect_depth_view_publish(uint32_t(record),view); + sfr::kinect_depth_view_publish(uint32_t(slot),view); + } + } } PPC_FUNC_IMPL(__imp__sub_82463808); diff --git a/src/nui_race_hooks.cpp b/src/nui_race_hooks.cpp index 8772833..4f387eb 100644 --- a/src/nui_race_hooks.cpp +++ b/src/nui_race_hooks.cpp @@ -4,7 +4,10 @@ #include "nui_race.h" #include "camera_input.h" #include "camera_race_motion.h" +#include "kinect_race_record.h" +#include "kinect_nui_record.h" #include "avatar_state.h" +#include "nui_analysis_trace.h" #include "motion_trace.h" #include #include @@ -12,6 +15,7 @@ #include #include #include +#include #include #include @@ -58,9 +62,10 @@ bool pad_racing() { return swapped; } bool camera_player(uint32_t player) { return sfr::camera_controls_player(player,sfr::camera_motion_active()); } -// Real sensors retain the title's native depth detectors when available. -// Skeleton-only backends keep separate calibration and gesture timers for -// every live body read by the title, including P2's non-manager record. +// A real Kinect's players are read by the title's own depth detectors, as on +// the console: its lean, crouch, jump and side results always stand, and +// nothing here stands in for them. This only notes, with SFR_KINECT_RECORD, +// what the detectors answered (sensor_row). bool sensor_steering = false; uint32_t sensor_record = 0; void write_camera_lean(uint32_t record,float scale=1.f,const sfr::CameraRaceMotion& motion=camera_motion) { @@ -70,55 +75,31 @@ void write_camera_lean(uint32_t record,float scale=1.f,const sfr::CameraRaceMoti store_float(record+640,1.f+std::max(lean,0.f)); store_float(record+644,1.f+std::max(-lean,0.f)); } -// A full lean of the sensor's body is this ratio minus one: the title reads -// up to 3.5 from the depth view, so the default uses its whole range (a -// full lean at 1 turned the board only a little). SFR_KINECT_LEAN_SCALE -// changes it, 0.1..3.5. -// With an opened depth stream, the title's own depth view makes the lean -// and its own -// detectors see the crouch, as on the console, so nothing here stands in for -// them. Without the depth images the skeleton does, as a fallback. -// SFR_KINECT_BODY_LEAN and SFR_KINECT_BODY_GESTURES (1 or 0) choose either way. -bool sensor_fallback(const char* setting) { - const char* value=std::getenv(setting); - if(value && *value) return *value!='0'; - return !sfr::nui_sensor_has_depth(); -} -bool sensor_body_lean() { - return sensor_fallback("SFR_KINECT_BODY_LEAN"); -} -// The title's own pair on the last manager update, before the body's lean. -float depth_lean_right=0,depth_lean_left=0; -float sensor_lean_scale() { - static const float scale=[] { - const char* text=std::getenv("SFR_KINECT_LEAN_SCALE"); - return std::clamp(text && *text?std::strtof(text,nullptr):3.5f,0.1f,3.5f); - }(); - return scale; -} -// A real Kinect's crouch: how far the hips drop below the calibrated stance -// (SFR_KINECT_CROUCH_DEPTH metres, default 0.15), measured above a floor -// fixed at calibration rather than the guessed ankles. The motion counts a -// crouch at 80% of the standing height, so the floor sits five drops below -// the hips while the stance is being learnt. -float sensor_floor_below_hips() { - static const float depth=[] { - const char* text=std::getenv("SFR_KINECT_CROUCH_DEPTH"); - return std::clamp(text && *text?std::strtof(text,nullptr):.15f,.05f,.35f); - }(); - return depth/.2f; -} -struct SensorMotion { - sfr::CameraRaceMotion motion; - uint64_t tick=0, epoch=0; - float floor=0; +struct SensorRow { + uint64_t epoch=0; uint32_t tracking_id=0; + // SFR_KINECT_RECORD (kinect_race_record.h): this update's row, written + // once the update's detectors have answered (the next update, or when + // the record is dropped). + sfr::KinectRaceRecord row; + bool pending=false; }; -std::unordered_map sensor_motions; +std::unordered_map sensor_rows; uint64_t sensor_epoch=0; uint32_t sensor_manager=0; -bool sensor_depth=false; -void update_motion(uint32_t record,uint64_t generation,sfr::CameraRaceMotion& motion,uint64_t& last_tick,float* floor=nullptr) { +void flush_race_row(SensorRow& state) { + if(!state.pending)return; + state.pending=false; + if(sfr::KinectRecordFile* file=sfr::kinect_record_race()) { + std::string text; + sfr::kinect_race_record_row(state.row,text); + file->write(std::move(text)); // queued: never waits for the disk + } +} +void flush_race_rows() { + for(auto& item:sensor_rows)flush_race_row(item.second); +} +void update_motion(uint32_t record,uint64_t generation,sfr::CameraRaceMotion& motion,uint64_t& last_tick) { const uint64_t now=sfr::camera_motion_clock_ns(); const float seconds=last_tick && now>last_tick?float(double(now-last_tick)*1e-9):1.f/60.f; last_tick=now; @@ -127,65 +108,48 @@ void update_motion(uint32_t record,uint64_t generation,sfr::CameraRaceMotion& mo }; const bool ready=motion.ready(); sfr::CameraRacePose pose{point(0),point(32),point(224),point(288),point(64),point(128),point(192),point(256),point(80),point(96),point(144),point(160),point(208),point(272)}; - if(floor) { - if(!ready)*floor=pose.hip[1]-sensor_floor_below_hips(); - pose.floor_y=*floor; - } - const bool crouched=motion.crouch(); motion.observe(pose,seconds,generation); if(motion.ready()!=ready) std::cerr<<"CAMERA_RACE_CALIBRATION ready="<=std::chrono::seconds(3)) { - if(lines(record + 768) : 0; + std::ostringstream line; + line << "NUI_TRACE_TURN stage=lean source=0x" << std::hex << source << " record=0x" << record << std::dec + << " tracking_id=" << (data && memory().readable(uint64_t(data) + 4, 4) ? memory().load(data + 4) : 0); + if (record && memory().readable(uint64_t(record) + 640, 8)) + line << " depth_right=" << load_float(record + 640) << " depth_left=" << load_float(record + 644); + trace_turn_samples(line, "depth_lean", source + 4); + trace_turn_samples(line, "tilt", source + 48); + std::cerr << line.str() << '\n'; +} +void trace_steer(uint32_t detector, uint32_t source, uint32_t results, uint32_t result) { + if (!sfr::nui_analysis_trace() || !detector || !source || !results || !memory().load(race_flag)) return; + static sfr::NuiTraceLimiter limiter("TURN", 5.0, 18000); + if (!memory().readable(detector, 116) || !memory().readable(source, 48) || !limiter.take()) return; + const uint32_t e = entry(results); + std::ostringstream line; + line << "NUI_TRACE_TURN stage=steer detector=0x" << std::hex << detector << " source=0x" << source << " record=0x" + << trace_record_of(source) << std::dec << " lean_in=" << load_float(source + 44) + << " filter_mean=" << load_float(detector + 112); + if (memory().readable(e, 84)) + line << " steer=" << int(int8_t(memory().load(e + 76))) << " steer78=" + << int(int8_t(memory().load(e + 78))) << " bits=" << (memory().load(e + 4) & 3u); + line << " result=" << result; + std::cerr << line.str() << '\n'; +} +} + // This original race consumer updates source->object through vtable[2], // then reads its body through vtable[1] before calculating lean. Keep that // order and the title's lean calculation. Controller lean bypasses only the // depth-input smoothing below, after the live reader identifies its player. SFR_INPUT_HOOK(sub_822C6200) { sfr::enter_function(ctx, "sub_822C6200", 0x822C6200); - RaceSourceScope scope(pad_racing() || sensor_steering ? ctx.r3.u32 : 0); - __imp__sub_822C6200(ctx, base); + const uint32_t source = ctx.r3.u32; + { + RaceSourceScope scope(pad_racing() ? ctx.r3.u32 : 0); + __imp__sub_822C6200(ctx, base); + } + trace_lean(source); + // SFR_KINECT_RECORD: the means of its two filters, for the nui.csv row. + if (!pad_racing() && nui_recording() && memory().readable(source, 92)) { + const int player = nui_player_of_record(body_of_source(ctx, base, source)); + if (player >= 0) sfr::kinect_nui_recorder().lean(uint32_t(player), load_float(source + 44), load_float(source + 88)); + } } // 82918418 is a shared accessor, so calls outside the live race source's @@ -541,13 +623,6 @@ SFR_INPUT_HOOK(sub_82918418) { const uint32_t object = ctx.r3.u32; __imp__sub_82918418(ctx, base); const uint32_t record = ctx.r3.u32; - // The depth view's worker can refill the pair between the manager's - // update and this read: each sensor player gets its own lean again here. - if (sensor_steering && live_race_source && record && - memory().load(live_race_source) == object) { - if(sensor_body_lean())write_camera_lean(record,sensor_lean_scale(),sensor_motion(record)); - return; - } if (pad_racing() && live_race_source && record && memory().load(live_race_source) == object) { const uint32_t player=player_of_record(record); @@ -615,16 +690,13 @@ SFR_INPUT_HOOK(sub_822B60F8) { sfr::enter_function(ctx,"sub_822B60F8",0x822B60F8); const uint32_t actor=ctx.r3.u32; __imp__sub_822B60F8(ctx,base); - // A webcam player's race, or a real Kinect's (the skeleton fallback's - // motion: did the rider take the crouch and jump it was given?). + // A webcam player's race: did the rider take the crouch and jump it was + // given? (A real Kinect's players are read by the title's own detectors.) static const bool enabled=sfr::motion_trace_enabled(); if(!enabled)return; const bool camera=pad_racing() && sfr::camera_motion_active(); - if(!camera && !sensor_steering)return; - // Read only: the manager's update keeps the sensor's motion current. - static const sfr::CameraRaceMotion none; - const auto known=camera?sensor_motions.end():sensor_motions.find(sensor_record); - const sfr::CameraRaceMotion& motion=camera?camera_motion:known!=sensor_motions.end()?known->second.motion:none; + if(!camera)return; + const sfr::CameraRaceMotion& motion=camera_motion; const uint32_t owner=memory().load(actor+1680); if(!owner)return; const uint32_t results=memory().load(owner+3220); @@ -660,38 +732,21 @@ void finish_camera_overthrow(uint32_t detector,uint32_t results,uint64_t& result camera_motion.consume_overthrow();result=1; } -// A skeleton-only Kinect backend cannot supply the native depth features -// used by crouch and the charge/jump that follow it. When the original -// does not recognize one, the current player's skeleton decides as the webcam -// motion does (the hips dropping below the calibrated stance, then rising). -// Only the skeleton-only fallback does this (sensor_fallback above). -void finish_sensor_detector(PPCContext& ctx,uint8_t* base,uint32_t address,uint32_t detector,uint32_t source, - uint32_t results) { +// A real Kinect's detectors answer from the title's own depth view and the +// answer always stands. With SFR_KINECT_RECORD only, the answer is noted in +// the update's row (original is what the title said, result what the race +// used: the same). +void finish_sensor_detector(PPCContext& ctx,uint8_t* base,uint32_t address,uint32_t source) { if(address!=0x822C8778 && address!=0x822CB840 && address!=0x822C9050 && address!=0x822CA6B0)return; - const bool bridge=sensor_fallback("SFR_KINECT_BODY_GESTURES"); - static const bool trace=sfr::motion_trace_enabled(); + if(!sfr::kinect_record_race())return; + const int column=sfr::kinect_race_detector_index(address); + if(column<0)return; const uint32_t record=body_of_source(ctx,base,source); if(!record)return; - auto& motion=sensor_motion(record); - const uint32_t original=ctx.r3.u32; - const bool wanted=address==0x822C9050?motion.jump():address==0x822CA6B0?false:motion.crouch(); - if(bridge && original!=1 && wanted) { - if(const auto bridged=camera_gesture(address,detector,results,motion))ctx.r3.u64=*bridged; + if(SensorRow* found=sensor_row(record); found && found->pending) { + found->row.original[size_t(column)]=int(ctx.r3.u32); + found->row.result[size_t(column)]=int(ctx.r3.u32); } - if(!trace)return; - static std::unordered_map> last; - const std::pair now{original,ctx.r3.u32}; - // A jump is a one-frame pulse: always written, so a lost one shows. - if(last.contains(address) && last[address]==now && !(address==0x822C9050 && motion.jump()))return; - last[address]=now; - static sfr::TraceRate rate(20); - if(!rate.allow())return; - // primary/secondary: the bits this detector's result entry holds now. - std::cerr<<"KINECT_GESTURE address=0x"<(entry(results)+4) - <<" secondary=0x"<(entry(results)+20)< +#include +#include +#include +#include +#include + +// NuiTransformSmooth (827701A0) as the title calls it: once for every new +// skeleton frame, from the Kinect manager's update (82438D80 at 82438E58), +// on its own copy of the frame (manager +16) with the parameters its +// constructor wrote at manager +2752 (82438A98: 0.5, 0.5, 0.5, 0.05, 0.04), +// before it turns the joints into its body records. So the title's menu +// cursor, its race detectors and its body measurements were all written for +// smoothed skeletons. +// +// The original does nothing here: its first test is the flag at 82B5D2AC, +// which only the original NuiInitialize (8276FD10) sets, and that never runs +// (nui_hooks.cpp answers NuiInitialize itself), so it answers E_UNEXPECTED +// (0x8000FFFF) and the title, which does not look at the answer, read its +// skeletons unsmoothed. Now the library's filter runs natively +// (nui_transform_smooth.h) with the parameters the title passes, on the +// title's frame, and the answer is S_OK. +// +// Only a real Kinect's skeletons (v1 or v2: nui_body_from_sensor) are +// smoothed. The console smoothed every skeleton the runtime tracked, but +// the emulated ones (the pad's player, a webcam's pose) are not a sensor's: +// the pad's are exact and step only as the pad does, and a webcam's pose is +// smoothed already (pose_smoothing.h); filtering them again would only add +// latency (a frame or so) to every menu and race input. They are left as +// they are, and the answer is S_OK all the same. +// +// A real skeleton is smoothed exactly once: SFR_KINECT_SMOOTH=1 (the +// backends' own filter, for sfr_kinect_probe and tests; off by default) +// leaves this one out. SFR_NUI_SMOOTH=0 runs the original instead (which +// does nothing, as before), to compare. +// +// SFR_NUI_ANALYSIS_TRACE=1 (nui_analysis_trace.h): at most once a second, +// NUI_TRACE_SMOOTH with the mode, the answer, the library's flag, the +// parameters and the tracked skeletons. + +namespace { +constexpr uint32_t nui_initialized_flag = 0x82B5D2AC; + +float guest_float(sfr::GuestMemory& memory, uint64_t address) { + return std::bit_cast(memory.load(address)); +} + +bool native_smoothing() { + static const bool native = [] { + const char* text = std::getenv("SFR_NUI_SMOOTH"); + return !text || *text != '0'; + }(); + return native; +} + +sfr::NuiFrameSmoother smoother; +} + +PPC_FUNC_IMPL(__imp__sub_827701A0); +SFR_INPUT_HOOK(sub_827701A0) { + sfr::enter_function(ctx, "sub_827701A0", 0x827701A0); + const uint32_t frame = ctx.r3.u32, parameters = ctx.r4.u32; + auto& memory = *sfr::active_memory; + const char* mode = "native"; + if (!native_smoothing()) { + mode = "original"; + __imp__sub_827701A0(ctx, base); + } else if (!sfr::nui_body_from_sensor()) { + mode = "emulated"; // left as it is (see above) + smoother.reset(); + ctx.r3.u64 = 0; + } else if (sfr::kinect_host_smoothing()) { + mode = "host"; // smoothed by the backend already + smoother.reset(); + ctx.r3.u64 = 0; + } else { + try { + ctx.r3.u64 = sfr::nui_transform_smooth_guest(memory, frame, parameters, smoother); + } catch (const std::exception& error) { + static bool told = false; + if (!told) std::cerr << "NUI_TRANSFORM_SMOOTH unreadable frame=0x" << std::hex << frame << std::dec << ": " + << error.what() << '\n'; + told = true; + ctx.r3.u64 = 0x80004003u; // E_POINTER + } + } + { + // Once, and whenever it changes: which way the title's skeletons go. + static char told = 0; + if (told != mode[0]) { + told = mode[0]; + std::cerr << "NUI_TRANSFORM_SMOOTH mode=" << mode << " result=0x" << std::hex << ctx.r3.u32 << std::dec + << (mode[0] == 'n' ? " (the library's filter, natively, with the title's parameters)" + : mode[0] == 'e' ? " (an emulated skeleton: not smoothed)" + : mode[0] == 'h' ? " (SFR_KINECT_SMOOTH=1 smoothed it in the backend)" + : " (SFR_NUI_SMOOTH=0: the original, which does nothing uninitialized)") + << '\n'; + } + } + static sfr::NuiTraceLimiter limiter("SMOOTH", 1.0, 4000); + if (!limiter.take()) return; + std::ostringstream line; + line << "NUI_TRACE_SMOOTH mode=" << mode << " result=0x" << std::hex << ctx.r3.u32 << std::dec << " initialized=" + << (memory.readable(nui_initialized_flag, 4) ? memory.load(nui_initialized_flag) : ~0u) + << " frame=0x" << std::hex << frame << " params=0x" << parameters << std::dec; + if (parameters && memory.readable(parameters, 20)) { + line << " smoothing=" << guest_float(memory, parameters) << " correction=" << guest_float(memory, parameters + 4) + << " prediction=" << guest_float(memory, parameters + 8) + << " jitter_radius=" << guest_float(memory, parameters + 12) + << " max_deviation_radius=" << guest_float(memory, parameters + 16); + } + if (frame && memory.readable(frame, sfr::nui_skeleton_frame_size)) { + uint32_t tracked = 0; + for (uint32_t slot = 0; slot < 6; ++slot) + tracked += memory.load(uint64_t(frame) + sfr::nui_skeleton_data_offset + + uint64_t(slot) * sfr::nui_skeleton_data_size) == sfr::nui_tracked; + line << " frame_number=" << memory.load(uint64_t(frame) + 8) << " tracked=" << tracked; + } + std::cerr << line.str() << '\n'; +} diff --git a/src/nui_transform_smooth.cpp b/src/nui_transform_smooth.cpp new file mode 100644 index 0000000..2f8c8fb --- /dev/null +++ b/src/nui_transform_smooth.cpp @@ -0,0 +1,87 @@ +#include "nui_transform_smooth.h" + +#include +#include + +namespace sfr { +namespace { +using Vec = std::array; +Vec add(const Vec& a, const Vec& b) { return {a[0] + b[0], a[1] + b[1], a[2] + b[2]}; } +Vec sub(const Vec& a, const Vec& b) { return {a[0] - b[0], a[1] - b[1], a[2] - b[2]}; } +Vec scale(const Vec& a, float s) { return {a[0] * s, a[1] * s, a[2] * s}; } +// a * s + b, as the library's vmaddfp. +Vec madd(const Vec& a, float s, const Vec& b) { return {a[0] * s + b[0], a[1] * s + b[1], a[2] * s + b[2]}; } +float length(const Vec& a) { return std::sqrt(a[0] * a[0] + a[1] * a[1] + a[2] * a[2]); } + +// The library's smallest jitter radius (820C12D8). +constexpr float minimum_jitter_radius = 0.0001f; +constexpr uint32_t joint_inferred = 1; +} + +Vec nui_smooth_joint(NuiSmoothJoint& history, const Vec& measured, uint32_t state, const NuiSmoothParameters& p) { + float jitter = p.jitter_radius < minimum_jitter_radius ? minimum_jitter_radius : p.jitter_radius; + float deviation = p.max_deviation_radius; + if (state == joint_inferred) { + jitter *= 2.0f; + deviation *= 2.0f; + } + // A joint at the origin has no position: its history starts again. + if (measured[0] == 0.0f && measured[1] == 0.0f && measured[2] == 0.0f) history.frames = 0; + Vec filtered, trend; + if (history.frames == 0) { + filtered = measured; + trend = {}; + history.frames = 1; + } else if (history.frames == 1) { + filtered = scale(add(measured, history.raw), 0.5f); + trend = madd(sub(filtered, history.filtered), p.correction, scale(history.trend, 1.0f - p.correction)); + history.frames = 2; + } else { + Vec pulled = measured; + const float moved = length(sub(measured, history.filtered)); + if (!(moved > jitter)) { + const float t = moved / jitter; + pulled = madd(measured, t, scale(history.filtered, 1.0f - t)); + } + filtered = madd(pulled, 1.0f - p.smoothing, scale(add(history.filtered, history.trend), p.smoothing)); + trend = madd(sub(filtered, history.filtered), p.correction, scale(history.trend, 1.0f - p.correction)); + } + Vec predicted = madd(trend, p.prediction, filtered); + const float lead = length(sub(predicted, measured)); + if (lead > deviation) { + const float t = deviation / lead; + predicted = madd(predicted, t, scale(measured, 1.0f - t)); + } + history.raw = measured; + history.filtered = filtered; + history.trend = trend; + return predicted; +} + +void nui_smooth_skeleton(NuiSmoothSkeleton& history, SkeletonJoints& joints, + const std::array& states, const NuiSmoothParameters& parameters) { + for (uint32_t j = 0; j < nui_joint_count; ++j) + joints[j] = nui_smooth_joint(history[j], joints[j], states[j], parameters); +} + +void NuiFrameSmoother::smooth(std::array& slots, const NuiSmoothParameters& parameters) { + for (size_t s = 0; s < slots.size(); ++s) { + Slot& slot = slots[s]; + if (slot.tracking_state != nui_tracked || slot.tracking_id != ids_[s]) { + history_[s] = {}; + ids_[s] = slot.tracking_state == nui_tracked ? slot.tracking_id : 0; + } + if (slot.tracking_state != nui_tracked) continue; + nui_smooth_skeleton(history_[s], slot.joints, slot.states, parameters); + } +} + +void NuiFrameSmoother::reset() { + history_ = {}; + ids_ = {}; + has_last = false; + last_frame = 0; + last = {}; +} + +} diff --git a/src/nui_transform_smooth.h b/src/nui_transform_smooth.h new file mode 100644 index 0000000..91b90f3 --- /dev/null +++ b/src/nui_transform_smooth.h @@ -0,0 +1,86 @@ +#pragma once +#include "nui_skeleton.h" + +#include +#include + +namespace sfr { +class GuestMemory; + +// NuiTransformSmooth, the NUI library's skeleton filter, as the title links +// it (827701A0, which runs 82779700 for each skeleton) and as the Kinect +// for Windows SDK 1.8 exports it from Kinect10.dll: Holt's double +// exponential smoothing of each joint, with a jitter radius and a cap on how +// far the prediction may lead the measurement. Read from the title's code: +// +// - Each of the six skeleton slots keeps a history of 20 joints (raw, +// filtered, trend, frames seen). A slot whose skeleton is not TRACKED (2) +// has its history cleared; only TRACKED skeletons are filtered. Here a +// slot's history is also cleared when its tracking id changes (another +// person in the slot), which the console's library does not check. +// - A joint at exactly (0, 0, 0) starts its history again and is passed on +// as it is; being that history's first frame, the next one is its mean +// with the origin (held back to the deviation radius), as in the +// library's code. A joint's tracking state matters only for its radii: inferred +// (1) joints get twice the jitter and deviation radius; not tracked (0) +// joints with a position are filtered like tracked ones. +// - The jitter radius is at least 0.0001 m. A skeleton's position (its +// centre) is not filtered; each joint's w becomes 1. +// - First frame: as measured, no trend. Second: the mean of the two +// measurements, trend corrected towards their difference. Then: inside +// the jitter radius the measurement is pulled towards the last output in +// proportion to how far it moved; filtered = (1 - smoothing) measurement +// + smoothing (last filtered + last trend); trend = correction (filtered - +// last filtered) + (1 - correction) last trend; output = filtered + +// prediction trend, drawn back to max deviation radius from the +// measurement. +struct NuiSmoothParameters { + float smoothing = 0.5f, correction = 0.5f, prediction = 0.5f; + float jitter_radius = 0.05f, max_deviation_radius = 0.04f; // metres +}; +struct NuiSmoothJoint { + std::array raw{}, filtered{}, trend{}; + uint32_t frames = 0; // 0, 1, then 2 for good +}; +using NuiSmoothSkeleton = std::array; + +// One joint of one frame: returns the output and advances its history. +std::array nui_smooth_joint(NuiSmoothJoint& history, const std::array& measured, uint32_t state, + const NuiSmoothParameters& parameters); +// A tracked skeleton's joints in place. +void nui_smooth_skeleton(NuiSmoothSkeleton& history, SkeletonJoints& joints, + const std::array& states, const NuiSmoothParameters& parameters); + +// The six slots of a skeleton frame, each with its own history. +class NuiFrameSmoother { +public: + struct Slot { + uint32_t tracking_state = 0, tracking_id = 0; + SkeletonJoints joints{}; + std::array states{}; + }; + // Smooths every TRACKED slot in place; clears the others' history. + void smooth(std::array& slots, const NuiSmoothParameters& parameters); + void reset(); + // The frame number nui_transform_smooth_guest smoothed last, and what + // that gave. + bool has_last = false; + uint32_t last_frame = 0; + std::array last{}; +private: + std::array history_{}; + std::array ids_{}; +}; + +// The parameters at a guest NUI_TRANSFORM_SMOOTH_PARAMETERS (five +// big-endian floats), or the library's defaults for address 0. +NuiSmoothParameters nui_smooth_parameters(GuestMemory& memory, uint32_t address); +// NuiTransformSmooth on the title's big-endian NUI_SKELETON_FRAME at frame +// (nui_skeleton.h's layout): S_OK (0), or E_POINTER for no frame. A frame +// with the same frame number as the last one smoothed (dwFrameNumber, +8; +// the skeleton hook hands the title its last frame again when the sensor +// had nothing newer) is not filtered a second time: each slot whose tracking +// id is unchanged gets that frame's output again. +uint32_t nui_transform_smooth_guest(GuestMemory& memory, uint32_t frame, uint32_t parameters, NuiFrameSmoother& smoother); + +} diff --git a/src/nui_transform_smooth_guest.cpp b/src/nui_transform_smooth_guest.cpp new file mode 100644 index 0000000..6b88a2c --- /dev/null +++ b/src/nui_transform_smooth_guest.cpp @@ -0,0 +1,68 @@ +#include "nui_transform_smooth.h" +#include "guest_memory.h" + +#include + +// NuiTransformSmooth on the title's own frame (nui_transform_smooth.h); apart +// from the filter so that a host program without guest memory can link it. +namespace sfr { +namespace { +constexpr uint32_t e_pointer = 0x80004003u; +float load_float(GuestMemory& memory, uint64_t address) { return std::bit_cast(memory.load(address)); } +void store_float(GuestMemory& memory, uint64_t address, float value) { + memory.store(address, std::bit_cast(value)); +} +} + +NuiSmoothParameters nui_smooth_parameters(GuestMemory& memory, uint32_t address) { + NuiSmoothParameters parameters; // the library's own defaults for none + if (!address) return parameters; + parameters.smoothing = load_float(memory, address); + parameters.correction = load_float(memory, uint64_t(address) + 4); + parameters.prediction = load_float(memory, uint64_t(address) + 8); + parameters.jitter_radius = load_float(memory, uint64_t(address) + 12); + parameters.max_deviation_radius = load_float(memory, uint64_t(address) + 16); + return parameters; +} + +uint32_t nui_transform_smooth_guest(GuestMemory& memory, uint32_t frame, uint32_t parameters, NuiFrameSmoother& smoother) { + if (!frame) return e_pointer; + memory.check_write(frame, nui_skeleton_frame_size); + const NuiSmoothParameters given = nui_smooth_parameters(memory, parameters); + const uint32_t frame_number = memory.load(uint64_t(frame) + 8); + std::array slots; + for (uint32_t s = 0; s < 6; ++s) { + const uint64_t data = uint64_t(frame) + nui_skeleton_data_offset + uint64_t(s) * nui_skeleton_data_size; + NuiFrameSmoother::Slot& slot = slots[s]; + slot.tracking_state = memory.load(data); + slot.tracking_id = memory.load(data + 4); + for (uint32_t j = 0; j < nui_joint_count; ++j) { + for (uint32_t axis = 0; axis < 3; ++axis) slot.joints[j][axis] = load_float(memory, data + 32 + 16 * j + 4 * axis); + slot.states[j] = memory.load(data + 352 + 4 * j); + } + } + if (smoother.has_last && frame_number == smoother.last_frame) { + // The same sensor frame again: its output again, not a second step. + for (uint32_t s = 0; s < 6; ++s) + if (slots[s].tracking_state == nui_tracked && smoother.last[s].tracking_state == nui_tracked && + slots[s].tracking_id == smoother.last[s].tracking_id) + slots[s].joints = smoother.last[s].joints; + } else { + smoother.smooth(slots, given); + smoother.has_last = true; + smoother.last_frame = frame_number; + smoother.last = slots; + } + for (uint32_t s = 0; s < 6; ++s) { + if (slots[s].tracking_state != nui_tracked) continue; + const uint64_t data = uint64_t(frame) + nui_skeleton_data_offset + uint64_t(s) * nui_skeleton_data_size; + for (uint32_t j = 0; j < nui_joint_count; ++j) { + for (uint32_t axis = 0; axis < 3; ++axis) + store_float(memory, data + 32 + 16 * j + 4 * axis, slots[s].joints[j][axis]); + store_float(memory, data + 32 + 16 * j + 12, 1.0f); + } + } + return 0; +} + +} diff --git a/tests/consecutive_failures_test.cpp b/tests/consecutive_failures_test.cpp new file mode 100644 index 0000000..308ba4a --- /dev/null +++ b/tests/consecutive_failures_test.cpp @@ -0,0 +1,30 @@ +#include "consecutive_failures.h" + +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +} + +int main() { + try { + uint32_t failures = 0; + for (uint32_t i = 1; i <= sfr::consecutive_failure_limit; ++i) + require(sfr::failure_within_limit(failures), "failures up to the limit are tolerated"); + require(!sfr::failure_within_limit(failures), "one more failure than the limit is not"); + require(!sfr::failure_within_limit(failures), "and it stays refused"); + failures = 0; // a success + for (uint32_t i = 1; i <= 3; ++i) require(sfr::failure_within_limit(failures, 3), "a custom limit is tolerated"); + require(!sfr::failure_within_limit(failures, 3), "a custom limit is exceeded"); + failures = 0; + require(sfr::failure_within_limit(failures, 3), "a success starts the count again"); + std::cout << "consecutive failures tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/tests/extract_guest_functions.cmake b/tests/extract_guest_functions.cmake new file mode 100644 index 0000000..4eb2872 --- /dev/null +++ b/tests/extract_guest_functions.cmake @@ -0,0 +1,38 @@ +# Copies generated guest functions out of the diagnostic's sources, for a +# test that runs them on their own (kinect_depth_view_guest_test): +# +# cmake -DDIR= -DOUT= -DFUNCTIONS=82439998,82439D80 -P extract_guest_functions.cmake +# +# Each function is its PPC_FUNC_IMPL(__imp__sub_
) body, from that +# line to the first line that is only "}". The generated code stays in out/ +# (it is the game's), so this runs at build time and writes into the build +# directory only. +if(NOT DIR OR NOT OUT OR NOT FUNCTIONS) + message(FATAL_ERROR "usage: cmake -DDIR=... -DOUT=... -DFUNCTIONS=a,b -P extract_guest_functions.cmake") +endif() +string(REPLACE "," ";" wanted "${FUNCTIONS}") +file(GLOB sources "${DIR}/ppc_recomp.*.cpp") +set(result "// Made by tests/extract_guest_functions.cmake from ${DIR}: generated code, not committed.\n") +foreach(function IN LISTS wanted) + set(found FALSE) + foreach(source IN LISTS sources) + file(READ "${source}" text) + string(FIND "${text}" "PPC_FUNC_IMPL(__imp__sub_${function}) {" start) + if(start GREATER -1) + string(SUBSTRING "${text}" ${start} -1 rest) + string(FIND "${rest}" "\n}\n" end) + if(end LESS 0) + message(FATAL_ERROR "sub_${function} in ${source} has no end") + endif() + math(EXPR end "${end} + 3") + string(SUBSTRING "${rest}" 0 ${end} body) + string(APPEND result "\n// ${source}\n${body}") + set(found TRUE) + break() + endif() + endforeach() + if(NOT found) + message(FATAL_ERROR "sub_${function} is not in ${DIR}") + endif() +endforeach() +file(WRITE "${OUT}" "${result}") diff --git a/tests/kinect_calibration_test.cpp b/tests/kinect_calibration_test.cpp deleted file mode 100644 index e971803..0000000 --- a/tests/kinect_calibration_test.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "kinect_calibration.h" -#include "kinect_sensor.h" -#include "nui_skeleton.h" - -#include -#include -#include - -namespace { -int failures = 0; -void check(bool ok, const char* what) { - if (!ok) { - std::printf("FAILED: %s\n", what); - ++failures; - } -} - -// A body whose hips are `height` above a level floor (y = 0) and whose trunk -// leans `degrees` to the side (positive x). -sfr::KinectBody body(float height, float degrees) { - sfr::KinectBody b; - b.joints[sfr::nui_joint::hip_center] = {0.0f, height, 2.5f}; - const float radians = degrees / 57.29578f; - b.joints[sfr::nui_joint::shoulder_center] = {0.5f * std::sin(radians), height + 0.5f * std::cos(radians), 2.5f}; - b.joint_states[sfr::nui_joint::hip_center] = 2; - b.joint_states[sfr::nui_joint::shoulder_center] = 2; - return b; -} - -constexpr std::array level_floor{0.0f, 1.0f, 0.0f, 0.0f}; - -// Feeds 30 frames a second for `seconds`, the body from `at(t)` (t 0..1). -double run(sfr::KinectCalibration& calibration, double now, double seconds, - const std::function& at) { - const int frames = int(seconds * 30); - for (int i = 0; i < frames; ++i) { - const sfr::KinectBody b = at(double(i) / frames); - calibration.observe(&b, level_floor, now); - now += 1.0 / 30.0; - } - return now; -} -} - -int main() { - using Step = sfr::KinectCalibrationStep; - sfr::KinectCalibration c; - check(c.step() == Step::idle, "starts idle"); - c.start(); - double t = 0; - // Standing still with a 1 cm sway: the stance is taken after 1.5 s. - t = run(c, t, 2.0, [](double p) { return body(0.90f + 0.005f * float(std::sin(p * 20)), 1.0f); }); - check(c.step() == Step::crouch, "standing still leads to the crouch"); - check(std::fabs(c.result().standing_height - 0.90f) < 0.01f, "standing height measured"); - // A 12 cm crouch, down and up again (Issue #97's player). - t = run(c, t, 1.0, [](double p) { return body(0.90f - 0.12f * float(std::sin(p * 3.14159)), 1.0f); }); - t = run(c, t, 0.3, [](double) { return body(0.90f, 1.0f); }); - check(c.step() == Step::jump, "the crouch leads to the jump"); - check(std::fabs(c.result().crouch_drop - 0.12f) < 0.01f, "crouch depth measured"); - check(c.result().crouch_cm == 8, "suggested depth is 70% of the crouch"); - // A 10 cm jump. - t = run(c, t, 0.6, [](double p) { return body(0.90f + 0.10f * float(std::sin(p * 3.14159)), 1.0f); }); - t = run(c, t, 0.3, [](double) { return body(0.90f, 1.0f); }); - check(c.step() == Step::lean_left, "the jump leads to the leans"); - check(std::fabs(c.result().jump_rise - 0.10f) < 0.01f, "jump height measured"); - // Leaning 15 degrees one way (from the 1 degree stance), then upright. - t = run(c, t, 1.0, [](double p) { return body(0.90f, 1.0f - 15.0f * float(std::sin(p * 3.14159))); }); - t = run(c, t, 0.3, [](double) { return body(0.90f, 1.0f); }); - check(c.step() == Step::lean_right, "the first lean leads to the second"); - check(std::fabs(c.result().lean_left - 15.0f) < 0.5f, "first lean measured"); - // Leaning the same way again does not count as the other side. - t = run(c, t, 1.0, [](double p) { return body(0.90f, 1.0f - 15.0f * float(std::sin(p * 3.14159))); }); - t = run(c, t, 0.3, [](double) { return body(0.90f, 1.0f); }); - check(c.step() == Step::lean_right, "the same side again is not the other lean"); - // Then 18 degrees the other way. - t = run(c, t, 1.0, [](double p) { return body(0.90f, 1.0f + 18.0f * float(std::sin(p * 3.14159))); }); - t = run(c, t, 0.3, [](double) { return body(0.90f, 1.0f); }); - check(c.step() == Step::done, "both leans finish it"); - check(std::fabs(c.result().lean_right - 18.0f) < 0.5f, "second lean measured"); - - // Nobody tracked: standing still never completes. - c.start(); - for (int i = 0; i < 90; ++i) c.observe(nullptr, level_floor, i / 30.0); - check(c.step() == Step::stand && !c.tracked(), "nobody tracked stays on standing"); - // Swaying more than the band keeps restarting the stance. - run(c, 3.0, 3.0, [](double p) { return body(0.90f + 0.05f * float(std::sin(p * 40)), 0.0f); }); - check(c.step() == Step::stand, "a body that does not stand still is not taken"); - - // A shallow crouch still gives the smallest depth; a deep one the largest. - check(sfr::KinectCalibration::crouch_cm_for(0.04f) == 5, "depth at least 5 cm"); - check(sfr::KinectCalibration::crouch_cm_for(0.80f) == 35, "depth at most 35 cm"); - // No floor from the sensor: the hips' own height still gives the drop. - c.start(); - const std::array no_floor{}; - double u = 0; - for (int i = 0; i < 60; ++i) { const auto b = body(0.30f, 0.0f); c.observe(&b, no_floor, u); u += 1 / 30.0; } - for (int i = 0; i < 30; ++i) { - const auto b = body(0.30f - 0.20f * float(std::sin(i / 30.0 * 3.14159)), 0.0f); - c.observe(&b, no_floor, u); - u += 1 / 30.0; - } - for (int i = 0; i < 10; ++i) { const auto b = body(0.30f, 0.0f); c.observe(&b, no_floor, u); u += 1 / 30.0; } - check(c.result().crouch_cm == 14, "without a floor the hips' height gives the depth"); - - if (!failures) std::printf("kinect calibration checks passed\n"); - return failures ? 1 : 0; -} diff --git a/tests/kinect_compare.cpp b/tests/kinect_compare.cpp index 512999c..46e0e2f 100644 --- a/tests/kinect_compare.cpp +++ b/tests/kinect_compare.cpp @@ -119,7 +119,11 @@ struct Compare { const float sx = float(area.right - area.left) / (colour_view ? 640.0f : 320.0f); const float sy = float(area.bottom - area.top) / (colour_view ? 480.0f : 240.0f); const auto place = [&](const std::array& p) { - const sfr::KinectImagePoint point = colour_view ? sfr::kinect_colour_point(p) : sfr::kinect_depth_point(p); + // A sensor that places points in its own images (a v2) first. + sfr::KinectImagePoint point; + const auto kind = colour_view ? sfr::KinectImageKind::colour : sfr::KinectImageKind::depth_and_player; + if (!sensor || !sensor->image_point(kind, p, point)) + point = colour_view ? sfr::kinect_colour_point(p) : sfr::kinect_depth_point(p); return POINT{area.left + LONG(point.x * sx), area.top + LONG(point.y * sy)}; }; HRGN clip = CreateRectRgn(area.left, area.top, area.right, area.bottom); diff --git a/tests/kinect_depth_compare.cpp b/tests/kinect_depth_compare.cpp new file mode 100644 index 0000000..0b013c1 --- /dev/null +++ b/tests/kinect_depth_compare.cpp @@ -0,0 +1,366 @@ +// How well a Kinect v2's depth for the game stands in for a real Kinect v1's: +// two depth recordings of the same movement at the same time (the game on +// the v1, sfr_kinect_probe on the v2; kinect_record_depth.h), compared +// feature by feature of the title's own depth view (kinect_depth_compare.h). +// +// sfr_kinect_depth_compare [--skeleton ] +// [--csv aligned.csv] [--report report.txt] [--max-lag-ms 1000] [--gap-ms 100] +// [--events ] +// +// Both files' images are stamped with the host's steady clock (steady_us), +// which every process of the machine shares, so they are first put on it as +// they are. What is left (each sensor's own latency, the v1's images reaching +// the game every fourth game frame) is found by cross-correlating the +// vertical centroid of the player (pass one's row), resampled every 10 ms, +// over lags of up to --max-lag-ms: the lag is how much later the v2's +// movement shows than the v1's. --skeleton does the same with the two +// skeleton recordings' SpineBase height, as a check of that lag. +// +// Then, at each v1 image, the v2's value at the same moment (plus the lag; +// linear between its two images around it, when they are at most --gap-ms +// apart and both have a player) and per feature: n, Pearson r, the least +// squares line v1 = slope * v2 + intercept, the RMS difference, the RMS left +// after the line, and the feature's own best lag. Also each sensor's share +// of pixels with a depth, its player pixels, and the share of images where +// either lost the player. The report (Traditional Chinese labels, English +// names) goes to the console and --report; --csv gets the aligned series. +// +// --events takes the game's race CSV of the v1 session: every crouch and +// jump the game's own detectors found (runs of 1 in crouch_original and +// jump_original, at the run's first row) is an event, and for both sensors +// the vertical centroid's and the top row's (min_row) amplitude around it: +// the mean over 2.0 to 0.5 s before as the baseline, the largest (crouch) or +// smallest (jump) value over the 1.2 s after less it, on the aligned series +// (kinect_event_amplitudes). Printed: the mean amplitudes, their ratio v2/v1 +// and how many events moved the same way on both. +// The last line, "SUMMARY key=value ...", gathers the headline numbers for +// scripts (rec_session.ps1 -Analyse). +#include "kinect_depth_compare.h" +#include "kinect_record.h" +#include "kinect_record_depth.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#endif + +namespace { +struct Recording { + std::string path, source; + std::vector samples; + uint32_t width = 0, height = 0; + bool truncated = false; +}; + +bool load(const std::string& path, Recording& out) { + sfr::KinectDepthReader reader; + if (!reader.open(path)) { + std::cerr << path << ": " << reader.error() << '\n'; + return false; + } + out.path = path; + out.width = reader.width(); + out.height = reader.height(); + sfr::KinectDepthRecordFrame frame; + while (reader.next(frame)) { + out.source = sfr::kinect_depth_source_name(frame.source); + out.samples.push_back(sfr::kinect_depth_sample(frame)); + } + out.truncated = reader.truncated(); + if (!reader.error().empty()) std::cerr << path << ": stopped at " << reader.error() << '\n'; + // Rising times, as the series need (a recorder writes them so). + for (size_t i = 1; i < out.samples.size(); ++i) + if (out.samples[i].steady_us < out.samples[i - 1].steady_us) { + std::cerr << path << ": images out of time order at " << i << '\n'; + return false; + } + return !out.samples.empty(); +} + +sfr::KinectSeries series(const Recording& r, size_t feature) { + sfr::KinectSeries s; + for (const auto& sample : r.samples) s.add(sample.steady_us, sfr::kinect_depth_feature(sample, feature)); + return s; +} + +// A skeleton recording's SpineBase height for one sensor, the nearest body +// of each frame (NaN without one). +sfr::KinectSeries skeleton_series(const std::string& path, const std::string& sensor, bool& ok) { + sfr::KinectSeries s; + std::ifstream in(path); + ok = bool(in); + std::string line; + int64_t time = INT64_MIN; + double y = NAN, z = INFINITY; + const auto flush = [&] { + if (time != INT64_MIN) s.add(time, y); + }; + while (std::getline(in, line)) { + sfr::KinectRecordRow row; + if (!sfr::kinect_parse_skeleton_row(line, row) || row.frame.sensor != sensor) continue; + if (row.frame.steady_us != time) { + flush(); + time = row.frame.steady_us; + y = NAN; + z = INFINITY; + } + const auto& joint = row.body.joints[0]; + if (row.has_body && joint.state == 2 && joint.position[2] < z) { + z = joint.position[2]; + y = joint.position[1]; + } + } + flush(); + return s; +} + +std::string format(const char* pattern, double value) { + if (!std::isfinite(value)) return "-"; + char text[64]; + std::snprintf(text, sizeof text, pattern, value); + return text; +} +} + +int main(int argc, char** argv) { +#ifdef _WIN32 + SetConsoleOutputCP(CP_UTF8); +#endif + std::vector paths, skeletons; + std::string csv, report_path, events_path; + double max_lag_ms = 1000, gap_ms = 100; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--skeleton" && i + 2 < argc) { + skeletons = {argv[i + 1], argv[i + 2]}; + i += 2; + } else if (arg == "--csv" && i + 1 < argc) { + csv = argv[++i]; + } else if (arg == "--report" && i + 1 < argc) { + report_path = argv[++i]; + } else if (arg == "--max-lag-ms" && i + 1 < argc) { + max_lag_ms = std::atof(argv[++i]); + } else if (arg == "--gap-ms" && i + 1 < argc) { + gap_ms = std::atof(argv[++i]); + } else if (arg == "--events" && i + 1 < argc) { + events_path = argv[++i]; + } else { + paths.push_back(arg); + } + } + if (paths.size() != 2 || !(max_lag_ms > 0) || !(gap_ms > 0)) { + std::cerr << "usage: sfr_kinect_depth_compare [--skeleton ]\n" + " [--csv aligned.csv] [--report report.txt] [--max-lag-ms 1000] [--gap-ms 100]\n" + " [--events ]\n"; + return 2; + } + Recording a, b; + if (!load(paths[0], a) || !load(paths[1], b)) { + std::cerr << "cannot read both depth recordings\n"; + return 1; + } + std::string text; + const auto line = [&](const std::string& s) { text += s + "\n"; }; + const auto describe = [&](const char* label, const Recording& r) { + const double seconds = double(r.samples.back().steady_us - r.samples.front().steady_us) / 1e6; + char out[512]; + std::snprintf(out, sizeof out, "%s %s: source=%s %ux%u images=%zu 長度 %.1f s (%.1f/s)%s", label, r.path.c_str(), + r.source.c_str(), r.width, r.height, r.samples.size(), seconds, + seconds > 0 ? double(r.samples.size() - 1) / seconds : 0.0, r.truncated ? " 最後一張不完整" : ""); + line(out); + }; + line("== Kinect 深度比較 (v1 = 標準答案, v2 = 合成深度) =="); + describe("v1", a); + describe("v2", b); + // The game falls back to a Kinect v2 when no v1 opens: then its file is + // no reference at all. + if (a.source != "v1") line("警告：第一個檔的來源是 " + a.source + "，不是真正的 Kinect v1（遊戲沒有開到 v1？），它不能當標準答案。"); + if (b.source != "v2") line("警告：第二個檔的來源是 " + b.source + "，不是 Kinect v2 合成的深度。"); + const int64_t overlap_start = std::max(a.samples.front().steady_us, b.samples.front().steady_us); + const int64_t overlap_end = std::min(a.samples.back().steady_us, b.samples.back().steady_us); + if (overlap_end <= overlap_start) { + line("兩個錄製在主機時鐘 (steady_us) 上沒有重疊：不是同一時間錄的，無法比較。"); + std::cout << text; + return 1; + } + line("主機時鐘重疊 overlap: " + format("%.1f", double(overlap_end - overlap_start) / 1e6) + " s"); + + // The lag from the vertical centroid. + const int64_t max_lag = int64_t(max_lag_ms * 1000), step = 10000, gap = int64_t(gap_ms * 1000); + const sfr::KinectLag lag = sfr::kinect_cross_correlate(series(a, 1), series(b, 1), max_lag, step); + const double lag_us = lag.found ? lag.lag_us : 0.0; + line("時間差 lag (centroid_row 互相關): " + + (lag.found ? format("%+.1f", lag_us / 1000.0) + " ms (v2 比 v1 晚為正), r=" + format("%.3f", lag.r) + + ", pairs=" + std::to_string(lag.pairs) + : std::string("找不到 (重疊的有玩家影像太少或重心沒有移動)，以 0 ms 對齊"))); + if (lag.found && std::fabs(lag_us) >= double(max_lag) - double(step)) + line(" 注意：時間差落在搜尋範圍邊緣，請加大 --max-lag-ms 再跑一次。"); + if (skeletons.size() == 2) { + bool ok_a = false, ok_b = false; + const sfr::KinectSeries sa = skeleton_series(skeletons[0], "v1", ok_a); + const sfr::KinectSeries sb = skeleton_series(skeletons[1], "v2", ok_b); + if (!ok_a || !ok_b) { + line("骨架檔讀不到：" + (!ok_a ? skeletons[0] : skeletons[1])); + } else { + const sfr::KinectLag skeleton = sfr::kinect_cross_correlate(sa, sb, max_lag, step); + line("骨架時間差 skeleton lag (spine_base_y): " + + (skeleton.found ? format("%+.1f", skeleton.lag_us / 1000.0) + " ms, r=" + format("%.3f", skeleton.r) + + ", pairs=" + std::to_string(skeleton.pairs) + : std::string("找不到")) + + " (v1 frames " + std::to_string(sa.t.size()) + ", v2 frames " + std::to_string(sb.t.size()) + ")"); + } + } + + // Each v1 image against the v2 at the same moment. + const size_t features = sfr::kinect_depth_compare_features.size(); + std::vector sb(features); + for (size_t f = 0; f < features; ++f) sb[f] = series(b, f); + std::vector> truth(features), other(features); + std::vector pair_times; // each pair's v1 image time + uint64_t pairs = 0, lost_a = 0, lost_b = 0, lost_either = 0; + double valid_a = 0, valid_b = 0, pixels_a = 0, pixels_b = 0; + std::FILE* out = csv.empty() ? nullptr : std::fopen(csv.c_str(), "w"); + if (!csv.empty() && !out) std::cerr << "cannot write " << csv << '\n'; + if (out) { + std::fprintf(out, "steady_us,v1_number,v2_steady_us"); + for (const auto& name : sfr::kinect_depth_compare_features) std::fprintf(out, ",v1_%s,v2_%s", name.c_str(), name.c_str()); + std::fprintf(out, "\n"); + } + // The v2's presence of a player at a moment: both images around it have one. + const sfr::KinectSeries b_pixels = series(b, 6); + for (const auto& s : a.samples) { + const int64_t at = s.steady_us + int64_t(std::llround(lag_us)); + if (at < b.samples.front().steady_us || at > b.samples.back().steady_us) continue; + const double b_invalid = sfr::kinect_series_at(sb[18], at, gap); + if (!std::isfinite(b_invalid)) continue; // a gap in the v2's images + ++pairs; + pair_times.push_back(s.steady_us); + valid_a += 1.0 - double(s.invalid); + valid_b += 1.0 - b_invalid; + const double b_player_pixels = sfr::kinect_series_at(b_pixels, at, gap); + const bool b_player = std::isfinite(b_player_pixels); + pixels_a += s.player ? double(s.features.centroid.pixels) : 0.0; + pixels_b += b_player ? b_player_pixels : 0.0; + lost_a += !s.player; + lost_b += !b_player; + lost_either += !s.player || !b_player; + if (out) std::fprintf(out, "%lld,%llu,%lld", (long long)s.steady_us, (unsigned long long)s.number, (long long)at); + for (size_t f = 0; f < features; ++f) { + const double x = sfr::kinect_depth_feature(s, f), y = sfr::kinect_series_at(sb[f], at, gap); + truth[f].push_back(x); + other[f].push_back(y); + if (out) { + // Empty for none. + const std::string cx = std::isfinite(x) ? format("%.9g", x) : "", cy = std::isfinite(y) ? format("%.9g", y) : ""; + std::fprintf(out, ",%s,%s", cx.c_str(), cy.c_str()); + } + } + if (out) std::fprintf(out, "\n"); + } + const bool csv_written = out != nullptr; + if (out) std::fclose(out); + out = nullptr; + if (!pairs) { + line("沒有可以配對的影像（v2 在 v1 的每張影像附近都沒有影像）。"); + std::cout << text; + return 1; + } + const auto percent = [](double value) { return format("%.1f%%", 100.0 * value); }; + line("配對影像 pairs: " + std::to_string(pairs) + " (v1 每張影像對 v2 同一時刻內插，間隔上限 " + format("%.0f", gap_ms) + " ms)"); + line("有效像素比例 valid ratio: v1 " + percent(valid_a / double(pairs)) + ", v2 " + percent(valid_b / double(pairs))); + line("玩家像素 player pixels (平均): v1 " + format("%.0f", pixels_a / double(pairs)) + ", v2 " + + format("%.0f", pixels_b / double(pairs))); + line("失去玩家 lost: v1 " + percent(double(lost_a) / double(pairs)) + ", v2 " + percent(double(lost_b) / double(pairs)) + + ", 任一台 either " + percent(double(lost_either) / double(pairs))); + line(""); + line("特徵 feature n r slope intercept rmse resid_rmse lag_ms mean_v1 mean_v2"); + line("（v1 = slope * v2 + intercept；rmse 是 v1 - v2；lag_ms 是該特徵自己的最佳時間差，已扣掉上面的整體時間差）"); + std::vector fits(features); + for (size_t f = 0; f < features; ++f) { + const sfr::KinectFit fit = fits[f] = sfr::kinect_fit(truth[f], other[f]); + sfr::KinectSeries shifted = sb[f]; + for (auto& t : shifted.t) t -= int64_t(std::llround(lag_us)); + const sfr::KinectLag own = sfr::kinect_cross_correlate(series(a, f), shifted, max_lag, step); + char row[512]; + std::snprintf(row, sizeof row, "%-20s %6zu %7s %10s %11s %11s %11s %8s %11s %11s", + sfr::kinect_depth_compare_features[f].c_str(), fit.n, format("%.3f", fit.r).c_str(), + format("%.4g", fit.slope).c_str(), format("%.4g", fit.intercept).c_str(), + format("%.4g", fit.rmse).c_str(), format("%.4g", fit.residual_rmse).c_str(), + own.found ? format("%+.0f", own.lag_us / 1000.0).c_str() : "-", + fit.n ? format("%.4g", fit.mean_truth).c_str() : "-", fit.n ? format("%.4g", fit.mean_other).c_str() : "-"); + line(row); + } + line(""); + line("說明：centroid/min/max 是像素欄列；count_* 是玩家在四個區塊的像素數（+640 下左 = 玩家右側，+644 下右）；" + "mean_* 與 mean_depth 是 毫米<<3（除以 8 為毫米）；lean 是遊戲由 +640/+644 算出的傾斜；" + "空的區塊與像素太少的影像不列入 mean_*。"); + line(""); + // Size and depth: the means over the pairs where both have a player. + const double pixel_ratio = fits[6].n ? fits[6].mean_other / fits[6].mean_truth : NAN; + const double depth_ratio = fits[17].n ? fits[17].mean_other / fits[17].mean_truth : NAN; + line("輪廓大小比 player pixel ratio v2/v1: " + format("%.3f", pixel_ratio) + "；平均深度比 mean depth ratio v2/v1: " + + format("%.3f", depth_ratio) + "（兩邊都有玩家的配對影像的平均）"); + std::string summary = "SUMMARY pairs=" + std::to_string(pairs); + const auto add = [&](const std::string& key, double value) { summary += " " + key + "=" + format("%.4g", value); }; + add("lag_ms", lag_us / 1000.0); + for (const auto& [name, f] : {std::pair{"lean", 11}, {"centroid_row", 1}, {"mean_depth", 17}}) { + add(std::string(name) + "_r", fits[f].r); + add(std::string(name) + "_slope", fits[f].slope); + } + add("pixel_ratio", pixel_ratio); + add("depth_ratio", depth_ratio); + if (!events_path.empty()) { + std::ifstream in(events_path); + std::vector rows; + std::string text_line; + while (std::getline(in, text_line)) { + sfr::KinectRaceRecord row; + if (sfr::kinect_parse_race_row(text_line, row)) rows.push_back(row); + } + if (!in.eof() || rows.empty()) { + line("事件 events: 讀不到 " + events_path + "（或沒有任何列）"); + } else { + line("遊戲判定的事件 events（" + events_path + "：crouch_original／jump_original 每段 1 的開頭；" + "基準 = 前 2.0–0.5 s 的平均，反應 = 之後 1.2 s 內的最大（蹲）／最小（跳）減基準，在對齊後的序列上）"); + line(" 事件 event 特徵 feature events measured v1 平均 v2 平均 v2/v1 同方向 same"); + // crouch_original is detector 0, jump_original 2 (kinect_race_detectors). + for (const auto& [event, detector, upward] : + {std::tuple{"crouch", 0, true}, {"jump", 2, false}}) { + const std::vector starts = sfr::kinect_detector_event_starts(rows, detector); + for (const auto& [feature, f] : {std::pair{"centroid_row", 1}, {"min_row", 3}}) { + const sfr::KinectEventAmplitudes a = + sfr::kinect_event_amplitudes(pair_times, truth[f], other[f], starts, upward); + char row_text[256]; + std::snprintf(row_text, sizeof row_text, " %-11s %-16s %6zu %8zu %11s %11s %8s %zu/%zu", event, feature, + a.events, a.measured, format("%.1f", a.truth).c_str(), format("%.1f", a.other).c_str(), + format("%.3f", a.ratio()).c_str(), a.same_direction, a.measured); + line(row_text); + const std::string key = std::string(event) + "_" + feature; + add(key + "_v1", a.truth); + add(key + "_v2", a.other); + add(key + "_ratio", a.ratio()); + summary += " " + key + "_same=" + std::to_string(a.same_direction) + "/" + std::to_string(a.measured); + } + } + } + } + if (csv_written) line("對齊後的序列 aligned series: " + csv); + line(summary); + std::cout << text; + if (!report_path.empty()) { + std::ofstream file(report_path, std::ios::binary); + file << text; + if (!file) std::cerr << "cannot write " << report_path << '\n'; + } + return 0; +} diff --git a/tests/kinect_depth_features.cpp b/tests/kinect_depth_features.cpp new file mode 100644 index 0000000..2255033 --- /dev/null +++ b/tests/kinect_depth_features.cpp @@ -0,0 +1,241 @@ +// The title's depth view, again, over a depth recording (kinect_record_depth.h, +// kinect_depth_features.h): per image and per player the numbers the game's +// own depth detectors read, without the game. +// +// sfr_kinect_depth_features [--csv out.csv] [--slot N] [--summary] +// [--validate -race.csv] [--show N] +// +// --csv writes one row per image and player slot with pixels (slot empty +// when the image has no player): the image's clocks and number, its share of +// pixels without a depth, its player pixels, then pass one (centroid, +// extents, the player's pixel count) and pass two (the four parts' counts, +// whether the rest was written, their mean depths in millimetres << 3, the +// near count and the average). --slot N only that skeleton slot (0..5, every +// image). --summary prints the file's images, span, rate, mean share +// without a depth and player pixels (also printed after --csv). +// +// --validate compares, for a recording the game made (SFR_KINECT_RECORD with +// SFR_KINECT_RECORD_DEPTH=1, so its -depth.bin and -race.csv +// are of the same images), what the game's depth view wrote for each image +// (the race CSV's view_frame and following columns) with what this tool +// computes for the image of that number and the same slot: matched by the +// image's number, which both files take from the same counter (the number +// the game sees in NUI_IMAGE_FRAME +8), so no clock is involved. A race row +// is written every game update and the view runs on every new image (the +// title asks for one every fourth frame), so rows repeat an image: each +// (image, slot) is checked once. It prints the exact-match rate field by +// field and overall, and the first --show N (default 10) differences. +// Exit status 0 when every compared field matched, 3 otherwise. +#include "kinect_depth_compare.h" +#include "kinect_depth_features.h" +#include "kinect_race_record.h" +#include "kinect_record_depth.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +void put_features(std::FILE* out, const sfr::KinectDepthFeatures& f) { + const auto& c = f.centroid; + const auto& s = f.split; + std::fprintf(out, ",%u,%u,%u,%u,%u,%u,%u,%u", f.slot, c.pixels, c.column, c.row, c.min_column, c.min_row, + c.max_column, c.max_row); + for (float v : s.counts) std::fprintf(out, ",%.9g", double(v)); + std::fprintf(out, ",%d", s.written ? 1 : 0); + if (s.written) { + for (float v : s.means) std::fprintf(out, ",%.9g", double(v)); + std::fprintf(out, ",%u,%.9g", s.near_count, double(s.average)); + } else { + std::fprintf(out, ",,,,,,"); + } +} + +int validate(const std::string& depth_path, const std::string& race_path, size_t show) { + // The game's own values, once per (image, slot). + std::ifstream in(race_path); + if (!in) { + std::cerr << "cannot read " << race_path << '\n'; + return 1; + } + std::map, sfr::KinectDepthViewRecord> game; + uint64_t rows = 0, view_rows = 0, inconsistent = 0; + std::string line; + while (std::getline(in, line)) { + sfr::KinectRaceRecord row; + if (!sfr::kinect_parse_race_row(line, row)) continue; + ++rows; + if (!row.view.known) continue; + ++view_rows; + const auto key = std::make_pair(row.view.frame, row.view.slot); + const auto [at, added] = game.emplace(key, row.view); + // The same image's values must be the same in every row. + if (!added && (at->second.counts != row.view.counts || at->second.centroid.column != row.view.centroid.column || + at->second.near_count != row.view.near_count)) + ++inconsistent; + } + if (game.empty()) { + std::cerr << race_path << ": " << rows << " race rows, none with the depth view's columns " + << "(a recording from before them, or no race with a real Kinect)\n"; + return 1; + } + sfr::KinectDepthReader reader; + if (!reader.open(depth_path)) { + std::cerr << depth_path << ": " << reader.error() << '\n'; + return 1; + } + // One pass over the images (a long recording does not fit in memory): + // each image the game's rows name is checked as it is read. + std::array compared{}, equal{}; + uint64_t checked = 0, all = 0, not_written = 0, images = 0; + std::set> seen; + size_t shown = 0; + sfr::KinectDepthRecordFrame image; + while (reader.next(image)) { + ++images; + for (auto at = game.lower_bound({image.number, 0}); at != game.end() && at->first.first == image.number; ++at) { + const auto& [key, view] = *at; + if (!seen.insert(key).second) continue; // an image number twice: the first is the game's + const sfr::KinectDepthFeatures tool = + sfr::kinect_depth_view_features(image.pixels.data(), image.width, image.height, key.second); + const sfr::KinectDepthViewCheck check = sfr::kinect_depth_view_check(view, tool); + ++checked; + if (!tool.split.written) ++not_written; + for (size_t i = 0; i < sfr::kinect_depth_view_field_count; ++i) { + compared[i] += check.compared[i]; + equal[i] += check.compared[i] && check.equal[i]; + } + if (check.all_equal()) { + ++all; + continue; + } + if (shown++ >= show) continue; + std::printf(" image %llu slot %u differs:", (unsigned long long)key.first, key.second); + for (size_t i = 0; i < sfr::kinect_depth_view_field_count; ++i) + if (check.compared[i] && !check.equal[i]) + std::printf(" %s game=%s tool=%s", sfr::kinect_depth_view_fields[i], check.game[i].c_str(), + check.tool[i].c_str()); + std::printf("\n"); + } + } + const uint64_t missing = game.size() - checked; + std::printf("validate: %s against %s\n", depth_path.c_str(), race_path.c_str()); + std::printf(" race rows %llu, with the depth view %llu; distinct (image, slot) %zu; depth images in the file %llu%s\n", + (unsigned long long)rows, (unsigned long long)view_rows, game.size(), (unsigned long long)images, + reader.truncated() ? " (the last cut short)" : ""); + std::printf(" matched by image number %llu; not in the depth file %llu (SFR_KINECT_RECORD_DEPTH_EVERY, its cap, " + "or images before it opened)\n", + (unsigned long long)checked, (unsigned long long)missing); + if (inconsistent) std::printf(" rows of one image disagreeing among themselves: %llu\n", (unsigned long long)inconsistent); + std::printf(" images too sparse for more than the zeroed counts: %llu (means, near, average not compared there)\n", + (unsigned long long)not_written); + for (size_t i = 0; i < sfr::kinect_depth_view_field_count; ++i) + if (compared[i]) + std::printf(" %-18s %llu/%llu exact (%.2f%%)\n", sfr::kinect_depth_view_fields[i], + (unsigned long long)equal[i], (unsigned long long)compared[i], + 100.0 * double(equal[i]) / double(compared[i])); + if (checked) + std::printf(" all fields exact: %llu/%llu (%.2f%%)\n", (unsigned long long)all, (unsigned long long)checked, + 100.0 * double(all) / double(checked)); + return checked && all == checked ? 0 : 3; +} +} + +int main(int argc, char** argv) { + std::string path, csv, race; + int only = -1; + bool summary = false; + size_t show = 10; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--csv" && i + 1 < argc) csv = argv[++i]; + else if (arg == "--slot" && i + 1 < argc) only = std::atoi(argv[++i]); + else if (arg == "--summary") summary = true; + else if (arg == "--validate" && i + 1 < argc) race = argv[++i]; + else if (arg == "--show" && i + 1 < argc) show = size_t(std::atoi(argv[++i])); + else if (path.empty()) path = arg; + else { + std::cerr << "unknown argument " << arg << '\n'; + return 2; + } + } + if (path.empty() || (only > 5)) { + std::cerr << "usage: sfr_kinect_depth_features [--csv out.csv] [--slot 0..5] [--summary]\n" + " [--validate -race.csv] [--show N]\n"; + return 2; + } + if (!race.empty()) return validate(path, race, show); + sfr::KinectDepthReader reader; + if (!reader.open(path)) { + std::cerr << path << ": " << reader.error() << '\n'; + return 1; + } + std::FILE* out = nullptr; + if (!csv.empty()) { + out = std::fopen(csv.c_str(), "w"); + if (!out) { + std::cerr << "cannot write " << csv << '\n'; + return 1; + } + std::fprintf(out, "number,steady_us,handed_us,sdk_time,sdk_frame,source,invalid_ratio,player_pixels,slot,pixels," + "centroid_col,centroid_row,min_col,min_row,max_col,max_row,count_lower_left,count_lower_right," + "count_upper_left,count_upper_right,written,mean_lower_left,mean_lower_right,mean_upper_left," + "mean_upper_right,near_count,mean_depth\n"); + } + sfr::KinectDepthRecordFrame frame; + uint64_t images = 0, with_player = 0; + double invalid = 0, player_pixels = 0; + int64_t first = 0, last = 0; + std::string source; + while (reader.next(frame)) { + if (!images) first = frame.steady_us; + last = frame.steady_us; + ++images; + source = sfr::kinect_depth_source_name(frame.source); + const sfr::KinectDepthImageStats stats = sfr::kinect_depth_image_stats(frame.pixels.data(), frame.pixels.size()); + uint32_t players = 0; + for (uint32_t index = 1; index <= 6; ++index) players += stats.players[index]; + invalid += stats.invalid; + player_pixels += players; + with_player += players > 0; + if (!out) continue; + const auto prefix = [&] { + std::fprintf(out, "%llu,%lld,%lld,%lld,%llu,%s,%.6f,%u", (unsigned long long)frame.number, + (long long)frame.steady_us, (long long)frame.handed_us, (long long)frame.sdk_time, + (unsigned long long)frame.sdk_frame, source.c_str(), double(stats.invalid), players); + }; + bool any = false; + for (uint32_t slot = 0; slot < 6; ++slot) { + if (only >= 0 ? int(slot) != only : !stats.players[slot + 1]) continue; + any = true; + prefix(); + put_features(out, sfr::kinect_depth_view_features(frame.pixels.data(), frame.width, frame.height, slot)); + std::fprintf(out, "\n"); + } + if (!any) { + prefix(); + std::fprintf(out, "%s\n", std::string(19, ',').c_str()); // no player: the slot's columns empty + } + } + if (out) std::fclose(out); + if (!reader.error().empty()) std::cerr << path << ": stopped at " << reader.error() << '\n'; + // The summary, always (--summary alone writes nothing else). + (void)summary; + const double seconds = images > 1 ? double(last - first) / 1e6 : 0; + std::printf("%s: %s, %ux%u, %llu images over %.1f s (%.1f a second)%s\n", path.c_str(), source.c_str(), + reader.width(), reader.height(), (unsigned long long)images, seconds, + seconds > 0 ? double(images - 1) / seconds : 0.0, reader.truncated() ? ", the last cut short" : ""); + if (images) + std::printf(" invalid_ratio %.2f%%, player pixels %.0f on average, images with a player %.1f%%\n", + 100.0 * invalid / double(images), player_pixels / double(images), + 100.0 * double(with_player) / double(images)); + if (out) std::printf(" features written to %s\n", csv.c_str()); + return images ? 0 : 1; +} diff --git a/tests/kinect_depth_resynth.cpp b/tests/kinect_depth_resynth.cpp new file mode 100644 index 0000000..41bcf48 --- /dev/null +++ b/tests/kinect_depth_resynth.cpp @@ -0,0 +1,119 @@ +// Makes the depth a Kinect v2 gives the game again, offline, from a raw +// recording of what it measured (kinect_record_raw.h: sfr_kinect_probe --raw, +// or the game with SFR_KINECT_RECORD_RAW=1), through any view: +// +// sfr_kinect_depth_resynth --view fit|sensor|player|ideal [--distance m] [--height m] +// [--follow s] [--deadband m] [--fit-scale k] --out +// +// The view state and the synthesis are the live sensor's own +// (KinectDepthViewer, kinect_v2_synthesize_depth; kinect_v2_resynth.h), so +// the same settings give the images the live path made. The output is a +// standard depth recording (SFRKDEP2, source 2: kinect_record_depth.h) that +// sfr_kinect_depth_features and sfr_kinect_depth_compare read as they read +// the probe's. Unset values are the environment variables' defaults +// (SFR_KINECT_V2_DEPTH_*); --fit-scale unset takes the recording's own lens. +#include "kinect_record_depth.h" +#include "kinect_record_raw.h" +#include "kinect_v2_resynth.h" + +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + std::string input, output; + const char* view = nullptr; + const char* distance = nullptr; + const char* height = nullptr; + const char* follow = nullptr; + const char* deadband = nullptr; + const char* fit_scale = nullptr; + bool usage = false; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + const bool more = i + 1 < argc; + if (arg == "--view" && more) view = argv[++i]; + else if (arg == "--distance" && more) distance = argv[++i]; + else if (arg == "--height" && more) height = argv[++i]; + else if (arg == "--follow" && more) follow = argv[++i]; + else if (arg == "--deadband" && more) deadband = argv[++i]; + else if (arg == "--fit-scale" && more) fit_scale = argv[++i]; + else if (arg == "--out" && more) output = argv[++i]; + else if (!arg.empty() && arg[0] != '-' && input.empty()) input = arg; + else usage = true; + } + const std::string name = view ? view : ""; + if (usage || input.empty() || output.empty() || + (name != "fit" && name != "sensor" && name != "player" && name != "ideal")) { + std::cerr << "usage: sfr_kinect_depth_resynth --view fit|sensor|player|ideal [--distance m] [--height m]\n" + " [--follow s] [--deadband m] [--fit-scale k] --out \n"; + return 2; + } + sfr::KinectRawReader reader; + if (!reader.open(input)) { + std::cerr << input << ": " << reader.error() << '\n'; + return 1; + } + const sfr::KinectDepthSettings settings = + sfr::KinectDepthSettings::parse(view, distance, height, fit_scale, follow, deadband); + sfr::KinectV2Resynth resynth(settings, reader.header()); + std::printf("%s: %ux%u, rays %s, lens focal %.1f\n", input.c_str(), reader.header().width, reader.header().height, + sfr::kinect_raw_rays_name(reader.header().rays_from), double(reader.header().lens[1])); + if (!resynth.warning().empty()) std::printf("note: %s\n", resynth.warning().c_str()); + std::printf("view=%s", settings.view_name()); + if (settings.mode == sfr::KinectDepthViewMode::fit) std::printf(" fit_scale=%.4f", double(resynth.viewer().fit_scale())); + if (settings.mode == sfr::KinectDepthViewMode::player || settings.mode == sfr::KinectDepthViewMode::ideal) + std::printf(" distance=%.2f height=%.2f", double(settings.distance), double(settings.height)); + if (settings.mode == sfr::KinectDepthViewMode::ideal) + std::printf(" follow=%.2f deadband=%.2f", double(settings.follow), double(settings.deadband)); + std::printf("\n"); + // A made image is never larger than the raw frame it comes from (153,600 + // bytes at most against about 434,000): the raw file's size bounds the + // output, and the free space warning means something. + // Offline: wait for the disk rather than drop an image. + sfr::KinectRecordQueue::process().set_blocking(true); + std::error_code error; + const uint64_t raw_bytes = std::filesystem::file_size(input, error); + sfr::KinectDepthRecordFile file(output, (error ? (64ull << 30) : raw_bytes) + (1ull << 20), 1); + sfr::KinectRawFrame frame; + sfr::KinectImage image; + uint64_t frames = 0, mapped = 0, written = 0; + double valid = 0, players = 0; + while (reader.next(frame)) { + const sfr::KinectDepthStats stats = resynth.make(frame, image); + ++frames; + mapped += (frame.flags & sfr::kinect_raw_flag_mapped) ? 1 : 0; + valid += stats.valid; + players += stats.players; + sfr::KinectDepthRecordFrame record; + record.steady_us = image.steady_us; + record.handed_us = image.steady_us; + record.sdk_time = image.sdk_time; + record.number = image.number; + record.source = sfr::KinectDepthSource::v2; + record.origin = image.view_origin; + record.z_scale = image.view_z_scale; + sfr::kinect_depth_frame_pixels(record, image.pixels.data(), image.width, image.height); + if (file.write(record)) ++written; + } + // Everything on disk before the summary (no time limit offline). + sfr::KinectRecordQueue::process().drain(&file, std::chrono::hours(24)); + if (reader.truncated()) std::printf("the last frame was cut short: left out\n"); + if (!reader.error().empty()) std::printf("stopped at a broken frame: %s\n", reader.error().c_str()); + const double n = frames ? double(frames) : 1.0; + std::printf("frames=%llu (mapper points %llu, lens %llu) written=%llu to %s; valid %.1f%%, player pixels %.0f\n", + (unsigned long long)frames, (unsigned long long)mapped, (unsigned long long)(frames - mapped), + (unsigned long long)written, output.c_str(), 100.0 * valid / n / (320.0 * 240.0), players / n); + if (settings.mode == sfr::KinectDepthViewMode::ideal) { + const sfr::KinectIdealState& state = resynth.viewer().ideal(); + const sfr::KinectDepthView& v = resynth.viewer().view(); + float pitch = 0, roll = 0; + sfr::kinect_floor_tilt(state.floor, pitch, roll); + std::printf("ideal camera at the end: %.3f,%.3f,%.3f level=%d pitch=%.1f roll=%.1f reference=%.3f,%.3f\n", + double(v.origin[0]), double(v.origin[1]), double(v.origin[2]), state.floor_known ? 1 : 0, + double(pitch), double(roll), double(state.reference[0]), double(state.reference[1])); + } + return frames && written == frames ? 0 : 1; +} diff --git a/tests/kinect_depth_test.cpp b/tests/kinect_depth_test.cpp new file mode 100644 index 0000000..1befafd --- /dev/null +++ b/tests/kinect_depth_test.cpp @@ -0,0 +1,434 @@ +// The depth recording's file format, the title's depth view features +// computed again from an image, and the comparison of two recordings +// (kinect_record_depth.h, kinect_depth_features.h, kinect_depth_compare.h). +#include "kinect_depth_compare.h" +#include "kinect_depth_features.h" +#include "kinect_race_record.h" +#include "kinect_record_depth.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} +bool near(double a, double b, double tolerance) { return std::fabs(a - b) <= tolerance; } + +constexpr uint32_t width = 320, height = 240; +uint16_t pixel(uint32_t millimetres, uint32_t player) { return uint16_t(millimetres << 3 | player); } + +std::vector blank() { return std::vector(size_t(width) * height, 0); } +void rectangle(std::vector& image, uint32_t x0, uint32_t x1, uint32_t y0, uint32_t y1, uint16_t value) { + for (uint32_t y = y0; y <= y1; ++y) + for (uint32_t x = x0; x <= x1; ++x) image[size_t(y) * width + x] = value; +} + +std::string read_file(const std::string& path) { + std::ifstream in(path, std::ios::binary); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} +uint32_t u32(const std::string& bytes, size_t at) { + return uint32_t(uint8_t(bytes[at])) | uint32_t(uint8_t(bytes[at + 1])) << 8 | uint32_t(uint8_t(bytes[at + 2])) << 16 | + uint32_t(uint8_t(bytes[at + 3])) << 24; +} + +sfr::KinectDepthRecordFrame frame_of(const std::vector& pixels, uint64_t number) { + sfr::KinectDepthRecordFrame f; + f.steady_us = 1000000 + int64_t(number) * 33333; + f.handed_us = f.steady_us + 5000; + f.sdk_time = int64_t(number) * 333333; + f.number = number; + f.sdk_frame = number + 100; + f.source = sfr::KinectDepthSource::v2; + f.origin = {0.0f, 0.5f, -1.5f}; + f.z_scale = 1.378f; + f.width = width; + f.height = height; + f.pixels = pixels; + return f; +} + +void header_layout() { + const std::string header = sfr::kinect_depth_file_header(320, 240); + require(header.size() == 32 && header.compare(0, 8, "SFRKDEP2") == 0, "magic"); + require(u32(header, 8) == 2 && u32(header, 12) == 32 && u32(header, 16) == 320 && u32(header, 20) == 240 && + u32(header, 24) == 1 && u32(header, 28) == 68, + "version, sizes, pixel format, frame header size"); + uint32_t w = 0, h = 0, frame_header = 0; + std::string why; + require(sfr::kinect_depth_parse_file_header(reinterpret_cast(header.data()), header.size(), w, h, + frame_header, why) && w == 320 && h == 240 && frame_header == 68, + "the header parses"); + std::string old = "SFRKDEP1" + std::string(24, '\0'); + require(!sfr::kinect_depth_parse_file_header(reinterpret_cast(old.data()), old.size(), w, h, + frame_header, why) && why.find("SFRKDEP1") != std::string::npos, + "an old SFRKDEP1 file is refused, saying so"); +} + +void frame_layout() { + auto image = blank(); + image[0] = 0x1234; + image[1] = 0xABCD; + const auto frame = frame_of(image, 7); + const std::string raw = sfr::kinect_depth_encode_frame(frame, false); + require(raw.size() == 68 + 320 * 240 * 2, "a raw frame: header and every pixel"); + require(raw.compare(0, 4, "KDF2") == 0 && u32(raw, 4) == 320 * 240 * 2, "frame magic and payload size"); + require(u32(raw, 8) == uint32_t(frame.steady_us) && u32(raw, 16) == uint32_t(frame.handed_us) && + u32(raw, 24) == uint32_t(frame.sdk_time) && u32(raw, 32) == 7 && u32(raw, 40) == 107, + "steady_us, handed_us, sdk_time, number, sdk_frame"); + require(uint8_t(raw[48]) == 2 && uint8_t(raw[49]) == 0, "source v2, raw"); + float z = 0, k = 0; + std::memcpy(&z, raw.data() + 60, 4); + std::memcpy(&k, raw.data() + 64, 4); + require(z == -1.5f && k == 1.378f, "the view"); + require(uint8_t(raw[68]) == 0x34 && uint8_t(raw[69]) == 0x12 && uint8_t(raw[70]) == 0xCD, "pixels little-endian"); + const std::string packed = sfr::kinect_depth_encode_frame(frame); + require(uint8_t(packed[49]) == 1 && packed.size() < raw.size(), "a mostly empty image goes as zero runs"); +} + +void zero_runs() { + std::vector pixels(200000, 0); + pixels[3] = 9; + pixels[199999] = 5; + std::vector tokens, back; + sfr::kinect_depth_zero_runs_encode(pixels.data(), pixels.size(), tokens); + // 3 zeros, 9, then 199995 zeros (65535 + 65535 + 65535 + 3390), 5. + require(tokens.size() == 2 + 1 + 8 + 1, "runs longer than 65535 split"); + require(sfr::kinect_depth_zero_runs_decode(tokens.data(), tokens.size(), pixels.size(), back) && back == pixels, + "and come back"); + require(!sfr::kinect_depth_zero_runs_decode(tokens.data(), tokens.size(), pixels.size() + 1, back), + "too few pixels is refused"); + const uint16_t broken[] = {0}; + require(!sfr::kinect_depth_zero_runs_decode(broken, 1, 1, back), "a run without its count is refused"); + const std::vector full(64, 3); + sfr::kinect_depth_zero_runs_encode(full.data(), full.size(), tokens); + require(tokens == full, "no zero: the pixels as they are"); +} + +void files_round_trip() { + const std::string path = "kinect_depth_test_round_trip.bin"; + std::remove(path.c_str()); + auto image = blank(); + rectangle(image, 100, 139, 50, 149, pixel(2000, 1)); + { + // Every second image kept; the cap fits three. + auto first = image; + first[1] = pixel(1001, 0); + const uint64_t frame_bytes = sfr::kinect_depth_encode_frame(frame_of(first, 1)).size(); + sfr::KinectDepthRecordFile file(path, 32 + 3 * frame_bytes, 2); + uint32_t written = 0; + for (uint64_t n = 1; n <= 9; ++n) { + auto changed = image; + changed[n] = pixel(1000 + uint32_t(n), 0); + written += file.write(frame_of(changed, n)); + } + require(written == 3 && file.frames_written() == 3, "every second image until the cap"); + require(!file.write(frame_of(image, 10)) && !file.write(frame_of(image, 11)), "a full file takes no more"); + } + sfr::KinectDepthReader reader; + require(reader.open(path) && reader.width() == 320 && reader.height() == 240, "the file opens"); + sfr::KinectDepthRecordFrame frame; + std::vector numbers; + while (reader.next(frame)) { + numbers.push_back(frame.number); + require(frame.pixels[frame.number] == pixel(1000 + uint32_t(frame.number), 0) && + frame.pixels[60 * width + 120] == pixel(2000, 1), + "the pixels come back"); + require(frame.source == sfr::KinectDepthSource::v2 && frame.sdk_frame == frame.number + 100 && + frame.handed_us == frame.steady_us + 5000 && frame.z_scale == 1.378f, + "the clocks and view come back"); + } + require(numbers == std::vector{1, 3, 5} && reader.error().empty() && !reader.truncated(), + "images 1, 3, 5 and a clean end"); + // A game ending mid-write: the last image cut short is left out. + std::string bytes = read_file(path); + bytes.resize(bytes.size() - 10); + { + std::ofstream out(path, std::ios::binary | std::ios::trunc); + out.write(bytes.data(), std::streamsize(bytes.size())); + } + require(reader.open(path), "the cut file opens"); + numbers.clear(); + while (reader.next(frame)) numbers.push_back(frame.number); + require(numbers == std::vector{1, 3} && reader.truncated(), "a frame cut short is left out"); + std::remove(path.c_str()); + { + sfr::KinectDepthRecordFile never("kinect_depth_test_never.bin", 1 << 20, 1); + } + std::ifstream none("kinect_depth_test_never.bin"); + require(!none, "no image, no file"); +} + +void naming_and_space() { + require(sfr::kinect_record_depth_path("rec\\run1\\v1game", "") == "rec\\run1\\v1game-depth.bin", "the game's file"); + require(sfr::kinect_record_depth_path("rec/v2", "v2") == "rec/v2-v2-depth.bin", "a probe's file per sensor"); + const uint64_t mb = 1024 * 1024; + require(sfr::kinect_depth_space_warning("x", 9000 * mb, 4096 * mb).empty(), "enough room: no warning"); + const std::string warning = sfr::kinect_depth_space_warning("x", 5000 * mb, 4096 * mb); + require(warning.find("NATIVE_KINECT_RECORD depth low disk") == 0 && warning.find("5000 MB") != std::string::npos, + "less than twice the cap: a warning"); +} + +void empty_image() { + const auto image = blank(); + const auto f = sfr::kinect_depth_view_features(image.data(), width, height, 0); + require(f.centroid.column == 160 && f.centroid.row == 120, "no pixel: the centre (82439CE8)"); + require(f.centroid.min_column == 320 && f.centroid.min_row == 240 && f.centroid.max_column == 0 && + f.centroid.max_row == 0 && f.centroid.pixels == 0, + "no pixel: the extents' starting values"); + require(!f.split.written && f.split.counts == std::array{}, "no pixel: counts zero, nothing else"); +} + +void rectangle_by_hand() { + // Player 1 (slot 0): columns 100..139, rows 50..149, 2000 mm; player 2 + // and a no-player depth around it, which slot 0 ignores. + auto image = blank(); + rectangle(image, 0, 319, 0, 239, pixel(3500, 0)); + rectangle(image, 100, 139, 50, 149, pixel(2000, 1)); + rectangle(image, 200, 209, 100, 109, pixel(1500, 2)); + const auto f = sfr::kinect_depth_view_features(image.data(), width, height, 0); + // 4000 pixels; columns sum 100 * (100 + ... + 139) = 478000, rows 40 * + // (50 + ... + 149) = 398000: 119.5 and 99.5, truncated. + require(f.centroid.pixels == 4000 && f.centroid.column == 119 && f.centroid.row == 99, "the centroid, truncated"); + // The largest column as the title has it: lanes e and e + 4 (column % 8) + // merged by their smaller largest (vminuw, 82439D30): columns 132..139 + // are lanes 4..7, 0..3, so the pairs give 132..135 and the largest is + // 135, not 139. + require(f.centroid.min_column == 100 && f.centroid.max_column == 135 && f.centroid.min_row == 50 && + f.centroid.max_row == 149, + "the extents"); + // Upper rows 50..98 (49), lower 99..149 (51); left columns 100..118 + // (19), right 119..139 (21). + require(f.split.written, "4000 >= 150: written"); + require(f.split.counts[sfr::lower_left] == 51 * 19 && f.split.counts[sfr::lower_right] == 51 * 21 && + f.split.counts[sfr::upper_left] == 49 * 19 && f.split.counts[sfr::upper_right] == 49 * 21, + "the four parts' counts"); + for (float mean : f.split.means) require(near(mean, 16000.0, 0.01), "every part at 2000 mm << 3"); + require(near(f.split.average, 16000.0, 0.01), "the average of the four"); + require(f.split.near_count == 4000, "every pixel within 75 mm of it"); + // The second player in slot 1. + const auto g = sfr::kinect_depth_view_features(image.data(), width, height, 1); + require(g.centroid.pixels == 100 && g.centroid.column == 204 && g.centroid.row == 104, "player 2 by its index"); + require(!g.split.written && g.split.counts[0] == 0, "100 pixels < 150: only the zeroed counts"); +} + +void near_count_by_hand() { + // Slot 2 (player 3): a 40x40 square at 2000 mm with its right half's + // lower rows 76 mm (608 units) further: those are not near the average. + auto image = blank(); + rectangle(image, 60, 99, 40, 79, pixel(2000, 3)); + const auto plain = sfr::kinect_depth_view_features(image.data(), width, height, 2); + require(plain.centroid.column == 79 && plain.centroid.row == 59 && plain.split.near_count == 1600, "the plain square"); + // The lower right part (rows 59..79, columns 79..99) at 2076 mm. + rectangle(image, 79, 99, 59, 79, pixel(2076, 3)); + const auto f = sfr::kinect_depth_view_features(image.data(), width, height, 2); + // The far pixels move the centroid by nothing (only positions count). + require(f.centroid.column == 79 && f.centroid.row == 59, "positions only"); + // Means: three parts at 16000, the lower right at 16608; average 16152: + // its pixels are 456 away (near), the others 152 (near). + require(near(f.split.means[sfr::lower_right], 16608.0, 0.01) && near(f.split.average, 16152.0, 0.01), + "the average of the means, not of the pixels"); + require(f.split.near_count == 1600, "all within 600 of 16152"); + // 2100 mm (16800): 648 away from (3 * 16000 + 16800) / 4 = 16200: not near. + rectangle(image, 79, 99, 59, 79, pixel(2100, 3)); + const auto g = sfr::kinect_depth_view_features(image.data(), width, height, 2); + require(near(g.split.average, 16200.0, 0.01) && g.split.near_count == 1600 - 21 * 21, "the far part is not near"); +} + +void image_stats() { + auto image = blank(); + rectangle(image, 0, 159, 0, 239, pixel(2000, 0)); + rectangle(image, 10, 19, 10, 19, pixel(2000, 4)); + const auto s = sfr::kinect_depth_image_stats(image.data(), image.size()); + require(near(s.invalid, 0.5, 1e-6) && s.players[4] == 100 && s.players[0] == image.size() - 100, "holes and players"); +} + +void series_and_fit() { + sfr::KinectSeries s; + s.add(0, 0.0); + s.add(100, 10.0); + s.add(300, NAN); + s.add(400, 40.0); + require(near(sfr::kinect_series_at(s, 50, 1000), 5.0, 1e-12) && sfr::kinect_series_at(s, 100, 1000) == 10.0, + "linear between samples, exact on one"); + require(std::isnan(sfr::kinect_series_at(s, 200, 1000)) && std::isnan(sfr::kinect_series_at(s, 500, 1000)) && + std::isnan(sfr::kinect_series_at(s, 50, 50)), + "none next to a missing sample, outside, or across a gap"); + // v1(t) = sin, v2 the same movement 120 ms later, twice as large and 5 up, + // at another rate and phase: the lag is found, then the line back. + sfr::KinectSeries a, b; + const auto movement = [](double seconds) { return std::sin(seconds * 2.1) + 0.4 * std::sin(seconds * 5.3 + 1); }; + for (int i = 0; i < 15 * 20; ++i) a.add(int64_t(i) * 66667, movement(i * 0.066667)); + for (int i = 0; i < 30 * 20; ++i) { + const int64_t t = 12345 + int64_t(i) * 33333; + b.add(t, 2.0 * movement(double(t - 120000) / 1e6) + 5.0); + } + const sfr::KinectLag lag = sfr::kinect_cross_correlate(a, b, 500000, 10000); + require(lag.found && near(lag.lag_us, 120000, 6000) && lag.r > 0.99, "the lag: 120 ms, got " + std::to_string(lag.lag_us)); + std::vector truth, other; + for (size_t i = 0; i < a.t.size(); ++i) { + truth.push_back(a.v[i]); + other.push_back(sfr::kinect_series_at(b, a.t[i] + int64_t(std::llround(lag.lag_us)), 100000)); + } + const sfr::KinectFit fit = sfr::kinect_fit(truth, other); + require(fit.n > 250 && fit.r > 0.995 && near(fit.slope, 0.5, 0.02) && near(fit.intercept, -2.5, 0.1), + "v1 = 0.5 v2 - 2.5"); + require(fit.residual_rmse < 0.05 && fit.rmse > 4, "the line explains it; the raw difference does not"); + const sfr::KinectFit exact = sfr::kinect_fit({1, 2, 3, 4}, {3, 5, 7, 9}); + require(exact.n == 4 && near(exact.slope, 0.5, 1e-12) && near(exact.intercept, -0.5, 1e-12) && + near(exact.r, 1.0, 1e-12) && near(exact.residual_rmse, 0.0, 1e-12) && near(exact.rmse, std::sqrt(13.5), 1e-12), + "an exact line"); + sfr::KinectSeries flat; + for (int i = 0; i < 100; ++i) flat.add(int64_t(i) * 10000, 1.0); + require(!sfr::kinect_cross_correlate(flat, flat, 100000, 10000).found, "a series that does not move has no lag"); +} + +void lean() { + require(sfr::kinect_depth_lean(0, 5) == 1.0 && sfr::kinect_depth_lean(4, 4) == 0.0, "zero +640, and upright"); + require(near(sfr::kinect_depth_lean(15, 10), 0.5, 1e-9) && near(sfr::kinect_depth_lean(10, 15), -0.5, 1e-9), + "right and left"); + require(sfr::kinect_depth_lean(100, 1) == 3.5 && sfr::kinect_depth_lean(1, 100) == -3.5 && + sfr::kinect_depth_lean(5, 0) == 3.5, + "at most 3.5"); +} + +void view_check() { + auto image = blank(); + rectangle(image, 100, 139, 50, 149, pixel(2000, 1)); + const auto tool = sfr::kinect_depth_view_features(image.data(), width, height, 0); + sfr::KinectDepthViewRecord game; + game.known = true; + game.slot = 0; + game.centroid = tool.centroid; + game.centroid.pixels = 0; // the game does not keep it + game.counts = tool.split.counts; + game.means = tool.split.means; + game.near_count = tool.split.near_count; + game.average = tool.split.average; + auto check = sfr::kinect_depth_view_check(game, tool); + require(check.all_equal() && check.compared[15], "the game's own values match"); + game.means[2] = std::nextafter(game.means[2], 1e9f); + check = sfr::kinect_depth_view_check(game, tool); + require(!check.all_equal() && !check.equal[12], "one ulp is a difference"); + // A sparse image: the game's means are an older image's, not compared. + auto sparse = blank(); + rectangle(sparse, 10, 19, 10, 19, pixel(2000, 1)); + const auto little = sfr::kinect_depth_view_features(sparse.data(), width, height, 0); + sfr::KinectDepthViewRecord stale = game; + stale.centroid = little.centroid; + stale.counts = {}; + check = sfr::kinect_depth_view_check(stale, little); + require(check.all_equal() && !check.compared[10] && !check.compared[15], "stale means are not compared"); + stale.counts[0] = 3; + require(!sfr::kinect_depth_view_check(stale, little).all_equal(), "but non-zero counts are a difference"); +} + +void race_columns() { + const size_t columns = sfr::kinect_record_fields(sfr::kinect_race_record_header()).size(); + require(columns == 14 + 19, "the race header: the old 14 columns and the depth view's 19"); + sfr::KinectRaceRecord row; + row.steady_us = 5; + row.record = 0x40002AE0; + row.view.known = true; + row.view.frame = 812; + row.view.steady_us = 4; + row.view.slot = 3; + row.view.centroid = {119, 99, 100, 50, 139, 149, 0}; + row.view.counts = {969, 1071, 931, 1029}; + row.view.means = {15999.999f, 16000.0f, 16000.0f, 65528.0f}; + row.view.near_count = 4000; + row.view.average = 16000.25f; + std::string text; + sfr::kinect_race_record_row(row, text); + sfr::KinectRaceRecord back; + require(sfr::kinect_parse_race_row(text, back) && back.view.known && back.view.frame == 812 && back.view.slot == 3, + "the view's columns read back"); + require(back.view.centroid.column == 119 && back.view.centroid.max_row == 149 && back.view.counts == row.view.counts && + back.view.means == row.view.means && back.view.near_count == 4000 && back.view.average == 16000.25f, + "every value exactly (floats through %.9g)"); + sfr::KinectRaceRecord unknown; + text.clear(); + sfr::kinect_race_record_row(unknown, text); + require(sfr::kinect_record_fields(text).size() == columns && sfr::kinect_parse_race_row(text, back) && !back.view.known, + "no depth view yet: empty columns"); + // A row from before the columns. + const std::string old = "1,2,3,4,1.5,1,1,1,,,2,2,,"; + require(sfr::kinect_parse_race_row(old, back) && !back.view.known && back.original[0] == 1, "an old row still loads"); + sfr::kinect_depth_view_publish(row.record, row.view); + sfr::KinectDepthViewRecord latest; + require(sfr::kinect_depth_view_latest(row.record, latest) && latest.frame == 812 && + !sfr::kinect_depth_view_latest(row.record + 800, latest), + "the newest view of a record, none of another"); +} + +void detector_events() { + // Two records; crouch runs 0 0 1 1 0 1 for one, 1 1 for the other, with + // updates where it was not asked (-1) in between. + std::vector rows; + const auto add = [&](int64_t t, uint32_t record, int crouch) { + sfr::KinectRaceRecord& row = rows.emplace_back(); + row.steady_us = t; + row.record = record; + row.original[0] = crouch; + }; + add(10, 1, 0); add(20, 1, 0); add(30, 1, 1); add(35, 2, 1); add(40, 1, -1); add(45, 2, 1); + add(50, 1, 1); add(60, 1, 0); add(70, 1, 1); + const auto starts = sfr::kinect_detector_event_starts(rows, 0); + require(starts == std::vector{30, 35, 70}, "the start of each run of 1, per record, not-asked skipped"); + require(sfr::kinect_detector_event_starts(rows, 2).empty(), "no jumps"); + // A series rising 10 at t = 0 on the v1, 6 on the v2; a second event + // that the v2 lost. + std::vector times; + std::vector truth, other; + for (int64_t t = -3000000; t <= 8000000; t += 100000) { + times.push_back(t); + const bool up = (t >= 200000 && t <= 700000), up2 = (t >= 5200000 && t <= 5600000); + truth.push_back(100 + (up ? 10 : 0) + (up2 ? -4 : 0)); + other.push_back(t > 4000000 ? std::nan("") : 50 + (up ? 6 : 0)); + } + const auto a = sfr::kinect_event_amplitudes(times, truth, other, {0, 5000000}, true); + require(a.events == 2 && a.measured == 1 && a.same_direction == 1, "only events both sensors give count"); + require(std::fabs(a.truth - 10) < 1e-9 && std::fabs(a.other - 6) < 1e-9 && std::fabs(a.ratio() - 0.6) < 1e-9, + "baseline mean, peak less it, the ratio"); + const auto down = sfr::kinect_event_amplitudes(times, truth, truth, {5000000}, false); + require(down.measured == 1 && std::fabs(down.truth + 4) < 1e-9, "a jump's response is the smallest value"); +} +} + +int main() { + const std::pair cases[] = { + {"header layout", header_layout}, + {"frame layout", frame_layout}, + {"zero runs", zero_runs}, + {"files round trip", files_round_trip}, + {"naming and space", naming_and_space}, + {"empty image", empty_image}, + {"rectangle by hand", rectangle_by_hand}, + {"near count by hand", near_count_by_hand}, + {"image stats", image_stats}, + {"series and fit", series_and_fit}, + {"lean", lean}, + {"view check", view_check}, + {"race columns", race_columns}, + {"detector events", detector_events}, + }; + unsigned failures = 0; + for (const auto& [name, test] : cases) { + try { + test(); + } catch (const std::exception& error) { + ++failures; + std::cerr << "FAIL " << name << ": " << error.what() << '\n'; + } + } + if (failures) return 1; + std::cout << "kinect_depth: all passed\n"; + return 0; +} diff --git a/tests/kinect_depth_view_guest_test.cpp b/tests/kinect_depth_view_guest_test.cpp new file mode 100644 index 0000000..7cfc7af --- /dev/null +++ b/tests/kinect_depth_view_guest_test.cpp @@ -0,0 +1,263 @@ +// The title's depth view as the recompiler generated it (82439998 and +// 82439D80, copied out of out/recomp/diagnostic at build time by +// extract_guest_functions.cmake), run on test images in a guest memory of +// its own, against the offline extractor (kinect_depth_features.h): every +// field the game writes must come out the same, floats bit for bit. Built +// with the diagnostic only (it needs the generated sources), and with the +// diagnostic's floating-point options for the generated code. +// +// The two functions read four constants from the game's image, given here +// as the decoded image has them: the vperm tables at 8217D100 and 8217D110 +// (horizontal sums), 0.25 at 8218FCE4 and 65528.0 x4 at 821AC140 (an empty +// part's mean, copied once into 83E54520 behind the flag word 83E54530). +#include "ppc_context.h" +#include "vector_integer.h" + +#include "kinect_depth_features.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#else +#include +#endif + +namespace sfr { +// What the generated code calls besides guest memory, as the diagnostic's +// hooks would but with no thread to schedule (diagnostic_hooks.h). +uint8_t* test_base = nullptr; +template inline void enter_function(C&, const char*, uint32_t) {} +template inline void guest_checkpoint(C&) {} +inline void load_vector_memory(uint32_t address, uint8_t (&destination)[16]) { + const uint8_t* source = test_base + (address & ~0xFu); + for (int i = 0; i < 16; ++i) destination[i] = source[15 - i]; +} +inline void store_vector_memory(uint32_t address, const uint8_t (&source)[16]) { + uint8_t* destination = test_base + (address & ~0xFu); + for (int i = 0; i < 16; ++i) destination[i] = source[15 - i]; +} +} +#define SFR_FAST_PATH() ((void)0) +// The register save and restore routines the functions branch to: nothing +// here depends on the registers they keep. +PPC_FUNC(__savegprlr_19) { (void)ctx; (void)base; } +PPC_FUNC(__restgprlr_19) { (void)ctx; (void)base; } + +#include "kinect_depth_view_guest.inc" + +namespace { +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr uint32_t width = 320, height = 240; +constexpr uint32_t stack_top = 0x7000F000, image_at = 0x40000000, manager_at = 0x50000000, record_at = 0x50001000; + +uint8_t* reserve() { +#ifdef _WIN32 + auto* base = static_cast(VirtualAlloc(nullptr, size_t(1) << 32, MEM_RESERVE, PAGE_NOACCESS)); + const auto commit = [&](uint32_t at, uint32_t size) { + require(VirtualAlloc(base + at, size, MEM_COMMIT, PAGE_READWRITE) != nullptr, "commit"); + }; + require(base != nullptr, "reserve 4 GB"); + commit(0x70000000, 0x10000); + commit(image_at, 0x40000); + commit(manager_at, 0x10000); + commit(0x8217D000, 0x1000); + commit(0x8218F000, 0x1000); + commit(0x821AC000, 0x1000); + commit(0x83E54000, 0x1000); + return base; +#else + void* base = mmap(nullptr, size_t(1) << 32, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + require(base != MAP_FAILED, "reserve 4 GB"); + return static_cast(base); +#endif +} + +void put32(uint8_t* base, uint32_t at, uint32_t value) { + for (int i = 0; i < 4; ++i) base[at + i] = uint8_t(value >> (24 - 8 * i)); +} +uint32_t get32(const uint8_t* base, uint32_t at) { + return uint32_t(base[at]) << 24 | uint32_t(base[at + 1]) << 16 | uint32_t(base[at + 2]) << 8 | base[at + 3]; +} +float getf(const uint8_t* base, uint32_t at) { + const uint32_t bits = get32(base, at); + float value = 0; + std::memcpy(&value, &bits, 4); + return value; +} +uint32_t bits(float value) { + uint32_t out = 0; + std::memcpy(&out, &value, 4); + return out; +} + +void constants(uint8_t* base) { + const uint8_t first[16] = {8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15}; + const uint8_t second[16] = {4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3}; + std::memcpy(base + 0x8217D100, first, 16); + std::memcpy(base + 0x8217D110, second, 16); + put32(base, 0x8218FCE4, 0x3E800000); // 0.25 + for (uint32_t i = 0; i < 4; ++i) put32(base, 0x821AC140 + 4 * i, 0x477FF800); // 65528.0 +} + +struct Guest { + sfr::KinectDepthCentroid centroid; + std::array record{}; // +640..+679 as words +}; + +Guest run(uint8_t* base, const std::vector& image, uint32_t slot) { + for (size_t i = 0; i < image.size(); ++i) { + base[image_at + 2 * i] = uint8_t(image[i] >> 8); // big-endian, as the game's buffer + base[image_at + 2 * i + 1] = uint8_t(image[i]); + } + std::memset(base + manager_at, 0xCD, 0x200); + for (uint32_t i = 0; i < 10; ++i) put32(base, record_at + 4 * i, 0xDEADBEEF); // what an older image left + // The host's floating-point state as a guest thread starts with it (its + // exceptions masked); the generated code turns flushing on and off in it, + // and it is put back after, for the extractor. + const uint32_t host_csr = simde_mm_getcsr(); + PPCContext ctx{}; + ctx.fpscr.loadFromHost(); + ctx.r1.u64 = stack_top; + ctx.r3.u64 = slot; + ctx.r4.u64 = manager_at; + ctx.r5.u64 = image_at; + ctx.r6.u64 = width; + ctx.r7.u64 = height; + __imp__sub_82439998(ctx, base); + Guest out; + const uint32_t at = manager_at + 8 * slot; + out.centroid.column = get32(base, at); + out.centroid.row = get32(base, at + 4); + out.centroid.min_column = get32(base, at + 48); + out.centroid.min_row = get32(base, at + 52); + out.centroid.max_column = get32(base, at + 96); + out.centroid.max_row = get32(base, at + 100); + PPCContext second{}; + second.fpscr.loadFromHost(); + second.r1.u64 = stack_top; + second.r3.u64 = slot; + second.r4.u64 = record_at; + second.r5.u64 = at; + second.r6.u64 = image_at; + second.r7.u64 = width; + second.r8.u64 = height; + __imp__sub_82439D80(second, base); + simde_mm_setcsr(host_csr); + for (uint32_t i = 0; i < 10; ++i) out.record[i] = get32(base, record_at + 4 * i); + return out; +} + +// What a game might be shown: up to three players as blobs of a few +// rectangles, with depth ramps, noise, holes, players without a depth, +// players at the image's edges, and now and then an image full of one +// player at the deepest value (the sums' saturation). +std::vector image_for(std::mt19937& random, uint32_t kind) { + std::vector image(size_t(width) * height, 0); + std::uniform_int_distribution any(0, 0xFFFF); + if (kind % 37 == 5) { + std::fill(image.begin(), image.end(), uint16_t(0xFFF8 | 1)); + return image; + } + if (kind % 41 == 7) { + for (auto& p : image) p = uint16_t(any(random)); + return image; + } + // A background. + std::uniform_int_distribution background(0, 3); + const uint32_t back = background(random); + for (uint32_t y = 0; y < height; ++y) + for (uint32_t x = 0; x < width; ++x) + if (back) image[size_t(y) * width + x] = uint16_t((2500 + x * back + y) << 3); + std::uniform_int_distribution players(0, 3), index(1, 6), size(1, 120), depth(400, 4100), share(0, 99); + const uint32_t count = kind % 11 == 0 ? 0 : players(random); + for (uint32_t p = 0; p < count; ++p) { + const uint32_t player = index(random); + const uint32_t base_depth = depth(random); + const uint32_t pieces = 1 + size(random) % 4; + for (uint32_t piece = 0; piece < pieces; ++piece) { + const uint32_t w = size(random) % (kind % 7 == 3 ? 6 : 90) + 1, h = size(random) % (kind % 7 == 3 ? 6 : 140) + 1; + const uint32_t x0 = std::uniform_int_distribution(0, width - 1)(random); + const uint32_t y0 = std::uniform_int_distribution(0, height - 1)(random); + for (uint32_t y = y0; y < std::min(height, y0 + h); ++y) + for (uint32_t x = x0; x < std::min(width, x0 + w); ++x) { + const uint32_t roll = share(random); + uint32_t mm = base_depth + (x - x0) * 3 + (y - y0) + (roll % 13); + if (roll < 3) mm = 0; // a player pixel without a depth + if (roll > 96) continue; // a hole + image[size_t(y) * width + x] = uint16_t(std::min(mm, 8191) << 3 | player); + } + } + } + // A few lone pixels of any player. + for (uint32_t i = 0; i < share(random) % 20; ++i) + image[std::uniform_int_distribution(0, image.size() - 1)(random)] = uint16_t(any(random)); + return image; +} +} + +int main() { + try { + uint8_t* base = reserve(); + sfr::test_base = base; + constants(base); + std::mt19937 random(20261019); + uint64_t compared = 0, written = 0, sparse = 0; + for (uint32_t kind = 0; kind < 600; ++kind) { + const auto image = image_for(random, kind); + for (uint32_t slot = 0; slot < 6; ++slot) { + const Guest guest = run(base, image, slot); + const sfr::KinectDepthFeatures tool = sfr::kinect_depth_view_features(image.data(), width, height, slot); + const std::string where = "image " + std::to_string(kind) + " slot " + std::to_string(slot) + ": "; + const auto& g = guest.centroid; + const auto& t = tool.centroid; + require(g.column == t.column && g.row == t.row, where + "centroid " + std::to_string(g.column) + "," + + std::to_string(g.row) + " vs " + std::to_string(t.column) + + "," + std::to_string(t.row)); + require(g.min_column == t.min_column && g.min_row == t.min_row && g.max_column == t.max_column && + g.max_row == t.max_row, + where + "extents " + std::to_string(g.min_column) + "," + std::to_string(g.min_row) + "-" + + std::to_string(g.max_column) + "," + std::to_string(g.max_row) + " vs " + + std::to_string(t.min_column) + "," + std::to_string(t.min_row) + "-" + + std::to_string(t.max_column) + "," + std::to_string(t.max_row)); + for (uint32_t i = 0; i < 4; ++i) + require(guest.record[i] == bits(tool.split.counts[i]), + where + "count " + std::to_string(i) + " " + std::to_string(getf(base, record_at + 4 * i)) + + " vs " + std::to_string(tool.split.counts[i])); + if (tool.split.written) { + ++written; + for (uint32_t i = 0; i < 4; ++i) + require(guest.record[4 + i] == bits(tool.split.means[i]), + where + "mean " + std::to_string(i) + " " + std::to_string(getf(base, record_at + 16 + 4 * i)) + + " vs " + std::to_string(tool.split.means[i])); + require(guest.record[8] == tool.split.near_count, + where + "near " + std::to_string(guest.record[8]) + " vs " + std::to_string(tool.split.near_count)); + require(guest.record[9] == bits(tool.split.average), where + "average"); + } else { + ++sparse; + for (uint32_t i = 4; i < 10; ++i) + require(guest.record[i] == 0xDEADBEEF, where + "a sparse image writes only the counts"); + } + ++compared; + } + } + require(written > 200 && sparse > 200, "the images exercise both outcomes"); + std::cout << "kinect_depth_view_guest: " << compared << " (image, slot) pairs identical to the generated code (" + << written << " written, " << sparse << " too sparse)\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "kinect_depth_view_guest_test: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/kinect_ideal_diagram_test.cpp b/tests/kinect_ideal_diagram_test.cpp new file mode 100644 index 0000000..060af18 --- /dev/null +++ b/tests/kinect_ideal_diagram_test.cpp @@ -0,0 +1,181 @@ +#include "kinect_ideal_diagram.h" + +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +bool close_to(float a, float b, float tolerance = 1e-3f) { return std::fabs(a - b) <= tolerance; } + +bool inside(const sfr::KinectDiagramPoint& p, float width, float height) { + return p.x >= 0 && p.x <= width && p.y >= 0 && p.y <= height; +} + +sfr::KinectIdealSnapshot two_players() { + sfr::KinectIdealSnapshot snapshot; + snapshot.valid = snapshot.placed = snapshot.floor_known = true; + // Players at 2.0 and 2.4 m, mean x 0.1: the camera 2.5 m behind their line. + snapshot.players = {{1, 0, -0.4f, 2.0f, 2.05f}, {2, 1, 0.6f, 2.4f, 2.45f}}; + snapshot.camera_x = 0.1f; + snapshot.camera_z = 2.2f - 2.5f; + snapshot.distance = 2.5f; + return snapshot; +} + +void names() { + require(std::string(sfr::kinect_depth_choice_name(sfr::KinectDepthChoiceView::ideal)) == "ideal" && + std::string(sfr::kinect_depth_choice_name(sfr::KinectDepthChoiceView::fit)) == "fit" && + std::string(sfr::kinect_depth_choice_name(sfr::KinectDepthChoiceView::sensor)) == "sensor", + "the environment's names"); + const sfr::KinectDepthChoice choice; + require(choice.view == sfr::KinectDepthChoiceView::ideal && choice.distance == 2.5f && choice.height == 1.0f && + choice.follow == 2.0f && choice.deadband == 0.15f && choice.skeleton_distance == 0, + "the defaults are Recompiled's"); +} + +void everything_fits() { + const float width = 300, height = 400; + const auto diagram = sfr::kinect_ideal_diagram(two_players(), width, height); + require(diagram.scale > 0, "drawn"); + require(diagram.scale * 5.0f <= width + 1e-3f, "at least 2.5 m each side"); + // The v2 at the bottom, in the middle; the virtual v1 behind it (lower). + const auto v2 = diagram.v2_view[0], v1 = diagram.v1_view[0]; + require(close_to(v2.x, width / 2) && inside(v2, width, height) && inside(v1, width, height), "both cameras fit"); + require(v1.y > v2.y, "the virtual v1 stands behind the v2, drawn below it"); + require(close_to(v1.x - v2.x, 0.1f * diagram.scale), "laterally on the players"); + for (const auto& player : diagram.players) require(inside(player.at, width, height), "every player fits"); + require(diagram.players[0].at.x < diagram.players[1].at.x, "+x drawn to the right, as the depth images"); + require(diagram.players[0].at.y > diagram.players[1].at.y, "nearer is lower"); + require(close_to(v1.y - diagram.players_y, 2.5f * diagram.scale), "the players' line distance in front of the v1"); + // The far corners spread at each camera's half angle. + const float v2_half = (diagram.v2_view[2].x - diagram.v2_view[1].x) / 2 / (v2.y - diagram.v2_view[1].y); + const float v1_half = (diagram.v1_view[2].x - diagram.v1_view[1].x) / 2 / (v1.y - diagram.v1_view[1].y); + require(close_to(v2_half, std::tan(35.3f * 3.14159265f / 180.0f), 1e-3f), "the v2's 70.6 degrees"); + require(close_to(v1_half, std::tan(28.5f * 3.14159265f / 180.0f), 1e-3f), "the v1's 57 degrees"); + require(close_to(v1.y - diagram.v1_near_y, 0.8f * diagram.scale) && close_to(v1.y - diagram.v1_far_y, 4.0f * diagram.scale), + "the v1's usable depths"); + require(close_to(diagram.v2_view[1].y, diagram.v1_view[1].y, 1e-2f) && close_to(diagram.v2_view[1].y, 0, 1e-2f), + "both views reach the drawing's top"); +} + +void numbers_and_notes() { + const auto diagram = sfr::kinect_ideal_diagram(two_players(), 300, 400); + const auto& one = diagram.players[0]; + const auto& two = diagram.players[1]; + require(one.slot == 0 && two.slot == 1, "1P and 2P"); + require(close_to(one.v1_depth, 2.3f) && close_to(two.v1_depth, 2.7f), "the depth the game sees"); + require(close_to(one.v2_depth, 2.05f) && close_to(two.v2_depth, 2.45f), "the v2's own"); + require(one.in_v1_view && one.in_v1_range && one.in_v2_view && two.in_v1_view && two.in_v2_view, "both seen"); + require(one.band_left < one.at.x && one.band_right > one.at.x, "inside the band of their depth"); + // At 2.3 m the v1 sees +-1.249 m about 0.1; the v2 at 2.0 m +-1.416 m. + const float scale = diagram.scale, middle = 150; + require(close_to(one.band_left, middle + (0.1f - 2.3f * std::tan(28.5f * 3.14159265f / 180.0f)) * scale, 0.05f) && + close_to(one.band_right, middle + (0.1f + 2.3f * std::tan(28.5f * 3.14159265f / 180.0f)) * scale, 0.05f), + "the band is the v1's view there, inside the v2's"); + // Far to the side: outside the v1, still in the v2. + auto snapshot = two_players(); + snapshot.players = {{3, 0, 1.75f, 2.6f, 2.6f}}; + auto side = sfr::kinect_ideal_diagram(snapshot, 300, 400).players[0]; + require(!side.in_v1_view && side.in_v2_view && side.in_v1_range, "outside the virtual v1's view"); + // Too far from the virtual v1. + snapshot.players = {{3, 0, 0.1f, 3.9f, 3.9f}}; + side = sfr::kinect_ideal_diagram(snapshot, 300, 400).players[0]; + require(side.in_v1_view && !side.in_v1_range && close_to(side.v1_depth, 4.2f), "beyond the v1's 4 m"); + // Too near: the camera is placed in front of the v2 (players far away) and + // someone steps up to it. + snapshot.camera_z = 1.0f; + snapshot.players = {{3, 0, 0.0f, 1.5f, 1.5f}}; + side = sfr::kinect_ideal_diagram(snapshot, 300, 400).players[0]; + require(!side.in_v1_range && close_to(side.v1_depth, 0.5f), "nearer than the v1's 0.8 m"); + snapshot.players = {{3, 0, 0.0f, 0.3f, 0.3f}}; + side = sfr::kinect_ideal_diagram(snapshot, 300, 400).players[0]; + require(!side.in_v1_view && !side.in_v2_view && side.band_left > side.band_right, "behind the camera: seen by none"); +} + +void grows_to_fit() { + sfr::KinectIdealSnapshot snapshot; + snapshot.valid = true; + // Nobody yet: the camera at the v2. + auto diagram = sfr::kinect_ideal_diagram(snapshot, 200, 200); + require(diagram.scale > 0 && diagram.players.empty(), "drawn without players"); + require(close_to(diagram.v1_view[0].y, diagram.v2_view[0].y) && diagram.v2_view[0].y < 200, "above the bottom edge"); + // A player far to the side and the camera far behind. + snapshot.camera_x = 1.0f; + snapshot.camera_z = -2.0f; + snapshot.players = {{1, 0, 3.4f, 3.0f, 3.0f}}; + diagram = sfr::kinect_ideal_diagram(snapshot, 400, 300); + require(inside(diagram.v1_view[0], 400, 300) && inside(diagram.players[0].at, 400, 300), "grown to fit"); + require(sfr::kinect_ideal_diagram(snapshot, 0, 300).scale == 0, "no room: nothing"); +} +} + +namespace { +void rulers() { + const float width = 320, height = 420, left = 30, bottom = 18; + const auto diagram = sfr::kinect_ideal_diagram(two_players(), width, height, left, bottom); + require(diagram.scale > 0, "drawn"); + // The useful range always shows, inside the strips left for the labels. + const auto near_left = diagram.place(-2.5f, 0.5f), far_right = diagram.place(2.5f, 4.5f); + require(near_left.x >= left - 1e-3f && far_right.x <= width + 1e-3f && far_right.y >= -1e-3f && near_left.y <= height - bottom, + "0.5-4.5 m deep and +-2.5 m wide in view, clear of the rulers"); + require(close_to(diagram.origin_x, left + (width - left) / 2), "the v2's axis in the middle of the drawing"); + // Depth: every half metre from the v2, a whole metre labelled. + bool one = false, two_and_a_half = false; + for (size_t i = 0; i < diagram.depth_ticks.size(); ++i) { + const auto& tick = diagram.depth_ticks[i]; + require(close_to(tick.at, diagram.place(0, tick.metres).y), "a depth tick at its distance from the v2"); + require(tick.whole == close_to(std::fmod(std::fabs(tick.metres), 1.0f), 0.0f), "whole metres marked"); + require(tick.at >= -1e-3f && tick.at <= height - bottom + 1e-3f, "inside the drawing"); + if (i) require(close_to(diagram.depth_ticks[i - 1].at - tick.at, 0.5f * diagram.scale, 1e-2f), "half a metre apart"); + one = one || (close_to(tick.metres, 1.0f) && tick.whole); + two_and_a_half = two_and_a_half || (close_to(tick.metres, 2.5f) && !tick.whole); + } + require(one && two_and_a_half && diagram.depth_ticks.back().metres >= 4.5f, "1 m labelled, 2.5 m a half, up to 4.5 m"); + require(close_to(diagram.place(0, 1.0f).y, diagram.v2_view[0].y - diagram.scale), "1 m is a metre above the v2"); + // Across: 0 on the v2's axis, every half metre. + bool zero = false; + for (const auto& tick : diagram.lateral_ticks) { + require(close_to(tick.at, diagram.origin_x + tick.metres * diagram.scale), "a lateral tick at its x"); + require(tick.at >= left - 1e-3f && tick.at <= width + 1e-3f, "inside the drawing"); + zero = zero || (tick.metres == 0 && tick.whole && close_to(tick.at, diagram.v2_view[0].x)); + } + require(zero && diagram.lateral_ticks.front().metres <= -2.5f && diagram.lateral_ticks.back().metres >= 2.5f, + "0 at the centre line, +-2.5 m"); + // The v1's usable depths on its own axis. + const auto v1 = diagram.v1_view[0]; + require(close_to(diagram.v1_near_mark.x, v1.x) && close_to(diagram.v1_far_mark.x, v1.x) && + close_to(v1.y - diagram.v1_near_mark.y, 0.8f * diagram.scale) && + close_to(v1.y - diagram.v1_far_mark.y, 4.0f * diagram.scale), + "0.8 m and 4.0 m from the virtual v1"); + require(close_to(diagram.v1_near_mark.y, diagram.v1_near_y) && close_to(diagram.v1_far_mark.y, diagram.v1_far_y), + "on its range lines"); +} +} + +int main() { + const std::pair cases[] = { + {"rulers", rulers}, + {"names", names}, + {"fits", everything_fits}, + {"numbers", numbers_and_notes}, + {"grows", grows_to_fit}, + }; + unsigned failures = 0; + for (const auto& [name, test] : cases) { + try { + test(); + } catch (const std::exception& error) { + ++failures; + std::cerr << "FAIL " << name << ": " << error.what() << '\n'; + } + } + if (failures) return 1; + std::cout << "kinect_ideal_diagram: all passed\n"; + return 0; +} diff --git a/tests/kinect_preview_layout_test.cpp b/tests/kinect_preview_layout_test.cpp new file mode 100644 index 0000000..4be06b4 --- /dev/null +++ b/tests/kinect_preview_layout_test.cpp @@ -0,0 +1,197 @@ +#include "kinect_preview_layout.h" +#include "kinect_v2_image.h" + +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +using sfr::KinectPanelRect; + +bool inside(const KinectPanelRect& r, int width, int height) { + return r.left >= 0 && r.top >= 0 && r.right <= width && r.bottom <= height && r.width() > 0 && r.height() > 0; +} + +double aspect(const KinectPanelRect& r) { return double(r.width()) / double(r.height()); } + +// The checks every layout shares: inside the client area, below the text, +// side by side without overlap, aligned at the top, one height. +void sane(const sfr::KinectPreviewLayout& layout, int width, int height) { + require(inside(layout.colour, width, height) && inside(layout.depth, width, height), "panels stay inside the client area"); + require(layout.colour.right <= layout.depth.left, "panels do not overlap"); + require(layout.colour.top == layout.depth.top && layout.colour.height() == layout.depth.height(), "same top, same height"); + const sfr::KinectPreviewLayoutInput defaults; + require(layout.colour.top >= defaults.top && layout.colour.bottom <= height - defaults.bottom, "clear of the text and the hint line"); +} + +void common_view_panels_are_the_same_size_and_shape() { + const double a = double(sfr::kinect_v2_preview_size(sfr::KinectImageKind::colour, sfr::KinectV2View::common).width) / + sfr::kinect_v2_preview_size(sfr::KinectImageKind::colour, sfr::KinectV2View::common).height; + require(std::fabs(a - 1.373) < 2e-3, "the common aspect"); + const int sizes[][2] = {{1360, 640}, {1920, 1080}, {744, 380}, {1000, 900}, {2560, 700}, {1600, 1200}}; + for (const auto& size : sizes) { + sfr::KinectPreviewLayoutInput input; + input.client_width = size[0]; + input.client_height = size[1]; + input.colour_aspect = a; + input.depth_aspect = 99; // ignored: same size + input.same_size = true; + const sfr::KinectPreviewLayout layout = sfr::kinect_preview_layout(input); + sane(layout, size[0], size[1]); + require(layout.colour.width() == layout.depth.width(), "same width"); + require(std::fabs(aspect(layout.colour) - a) < 0.02 && std::fabs(aspect(layout.depth) - a) < 0.02, "aspect kept"); + } +} + +void original_view_panels_follow_their_own_pictures() { + const int sizes[][2] = {{1360, 640}, {1920, 1080}, {744, 380}, {1000, 900}, {2560, 700}}; + for (const auto& size : sizes) { + sfr::KinectPreviewLayoutInput input; + input.client_width = size[0]; + input.client_height = size[1]; + input.colour_aspect = 16.0 / 9.0; + input.depth_aspect = 512.0 / 424.0; + const sfr::KinectPreviewLayout layout = sfr::kinect_preview_layout(input); + sane(layout, size[0], size[1]); + require(std::fabs(aspect(layout.colour) - 16.0 / 9.0) < 0.05, "colour 16:9"); + require(std::fabs(aspect(layout.depth) - 512.0 / 424.0) < 0.03, "depth 512:424"); + } +} + +void panels_shrink_uniformly_and_use_the_space() { + sfr::KinectPreviewLayoutInput input; + input.same_size = true; + input.colour_aspect = 1.373; + // Wide window: the height limits (client height - text - hint). + input.client_width = 2560; + input.client_height = 700; + sfr::KinectPreviewLayout layout = sfr::kinect_preview_layout(input); + require(layout.colour.height() == 700 - input.top - input.bottom, "the height limits a wide window"); + // Tall window: the width limits, and a narrower window gets smaller panels. + input.client_width = 1000; + input.client_height = 1200; + const sfr::KinectPreviewLayout narrow = sfr::kinect_preview_layout(input); + require(narrow.depth.right <= 1000 - input.gap && narrow.depth.right > 1000 - input.gap - 3, "the width is used"); + input.client_width = 1360; + const sfr::KinectPreviewLayout wider = sfr::kinect_preview_layout(input); + require(wider.colour.width() > narrow.colour.width() && wider.colour.height() > narrow.colour.height(), "bigger window, bigger panels"); + // Nothing fits: empty panels, not negative ones. + input.client_width = 30; + input.client_height = 100; + layout = sfr::kinect_preview_layout(input); + require(layout.colour.width() == 0 && layout.depth.width() == 0 && layout.colour.height() == 0, "no room, no panels"); +} + +void principal_points_land_on_the_panels_centres() { + // The window scales a preview image to its panel by the ratio of their + // sizes; the principal point (the lens axis) must end at the panel's centre. + using sfr::KinectImageKind; + using sfr::KinectV2View; + for (const KinectV2View view : {KinectV2View::common, KinectV2View::original}) { + const sfr::KinectImageRect colour_image = sfr::kinect_v2_picture_rect(KinectImageKind::colour, view); + const sfr::KinectImageRect depth_image = sfr::kinect_v2_picture_rect(KinectImageKind::depth_and_player, view); + sfr::KinectPreviewLayoutInput input; + input.client_width = 1360; + input.client_height = 640; + input.colour_aspect = colour_image.width / colour_image.height; + input.depth_aspect = depth_image.width / depth_image.height; + input.same_size = view == KinectV2View::common; + const sfr::KinectPreviewLayout layout = sfr::kinect_preview_layout(input); + sane(layout, 1360, 640); + const sfr::KinectImagePoint colour = sfr::kinect_v2_colour_to_preview(960, 540, view); + const sfr::KinectImagePoint depth = sfr::kinect_v2_depth_to_preview(256, 212, view); + const double colour_x = layout.colour.left + colour.x * layout.colour.width() / colour_image.width; + const double colour_y = layout.colour.top + colour.y * layout.colour.height() / colour_image.height; + const double depth_x = layout.depth.left + depth.x * layout.depth.width() / depth_image.width; + const double depth_y = layout.depth.top + depth.y * layout.depth.height() / depth_image.height; + require(std::fabs(colour_x - (layout.colour.left + layout.colour.right) / 2.0) < 1.5 && + std::fabs(colour_y - (layout.colour.top + layout.colour.bottom) / 2.0) < 1.5, "colour axis at the panel centre"); + require(std::fabs(depth_x - (layout.depth.left + layout.depth.right) / 2.0) < 1.5 && + std::fabs(depth_y - (layout.depth.top + layout.depth.bottom) / 2.0) < 1.5, "depth axis at the panel centre"); + } +} + +void a_v1_keeps_its_4_by_3_panels() { + sfr::KinectPreviewLayoutInput input; // the defaults are 4:3 + input.client_width = 1360; + input.client_height = 640; + const sfr::KinectPreviewLayout layout = sfr::kinect_preview_layout(input); + sane(layout, 1360, 640); + require(std::fabs(aspect(layout.colour) - 4.0 / 3.0) < 0.01 && std::fabs(aspect(layout.depth) - 4.0 / 3.0) < 0.01, "4:3"); + require(layout.colour.width() == layout.depth.width(), "equal"); +} + +bool within(const KinectPanelRect& inner, const KinectPanelRect& outer) { + return inner.left >= outer.left && inner.top >= outer.top && inner.right <= outer.right && inner.bottom <= outer.bottom; +} + +bool apart(const KinectPanelRect& a, const KinectPanelRect& b) { + return a.right <= b.left || b.right <= a.left || a.bottom <= b.top || b.bottom <= a.top; +} + +void recompiled_grid_and_diagram() { + for (const auto& [width, height] : {std::pair{1360, 820}, std::pair{1360, 640}, std::pair{760, 420}, std::pair{2400, 700}}) { + sfr::KinectRecompiledLayoutInput input; + input.client_width = width; + input.client_height = height; + const auto layout = sfr::kinect_preview_recompiled_layout(input); + const KinectPanelRect pictures[] = {layout.v2_colour, layout.v2_depth, layout.v1_colour, layout.v1_depth, layout.diagram}; + for (const auto& picture : pictures) require(inside(picture, width, height), "every area inside the window"); + for (size_t i = 0; i < 5; ++i) + for (size_t j = i + 1; j < 5; ++j) require(apart(pictures[i], pictures[j]), "no two areas overlap"); + require(within(layout.v2_colour, layout.colour_cells[0]) && within(layout.v1_colour, layout.colour_cells[1]) && + within(layout.v2_depth, layout.depth_cells[0]) && within(layout.v1_depth, layout.depth_cells[1]), + "each picture in its cell"); + require(std::fabs(aspect(layout.v2_colour) - 16.0 / 9.0) < 0.02 && std::fabs(aspect(layout.v2_depth) - 512.0 / 424.0) < 0.02 && + std::fabs(aspect(layout.v1_colour) - 4.0 / 3.0) < 0.02 && std::fabs(aspect(layout.v1_depth) - 4.0 / 3.0) < 0.02, + "true aspects"); + require(layout.colour_cells[0].left == layout.colour_cells[1].left && layout.depth_cells[0].left == layout.depth_cells[1].left && + layout.colour_cells[0].right <= layout.depth_cells[0].left, + "colour above colour, depth above depth"); + require(layout.colour_cells[1].top - layout.colour_cells[0].bottom >= input.caption && layout.v2_colour.top >= input.top && + layout.v1_colour.top - layout.v2_depth.bottom >= 30 && layout.v1_depth.top - layout.v2_depth.bottom >= 30, + "a caption line, and room above it, over each row"); + require(layout.diagram.left >= layout.depth_cells[0].right && layout.diagram.top == input.top && + layout.diagram.bottom == layout.colour_cells[1].bottom && layout.diagram.right == width - input.gap && + layout.diagram.width() >= (width - 4 * input.gap) / 3, + "the diagram: a third of the width or more, the grid's full height, on the right"); + // Every picture as wide as its column, at the top of its cell. + for (const auto& [picture, cell] : {std::pair{layout.v2_colour, layout.colour_cells[0]}, std::pair{layout.v1_colour, layout.colour_cells[1]}, + std::pair{layout.v2_depth, layout.depth_cells[0]}, std::pair{layout.v1_depth, layout.depth_cells[1]}}) + require(picture.left == cell.left && picture.right == cell.right && picture.top == cell.top, "the column's width, at the top"); + require(layout.v2_colour.width() == layout.v2_depth.width() && layout.v1_colour.width() == layout.v2_colour.width(), + "one width for both columns"); + // The height is used: limited by it, or by two thirds of the width. + const bool tall = layout.v1_depth.bottom >= height - input.bottom - 2; + const int grid_most = (width - 4 * input.gap) - (width - 4 * input.gap) / 3; + const bool wide = layout.depth_cells[0].right >= 2 * input.gap + 2 * (grid_most / 2) - 2; + require(tall || wide, "as large as the window allows"); + } + sfr::KinectRecompiledLayoutInput tiny; + tiny.client_width = 60; + tiny.client_height = 100; + require(sfr::kinect_preview_recompiled_layout(tiny).diagram.width() == 0, "nothing fits: nothing"); +} +} + +int main() { + try { + common_view_panels_are_the_same_size_and_shape(); + original_view_panels_follow_their_own_pictures(); + panels_shrink_uniformly_and_use_the_space(); + principal_points_land_on_the_panels_centres(); + a_v1_keeps_its_4_by_3_panels(); + recompiled_grid_and_diagram(); + } catch (const std::exception& error) { + std::cerr << "kinect_preview_layout_test: " << error.what() << '\n'; + return 1; + } + std::cout << "kinect_preview_layout_test passed\n"; + return 0; +} diff --git a/tests/kinect_probe.cpp b/tests/kinect_probe.cpp index f6abb0a..cc5f9a6 100644 --- a/tests/kinect_probe.cpp +++ b/tests/kinect_probe.cpp @@ -1,44 +1,305 @@ // Checks a real Kinect without starting the game: opens the first sensor // through the Kinect for Windows runtime and prints what it tracks for ten -// seconds (sfr_kinect_probe [seconds]). +// seconds. +// +// sfr_kinect_probe [seconds] [--v1 | --v2 | --both] [--record prefix [--depth] [--raw]] [--images [prefix]] [--game-depth] +// [--smooth] +// +// --v1 or --v2 opens that kind only, --both opens a Kinect v1 and a Kinect +// v2 together (each on its own runtime) and carries on with whichever +// opened, saying which. --record writes every skeleton frame of every open +// sensor to -skeleton.csv (kinect_record.h), each stamped with the +// host's steady clock and the sensor's own timestamp, with the share of its +// depth image that has no depth (a v2's depth reader is opened for it). +// --depth (or SFR_KINECT_RECORD_DEPTH=1) also writes every depth image each +// sensor gives the game, as the game would get it, to +// -v1-depth.bin and -v2-depth.bin (kinect_record_depth.h): +// a v1's own depth and player index, the one a v2 makes (its fit view). +// --raw (or SFR_KINECT_RECORD_RAW=1) also writes what a Kinect v2 measured for +// each image it makes, to -v2raw.bin (kinect_record_raw.h), so the +// depth for the game can be made again offline through any view +// (sfr_kinect_depth_resynth); the sensor writes it from the frames it makes the +// image from. +// With --images it also saves the first colour and depth image each sensor +// gives, as the launcher's preview shows them, to colour.bmp and +// depth.bmp (prefix "kinect_" by default, with "v1_"/"v2_" added +// when both are open), and prints where the head falls in each. +// --smooth (or SFR_KINECT_SMOOTH=1) smooths the skeletons in the backends +// with the NUI library's filter (NuiTransformSmooth, kinect_smoothing.h), as +// the game smooths them itself; recordings still take them before that. +// --game-depth reads the depth a Kinect v2 makes for the game (always made, +// kinect_v2_depth.h) as the game does (game_image): twice a +// second how many images were made, their share of pixels with a depth and +// the players' pixels, and with --images the first one as +// game_depth.bmp. +#include "kinect_record.h" +#include "kinect_record_depth.h" +#include "kinect_record_raw.h" #include "kinect_sensor.h" +#include #include +#include #include +#include #include +#include #include #include +#include -int main(int argc, char** argv) { - const int seconds = argc > 1 ? std::atoi(argv[1]) : 10; - std::string why; - auto sensor = sfr::KinectSensor::open(&why); - if (!sensor) { - std::cout << "no sensor: " << why << '\n'; - return 1; +namespace { +// 32-bit B, G, R, unused rows, top-down. +bool write_bmp(const std::string& path, const std::vector& bgrx, uint32_t width, uint32_t height) { + std::ofstream out(path, std::ios::binary); + if (!out) return false; + const uint32_t pixels = width * height * 4, size = 54 + pixels; + uint8_t header[54] = {}; + header[0] = 'B'; header[1] = 'M'; + const auto put32 = [&](int at, uint32_t value) { + header[at] = uint8_t(value); header[at + 1] = uint8_t(value >> 8); + header[at + 2] = uint8_t(value >> 16); header[at + 3] = uint8_t(value >> 24); + }; + put32(2, size); put32(10, 54); put32(14, 40); + put32(18, width); + put32(22, uint32_t(-int32_t(height))); // negative: rows top-down + header[26] = 1; header[28] = 32; + put32(34, pixels); + out.write(reinterpret_cast(header), sizeof header); + out.write(reinterpret_cast(bgrx.data()), pixels); + return bool(out); +} + +// Depth and player index as the preview shades it: near bright, nothing +// measured black, players tinted red. +std::vector shade_depth(const sfr::KinectImage& depth) { + std::vector out(size_t(depth.width) * depth.height * 4); + for (size_t i = 0; i < size_t(depth.width) * depth.height; ++i) { + const uint16_t value = uint16_t(depth.pixels[2 * i] | depth.pixels[2 * i + 1] << 8); + const uint32_t millimetres = value >> 3, player = value & 7; + const uint32_t shade = millimetres == 0 ? 0 : + 255 - std::min(255, uint32_t(std::max(0, int(millimetres) - 500)) * 255 / 3500); + out[4 * i] = uint8_t(player ? shade / 3 : shade); + out[4 * i + 1] = uint8_t(player ? shade / 3 : shade); + out[4 * i + 2] = uint8_t(shade); } - std::cout << sensor->model() << " open; stand in front of it" << std::endl; + return out; +} + +struct Probe { + std::unique_ptr sensor; + std::string tag; // "v1", "v2", or empty for the first found + std::string prefix; sfr::KinectFrame frame; + sfr::KinectImage colour, depth, game_depth, recorded_depth; + bool colour_saved = false, depth_saved = false, game_depth_saved = false; + uint64_t game_depth_frames = 0, depth_recorded = 0; + uint64_t shown = 0, frames = 0, with_body = 0; +}; +} + +int main(int argc, char** argv) { + int seconds = 10; + bool images = false; + std::string prefix = "kinect_", record; + int models = 0; // 0 the first found, 1 v1, 2 v2, 3 both + bool game_depth = false; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--images") { + images = true; + if (i + 1 < argc && argv[i + 1][0] != '-' && std::atoi(argv[i + 1]) == 0) prefix = argv[++i]; + } else if (arg == "--record" && i + 1 < argc) { + record = argv[++i]; + } else if (arg == "--v1") { + models = 1; + } else if (arg == "--v2") { + models = 2; + } else if (arg == "--both") { + models = 3; + } else if (arg == "--game-depth") { + game_depth = true; + } else if (arg == "--depth") { + sfr::kinect_record_depth_enable(); + } else if (arg == "--raw") { + sfr::kinect_record_raw_enable(); + } else if (arg == "--smooth") { + // Read by each backend as it opens (kinect_host_smoothing). +#ifdef _WIN32 + _putenv_s("SFR_KINECT_SMOOTH", "1"); +#else + setenv("SFR_KINECT_SMOOTH", "1", 1); +#endif + } else { + seconds = std::atoi(argv[i]); + } + } + // Before any sensor opens: their workers check once. + if (!record.empty()) sfr::kinect_record_start(record); + const bool recording = !sfr::kinect_record_prefix().empty(); + std::vector probes; + if (models == 0) { + std::string why; + auto sensor = sfr::KinectSensor::open(&why); + if (!sensor) { + std::cout << "no sensor: " << why << '\n'; + return 1; + } + probes.push_back({std::move(sensor), "", prefix}); + } else { + // Each kind on its own: one missing (or a runtime that will not + // share the machine with the other) leaves the other to record. + for (int model = 1; model <= 2; ++model) { + if (!(models & model)) continue; + const std::string tag = model == 1 ? "v1" : "v2"; + std::string why; + auto sensor = sfr::KinectSensor::open_model(model, &why); + std::cout << "Kinect " << tag << ": " << (sensor ? "open" : "not open (" + why + ")") << std::endl; + if (sensor) probes.push_back({std::move(sensor), tag, models == 3 ? prefix + tag + "_" : prefix}); + } + if (probes.empty()) { + std::cout << "no sensor opened" << '\n'; + return 1; + } + } + for (const Probe& probe : probes) + std::cout << probe.sensor->model() << " open; stand in front of it" + << (game_depth ? std::string(" (depth for the game: ") + + (probe.sensor->has_depth_stream() ? "yes" : "no") + ")" + : std::string()) + << std::endl; + if (recording) + std::cout << "recording every frame to " << sfr::kinect_record_prefix() << "-skeleton.csv for " << seconds + << " s" << std::endl; + // Each sensor's depth file, named after its kind. + const bool recording_depth = recording && sfr::kinect_record_depth_enabled(); + std::vector depth_files; + for (const Probe& probe : probes) { + const std::string tag = probe.tag.empty() ? (std::string(probe.sensor->model()) == "Kinect v2" ? "v2" : "v1") + : probe.tag; + sfr::KinectDepthRecordFile* file = recording_depth ? sfr::kinect_record_depth_file(tag) : nullptr; + depth_files.push_back(file); + if (file) + std::cout << "recording the depth images the game would get to " << file->path() + << (probe.sensor->has_depth_stream() ? "" : " (this sensor has no depth for the game: none)") + << std::endl; + } + // A Kinect v2's raw frames: written by the sensor itself. + const bool recording_raw = recording && sfr::kinect_record_raw_enabled(); + if (recording_raw) + std::cout << "recording what a Kinect v2 measures for each depth image it makes to " + << sfr::kinect_record_raw_path(sfr::kinect_record_prefix()) << std::endl; const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(seconds); - uint64_t shown = 0; while (std::chrono::steady_clock::now() < end) { - if (!sensor->next(frame)) { - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - continue; - } - const float gravity_y = frame.gravity[1]; - const float tilt = sfr::kinect_level(frame); - if (frame.number - shown < 15) continue; // twice a second - shown = frame.number; - std::cout << "frame " << frame.number << " floor=" << frame.floor_plane[1] << ',' << frame.floor_plane[3] - << " gravity_y=" << gravity_y << " level=" << tilt - << " bodies=" << frame.bodies.size(); - for (const auto& body : frame.bodies) { - const auto& hand = body.joints[sfr::nui_joint::hand_right]; - std::cout << " [id " << body.tracking_id << " at " << body.position[0] << ',' << body.position[2] - << " right hand " << hand[0] << ',' << hand[1] << ',' << hand[2] << ']'; + bool any = false; + for (size_t index = 0; index < probes.size(); ++index) { + Probe& probe = probes[index]; + auto& sensor = probe.sensor; + // The depth image the game would be handed (game_image: a v1's + // own, a v2's made one), every one, into its file. + if (sfr::KinectDepthRecordFile* file = depth_files[index]; + file && sensor->has_depth_stream() && + sensor->game_image(sfr::KinectImageKind::depth_and_player, probe.recorded_depth) && + probe.recorded_depth.bytes_per_pixel == 2) { + const sfr::KinectImage& image = probe.recorded_depth; + sfr::KinectDepthRecordFrame record; + record.steady_us = image.steady_us; + record.handed_us = sfr::kinect_record_now_us(); + record.sdk_time = image.sdk_time; + record.number = image.number; + record.sdk_frame = image.sdk_frame; + record.source = std::string(sensor->model()) == "Kinect v2" ? sfr::KinectDepthSource::v2 + : sfr::KinectDepthSource::v1; + record.origin = image.view_origin; + record.z_scale = image.view_z_scale; + sfr::kinect_depth_frame_pixels(record, image.pixels.data(), image.width, image.height); + if (file->write(record)) ++probe.depth_recorded; + any = true; + } + // Recording keeps a v2's depth reader open (its share of pixels + // without a depth goes in the file); a v1 has it open already. + const bool want_depth = images || recording; + if (want_depth && sensor->image(sfr::KinectImageKind::depth_and_player, probe.depth) && images && + !probe.depth_saved && probe.depth.bytes_per_pixel == 2) { + const std::string path = probe.prefix + "depth.bmp"; + probe.depth_saved = write_bmp(path, shade_depth(probe.depth), probe.depth.width, probe.depth.height); + std::cout << probe.tag << (probe.tag.empty() ? "" : " ") << "depth " << probe.depth.width << 'x' + << probe.depth.height << (probe.depth_saved ? " saved to " : " NOT saved to ") << path << std::endl; + } + if (images && !probe.colour_saved && sensor->image(sfr::KinectImageKind::colour, probe.colour) && + probe.colour.bytes_per_pixel == 4) { + const std::string path = probe.prefix + "colour.bmp"; + probe.colour_saved = write_bmp(path, probe.colour.pixels, probe.colour.width, probe.colour.height); + std::cout << probe.tag << (probe.tag.empty() ? "" : " ") << "colour " << probe.colour.width << 'x' + << probe.colour.height << (probe.colour_saved ? " saved to " : " NOT saved to ") << path + << std::endl; + } + if (game_depth && sensor->has_depth_stream() && + sensor->game_image(sfr::KinectImageKind::depth_and_player, probe.game_depth)) { + ++probe.game_depth_frames; + if (images && !probe.game_depth_saved) { + const std::string path = probe.prefix + "game_depth.bmp"; + probe.game_depth_saved = write_bmp(path, shade_depth(probe.game_depth), probe.game_depth.width, + probe.game_depth.height); + std::cout << probe.tag << (probe.tag.empty() ? "" : " ") << "game depth " << probe.game_depth.width + << 'x' << probe.game_depth.height << (probe.game_depth_saved ? " saved to " : " NOT saved to ") + << path << std::endl; + } + } + if (!sensor->next(probe.frame)) continue; + any = true; + sfr::KinectFrame& frame = probe.frame; + ++probe.frames; + if (!frame.bodies.empty()) ++probe.with_body; + const float gravity_y = frame.gravity[1]; + const sfr::KinectFrame& given = frame; + if (frame.number - probe.shown < 15) continue; // twice a second + probe.shown = frame.number; + if (!probe.tag.empty()) std::cout << probe.tag << ' '; + std::cout << "frame " << frame.number << " floor=" << frame.floor_plane[1] << ',' << frame.floor_plane[3] + << " gravity_y=" << gravity_y << " bodies=" << frame.bodies.size(); + if (game_depth && !probe.game_depth.pixels.empty()) { + // The newest image the game would get: made, with a depth, players. + uint32_t valid = 0, players = 0; + for (size_t i = 0; i + 1 < probe.game_depth.pixels.size(); i += 2) { + const uint16_t value = uint16_t(probe.game_depth.pixels[i] | probe.game_depth.pixels[i + 1] << 8); + if (value) ++valid; + if (value & 7) ++players; + } + std::cout << " game_depth_frames=" << probe.game_depth_frames + << " valid=" << double(valid) / double(probe.game_depth.pixels.size() / 2) + << " player_pixels=" << players; + } + for (size_t b = 0; b < frame.bodies.size(); ++b) { + const auto& body = frame.bodies[b]; + const auto& hand = body.joints[sfr::nui_joint::hand_right]; + std::cout << " [id " << body.tracking_id << " at " << body.position[0] << ',' << body.position[2] + << " right hand " << hand[0] << ',' << hand[1] << ',' << hand[2]; + // Where the preview draws the head, when the sensor places it + // itself (a v2): depth image 320x240, colour image 640x480. + sfr::KinectImagePoint in_depth, in_colour; + const auto& head = given.bodies[b].joints[sfr::nui_joint::head]; + if (images && sensor->image_point(sfr::KinectImageKind::depth_and_player, head, in_depth) && + sensor->image_point(sfr::KinectImageKind::colour, head, in_colour)) + std::cout << " head depth " << in_depth.x << ',' << in_depth.y << " colour " << in_colour.x << ',' + << in_colour.y; + std::cout << ']'; + } + std::cout << std::endl; } + if (!any) std::this_thread::sleep_for(std::chrono::milliseconds(3)); + } + for (const Probe& probe : probes) { + std::cout << probe.sensor->model() << ": " << probe.frames << " frames, " << probe.with_body << " with a body"; + if (images) + std::cout << "; images: colour " << (probe.colour_saved ? "saved" : "none") << ", depth " + << (probe.depth_saved ? "saved" : "none"); + if (game_depth) std::cout << "; depth images for the game: " << probe.game_depth_frames; + if (recording_depth) std::cout << "; depth images recorded: " << probe.depth_recorded; std::cout << std::endl; } + if (recording_raw) + if (const sfr::KinectRawRecordFile* raw = sfr::kinect_record_raw_file()) + std::cout << "Kinect v2 raw frames recorded: " << raw->frames_written() << " to " << raw->path() << std::endl; return 0; } diff --git a/tests/kinect_record_summary.cpp b/tests/kinect_record_summary.cpp new file mode 100644 index 0000000..37e513a --- /dev/null +++ b/tests/kinect_record_summary.cpp @@ -0,0 +1,117 @@ +// Compares two Kinect recordings of a player standing still +// (kinect_record.h): one with a sensor alone, one with both sensors on, to +// see whether the other sensor's light makes it worse. +// +// sfr_kinect_record_summary -skeleton.csv -skeleton.csv [--sensor v1|v2] +// +// For each sensor in both files (or the one named) it prints, alone and with +// both: joint jitter (SpineBase, SpineShoulder, both hands; millimetres RMS), +// the share of joints tracked, the share of depth pixels without a depth and +// how often the player was lost, then checks them against the plan's limits: +// jitter up by less than 20%, tracked down by less than 5 points, depth +// holes up by less than 10 points, and the player never lost. +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { +bool load(const std::string& path, std::vector& rows, std::set& sensors) { + std::ifstream in(path); + if (!in) return false; + std::string line; + while (std::getline(in, line)) { + sfr::KinectRecordRow row; + if (!sfr::kinect_parse_skeleton_row(line, row)) continue; + sensors.insert(row.frame.sensor); + rows.push_back(std::move(row)); + } + return true; +} + +void print(const char* label, const sfr::KinectRecordStats& s) { + std::printf(" %-6s frames=%llu with_body=%llu jitter_mm=", label, (unsigned long long)s.frames, + (unsigned long long)s.with_body); + for (size_t k = 0; k < s.jitter.size(); ++k) + std::printf("%s%.2f", k ? "/" : "", double(s.jitter[k]) * 1000.0); + std::printf(" tracked=%.1f%%", double(s.tracked) * 100.0); + if (s.depth_invalid >= 0) std::printf(" depth_invalid=%.1f%%", double(s.depth_invalid) * 100.0); + else std::printf(" depth_invalid=none"); + std::printf(" lost=%u clipped_frames=%llu\n", s.lost, (unsigned long long)s.clipped); +} +} + +int main(int argc, char** argv) { + std::vector paths; + std::string only; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "--sensor" && i + 1 < argc) only = argv[++i]; + else paths.push_back(arg); + } + if (paths.size() != 2) { + std::cerr << "usage: sfr_kinect_record_summary [--sensor v1|v2]\n"; + return 2; + } + std::vector alone, both; + std::set alone_sensors, both_sensors; + if (!load(paths[0], alone, alone_sensors) || !load(paths[1], both, both_sensors)) { + std::cerr << "cannot read the recordings\n"; + return 1; + } + bool all_pass = true, any = false; + for (const std::string& sensor : alone_sensors) { + if (!only.empty() && sensor != only) continue; + if (!both_sensors.count(sensor)) continue; + any = true; + const auto a = sfr::kinect_record_stats(alone, sensor), b = sfr::kinect_record_stats(both, sensor); + std::printf("Kinect %s (jitter: spine_base/spine_shoulder/hand_left/hand_right)\n", sensor.c_str()); + print("alone", a); + print("both", b); + bool pass = true; + if (!a.with_body || !b.with_body) { + // Nothing to compare: not a pass. + std::printf(" nobody tracked in %s: nothing measured\n FAIL\n", !a.with_body ? "the first" : "the second"); + all_pass = false; + continue; + } + const char* names[4] = {"spine_base", "spine_shoulder", "hand_left", "hand_right"}; + for (size_t k = 0; k < 4; ++k) { + if (!std::isfinite(a.jitter[k]) || !std::isfinite(b.jitter[k]) || !(a.jitter[k] > 0)) { + std::printf(" jitter %-14s not measured\n", names[k]); + continue; + } + const double rise = (double(b.jitter[k]) / double(a.jitter[k]) - 1.0) * 100.0; + const bool ok = rise < 20.0; + pass = pass && ok; + std::printf(" jitter %-14s %+6.1f%% (limit +20%%) %s\n", names[k], rise, ok ? "ok" : "FAIL"); + } + const double drop = (double(a.tracked) - double(b.tracked)) * 100.0; + const bool tracked_ok = drop < 5.0; + std::printf(" tracked drop %+6.1f points (limit 5) %s\n", drop, tracked_ok ? "ok" : "FAIL"); + pass = pass && tracked_ok; + if (a.depth_invalid >= 0 && b.depth_invalid >= 0) { + const double rise = (double(b.depth_invalid) - double(a.depth_invalid)) * 100.0; + const bool ok = rise < 10.0; + std::printf(" depth holes rise %+6.1f points (limit 10) %s\n", rise, ok ? "ok" : "FAIL"); + pass = pass && ok; + } else { + std::printf(" depth holes not recorded\n"); + } + const bool kept = b.lost == 0; + std::printf(" lost with both %u (limit 0) %s\n", b.lost, kept ? "ok" : "FAIL"); + pass = pass && kept; + std::printf(" %s\n", pass ? "PASS" : "FAIL"); + all_pass = all_pass && pass; + } + if (!any) { + std::cerr << "no sensor is in both recordings\n"; + return 1; + } + return all_pass ? 0 : 3; +} diff --git a/tests/kinect_record_test.cpp b/tests/kinect_record_test.cpp new file mode 100644 index 0000000..e9d86bf --- /dev/null +++ b/tests/kinect_record_test.cpp @@ -0,0 +1,408 @@ +#include "kinect_nui_record.h" +#include "kinect_race_record.h" +#include "kinect_record.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +bool near(float a, float b, float tolerance = 1e-5f) { return std::fabs(a - b) <= tolerance; } + +std::vector lines(const std::string& text) { + std::vector out; + size_t start = 0; + while (start < text.size()) { + const size_t end = text.find('\n', start); + out.push_back(text.substr(start, end - start)); + start = end == std::string::npos ? text.size() : end + 1; + } + return out; +} + +void skeleton_rows_round_trip() { + const size_t columns = sfr::kinect_record_fields(sfr::kinect_record_skeleton_header()).size(); + require(columns == 22 + 4 * 25 + 1, "the header names every column"); + require(sfr::kinect_record_skeleton_header().find(",clipped_edges\n") != std::string::npos, "clipped_edges is the last column"); + + // A v2 body: 25 joints, a 64-bit id, hand states, no centre of its own. + sfr::KinectRecordFrame v2; + v2.steady_us = 123456789; + v2.sensor = "v2"; + v2.frame = 41; + v2.sdk_time = 987654321012LL; + v2.floor = {0.01f, 0.99f, 0.1f, 0.8f}; + v2.depth_invalid = 0.125f; + v2.depth_frame = 7; + v2.depth_steady_us = 123450000; + sfr::KinectRecordBody body; + body.sensor_index = 4; + body.tracking_id = 72057594037931234ULL; + body.hand_states = {2, 3}; + body.clipped_edges = 6; // left and top + for (uint32_t j = 0; j < 25; ++j) body.joints[j] = {{0.01f * j, -0.5f + 0.02f * j, 2.0f + 0.001f * j}, int(j % 3)}; + v2.bodies = {body}; + std::string text; + sfr::kinect_record_skeleton_rows(v2, text); + auto rows = lines(text); + require(rows.size() == 1, "one row per body"); + require(sfr::kinect_record_fields(rows[0]).size() == columns, "a body row has every column"); + sfr::KinectRecordRow parsed; + require(sfr::kinect_parse_skeleton_row(rows[0], parsed) && parsed.has_body, "a v2 row reads back"); + require(parsed.frame.sensor == "v2" && parsed.frame.frame == 41 && parsed.frame.sdk_time == 987654321012LL && + parsed.frame.steady_us == 123456789, "the frame's clocks read back"); + require(near(parsed.frame.depth_invalid, 0.125f) && parsed.frame.depth_frame == 7, "the depth quality reads back"); + require(parsed.body.tracking_id == body.tracking_id && parsed.body.sensor_index == 4, "the whole 64-bit id"); + require(!parsed.body.has_position && parsed.body.hand_states[0] == 2 && parsed.body.hand_states[1] == 3, "hands"); + require(parsed.body.clipped_edges == 6, "the clipped edges read back"); + { + // A recording from before the column (no trailing field) still loads, as unknown. + std::string old_row = rows[0]; + old_row.erase(old_row.rfind(',')); + sfr::KinectRecordRow older; + require(sfr::kinect_parse_skeleton_row(old_row, older) && older.has_body && older.body.clipped_edges == -1 && + older.body.hand_states[1] == 3, "an older row without the column loads"); + } + for (uint32_t j = 0; j < 25; ++j) + require(parsed.body.joints[j].state == int(j % 3) && near(parsed.body.joints[j].position[1], -0.5f + 0.02f * j, 1e-5f), + "every joint and its state"); + + // A v1 skeleton: its joints in the v2 joints alike, Neck and the hands' tips empty. + std::array, 20> v1_joints{}; + std::array v1_states{}; + for (int j = 0; j < 20; ++j) { + v1_joints[j] = {float(j), 0, 2}; + v1_states[j] = j == 7 ? 1 : 2; + } + sfr::KinectRecordFrame v1; + v1.steady_us = 5; + v1.sensor = "v1"; + v1.gravity = {0, 1, 0}; + sfr::KinectRecordBody old; + old.has_position = true; + old.position = {0.1f, 0.2f, 2.1f}; + sfr::kinect_record_from_v1(v1_joints, v1_states, old); + require(old.joints[20].position[0] == 2.0f && old.joints[20].state == 2, "v1's shoulder centre is SpineShoulder"); + require(old.joints[2].state < 0 && old.joints[21].state < 0 && old.joints[24].state < 0, "v1 has no neck or fingertips"); + require(old.joints[0].position[0] == 0.0f && old.joints[19].position[0] == 19.0f, "the rest keep their numbers"); + require(old.joints[7].state == 1, "an inferred v1 joint stays inferred"); + v1.bodies = {old}; + text.clear(); + sfr::kinect_record_skeleton_rows(v1, text); + require(sfr::kinect_parse_skeleton_row(lines(text)[0], parsed) && parsed.body.has_position && + near(parsed.body.position[2], 2.1f) && parsed.body.joints[2].state < 0 && parsed.frame.depth_invalid < 0, + "a v1 row reads back in the same schema"); + require(parsed.body.clipped_edges == -1 && sfr::kinect_record_fields(lines(text)[0]).back().empty(), + "a v1 row leaves clipped_edges empty (unknown)"); + + // Nobody tracked: still a row, so lost tracking shows. + sfr::KinectRecordFrame empty; + empty.sensor = "v2"; + empty.steady_us = 9; + text.clear(); + sfr::kinect_record_skeleton_rows(empty, text); + rows = lines(text); + require(rows.size() == 1 && sfr::kinect_record_fields(rows[0]).size() == columns, "an empty frame is one full row"); + require(sfr::kinect_parse_skeleton_row(rows[0], parsed) && !parsed.has_body, "with no body"); + require(!sfr::kinect_parse_skeleton_row(sfr::kinect_record_skeleton_header(), parsed), "the header is not a row"); +} + +void depth_holes() { + // v1: depth above three player bits; 0, "too far" and "unknown" have none. + const uint16_t v1[6] = {0, uint16_t(2000 << 3 | 1), uint16_t(0x0FFF << 3), uint16_t(0x1FFF << 3), uint16_t(1500 << 3), 3}; + require(near(sfr::kinect_depth_invalid_ratio(v1, 6, 3), 4.0f / 6.0f), "v1 holes"); + const uint16_t v2[4] = {0, 2000, 0, 800}; + require(near(sfr::kinect_depth_invalid_ratio(v2, 4, 0), 0.5f), "v2 holes are zero millimetres"); + require(sfr::kinect_depth_invalid_ratio(nullptr, 0, 0) < 0, "no image, no ratio"); +} + +void race_rows_round_trip() { + // The detectors' 14 columns, then the depth view's 19 (kinect_depth_test). + require(sfr::kinect_record_fields(sfr::kinect_race_record_header()).size() == 14 + 19, "the race header"); + // A session as the game records it: the title's depth lean pair and what + // its detectors answered, which the race used as they were. + std::string text; + for (int i = 0; i < 5; ++i) { + sfr::KinectRaceRecord row; + row.steady_us = int64_t(i) * 33333; + row.generation = uint64_t(i + 1); + row.record = 0x71730000; + row.tracking_id = 9; + row.depth_right = 1.25f; + row.depth_left = 1.0f; + row.original[0] = i < 3 ? 2 : 1; // the title's own crouch detector + row.result[0] = row.original[0]; + row.original[2] = 2; + row.result[2] = 2; + sfr::kinect_race_record_row(row, text); + } + const auto rows = lines(text); + require(rows.size() == 5, "a row per update"); + sfr::KinectRaceRecord row; + require(sfr::kinect_parse_race_row(rows[4], row), "every race row reads back"); + require(row.generation == 5 && row.record == 0x71730000 && row.tracking_id == 9 && near(row.depth_right, 1.25f) && + near(row.depth_left, 1.0f), + "the title's depth lean pair is kept"); + require(row.original[0] == 1 && row.result[0] == 1 && row.original[2] == 2 && row.original[1] == -1 && + row.result[3] == -1, + "the detectors' answers and the ones not asked"); + require(!sfr::kinect_parse_race_row(sfr::kinect_race_record_header(), row), "the header is not a row"); +} + +void stats_measure_jitter_and_losses() { + // 30 frames still, the hips alternating 2 mm: their noise is 1 mm. + std::vector rows; + for (int f = 0; f < 30; ++f) { + sfr::KinectRecordRow row; + row.frame.sensor = "v1"; + row.frame.frame = uint64_t(f + 1); + row.frame.steady_us = int64_t(f) * 33333; + row.frame.depth_invalid = 0.2f; + row.has_body = f != 20; // lost once + row.body.tracking_id = 5; + for (auto& joint : row.body.joints) joint = {{0, 0, 2}, 2}; + row.body.joints[2].state = -1; // a v1: no neck + row.body.joints[11].state = f % 2 ? 1 : 2; // the right hand half inferred + row.body.joints[0].position[0] = f % 2 ? 0.002f : 0.0f; + rows.push_back(row); + } + // Another sensor's rows do not count. + sfr::KinectRecordRow other = rows[0]; + other.frame.sensor = "v2"; + other.body.joints[0].position[0] = 5; + rows.insert(rows.begin() + 3, other); + const sfr::KinectRecordStats stats = sfr::kinect_record_stats(rows, "v1"); + require(stats.frames == 30 && stats.with_body == 29 && stats.lost == 1, "frames, bodies and one loss"); + require(near(stats.jitter[0], 0.002f / std::sqrt(2.0f), 1e-6f), "frame-to-frame jitter"); + require(near(stats.jitter[1], 0.0f) && std::isnan(stats.jitter[3]), "still joints and a hand never tracked twice"); + require(near(stats.tracked, (24.0f - 0.5f) / 24.0f, 0.02f), "the tracked share of the joints a v1 has"); + require(near(stats.depth_invalid, 0.2f), "mean depth holes"); +} + +void files_stay_bounded() { + const std::string path = "kinect_record_test_bounded.csv"; + { + sfr::KinectRecordFile file(path, "a,b\n", 20); + require(file.write("1,2\n") && file.write("3,4\n"), "rows within the limit"); + require(!file.write("5,6,7,8,9,10\n") && !file.write("1,1\n"), "a full file takes nothing more"); + } + std::ifstream in(path); + const std::string written((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + require(written == "a,b\n1,2\n3,4\n", "header and whole rows only"); + in.close(); + std::remove(path.c_str()); +} +} + +// A sink for the record queue: keeps what it is given, and can be stalled +// (a disk that does not answer) until released. +struct FakeSink : sfr::KinectRecordSink { + std::mutex lock; + std::condition_variable changed; + bool stalled = false, inside = false; + std::vector written; + int flushes = 0; + void record_write(std::string& bytes) override { + std::unique_lock guard(lock); + inside = true; + changed.notify_all(); + changed.wait(guard, [&] { return !stalled; }); + written.push_back(bytes); + inside = false; + } + void record_flush() override { + std::lock_guard guard(lock); + ++flushes; + } + void stall(bool on) { + { + std::lock_guard guard(lock); + stalled = on; + } + changed.notify_all(); + } + void wait_inside() { + std::unique_lock guard(lock); + changed.wait(guard, [&] { return inside; }); + } + size_t count() { + std::lock_guard guard(lock); + return written.size(); + } +}; + +void queue_keeps_order() { + sfr::KinectRecordQueue queue(1 << 20); + FakeSink a, b; + for (int i = 0; i < 500; ++i) { + require(queue.push(&a, "a" + std::to_string(i)) && queue.push(&b, "b" + std::to_string(i)), "taken"); + } + require(queue.drain(&a, std::chrono::seconds(10)) && queue.drain(&b, std::chrono::seconds(10)), "drained"); + require(a.written.size() == 500 && b.written.size() == 500, "every item written"); + for (int i = 0; i < 500; ++i) + require(a.written[i] == "a" + std::to_string(i) && b.written[i] == "b" + std::to_string(i), "in the order queued"); + require(a.flushes > 0 && queue.dropped() == 0 && queue.queued_bytes() == 0, "flushed, nothing dropped or left"); +} + +void queue_never_waits_for_a_stalled_disk() { + sfr::KinectRecordQueue queue(1000); + FakeSink sink; + sink.stall(true); + require(queue.push(&sink, std::string(300, 'x')), "the first item"); + sink.wait_inside(); // the writer is stuck in it + uint32_t taken = 0, dropped = 0; + double slowest_us = 0; + for (int i = 0; i < 200; ++i) { + const auto before = std::chrono::steady_clock::now(); + const bool ok = queue.push(&sink, std::string(300, 'y')); + const double us = std::chrono::duration(std::chrono::steady_clock::now() - before).count(); + slowest_us = std::max(slowest_us, us); + (ok ? taken : dropped) += 1; + } + std::cout << "kinect_record: the slowest push with the disk stalled took " << slowest_us << " us\n"; + // 300 bytes being written count until written: two more fit in 1000. + require(taken == 2 && dropped == 198 && queue.dropped() == 198, "the budget holds; the rest dropped and counted"); + require(slowest_us < 50000, "a push never waits for the disk"); + sink.stall(false); + require(queue.drain(&sink, std::chrono::seconds(10)) && sink.count() == 3, "what was taken is written"); + // A drain that runs out of time drops what has not started. + sink.stall(true); + queue.push(&sink, "1"); + sink.wait_inside(); + queue.push(&sink, "2"); + queue.push(&sink, "3"); + std::thread release([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + sink.stall(false); + }); + const bool drained = queue.drain(&sink, std::chrono::milliseconds(50)); + release.join(); + require(!drained && sink.count() == 4 && queue.dropped() == 200, "out of time: the queued ones dropped"); +} + +void file_written_on_destruction() { + const std::string path = "kinect_record_test_queued.csv"; + std::string expected = "h\n"; + { + sfr::KinectRecordQueue queue(64 << 20); + sfr::KinectRecordFile file(path, "h\n", 1 << 20, &queue); + for (int i = 0; i < 1000; ++i) { + const std::string row = std::to_string(i) + ",x\n"; + expected += row; + require(file.write(row), "taken"); + } + } // the file drains, then closes + std::ifstream in(path, std::ios::binary); + const std::string written((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + require(written == expected, "every row, in order, after the header"); + in.close(); + std::remove(path.c_str()); +} + +void nui_rows_round_trip() { + const std::string header = sfr::kinect_nui_record_header(); + const size_t columns = sfr::kinect_record_fields(header.substr(0, header.size() - 1)).size(); + require(columns == 6 + 14 + 24 + 2 + 4 * sfr::kinect_nui_detector_count + 2 + 34, "the header names every column"); + require(header.find(",right_cursor_x,") != std::string::npos && header.find(",steer_value,") != std::string::npos && + header.find(",calib_132\n") != std::string::npos, + "cursor, detector and calibration columns"); + for (const auto& detector : sfr::kinect_nui_detectors) + require(sfr::kinect_nui_detector_index(detector.address) >= 0, "every detector has its columns"); + require(sfr::kinect_nui_detector_index(0x822C9050) == 0 && sfr::kinect_nui_detector_index(0x12345678) < 0, "by address"); + + // The recorder: several calls of one detector in an update keep the last + // answer and every bit; taking a row starts the next one empty. + sfr::KinectNuiRecorder recorder; + recorder.detector(1, 0x822C8778, 2, 0, 0x2000, false, 0); + recorder.detector(1, 0x822C8778, 1, 0x7000, 0, false, 0); + recorder.detector(1, 0x822C9BF0, 1, 0x00400100, 0, true, 75.0f); + recorder.detector(0, 0x822C9050, 2, 0, 0, false, 0); + recorder.lean(1, -1.25f, 0.5f); + std::array words{}; + for (uint32_t i = 0; i < words.size(); ++i) words[i] = std::bit_cast(0.25f * float(i)); + words[28] = 161; // +112, the centroid column + words[29] = uint32_t(-3); + recorder.calibration(1, "body", words); + sfr::KinectNuiRecord row; + row.steady_us = 42; + row.update = 1234; + row.player = 1; + row.tracking_id = 77; + row.body_slot = 3; + row.racing = true; + row.has_cursor = true; + row.hands[1] = {8, 101.5f, 0.42f, 600.0f, 300.0f, 590.25f, 310.5f}; + row.has_joints = true; + for (uint32_t i = 0; i < sfr::kinect_nui_joint_count; ++i) row.joints[i] = {-0.1f * float(i), 0.2f, -2.5f}; + recorder.take(1, row); + require(row.detectors[1].result == 1 && row.detectors[1].primary == 0x7000 && row.detectors[1].secondary == 0x2000 && + row.detectors[1].calls == 2, + "the last answer, every bit"); + require(row.detectors[0].result < 0, "another player's answer stays theirs"); + std::string text; + sfr::kinect_nui_record_row(row, text); + require(text.back() == '\n' && sfr::kinect_record_fields(text.substr(0, text.size() - 1)).size() == columns, + "a row has every column"); + sfr::KinectNuiRecord parsed; + require(sfr::kinect_parse_nui_row(text.substr(0, text.size() - 1), parsed), "a row reads back"); + require(parsed.update == 1234 && parsed.player == 1 && parsed.tracking_id == 77 && parsed.body_slot == 3 && parsed.racing, + "who and when"); + require(parsed.has_cursor && parsed.hands[1].flags == 8 && near(parsed.hands[1].theta, 101.5f) && + near(parsed.hands[1].cursor_y, 310.5f) && near(parsed.hands[1].r, 0.42f), + "the cursor"); + require(parsed.has_joints && near(parsed.joints[7][0], -0.7f) && near(parsed.joints[7][2], -2.5f), "the joints"); + require(parsed.has_lean && near(parsed.lean_depth, -1.25f) && near(parsed.lean_tilt, 0.5f), "the lean pair"); + require(parsed.detectors[1].result == 1 && parsed.detectors[1].primary == 0x7000 && !parsed.detectors[1].has_value, + "a detector's answer"); + require(parsed.detectors[10].result == 1 && parsed.detectors[10].has_value && near(parsed.detectors[10].value, 75.0f), + "the brake's strength"); + require(parsed.detectors[0].result == -1 && !parsed.has_wipe_y, "a detector not asked is empty"); + require(parsed.calibration_step == "body" && parsed.calibration[28] == 161 && int32_t(parsed.calibration[29]) == -3 && + near(std::bit_cast(parsed.calibration[33]), 8.25f), + "the calibration, whole numbers at +112..+120"); + sfr::KinectNuiRecord next; + recorder.take(1, next); + require(next.detectors[1].result < 0 && !next.has_lean && next.calibration_step.empty(), "the next row starts empty"); + // A row with nothing but its clocks still has every column. + sfr::KinectNuiRecord bare; + std::string bare_text; + sfr::kinect_nui_record_row(bare, bare_text); + sfr::KinectNuiRecord bare_parsed; + require(sfr::kinect_parse_nui_row(bare_text.substr(0, bare_text.size() - 1), bare_parsed) && !bare_parsed.has_cursor && + !bare_parsed.has_joints && bare_parsed.calibration_step.empty(), + "an empty row reads back empty"); + require(!sfr::kinect_parse_nui_row(header.substr(0, header.size() - 1), bare_parsed), "the header is not a row"); +} + +int main() { + try { + nui_rows_round_trip(); + queue_keeps_order(); + queue_never_waits_for_a_stalled_disk(); + file_written_on_destruction(); + skeleton_rows_round_trip(); + depth_holes(); + race_rows_round_trip(); + stats_measure_jitter_and_losses(); + files_stay_bounded(); + } catch (const std::exception& error) { + std::cerr << "kinect_record_test: " << error.what() << '\n'; + return 1; + } + std::cout << "kinect_record_test passed\n"; + return 0; +} diff --git a/tests/kinect_sensor_test.cpp b/tests/kinect_sensor_test.cpp index 0ef74e1..7bf343e 100644 --- a/tests/kinect_sensor_test.cpp +++ b/tests/kinect_sensor_test.cpp @@ -6,8 +6,11 @@ #include #include #include +#include #include +#include #include +#include #include #include @@ -96,6 +99,81 @@ int main() { "empty tracking releases selection and a later body keeps its sensor slot"); } + { + // The title keeps the TRACKED skeletons' slots in a two-entry + // array (82438EB0 writes every TRACKED slot to manager +7740 + 4n): + // a third overwrites +7748, a fourth the skeleton thread's stop + // flag (+7752). Whatever a Kinect v2 sees -- three to six bodies, + // coming, going and changing places, in any order, even ids it + // repeats -- at most two skeletons may reach the title, each once, + // in a slot of its own; the players stay put while tracked. + sfr::KinectPlayerSlots slots_under_test; + std::mt19937 random(2024); + std::vector present; // tracking ids in front of the sensor + uint32_t next_id = 1000, frames_checked = 0, most = 0; + std::array before{}; + const auto tracked_in_guest_frame = [](const sfr::KinectFrame& frame) { + // As the skeleton hook writes them (write_kinect_body): each + // body in its own sensor slot of a cleared frame. + sfr::GuestMemory guest; + guest.map(0x30000, 0x1000); + sfr::NuiSkeletonEmulation::write_header(guest, 0x30000, 1, 0); + sfr::NuiSkeletonEmulation writer; + for (const sfr::KinectBody& b : frame.bodies) + writer.write_joints(guest, 0x30000, b.sensor_index % 6, b.tracking_id, b.joints, b.joint_states, b.position); + uint32_t tracked = 0; + for (uint32_t slot = 0; slot < 6; ++slot) + tracked += guest.load(0x30000 + sfr::nui_skeleton_data_offset + slot * sfr::nui_skeleton_data_size) == + sfr::nui_tracked; + return tracked; + }; + for (uint32_t step = 0; step < 3000; ++step) { + // Someone leaves, someone steps in, keeping three to six around + // most of the time (and now and then nobody at all). + if (step % 500 == 499) present.clear(); + if (!present.empty() && random() % 4 == 0) present.erase(present.begin() + random() % present.size()); + while (present.size() < 3 + random() % 4 && present.size() < 6) present.push_back(next_id++); + std::vector bodies; + std::vector free_slots = {0, 1, 2, 3, 4, 5}; + std::shuffle(free_slots.begin(), free_slots.end(), random); + for (size_t i = 0; i < present.size(); ++i) { + auto b = body(present[i], float(i)); + b.sensor_index = free_slots[i]; + bodies.push_back(b); + } + // The sensor's order changes from frame to frame; once in a + // while a v2 hands the same (shortened) id over twice. + std::shuffle(bodies.begin(), bodies.end(), random); + if (step % 97 == 0 && bodies.size() >= 2) { + bodies.push_back(bodies.front()); + bodies.back().sensor_index = (bodies.front().sensor_index + 1) % 6; + } + sfr::KinectFrame frame; + frame.bodies = bodies; + slots_under_test.retain(frame); + ++frames_checked; + most = (std::max)(most, uint32_t(frame.bodies.size())); + require(frame.bodies.size() <= 2, "never more than two bodies reach the title"); + require(tracked_in_guest_frame(frame) <= 2, "never more than two TRACKED skeletons in the title's frame"); + if (frame.bodies.size() == 2) { + require(frame.bodies[0].tracking_id != frame.bodies[1].tracking_id, "each player once"); + require(frame.bodies[0].sensor_index != frame.bodies[1].sensor_index, "each in a slot of its own"); + } + for (const sfr::KinectBody& kept : frame.bodies) + require(std::find(present.begin(), present.end(), kept.tracking_id) != present.end(), + "only someone who is there"); + // A player who is still there is still a player. + for (uint32_t id : before) + if (id && std::find(present.begin(), present.end(), id) != present.end()) + require(std::any_of(frame.bodies.begin(), frame.bodies.end(), + [&](const sfr::KinectBody& kept) { return kept.tracking_id == id; }), + "a tracked player keeps playing while newcomers come and go"); + before = {}; + for (size_t i = 0; i < frame.bodies.size() && i < 2; ++i) before[i] = frame.bodies[i].tracking_id; + } + require(frames_checked == 3000 && most == 2, "two players were seen, never more"); + } + // The frame the title reads: the sensor's floor, its joint states, // and the body's own centre. constexpr uint32_t address = 0x10000; @@ -219,40 +297,6 @@ int main() { require(sfr::kinect_open_failure("initialize-0x8007048F", "no-sensor") == "initialize-0x8007048F", "a v1 that will not start says more than a missing v2"); - { - // A sensor pitched up by 15 degrees sees an upright player leaning - // back: its up is (0, cos, sin) in its own space. Levelled, the - // body is upright again and gravity is straight down. - const float t = 15.0f * 3.14159265f / 180.0f, c = std::cos(t), s = std::sin(t); - const auto seen = [&](float y, float z) { return std::array{0.0f, y * c - z * s, y * s + z * c}; }; - sfr::KinectFrame tilted; - tilted.gravity = {0.0f, c, s}; - tilted.floor_plane = {0.0f, c, s, 0.54f}; - sfr::KinectBody rider; - rider.tracking_id = 5; - for (auto& joint : rider.joints) joint = seen(0.0f, 2.2f); - rider.joints[sfr::nui_joint::hip_center] = seen(0.0f, 2.2f); - rider.joints[sfr::nui_joint::head] = seen(0.7f, 2.2f); - rider.position = seen(0.0f, 2.2f); - tilted.bodies = {rider}; - require(near(sfr::kinect_level(tilted), 15.0f), "the tilt taken out is the sensor's pitch"); - const auto& hip = tilted.bodies[0].joints[sfr::nui_joint::hip_center]; - const auto& head = tilted.bodies[0].joints[sfr::nui_joint::head]; - require(near(head[2], hip[2]) && near(head[1] - hip[1], 0.7f), "a levelled body stands upright"); - require(near(tilted.gravity[1], 1.0f) && near(tilted.floor_plane[1], 1.0f) && - near(tilted.floor_plane[2], 0.0f) && near(tilted.floor_plane[3], 0.54f), - "gravity and the floor are straight up after levelling"); - - sfr::KinectFrame upside; - upside.gravity = {0.0f, -1.0f, 0.0f}; - upside.bodies = {rider}; - require(sfr::kinect_level(upside) == 0.0f && upside.bodies[0].joints[3] == rider.joints[3], - "gravity the other way up is not a tilt to take out"); - sfr::KinectFrame none; - none.bodies = {rider}; - require(sfr::kinect_level(none) == 0.0f, "a frame without gravity is left alone"); - } - { using sfr::KinectStep; using sfr::KinectStepState; @@ -317,9 +361,60 @@ int main() { require(body.joints[joint::hand_right][1] < body.joints[joint::elbow_right][1], "the hand below the elbow"); } + // The launcher's Kinect choice (SFR_KINECT_MODEL): 2 is the Kinect + // v2; unset, 1 or anything else is the Xbox 360 Kinect. + { +#ifdef _WIN32 + const auto set_model = [](const char* value) { _putenv_s("SFR_KINECT_MODEL", value); }; +#else + const auto set_model = [](const char* value) { + if (*value) setenv("SFR_KINECT_MODEL", value, 1); + else unsetenv("SFR_KINECT_MODEL"); + }; +#endif + set_model(""); + require(sfr::kinect_selected_model() == 1, "no model chosen is the Xbox 360 Kinect"); + set_model("1"); + require(sfr::kinect_selected_model() == 1, "model 1 is the Xbox 360 Kinect"); + set_model("2"); + require(sfr::kinect_selected_model() == 2, "model 2 is the Kinect v2"); + set_model("3"); + require(sfr::kinect_selected_model() == 1, "an unknown model is the Xbox 360 Kinect"); + set_model(""); + } + + { + // Which edges cut a body: names, and what the player should do. + using namespace sfr; + require(kinect_edges_text(0) == "none", "no edge is none"); + require(kinect_edges_text(kinect_edge::top) == "top", "top"); + require(kinect_edges_text(kinect_edge::left | kinect_edge::top) == "top+left", "edges in a fixed order"); + require(kinect_edges_text(kinect_edge::all) == "top+bottom+left+right", "all four"); + require(kinect_edge::right == 0x1 && kinect_edge::left == 0x2 && kinect_edge::top == 0x4 && kinect_edge::bottom == 0x8, + "the SDK's FrameEdges values"); + require(kinect_edges_move(0) == 0, "nothing cut, nothing to do"); + require(kinect_edges_move(kinect_edge::top) == (kinect_edge_move::step_back | kinect_edge_move::tilt_up), "top: back or tilt up"); + require(kinect_edges_move(kinect_edge::bottom) == (kinect_edge_move::step_back | kinect_edge_move::tilt_down), + "bottom: back or tilt down"); + // The picture's left is the player's right (facing the sensor): step to the player's own left. + require(kinect_edges_move(kinect_edge::left) == kinect_edge_move::player_left, "picture left: the player's left"); + require(kinect_edges_move(kinect_edge::right) == kinect_edge_move::player_right, "picture right: the player's right"); + require(kinect_edges_move(kinect_edge::left | kinect_edge::right) == kinect_edge_move::step_back, "both sides: step back"); + require(kinect_edges_move(kinect_edge::top | kinect_edge::bottom) == kinect_edge_move::step_back, "both ends: step back"); + require(kinect_edges_move(kinect_edge::top | kinect_edge::left) == + (kinect_edge_move::step_back | kinect_edge_move::tilt_up | kinect_edge_move::player_left), "a corner needs both"); + KinectBody body; + require(body.clipped_edges == 0, "a body (a v1's) is not cut by default"); + } + std::string why; - if (!sfr::KinectSensor::supported()) + if (!sfr::KinectSensor::supported()) { require(!sfr::KinectSensor::open(&why) && !why.empty(), "a platform without a runtime says why"); + why.clear(); + require(!sfr::KinectSensor::open_selected(1, &why) && !why.empty(), "the Xbox 360 choice says why it found nothing"); + why.clear(); + require(!sfr::KinectSensor::open_selected(2, &why) && !why.empty(), "the Kinect v2 choice says why it found nothing"); + } } catch (const std::exception& error) { std::cerr << "kinect_sensor_test: " << error.what() << '\n'; return 1; diff --git a/tests/kinect_smoothing_test.cpp b/tests/kinect_smoothing_test.cpp index f5ed283..f49b39a 100644 --- a/tests/kinect_smoothing_test.cpp +++ b/tests/kinect_smoothing_test.cpp @@ -60,14 +60,41 @@ void a_move_is_followed_without_running_ahead() { require(std::abs(smoothed_x(inferred, 0.5f, 1) - 0.5f) <= 0.08f + 1e-5f, "an inferred joint has twice the radius"); } -void untracked_joints_and_new_bodies_start_over() { +void lost_joints_and_new_bodies_start_over() { + // As the NUI library does: a joint at the origin (no position) passes + // as it is and starts its history over; a joint called not tracked that + // still has a position is filtered like any other. sfr::KinectJointSmoother smoother; - for (int i = 0; i < 10; ++i) smoothed_x(smoother, 0.0f); - require(smoothed_x(smoother, 0.9f, 0) == 0.9f, "a joint that is not tracked passes through"); - require(smoothed_x(smoother, 0.9f) == 0.9f, "and starts over when it is seen again"); + for (int i = 0; i < 10; ++i) smoothed_x(smoother, 0.2f); + auto lost = frame_with(0.0f); + for (auto& joint : lost.bodies[0].joints) joint = {0.0f, 0.0f, 0.0f}; + smoother.apply(lost); + require(lost.bodies[0].joints[3] == (std::array{0.0f, 0.0f, 0.0f}), "a joint without a position passes"); + sfr::KinectJointSmoother untracked; + smoothed_x(untracked, 0.0f, 0); + require(std::abs(smoothed_x(untracked, 0.1f, 0) - 0.0625f) < 1e-6f, "a joint not tracked but placed is filtered"); for (int i = 0; i < 10; ++i) smoothed_x(smoother, 0.0f, 2, 8); require(smoothed_x(smoother, 0.3f, 2, 9) == 0.3f, "a new body starts from its own measurement"); } + +void the_same_filter_as_the_games() { + // The backends' filter (SFR_KINECT_SMOOTH=1) is the game's own + // NuiTransformSmooth step for step. + sfr::KinectJointSmoother host; + sfr::NuiFrameSmoother game; + for (int i = 0; i < 60; ++i) { + const float x = 0.3f * std::sin(float(i) * 0.3f) + (i % 3 ? 0.004f : -0.006f); + auto frame = frame_with(x, i % 9 == 0 ? 1u : 2u); + std::array slots{}; + slots[2].tracking_state = 2; + slots[2].tracking_id = 7; + slots[2].joints = frame.bodies[0].joints; + slots[2].states = frame.bodies[0].joint_states; + host.apply(frame); + game.smooth(slots, {}); + require(frame.bodies[0].joints == slots[2].joints, "the host's smoothing is the game's"); + } +} } int main() { @@ -75,7 +102,8 @@ int main() { a_still_joint_stays_put(); noise_is_damped(); a_move_is_followed_without_running_ahead(); - untracked_joints_and_new_bodies_start_over(); + lost_joints_and_new_bodies_start_over(); + the_same_filter_as_the_games(); std::cout << "Kinect smoothing checks passed\n"; return 0; } catch (const std::exception& error) { diff --git a/tests/kinect_v2_depth_test.cpp b/tests/kinect_v2_depth_test.cpp new file mode 100644 index 0000000..053a8b9 --- /dev/null +++ b/tests/kinect_v2_depth_test.cpp @@ -0,0 +1,1281 @@ +#include "kinect_record_raw.h" +#include "kinect_v2_depth.h" +#include "kinect_v2_image.h" +#include "kinect_v2_resynth.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +bool close_to(float a, float b, float tolerance = 1e-3f) { return std::fabs(a - b) <= tolerance; } + +uint16_t depth_at(const sfr::KinectImage& image, uint32_t x, uint32_t y) { + const uint8_t* p = &image.pixels[(size_t(y) * image.width + x) * 2]; + return uint16_t(p[0] | p[1] << 8); +} + +// Camera-space points, three floats each. +struct Points { + std::vector xyz; + std::vector body; + void add(float x, float y, float z, uint8_t index = sfr::kinect_v2_no_body) { + xyz.insert(xyz.end(), {x, y, z}); + body.push_back(index); + } + // A point that lands on pixel (column, row) of the virtual v1 at depth z. + void at_pixel(float column, float row, float z, uint8_t index = sfr::kinect_v2_no_body) { + add((column - 160.0f) * z / sfr::kinect_v1_depth_focal, -(row - 120.0f) * z / sfr::kinect_v1_depth_focal, z, + index); + } + sfr::KinectDepthStats make(sfr::KinectImage& out, const sfr::KinectDepthView& view = {}) const { + return sfr::kinect_v2_synthesize_depth(xyz.data(), body.data(), body.size(), view, out); + } +}; + +void packing_is_the_v1s() { + // NuiImageCamera.h: millimetres << NUI_IMAGE_PLAYER_INDEX_SHIFT (3), the + // player in NUI_IMAGE_PLAYER_INDEX_MASK. + require(sfr::kinect_v1_depth_pixel(2500, 1) == uint16_t(2500 << 3 | 1), "depth and player packed"); + require(sfr::kinect_v1_depth_pixel(2500, 0) == uint16_t(2500 << 3), "no player"); + require(sfr::kinect_v1_depth_pixel(800, 6) == uint16_t(800 << 3 | 6), "nearest a v1 measures"); + require(sfr::kinect_v1_depth_pixel(4000, 2) == uint16_t(4000 << 3 | 2), "farthest a v1 measures"); + // Without DISTINCT_OVERFLOW_DEPTH_VALUES too near and too far are no value. + require(sfr::kinect_v1_depth_pixel(799, 1) == sfr::kinect_v1_depth_no_value, "too near is no value"); + require(sfr::kinect_v1_depth_pixel(4001, 1) == sfr::kinect_v1_depth_no_value, "too far is no value"); + require(sfr::kinect_v1_depth_pixel(0, 0) == 0, "unknown is no value"); + require(sfr::kinect_v1_depth_no_value == 0, "NUI_IMAGE_DEPTH_NO_VALUE"); +} + +void projection_is_the_nominal_lens() { + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project({0, 0, 2}, x, y) && close_to(x, 160) && close_to(y, 120), "the axis is the centre"); + // NuiTransformSkeletonToDepthImage: 160 + X * 285.63 / Z, 120 - Y * 285.63 / Z. + require(sfr::kinect_v1_depth_project({0.5f, 0.25f, 2.5f}, x, y), "a point in front"); + require(close_to(x, 160 + 0.5f * 285.63f / 2.5f) && close_to(y, 120 - 0.25f * 285.63f / 2.5f), "nominal focal length"); + require(!sfr::kinect_v1_depth_project({0, 0, 0}, x, y) && !sfr::kinect_v1_depth_project({0, 0, -1}, x, y), + "nothing at or behind the camera"); +} + +void points_land_where_projected() { + Points points; + points.at_pixel(100, 50, 2.0f, 0); + points.at_pixel(300, 200, 3.5f, sfr::kinect_v2_no_body); + sfr::KinectImage image; + const sfr::KinectDepthStats stats = points.make(image); + require(image.width == 320 && image.height == 240 && image.bytes_per_pixel == 2 && image.pixels.size() == 320 * 240 * 2, + "a v1's 320x240, two bytes a pixel"); + require(depth_at(image, 100, 50) == uint16_t(2000 << 3 | 1), "body 0 is player 1"); + require(depth_at(image, 300, 200) == uint16_t(3500 << 3), "no body is player 0"); + require(depth_at(image, 101, 50) == 0 && depth_at(image, 0, 0) == 0, "everything else unknown"); + require(stats.valid == 2 && stats.players == 1, "counts"); +} + +void nearest_wins() { + Points points; + points.at_pixel(40, 40, 3.0f, sfr::kinect_v2_no_body); // the wall behind + points.at_pixel(40, 40, 1.5f, 2); // a hand in front + points.at_pixel(40, 40, 2.0f, sfr::kinect_v2_no_body); + sfr::KinectImage image; + points.make(image); + require(depth_at(image, 40, 40) == uint16_t(1500 << 3 | 3), "the nearest point and its player"); + // And the other way round in arrival order. + Points reversed; + reversed.at_pixel(40, 40, 1.5f, 2); + reversed.at_pixel(40, 40, 3.0f, sfr::kinect_v2_no_body); + reversed.make(image); + require(depth_at(image, 40, 40) == uint16_t(1500 << 3 | 3), "a farther point does not overwrite"); +} + +void holes_filled_conservatively() { + Points points; + // A one-pixel gap between two that agree: filled with their mean. + points.at_pixel(10, 10, 2.00f, 0); + points.at_pixel(12, 10, 2.04f, 0); + // Between two that do not (an edge): left. + points.at_pixel(20, 10, 2.0f, 0); + points.at_pixel(22, 10, 3.0f, sfr::kinect_v2_no_body); + // Between two players: left. + points.at_pixel(30, 10, 2.0f, 0); + points.at_pixel(32, 10, 2.0f, 1); + // Vertically too. + points.at_pixel(50, 20, 2.5f); + points.at_pixel(50, 22, 2.5f); + // A two-pixel gap: left. + points.at_pixel(60, 10, 2.0f); + points.at_pixel(63, 10, 2.0f); + sfr::KinectImage image; + const sfr::KinectDepthStats stats = points.make(image); + require(depth_at(image, 11, 10) == uint16_t(2020 << 3 | 1), "a gap between agreeing pixels is filled"); + require(depth_at(image, 21, 10) == 0, "an edge is not filled"); + require(depth_at(image, 31, 10) == 0, "two players are not joined"); + require(depth_at(image, 50, 21) == uint16_t(2500 << 3), "vertical gap filled"); + require(depth_at(image, 61, 10) == 0 && depth_at(image, 62, 10) == 0, "wider gaps stay unknown"); + require(stats.filled == 2, "two filled"); +} + +void out_of_range_is_no_value() { + Points points; + points.at_pixel(5, 5, 0.5f, 0); // too near for a v1 + points.at_pixel(6, 5, 4.5f, 0); // too far + points.add(0, 0, 0); // v2 measured nothing + points.add(0, 0, -INFINITY); // the mapper's no point + points.add(NAN, 0, 2); + sfr::KinectImage image; + const sfr::KinectDepthStats stats = points.make(image); + require(depth_at(image, 5, 5) == 0 && depth_at(image, 6, 5) == 0 && depth_at(image, 160, 120) == 0, + "outside the v1's range, or no point, is no value"); + require(stats.valid == 0 && stats.players == 0, "nothing valid"); +} + +void view_moves_the_camera() { + // The virtual camera 1.5 m behind the v2: a point 1 m in front of the v2 + // is 2.5 m from it, and a point to the side comes towards the centre. + sfr::KinectDepthView view; + view.origin = {0, 0, -1.5f}; + const auto seen = sfr::kinect_depth_view_point(view, {0.4f, 0.1f, 1.0f}); + require(close_to(seen[0], 0.4f) && close_to(seen[1], 0.1f) && close_to(seen[2], 2.5f), "moved along z"); + const auto back = sfr::kinect_depth_view_unpoint(view, seen); + require(close_to(back[2], 1.0f), "and back"); + Points points; + points.add(0.4f, 0.1f, 1.0f, 0); + sfr::KinectImage image; + points.make(image, view); + float x = 0, y = 0; + sfr::kinect_v1_depth_project(seen, x, y); + require(depth_at(image, uint32_t(std::lround(x)), uint32_t(std::lround(y))) == uint16_t(2500 << 3 | 1), + "the image is the moved camera's"); + + // The skeletons and the floor move with it. + sfr::KinectFrame frame; + frame.floor_plane = {0, 1, 0, 0.9f}; // the floor 0.9 m below the v2 + frame.gravity = {0, 1, 0}; + sfr::KinectBody& body = frame.bodies.emplace_back(); + body.position = {0.1f, 0.0f, 1.2f}; + body.joints[0] = {0.1f, 0.0f, 1.2f}; + body.hand_tips[0] = {}; // not seen: stays none + view.origin = {0, 0.2f, -1.3f}; + sfr::kinect_depth_view_frame(view, frame); + require(close_to(body.position[2], 2.5f) && close_to(body.position[1], -0.2f), "skeleton centre moved"); + require(close_to(body.joints[0][2], 2.5f), "joints moved"); + require(body.hand_tips[0][0] == 0 && body.hand_tips[0][2] == 0, "an unseen point stays none"); + // The floor at y = -0.9 in the v2 is y = -1.1 to a camera 0.2 m higher. + require(close_to(frame.floor_plane[3], 1.1f), "floor plane moved"); + require(close_to(frame.gravity[1], 1.0f), "gravity is a direction"); +} + +sfr::KinectFrame frame_with(std::initializer_list>> bodies, + std::array floor = {}) { + sfr::KinectFrame frame; + frame.floor_plane = floor; + for (const auto& [id, position] : bodies) { + sfr::KinectBody& body = frame.bodies.emplace_back(); + body.tracking_id = id; + body.position = position; + } + return frame; +} + +void settings_from_text() { + const auto off = sfr::KinectDepthSettings::parse(nullptr, nullptr, nullptr); + require(off.mode == sfr::KinectDepthViewMode::fit && !off.player_view && off.fit_scale == 0 && + close_to(off.distance, 2.5f) && close_to(off.height, 0.8f), + "defaults: fit, the scale from the lens"); + require(std::string(off.view_name()) == "fit", "named fit"); + const auto set = sfr::KinectDepthSettings::parse("sensor", "3", "1.2"); + require(set.mode == sfr::KinectDepthViewMode::sensor && !set.player_view && close_to(set.distance, 3) && + close_to(set.height, 1.2f), + "set"); + const auto odd = sfr::KinectDepthSettings::parse("player", "9", "x"); + require(odd.player_view && odd.mode == sfr::KinectDepthViewMode::player && close_to(odd.distance, 4) && + close_to(odd.height, 0.8f), + "clamped"); + require(close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, "1.5").fit_scale, 1.5f) && + close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, "3").fit_scale, 2.0f) && + close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, "0.5").fit_scale, 1.0f), + "fit scale set and clamped to 1..2"); + require(sfr::KinectDepthSettings::parse("odd", nullptr, nullptr).mode == sfr::KinectDepthViewMode::fit, + "an unknown view is fit"); +} + +void latch_holds_still() { + sfr::KinectDepthSettings settings; + settings.mode = sfr::KinectDepthViewMode::player; + settings.player_view = true; + sfr::KinectDepthViewLatch latch(settings); + double t = 0; + const auto step = [&](const sfr::KinectFrame& frame) { + t += 1.0 / 30.0; + return latch.update(frame, t); + }; + // Nobody: the v2's own place. + require(close_to(step(frame_with({})).origin[2], 0), "nobody, nothing moved"); + // Someone at 1.2 m: placed at once, latched after settling. + sfr::KinectDepthView view = step(frame_with({{7, {0.0f, 0.0f, 1.2f}}})); + require(close_to(view.origin[2], 1.2f - 2.5f) && !latch.latched(), "first frame places it"); + for (uint32_t i = 1; i < sfr::KinectDepthViewLatch::settle_frames; ++i) view = step(frame_with({{7, {0.0f, 0.0f, 1.2f}}})); + require(latch.latched() && close_to(view.origin[2], -1.3f), "latched at the settled place"); + // Stepping aside, crouching, stepping forward: the camera stays. + view = step(frame_with({{7, {0.5f, -0.3f, 1.0f}}})); + require(close_to(view.origin[0], 0) && close_to(view.origin[1], 0) && close_to(view.origin[2], -1.3f), + "the player's moves are not followed"); + // A second player stepping in does not move it either. + view = step(frame_with({{7, {0.0f, 0.0f, 1.2f}}, {9, {0.0f, 0.0f, 0.8f}}})); + require(close_to(view.origin[2], -1.3f), "a second player does not move it"); + // A short gap keeps it; a second away lets it go, and the next person + // places it again. + for (int i = 0; i < 10; ++i) view = step(frame_with({})); + require(latch.latched() && close_to(view.origin[2], -1.3f), "a third of a second away keeps it"); + for (int i = 0; i < 30; ++i) step(frame_with({})); + require(!latch.latched(), "a second away lets it go"); + view = step(frame_with({{11, {0.3f, 0.0f, 2.0f}}, {12, {-0.1f, 0.0f, 1.6f}}})); + require(close_to(view.origin[2], 1.6f - 2.5f), "the most central body places it again"); +} + +void latch_settles_on_the_mean() { + sfr::KinectDepthSettings settings; + settings.mode = sfr::KinectDepthViewMode::player; + settings.player_view = true; + sfr::KinectDepthViewLatch latch(settings); + double t = 0; + sfr::KinectDepthView view; + // Walking in: 1.0 m to about 1.28 m over the settling frames. + double sum = 0; + for (uint32_t i = 0; i < sfr::KinectDepthViewLatch::settle_frames; ++i) { + const float z = 1.0f + 0.02f * float(i); + sum += z; + view = latch.update(frame_with({{3, {0, 0, z}}}), t += 1.0 / 30.0); + } + require(latch.latched() && close_to(view.origin[2], float(sum / sfr::KinectDepthViewLatch::settle_frames) - 2.5f), + "the mean of the settling frames"); +} + +void latch_height_from_the_floor() { + sfr::KinectDepthSettings settings; + settings.mode = sfr::KinectDepthViewMode::player; + settings.player_view = true; + settings.height = 0.8f; + const sfr::KinectDepthViewLatch latch(settings); + // The v2 1.4 m above the floor (on top of a TV), level; the hips 0.95 m + // above the floor at y = -0.45, 1.2 m away. + const auto view = latch.place({0.2f, -0.45f, 1.2f}, {0, 1, 0, 1.4f}); + require(close_to(view.origin[2], -1.3f), "distance"); + // The reference sits 0.15 m above a camera 0.8 m over the floor: the + // camera comes down 0.6 m. + require(close_to(view.origin[1], -0.6f), "height above the floor"); + require(close_to(view.origin[0], 0), "never sideways"); + const auto seen = sfr::kinect_depth_view_point(view, {0.2f, -0.45f, 1.2f}); + require(close_to(seen[1], 0.15f) && close_to(seen[2], 2.5f), "the reference where a v1 would see it"); + // Without a floor, the height stays the v2's. + require(close_to(latch.place({0.2f, -0.45f, 1.2f}, {}).origin[1], 0), "no floor, no height change"); +} + +sfr::KinectDepthView fit_view(float k) { + sfr::KinectDepthView view; + view.z_scale = k; + return view; +} + +void fit_scale_from_the_lens() { + // 285.63 / (366 * 240 / 424) = 1.3788. + require(close_to(sfr::kinect_v2_fit_scale(366.0f), 285.63f / (366.0f * 240.0f / 424.0f)) && + close_to(sfr::kinect_v2_fit_scale(366.0f), 1.379f, 1e-3f), + "the typical v2's stretch"); + require(close_to(sfr::kinect_v2_fit_scale(0), sfr::kinect_v2_fit_scale(366.0f)), "no lens: the typical one"); + const float k = sfr::kinect_v2_fit_scale(366.0f); + require(close_to(sfr::kinect_depth_true_minimum(k), 0.8f / k) && close_to(sfr::kinect_depth_true_maximum(k), 4.0f / k), + "the true range"); + require(close_to(sfr::kinect_depth_true_minimum(k), 0.58f, 0.01f) && close_to(sfr::kinect_depth_true_maximum(k), 2.90f, 0.01f), + "about 0.58 to 2.90 m"); +} + +void fit_shows_the_whole_v2_view() { + const float k = sfr::kinect_v2_fit_scale(366.0f); + const auto view = fit_view(k); + // The v2's corner rays (pixel (0, 0) and (511, 423) through its typical + // lens) land inside the v1's 320x240; at sensor they do not. + for (const auto& corner : {std::array{0, 0}, std::array{511, 423}, std::array{0, 423}}) { + const float z = 2.0f; + const std::array p{(corner[0] - 256.0f) * z / 366.0f, -(corner[1] - 212.0f) * z / 366.0f, z}; + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, p), x, y), "projected"); + require(x >= -0.5f && x < 320.5f && y >= -0.5f && y < 240.5f, "a corner of the v2's view inside the v1's image"); + sfr::kinect_v1_depth_project(p, x, y); + require(x < 0 || x >= 320 || y < 0 || y >= 240, "outside without the stretch"); + } + // The top and bottom rows of the v2 fill the v1's rows. + float x = 0, y = 0; + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, {0, 212.0f * 2.0f / 366.0f, 2.0f}), x, y); + require(close_to(y, 0.0f, 0.01f), "the v2's top row is the v1's"); +} + +void fit_depth_matches_size() { + const float k = 1.4f; + // A person 1.8 m tall at 1.5 m through the fit view looks as tall as + // one at 2.1 m through the v1, and its depth is 2.1 m. + const auto view = fit_view(k); + float x0 = 0, top = 0, bottom = 0; + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, {0, 0.9f, 1.5f}), x0, top); + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, {0, -0.9f, 1.5f}), x0, bottom); + float v1_top = 0, v1_bottom = 0; + sfr::kinect_v1_depth_project({0, 0.9f, 2.1f}, x0, v1_top); + sfr::kinect_v1_depth_project({0, -0.9f, 2.1f}, x0, v1_bottom); + require(close_to(bottom - top, v1_bottom - v1_top), "the same size as at k times the distance"); + Points points; + points.at_pixel(0, 0, 1.0f); // replaced below + points.xyz = {0.0f, 0.0f, 1.5f}; + points.body = {0}; + sfr::KinectImage image; + points.make(image, view); + require(depth_at(image, 160, 120) == uint16_t(2100 << 3 | 1), "with that depth"); +} + +void fit_range_clamps() { + const auto view = fit_view(1.4f); + Points points; + points.add(0.0f, 0.0f, 0.55f, 0); // 0.77 m stretched: too near + points.add(-0.1f, 0.0f, 0.60f, 0); // 0.84 m: in + points.add(0.2f, 0.0f, 2.80f, 0); // 3.92 m: in + points.add(0.4f, 0.0f, 2.90f, 0); // 4.06 m: too far + sfr::KinectImage image; + const sfr::KinectDepthStats stats = points.make(image, view); + require(stats.valid == 2, "only the stretched 800..4000 mm"); + float x = 0, y = 0; + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, {0.0f, 0.0f, 0.55f}), x, y); + require(depth_at(image, uint32_t(std::lround(x)), uint32_t(std::lround(y))) == 0, "too near is no value"); +} + +void fit_moves_skeleton_and_floor() { + const float k = 1.4f; + const auto view = fit_view(k); + sfr::KinectFrame frame; + // A floor 1 m below a v2 tilted down a little: n = (0, cos, sin). + const float c = std::cos(0.2f), s = std::sin(0.2f); + frame.floor_plane = {0, c, s, 1.0f}; + frame.gravity = {0, c, s}; + sfr::KinectBody& body = frame.bodies.emplace_back(); + body.position = {0.1f, -0.2f, 1.5f}; + body.joints[3] = {0.0f, 0.5f, 1.4f}; + body.hand_tips[1] = {0.3f, 0.1f, 1.2f}; + // A point on the floor before is on it after. + const std::array on_floor{0.4f, -(1.0f + s * 2.0f) / c, 2.0f}; + sfr::kinect_depth_view_frame(view, frame); + require(close_to(body.position[2], 1.5f * k) && close_to(body.position[0], 0.1f) && close_to(body.position[1], -0.2f), + "the skeleton's centre stretched"); + require(close_to(body.joints[3][2], 1.4f * k) && close_to(body.hand_tips[1][2], 1.2f * k), "joints and tips"); + require(body.joints[0] == std::array{} && body.thumbs[0] == std::array{}, "none stays none"); + const auto& n = frame.floor_plane; + require(close_to(std::sqrt(n[0] * n[0] + n[1] * n[1] + n[2] * n[2]), 1.0f), "a unit normal"); + const auto seen = sfr::kinect_depth_view_point(view, on_floor); + require(close_to(n[0] * seen[0] + n[1] * seen[1] + n[2] * seen[2] + n[3], 0.0f), "the floor goes through the same space"); + require(close_to(frame.gravity[1], n[1]) && close_to(frame.gravity[2], n[2]), "gravity the new floor normal"); + // And back: what the preview does with a handed-out point. + const auto back = sfr::kinect_depth_view_unpoint(view, body.joints[3]); + require(close_to(back[2], 1.4f) && close_to(back[1], 0.5f), "undone"); +} + +void fit_of_one_is_sensor() { + Points points; + points.add(0.3f, -0.2f, 2.0f, 1); + points.add(-0.5f, 0.4f, 3.1f, sfr::kinect_v2_no_body); + sfr::KinectImage fit, sensor; + points.make(fit, fit_view(1.0f)); + points.make(sensor, {}); + require(fit.pixels == sensor.pixels, "a stretch of one is the sensor view"); + sfr::KinectFrame a, b; + a.floor_plane = b.floor_plane = {0, 1, 0, 0.8f}; + a.gravity = b.gravity = {0, 1, 0}; + a.bodies.emplace_back().position = {0.1f, 0.2f, 1.9f}; + b.bodies = a.bodies; + sfr::kinect_depth_view_frame(fit_view(1.0f), a); + require(a.floor_plane == b.floor_plane && a.bodies[0].position == b.bodies[0].position, "nothing moved"); +} + +void sensor_view_stays() { sfr::KinectDepthSettings settings; + settings.player_view = false; + sfr::KinectDepthViewLatch latch(settings); + for (int i = 0; i < 30; ++i) { + const auto& view = latch.update(frame_with({{1, {0.0f, 0.0f, 1.0f}}}, {0, 1, 0, 1}), i / 30.0); + require(view.origin == std::array{}, "the sensor view is the v2's own place"); + } +} + +void lens_fallback_matches_estimate() { + // A typical v2 lens: the inverse of kinect_v2_depth_estimate. + std::vector depth(sfr::kinect_v2_depth_width * sfr::kinect_v2_depth_height, 0); + depth[100 * sfr::kinect_v2_depth_width + 300] = 2000; + std::vector points; + sfr::kinect_v2_depth_points_from_lens(depth.data(), sfr::kinect_v2_depth_width, sfr::kinect_v2_depth_height, points); + const float* p = &points[3 * (100 * sfr::kinect_v2_depth_width + 300)]; + require(close_to(p[2], 2.0f) && close_to(p[0], (300 - 256) * 2.0f / 366.0f) && close_to(p[1], -(100 - 212) * 2.0f / 366.0f), + "unprojected through the typical lens"); + require(points[2] == 0, "no depth, no point"); +} + +// A floor below a v2 that looks down by pitch and is rolled by roll +// (radians), d metres above it: its normal in the v2's space. +std::array tilted_floor(float pitch, float roll, float d) { + // Up in a level camera is (0, 1, 0); the camera pitched down sees it + // leaning back towards itself (-z), rolled to its +x. + const float nx = std::sin(roll) * std::cos(pitch), ny = std::cos(roll) * std::cos(pitch), nz = -std::sin(pitch); + return {nx, ny, nz, d}; +} + +std::array times(const std::array& r, const std::array& p) { + return {r[0] * p[0] + r[1] * p[1] + r[2] * p[2], r[3] * p[0] + r[4] * p[1] + r[5] * p[2], + r[6] * p[0] + r[7] * p[1] + r[8] * p[2]}; +} + +void levelling_takes_out_a_known_tilt() { + const float pitch = 15.0f * 3.14159265f / 180.0f, roll = 4.0f * 3.14159265f / 180.0f; + const auto floor = tilted_floor(pitch, roll, 1.3f); + require(sfr::kinect_floor_known(floor), "a tilted floor is a floor"); + const auto r = sfr::kinect_level_rotation(floor); + const auto up = times(r, {floor[0], floor[1], floor[2]}); + require(close_to(up[0], 0, 1e-5f) && close_to(up[1], 1, 1e-5f) && close_to(up[2], 0, 1e-5f), "the normal turned straight up"); + // A rotation: rows of unit length, at right angles. + for (int i = 0; i < 3; ++i) + for (int j = 0; j < 3; ++j) { + const float dot = r[3 * i] * r[3 * j] + r[3 * i + 1] * r[3 * j + 1] + r[3 * i + 2] * r[3 * j + 2]; + require(close_to(dot, i == j ? 1.0f : 0.0f, 1e-5f), "orthonormal"); + } + // No yaw: the optical axis keeps its heading (no sideways part). + const auto forward = times(r, {0, 0, 1}); + require(close_to(forward[0], 0, 1e-6f) && forward[2] > 0.9f, "the heading stays"); + // A point on the floor is d below the levelled camera. + const std::array on_floor{0.7f, 0, 2.4f}; + std::array p = on_floor; + // y from n.p + d = 0. + p[1] = -(floor[3] + floor[0] * p[0] + floor[2] * p[2]) / floor[1]; + require(close_to(times(r, p)[1], -1.3f, 1e-4f), "the floor is level, d below"); + float tilt_pitch = 0, tilt_roll = 0; + sfr::kinect_floor_tilt(floor, tilt_pitch, tilt_roll); + require(close_to(tilt_pitch, 15.0f, 0.01f) && close_to(tilt_roll, 4.0f, 0.2f), "the tilt for the log"); + // No floor: no turn. + const auto identity = sfr::kinect_level_rotation({0, 0, 0, 0}); + require(identity == std::array{1, 0, 0, 0, 1, 0, 0, 0, 1} && !sfr::kinect_floor_known({0, 0, 0, 0}) && + !sfr::kinect_floor_known({0, -1, 0, 1}), + "an unknown floor is not levelled"); +} + +sfr::KinectDepthSettings ideal_settings() { + return sfr::KinectDepthSettings::parse("ideal", nullptr, nullptr); +} + +void ideal_settings_from_text() { + const auto s = ideal_settings(); + require(s.mode == sfr::KinectDepthViewMode::ideal && std::string(s.view_name()) == "ideal" && !s.player_view, + "named ideal"); + require(close_to(s.distance, 2.5f) && close_to(s.height, 1.0f) && close_to(s.follow, 2.0f) && close_to(s.deadband, 0.15f), + "defaults: 2.5 m, 1.0 m up, 2 s, 0.15 m"); + const auto set = sfr::KinectDepthSettings::parse("ideal", "3", "0.6", nullptr, "50", "-1"); + require(close_to(set.distance, 3) && close_to(set.height, 0.6f) && close_to(set.follow, 20) && close_to(set.deadband, 0), + "set and clamped"); + require(close_to(sfr::KinectDepthSettings::parse("player", nullptr, nullptr).height, 0.8f), "the player view keeps 0.8 m"); +} + +// Steps an ideal state at 30 frames a second. +struct IdealRun { + sfr::KinectDepthSettings settings = ideal_settings(); + sfr::KinectIdealState state; + double t = 0; + const sfr::KinectIdealState& step(const sfr::KinectFrame& frame, int frames = 1) { + for (int i = 0; i < frames; ++i) { + t += 1.0 / 30.0; + state = sfr::kinect_ideal_step(state, settings, t, frame); + } + return state; + } + sfr::KinectDepthView view() const { return sfr::kinect_ideal_view(state, settings); } +}; + +void ideal_follows_slowly() { + const std::array level{0, 1, 0, 1.0f}; // the v2 1 m above the floor, level + IdealRun run; + // Nobody yet: in front of the v2's own place, at the height. + run.step(frame_with({}, level)); + require(!run.state.placed && close_to(run.view().origin[2], 0) && close_to(run.view().origin[1], 0), "nobody yet"); + // The first body is taken at once. + run.step(frame_with({{1, {0.3f, -0.2f, 2.0f}}}, level)); + require(run.state.placed && close_to(run.state.reference[0], 0.3f) && close_to(run.state.reference[1], 2.0f), + "the first body snaps"); + auto view = run.view(); + require(close_to(view.origin[0], 0.3f) && close_to(view.origin[1], 0.0f) && close_to(view.origin[2], -0.5f), + "the camera 2.5 m in front, 1 m above the floor"); + // Leaning and small steps inside the dead band: the camera stays. + run.step(frame_with({{1, {0.42f, -0.2f, 1.9f}}}, level), 60); + require(close_to(run.state.reference[0], 0.3f, 1e-6f) && close_to(run.state.reference[1], 2.0f, 1e-6f), + "inside the dead band nothing moves"); + // A step of 1 m held: only what is beyond the band, with the time constant. + run.step(frame_with({{1, {1.3f, -0.2f, 2.0f}}}, level), 60); // two seconds, one time constant + const float expected = 1.3f - 0.15f - 0.85f * std::exp(-1.0f); + require(close_to(run.state.reference[0], expected, 2e-3f), "glides: one time constant, 63% of the way"); + run.step(frame_with({{1, {1.3f, -0.2f, 2.0f}}}, level), 600); + require(close_to(run.state.reference[0], 1.15f, 1e-3f), "and settles at the edge of the band"); + // Height never follows the player. + run.step(frame_with({{1, {1.3f, 0.4f, 2.0f}}}, level), 30); + require(close_to(run.view().origin[1], 0.0f), "a hop does not lift the camera"); + // Nobody: held. The next body, wherever, snaps. + const auto held = run.state.reference; + run.step(frame_with({}, level), 90); + require(run.state.reference == held && !run.state.had_bodies, "nobody holds the camera"); + run.step(frame_with({{4, {-0.6f, -0.2f, 3.0f}}}, level)); + require(close_to(run.state.reference[0], -0.6f) && close_to(run.state.reference[1], 3.0f), "the next body snaps"); +} + +void ideal_two_players() { + const std::array level{0, 1, 0, 1.0f}; + IdealRun run; + // Two players: their mean. + run.step(frame_with({{1, {-0.5f, 0, 2.2f}}, {2, {0.5f, 0, 2.6f}}}, level)); + require(close_to(run.state.reference[0], 0.0f) && close_to(run.state.reference[1], 2.4f) && run.state.bodies == 2, + "two players: the mean"); + // One leaves: glides (no jump) towards the one left. + run.step(frame_with({{2, {0.5f, 0, 2.6f}}}, level)); + require(run.state.reference[0] > 0.0f && run.state.reference[0] < 0.05f, "a player leaving glides, not jumps"); + run.step(frame_with({{2, {0.5f, 0, 2.6f}}}, level), 900); + require(close_to(run.state.reference[0], 0.35f, 1e-3f) && close_to(run.state.reference[1], 2.45f, 1e-3f), + "towards the one left, up to the band"); +} + +void ideal_is_deterministic() { + const auto floor = tilted_floor(0.2f, 0.03f, 1.4f); + const auto play = [&] { + IdealRun run; + std::vector views; + for (int i = 0; i < 300; ++i) { + const float x = 0.6f * std::sin(i * 0.05f), z = 2.0f + 0.3f * std::cos(i * 0.03f); + if (i % 70 < 60) + run.step(frame_with({{1, {x, -0.3f, z}}, {2, {x + 0.8f, -0.3f, z + 0.2f}}}, i < 20 ? std::array{} : floor)); + else + run.step(frame_with({}, floor)); + views.push_back(run.view()); + } + return views; + }; + const auto a = play(), b = play(); + require(a.size() == b.size(), "same length"); + for (size_t i = 0; i < a.size(); ++i) + require(std::memcmp(&a[i].origin, &b[i].origin, sizeof a[i].origin) == 0 && + std::memcmp(&a[i].rotation, &b[i].rotation, sizeof a[i].rotation) == 0 && a[i].rotated == b[i].rotated, + "the same inputs give the same camera, bit for bit"); +} + +void ideal_without_a_floor() { + IdealRun run; + run.step(frame_with({{1, {0.2f, -0.1f, 1.8f}}})); + auto view = run.view(); + require(!view.rotated && !run.state.floor_known && close_to(view.origin[1], 0) && close_to(view.origin[2], 1.8f - 2.5f), + "no floor: no turn, the sensor's own height"); + // The floor turns up (a tilt): levelled from now, the reference taken again. + const auto floor = tilted_floor(0.25f, 0.0f, 1.5f); + run.step(frame_with({{1, {0.2f, -0.1f, 1.8f}}}, floor)); + view = run.view(); + const auto r = sfr::kinect_level_rotation(floor); + const auto expected = times(r, {0.2f, -0.1f, 1.8f}); + require(view.rotated && close_to(run.state.reference[0], expected[0]) && close_to(run.state.reference[1], expected[2]), + "levelled and placed again at once"); + require(close_to(view.origin[1], -1.5f + 1.0f), "the camera 1 m above the floor"); + // A frame without a floor keeps the one found. + run.step(frame_with({{1, {0.2f, -0.1f, 1.8f}}})); + require(run.state.floor_known && close_to(run.state.floor[3], 1.5f), "a lost floor keeps the last"); +} + +void ideal_is_a_true_v1() { + // The v2 on top of a TV, 1.5 m above the floor, looking down 12 degrees; + // the player 2.0 m away. The ideal camera stands 2.5 m in front of the + // player, 1.0 m above the floor, level: what it sees is what a real v1 + // there sees, not stretched. + const float pitch = 12.0f * 3.14159265f / 180.0f; + const auto floor = tilted_floor(pitch, 0.0f, 1.5f); + const auto r = sfr::kinect_level_rotation(floor); + // The world (levelled v2 space) to the v2's own: the transpose. + const auto to_sensor = [&](const std::array& w) { + return std::array{r[0] * w[0] + r[3] * w[1] + r[6] * w[2], r[1] * w[0] + r[4] * w[1] + r[7] * w[2], + r[2] * w[0] + r[5] * w[1] + r[8] * w[2]}; + }; + // The player's hips 0.95 m above the floor, 2.0 m ahead, 0.1 m aside. + const std::array hips_world{0.1f, -1.5f + 0.95f, 2.0f}; + IdealRun run; + run.step(frame_with({{1, to_sensor(hips_world)}}, floor)); + const auto view = run.view(); + // A real v1 at (0.1, -0.5, -0.5) in the world, level. + const std::array v1{0.1f, -1.5f + 1.0f, 2.0f - 2.5f}; + require(close_to(view.origin[0], v1[0]) && close_to(view.origin[1], v1[1]) && close_to(view.origin[2], v1[2]), + "the camera where a v1 would stand"); + // A head-height point of the player at true distance 2.0: depth 2.5 m from + // the v1, and on the pixel the v1's lens puts it. + const std::array head_world{0.25f, -1.5f + 1.6f, 2.0f}; + const std::array seen{head_world[0] - v1[0], head_world[1] - v1[1], head_world[2] - v1[2]}; + Points points; + const auto p = to_sensor(head_world); + points.add(p[0], p[1], p[2], 0); + sfr::KinectImage image; + points.make(image, view); + float x = 0, y = 0; + sfr::kinect_v1_depth_project(seen, x, y); + require(close_to(seen[2], 2.5f), "2.5 m from the virtual v1"); + require(depth_at(image, uint32_t(std::lround(x)), uint32_t(std::lround(y))) == uint16_t(2500 << 3 | 1), + "the v1's pixel, the true depth from it (no stretch)"); + // A 1.8 m person at 2.5 m from it: as tall as through a v1 (285.63 * 1.8 / 2.5). + float x0 = 0, top = 0, bottom = 0; + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, to_sensor({0.1f, -1.5f + 1.8f, 2.0f})), x0, top); + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, to_sensor({0.1f, -1.5f, 2.0f})), x0, bottom); + require(close_to(bottom - top, 285.63f * 1.8f / 2.5f, 0.05f), "a v1's size at the distance"); +} + +void ideal_moves_skeleton_and_floor() { + const auto floor = tilted_floor(0.2f, -0.05f, 1.2f); + IdealRun run; + sfr::KinectFrame frame = frame_with({{1, {0.1f, -0.2f, 1.7f}}}, floor); + frame.gravity = {floor[0], floor[1], floor[2]}; + run.step(frame); + const auto view = run.view(); + sfr::KinectBody& body = frame.bodies[0]; + body.joints[3] = {0.0f, 0.5f, 1.6f}; // the head + body.hand_tips[1] = {0.3f, 0.1f, 1.2f}; + const std::array head = body.joints[3]; + sfr::kinect_depth_view_frame(view, frame); + // The joint lands on the depth image where the same point of the scene does. + Points points; + points.add(head[0], head[1], head[2], 0); + sfr::KinectImage image; + points.make(image, view); + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project(body.joints[3], x, y), "the head projects"); + const uint16_t value = depth_at(image, uint32_t(std::lround(x)), uint32_t(std::lround(y))); + require((value & 7) == 1 && close_to(float(value >> 3) / 1000.0f, body.joints[3][2], 0.001f), + "skeleton and depth agree"); + require(body.thumbs[0] == std::array{} && body.joints[0] == std::array{}, "none stays none"); + // The floor: level, the camera height below it. + const auto& n = frame.floor_plane; + require(close_to(n[0], 0, 1e-4f) && close_to(n[1], 1, 1e-4f) && close_to(n[2], 0, 1e-4f) && close_to(n[3], 1.0f, 1e-4f), + "the floor level, 1 m below the camera"); + require(close_to(frame.gravity[1], 1.0f, 1e-4f) && close_to(frame.gravity[2], 0, 1e-4f), "gravity straight up"); + // And back, as the preview does. + const auto back = sfr::kinect_depth_view_unpoint(view, body.joints[3]); + require(close_to(back[0], head[0], 1e-5f) && close_to(back[1], head[1], 1e-5f) && close_to(back[2], head[2], 1e-5f), + "the preview's way back"); +} + +void viewer_keeps_the_old_views() { + // fit, sensor and player through the viewer are what they were. + sfr::KinectDepthSettings fit = sfr::KinectDepthSettings::parse("fit", nullptr, nullptr); + sfr::KinectDepthViewer fit_viewer(fit); + fit_viewer.set_fit_scale(1.25f); + require(fit_viewer.body_frame(frame_with({{1, {0, 0, 1}}}), 1.0).z_scale == 1.25f && + fit_viewer.depth_frame(1.0).z_scale == 1.25f && fit_viewer.view().origin == std::array{}, + "fit"); + sfr::KinectDepthViewer sensor(sfr::KinectDepthSettings::parse("sensor", nullptr, nullptr)); + require(sensor.body_frame(frame_with({{1, {0, 0, 1}}}), 1.0).origin == std::array{} && + sensor.depth_frame(1.1).z_scale == 1.0f && !sensor.depth_frame(1.1).rotated, + "sensor"); + const auto player_settings = sfr::KinectDepthSettings::parse("player", nullptr, nullptr); + sfr::KinectDepthViewer player(player_settings); + sfr::KinectDepthViewLatch latch(player_settings); + for (int i = 0; i < 40; ++i) { + const auto frame = frame_with({{7, {0.01f * i, 0.0f, 1.2f + 0.01f * i}}}, {0, 1, 0, 1.4f}); + const auto& a = player.body_frame(frame, i / 30.0); + const auto& b = latch.update(frame, i / 30.0); + require(a.origin == b.origin && player.depth_frame(i / 30.0).origin == b.origin, "player: the latch's view"); + } +} + +bool same_view(const sfr::KinectDepthView& a, const sfr::KinectDepthView& b) { + return a.origin == b.origin && a.z_scale == b.z_scale && a.rotated == b.rotated && a.rotation == b.rotation; +} + +void skeleton_knobs() { + // Unset: sensor, player and ideal hand the skeleton over through the + // depth's view; fit hands it over in real metres while its depth is + // stretched by k (the title never projects the skeleton into the depth). + const auto frame = frame_with({{1, {0.1f, -0.2f, 1.9f}}}, tilted_floor(0.2f, 0.0f, 1.2f)); + for (const char* name : {"fit", "sensor", "player", "ideal"}) { + sfr::KinectDepthViewer viewer(sfr::KinectDepthSettings::parse(name, nullptr, nullptr)); + viewer.set_fit_scale(1.379f); + viewer.body_frame(frame, 1.0); + viewer.depth_frame(1.0); + if (std::string_view(name) == "fit") { + require(close_to(viewer.view().z_scale, 1.379f) && close_to(viewer.skeleton_view().z_scale, 1.0f) && + viewer.skeleton_view().origin == viewer.view().origin, + "unset, fit's skeleton is in real metres and only its depth is stretched"); + } else { + require(same_view(viewer.skeleton_view(), viewer.view()), "unset knobs change nothing for the other views"); + } + } + { + const auto settings = sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, nullptr); + require(close_to(settings.skeleton_stretch, 1.0f), "the default skeleton stretch is 1.0"); + sfr::KinectDepthViewer viewer(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "depth")); + viewer.set_fit_scale(1.379f); + viewer.body_frame(frame, 1.0); + require(viewer.settings().skeleton_stretch == 0.0f && same_view(viewer.skeleton_view(), viewer.view()), + "\"depth\" stretches the skeleton with the depth's k, as before"); + require(close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "1.6").skeleton_stretch, 1.6f) && + close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "7").skeleton_stretch, 2.0f) && + close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "x").skeleton_stretch, 1.0f), + "other stretches up to 2.0 still work; nonsense is the default"); + } + const auto set = sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "1.0", "9"); + require(close_to(set.skeleton_stretch, 1.0f) && close_to(set.skeleton_distance, 4.0f), "parsed and clamped"); + require(close_to(sfr::KinectDepthSettings::parse("fit", nullptr, nullptr, nullptr, nullptr, nullptr, "0.5").skeleton_stretch, 1.0f), + "the stretch is at least 1"); + + // fit with the skeleton unstretched: the depth keeps k, the skeleton is + // the sensor's; a hand's travel through a perspective divide is k times + // as large (X / Z against X / (k Z)). + sfr::KinectDepthViewer fit(set); + fit.set_fit_scale(1.379f); + fit.body_frame(frame, 1.0); + const auto skeleton = fit.skeleton_view(); + require(close_to(fit.view().z_scale, 1.379f) && close_to(skeleton.z_scale, 1.0f), "depth stretched, skeleton not"); + float x_depth = 0, x_skeleton = 0, y = 0; + const std::array hand{0.3f, 0.1f, 1.6f}; + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(fit.view(), hand), x_depth, y); + sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(skeleton, hand), x_skeleton, y); + require(close_to((x_skeleton - 160.0f) / (x_depth - 160.0f), 1.379f, 1e-3f), "cursor travel scaled by Z / Z'"); + sfr::KinectFrame moved = frame; + sfr::kinect_depth_view_frame(skeleton, moved); + require(moved.bodies[0].position == frame.bodies[0].position, "the skeleton in the sensor's own coordinates"); + const auto back = sfr::kinect_depth_view_unpoint(skeleton, sfr::kinect_depth_view_point(skeleton, hand)); + require(close_to(back[2], hand[2]), "the preview's way back"); + + // ideal with the skeleton's camera at 2.0 m: the depth keeps 2.5 m. + const auto ideal_set = sfr::KinectDepthSettings::parse("ideal", nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, "2.0"); + sfr::KinectDepthViewer ideal(ideal_set); + ideal.body_frame(frame, 1.0); + ideal.depth_frame(1.0); + const auto ideal_skeleton = ideal.skeleton_view(); + const auto in_depth = sfr::kinect_depth_view_point(ideal.view(), frame.bodies[0].position); + const auto in_skeleton = sfr::kinect_depth_view_point(ideal_skeleton, frame.bodies[0].position); + require(close_to(in_depth[2], 2.5f, 1e-4f) && close_to(in_skeleton[2], 2.0f, 1e-4f) && + close_to(in_depth[0], in_skeleton[0]) && close_to(in_depth[1], in_skeleton[1]), + "only the skeleton's camera comes closer"); + const auto ideal_back = sfr::kinect_depth_view_unpoint(ideal_skeleton, in_skeleton); + require(close_to(ideal_back[0], frame.bodies[0].position[0], 1e-5f) && close_to(ideal_back[2], frame.bodies[0].position[2], 1e-5f), + "undone for the preview"); +} + +// A synthetic raw recording: a wall, a player moving across it, a tilted +// floor, the typical lens's rays as the table. +sfr::KinectRawHeader synthetic_header() { + sfr::KinectRawHeader header; + header.rays_from = sfr::KinectRawRays::sensor_table; + header.lens = {366.0f, 366.0f, 256.0f, 212.0f, 0, 0, 0}; + sfr::kinect_v2_depth_rays_from_lens(366.0f, 366.0f, 256.0f, 212.0f, 512, 424, header.rays); + return header; +} + +sfr::KinectRawFrame synthetic_frame(int i) { + constexpr uint32_t width = 512, height = 424; + sfr::KinectRawFrame frame; + frame.steady_us = 1000000 + int64_t(i) * 33333; + frame.depth_time = 50000000 + int64_t(i) * 333333; + // A body frame most times, and some depth images without a new one. + frame.body_time = i % 7 == 3 ? 50000000 + int64_t(i - 1) * 333333 - 1000 : 50000000 + int64_t(i) * 333333 - 1000; + frame.number = uint64_t(i) + 1; + frame.flags = i % 11 == 5 ? 0 : sfr::kinect_raw_flag_mapped; + frame.floor = tilted_floor(0.18f, 0.02f, 1.3f); + frame.depth.assign(size_t(width) * height, 0); + frame.body_index.assign(size_t(width) * height, sfr::kinect_v2_no_body); + const int left = 180 + 3 * (i % 40), top = 60 + (i % 9), right = left + 110, bottom = 400 - (i % 5); + const float person_z = 1.9f + 0.01f * float(i % 13); + for (uint32_t y = 0; y < height; ++y) + for (uint32_t x = 0; x < width; ++x) { + const size_t at = size_t(y) * width + x; + if ((x * 7 + y * 3 + uint32_t(i)) % 97 == 0) continue; // holes + const bool person = int(x) >= left && int(x) < right && int(y) >= top && int(y) < bottom; + frame.depth[at] = uint16_t(person ? std::lround(person_z * 1000.0f) + (x % 5) : 3400 + (y % 50)); + if (person) frame.body_index[at] = 0; + } + if (i % 23 < 20) { + sfr::KinectRawBody& body = frame.bodies.emplace_back(); + body.tracking_id = 77; + body.body_index = 0; + body.spine_state = 2; + // Its SpineBase through the same lens (the middle of the box). + const float cx = (left + right) / 2.0f, cy = (top + bottom) / 2.0f + 30.0f; + body.spine_base = {(cx - 256.0f) * person_z / 366.0f, -(cy - 212.0f) * person_z / 366.0f, person_z}; + if (i > 30) { + sfr::KinectRawBody& second = frame.bodies.emplace_back(body); + second.tracking_id = 78; + second.body_index = 3; + second.spine_base[0] += 0.7f; + } + } + return frame; +} + +std::string temp_file(const char* name) { return (std::filesystem::temp_directory_path() / name).string(); } + +bool write_file(const std::string& path, const std::string& bytes) { + std::FILE* file = std::fopen(path.c_str(), "wb"); + if (!file) return false; + const bool ok = std::fwrite(bytes.data(), 1, bytes.size(), file) == bytes.size(); + std::fclose(file); + return ok; +} + +void raw_runs_round_trip() { + std::vector pixels(200000, 255); // longer than one run can hold + for (size_t i = 70000; i < 70100; ++i) pixels[i] = 2; + pixels[199999] = 0; + std::vector> runs; + sfr::kinect_raw_runs_encode(pixels.data(), pixels.size(), runs); + require(runs.size() == 6 && runs[0] == std::make_pair(uint8_t(255), uint16_t(65535)), "runs of at most 65535"); + std::vector back; + require(sfr::kinect_raw_runs_decode(runs, pixels.size(), back) && back == pixels, "runs decode to the pixels"); + require(!sfr::kinect_raw_runs_decode(runs, pixels.size() - 1, back), "too many pixels is broken"); + runs.back().second = 0; + require(!sfr::kinect_raw_runs_decode(runs, pixels.size(), back), "an empty run is broken"); +} + +void raw_file_round_trip() { + const sfr::KinectRawHeader header = synthetic_header(); + std::string bytes = sfr::kinect_raw_file_header(header); + require(bytes.size() == 64 + 8 * 512 * 424 && bytes.compare(0, 8, "SFRKRAW1") == 0, "header size and magic"); + std::vector frames; + for (int i = 0; i < 3; ++i) { + frames.push_back(synthetic_frame(i + 30)); + bytes += sfr::kinect_raw_encode_frame(frames.back()); + } + // A frame cut short at the end. + const std::string last = sfr::kinect_raw_encode_frame(synthetic_frame(40)); + bytes += last.substr(0, last.size() / 2); + const std::string path = temp_file("sfr_kinect_raw_round_trip.bin"); + require(write_file(path, bytes), "written"); + sfr::KinectRawReader reader; + require(reader.open(path), "opened"); + require(reader.header().width == 512 && reader.header().height == 424 && + reader.header().rays_from == sfr::KinectRawRays::sensor_table && reader.header().lens == header.lens && + reader.header().rays == header.rays, + "the header and its rays back"); + sfr::KinectRawFrame frame; + for (const auto& given : frames) { + require(reader.next(frame), "a frame back"); + require(frame.steady_us == given.steady_us && frame.depth_time == given.depth_time && + frame.body_time == given.body_time && frame.number == given.number && frame.flags == given.flags && + frame.floor == given.floor, + "its clocks, number, flags and floor"); + require(frame.bodies.size() == given.bodies.size(), "its bodies"); + for (size_t b = 0; b < given.bodies.size(); ++b) + require(frame.bodies[b].tracking_id == given.bodies[b].tracking_id && + frame.bodies[b].body_index == given.bodies[b].body_index && + frame.bodies[b].spine_state == given.bodies[b].spine_state && + frame.bodies[b].spine_base == given.bodies[b].spine_base && frame.bodies[b].tracked == 1, + "each body"); + require(frame.depth == given.depth && frame.body_index == given.body_index, "depth and body index"); + } + require(!reader.next(frame) && reader.truncated() && reader.error().empty(), "the cut frame left out"); + reader.close(); + std::filesystem::remove(path); + // Not a raw file. + const std::string other = temp_file("sfr_kinect_raw_other.bin"); + require(write_file(other, std::string("SFRKDEP2") + std::string(100, '\0')), "written"); + require(!reader.open(other) && !reader.error().empty(), "another file is refused"); + reader.close(); + std::filesystem::remove(other); +} + +void raw_rays_orientation() { + sfr::KinectRawHeader header = synthetic_header(); + bool flip_x = true, flip_y = true; + require(sfr::kinect_raw_rays_orientation(header, flip_x, flip_y) && !flip_x && !flip_y, "camera space"); + for (size_t i = 1; i < header.rays.size(); i += 2) header.rays[i] = -header.rays[i]; + require(sfr::kinect_raw_rays_orientation(header, flip_x, flip_y) && !flip_x && flip_y, "y down is seen"); + header.rays.assign(header.rays.size(), 0.0f); + require(!sfr::kinect_raw_rays_orientation(header, flip_x, flip_y), "zeros are unusable"); +} + +void resynth_equals_online() { + const sfr::KinectRawHeader header = synthetic_header(); + for (const char* name : {"fit", "sensor", "player", "ideal"}) { + const sfr::KinectDepthSettings settings = sfr::KinectDepthSettings::parse(name, nullptr, nullptr); + // Online, as the live sensor goes: body frames as they come, then each + // depth image through the viewer, points from the mapper (here the + // table's rays, or the typical lens when it had none), and the + // recorder's frame from what it used. + sfr::KinectDepthViewer viewer(settings); + viewer.set_fit_scale(sfr::kinect_v2_fit_scale(header.lens[1])); + std::string bytes = sfr::kinect_raw_file_header(header); + std::vector online; + int64_t last_body = 0; + std::vector points; + for (int i = 0; i < 120; ++i) { + const sfr::KinectRawFrame given = synthetic_frame(i); + if (given.body_time != last_body) { + sfr::KinectFrame body_frame; + body_frame.floor_plane = given.floor; + for (const auto& raw : given.bodies) { + sfr::KinectBody& body = body_frame.bodies.emplace_back(); + body.tracking_id = uint32_t(raw.tracking_id); + body.sensor_index = raw.body_index; + body.position = raw.spine_base; + body.joint_states[0] = raw.spine_state; + body.joints[5] = {1, 2, 3}; // what the view does not keep + } + viewer.body_frame(body_frame, double(given.body_time) * 1e-7); + last_body = given.body_time; + } + const sfr::KinectDepthView view = viewer.depth_frame(double(given.depth_time) * 1e-7); + if (given.flags & sfr::kinect_raw_flag_mapped) + sfr::kinect_v2_depth_points_from_rays(given.depth.data(), header.rays.data(), given.depth.size(), points); + else + sfr::kinect_v2_depth_points_from_lens(given.depth.data(), 512, 424, points); + sfr::KinectImage made; + sfr::kinect_v2_synthesize_depth(points.data(), given.body_index.data(), given.depth.size(), view, made); + online.push_back(made); + sfr::KinectRawFrame recorded = given; + sfr::kinect_raw_set_bodies(viewer.bodies(), recorded); + bytes += sfr::kinect_raw_encode_frame(recorded); + } + const std::string path = temp_file("sfr_kinect_raw_resynth.bin"); + require(write_file(path, bytes), "written"); + sfr::KinectRawReader reader; + require(reader.open(path), "opened"); + sfr::KinectV2Resynth resynth(settings, reader.header()); + require(resynth.warning().empty(), "the rays are fine"); + sfr::KinectRawFrame frame; + sfr::KinectImage image; + size_t index = 0; + uint32_t players = 0; + while (reader.next(frame)) { + resynth.make(frame, image); + require(index < online.size() && image.pixels == online[index].pixels, + (std::string(name) + ": the offline image is the online one, bit for bit").c_str()); + require(image.number == frame.number && image.steady_us == frame.steady_us, "its number and clock"); + for (size_t p = 0; p < image.pixels.size(); p += 2) players += (image.pixels[p] & 7) ? 1 : 0; + ++index; + } + require(index == online.size(), "every frame"); + require(players > 0, (std::string(name) + ": the player is in the images").c_str()); + reader.close(); + std::filesystem::remove(path); + } +} + +// The launcher's choice becomes what the game would read from the same values. +void choice_is_the_games_settings() { + sfr::KinectDepthChoice choice; + choice.distance = 3.0f; + choice.height = 0.95f; + choice.follow = 3.5f; + choice.deadband = 0.0f; + choice.skeleton_distance = 1.8f; + const auto made = sfr::kinect_depth_settings(choice); + const auto read = sfr::KinectDepthSettings::parse("ideal", "3.00", "0.95", nullptr, "3.50", "0.00", nullptr, "1.80"); + require(made.mode == read.mode && made.distance == read.distance && made.height == read.height && + made.follow == read.follow && made.deadband == read.deadband && + made.skeleton_distance == read.skeleton_distance && made.skeleton_stretch == read.skeleton_stretch && + made.fit_scale == read.fit_scale, + "Recompiled: the same settings as the game's environment"); + const auto defaults = sfr::kinect_depth_settings({}); + require(defaults.mode == sfr::KinectDepthViewMode::ideal && close_to(defaults.height, 1.0f) && + close_to(defaults.distance, 2.5f) && defaults.skeleton_distance == 0, + "the default choice is ideal's defaults"); + choice.view = sfr::KinectDepthChoiceView::fit; + const auto fit = sfr::kinect_depth_settings(choice); + require(fit.mode == sfr::KinectDepthViewMode::fit && !fit.player_view && fit.fit_scale == 0 && + fit.skeleton_stretch == 1.0f && close_to(fit.distance, 3.0f), + "fit, with the lens's stretch and the tuning kept"); + choice.view = sfr::KinectDepthChoiceView::sensor; + require(sfr::kinect_depth_settings(choice).mode == sfr::KinectDepthViewMode::sensor, "sensor"); + choice.distance = 9.0f; + require(close_to(sfr::kinect_depth_settings(choice).distance, 4.0f), "clamped as the game clamps"); +} + +// Settings changed while running: the ideal state is kept, the views follow. +void viewer_takes_new_settings() { + const std::array level{0, 1, 0, 1.0f}; + sfr::KinectDepthSettings settings = sfr::kinect_depth_settings({}); + settings.mode = sfr::KinectDepthViewMode::fit; + sfr::KinectDepthViewer viewer(settings); + viewer.body_frame(frame_with({{1, {0.4f, 0, 2.0f}}}, level), 0.0); + const sfr::KinectDepthView fit_view = viewer.depth_frame(0.0); + require(fit_view.z_scale > 1.3f && fit_view.origin == std::array{}, "fit makes fit"); + require(viewer.ideal().placed && close_to(viewer.ideal().reference[0], 0.4f) && + close_to(viewer.ideal_view().origin[2], 2.0f - 2.5f), + "the ideal camera is kept in fit too"); + settings.mode = sfr::KinectDepthViewMode::ideal; + settings.distance = 3.0f; + viewer.set_settings(settings); + require(close_to(viewer.view().origin[0], 0.4f) && close_to(viewer.view().origin[2], 2.0f - 3.0f) && + viewer.view().z_scale == 1.0f && viewer.view().rotated, + "ideal at once, from the state already kept, at the new distance"); + viewer.body_frame(frame_with({{1, {0.4f, 0, 2.0f}}}, level), 1.0 / 30); + require(close_to(viewer.depth_frame(1.0 / 30).origin[2], -1.0f), "and on"); + settings.mode = sfr::KinectDepthViewMode::sensor; + viewer.set_settings(settings); + require(viewer.view().origin == std::array{} && viewer.view().z_scale == 1.0f, "sensor"); + settings.mode = sfr::KinectDepthViewMode::fit; + viewer.set_fit_scale(1.5f); + viewer.set_settings(settings); + require(close_to(viewer.view().z_scale, 1.5f), "fit keeps the stretch the lens gave"); +} + +// The top-down snapshot is the ideal view's own camera and the bodies in its +// levelled space. +void ideal_snapshot_is_the_view() { + const float pitch = 10.0f * 3.14159265f / 180.0f; + const auto floor = tilted_floor(pitch, 0, 1.2f); + IdealRun run; + const auto frame = frame_with({{7, {-0.5f, -0.3f, 2.0f}}, {9, {0.7f, -0.3f, 2.6f}}}, floor); + run.step(frame); + const auto snapshot = sfr::kinect_ideal_snapshot(frame, run.state, run.settings); + const auto view = run.view(); + require(snapshot.valid && snapshot.floor_known && snapshot.placed && close_to(snapshot.camera_x, view.origin[0]) && + close_to(snapshot.camera_z, view.origin[2]) && close_to(snapshot.distance, 2.5f), + "the camera is the view's"); + require(snapshot.players.size() == 2 && snapshot.players[0].tracking_id == 7 && snapshot.players[0].slot == -1, + "every body, in order, no slot"); + for (size_t i = 0; i < 2; ++i) { + const auto seen = sfr::kinect_depth_view_point(view, frame.bodies[i].position); + require(close_to(snapshot.players[i].z - snapshot.camera_z, seen[2], 1e-4f) && + close_to(snapshot.players[i].x - snapshot.camera_x, seen[0], 1e-4f), + "a player's depth to the camera is the depth the game sees"); + require(close_to(snapshot.players[i].sensor_z, frame.bodies[i].position[2]), "and the v2's own depth"); + } + require(close_to((snapshot.players[0].z + snapshot.players[1].z) / 2 - snapshot.camera_z, 2.5f, 1e-4f), + "the camera is distance behind the players' mean"); + const auto nobody = sfr::kinect_ideal_snapshot(frame_with({{1, {0, 0, 0}}}), {}, run.settings); + require(nobody.players.empty() && !nobody.placed && close_to(nobody.camera_z, 0.0f), "a point of none is nobody"); +} + +// The simulated v1's colour: each pixel of the depth the game gets taken +// back into the sensor's space through the very view it was made with, so +// its colour is the colour of what that pixel shows. +void v1_depth_back_to_the_sensor() { + // A level-making turn and a camera moved off the v2, as ideal makes. + const auto floor = tilted_floor(12.0f * 3.14159265f / 180.0f, 3.0f * 3.14159265f / 180.0f, 1.1f); + IdealRun run; + run.step(frame_with({{1, {0.6f, -0.2f, 1.8f}}}, floor)); + const sfr::KinectDepthView view = run.view(); + require(view.rotated && !close_to(view.origin[0], 0) && !close_to(view.origin[2], 0), "a turned, moved camera"); + // A wall 2.6 m away in the sensor's space, densely sampled, and one known + // point on it (a player's pixel) that lands on a pixel of its own. + Points wall; + for (float x = -1.8f; x <= 1.8f; x += 0.004f) + for (float y = -1.2f; y <= 1.2f; y += 0.004f) wall.add(x, y, 2.6f + 0.1f * x); + const std::array known{0.35f, 0.15f, 2.6f + 0.1f * 0.35f}; + sfr::KinectImage image; + wall.make(image, view); + image.view_origin = view.origin; + image.view_z_scale = view.z_scale; + image.view_rotated = view.rotated; + image.view_rotation = view.rotation; + const sfr::KinectDepthView back = sfr::kinect_image_view(image); + require(back.origin == view.origin && back.rotated && back.rotation == view.rotation, "the image's own view"); + std::vector points; + sfr::kinect_v1_depth_points(image, back, points); + require(points.size() == size_t(320 * 240 * 3), "a point a pixel"); + size_t valid = 0; + for (uint32_t row = 0; row < 240; ++row) + for (uint32_t column = 0; column < 320; ++column) { + const size_t at = size_t(row) * 320 + column; + const float* p = &points[3 * at]; + if (depth_at(image, column, row) == 0) { + require(p[0] == 0 && p[1] == 0 && p[2] == 0, "no value, no point"); + continue; + } + ++valid; + // On the wall (to the millimetre the image keeps)... + require(std::fabs(p[2] - (2.6f + 0.1f * p[0])) < 0.004f, "the point is on the wall"); + // ...and seen by the virtual camera on that same pixel. + const auto seen = sfr::kinect_depth_view_point(view, {p[0], p[1], p[2]}); + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project(seen, x, y) && std::fabs(x - float(column)) < 0.51f && + std::fabs(y - float(row)) < 0.51f, + "the point lands on its own pixel"); + } + require(valid > 20000, "the wall is in the image"); + // The known point's pixel gives back the known point, within half a + // pixel across, and its colour position is the known point's. + const auto seen = sfr::kinect_depth_view_point(view, known); + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project(seen, x, y), "the known point is in front"); + const size_t at = size_t(std::floor(y + 0.5f)) * 320 + size_t(std::floor(x + 0.5f)); + const std::array found{points[3 * at], points[3 * at + 1], points[3 * at + 2]}; + const float pixel = 3.3f / sfr::kinect_v1_depth_focal; // a pixel across at that depth + require(std::fabs(found[0] - known[0]) < pixel && std::fabs(found[1] - known[1]) < pixel && + std::fabs(found[2] - known[2]) < 0.005f, + "the known point comes back"); + const auto colour_known = sfr::kinect_v2_colour_estimate(known, sfr::KinectV2View::original); + const auto colour_found = sfr::kinect_v2_colour_estimate(found, sfr::KinectV2View::original); + require(colour_known.visible && colour_found.visible && std::fabs(colour_known.x - colour_found.x) < 1.0f && + std::fabs(colour_known.y - colour_found.y) < 1.0f, + "and the same colour pixel (640x360)"); + // Without a camera move (the sensor view) the point is the pixel's ray. + sfr::KinectImage flat; + Points one; + one.at_pixel(200, 60, 2.0f); + one.make(flat); + sfr::kinect_v1_depth_points(flat, sfr::kinect_image_view(flat), points); + const float* p = &points[3 * (60 * 320 + 200)]; + require(close_to(p[0], 40.0f * 2.0f / sfr::kinect_v1_depth_focal) && close_to(p[1], 60.0f * 2.0f / sfr::kinect_v1_depth_focal) && + close_to(p[2], 2.0f), + "the sensor view: the pixel's own ray"); +} + +// Nobody tracked yet: the ideal camera is the v2's own place laterally and +// in depth, level, the height above the floor, so it sees the middle of the +// v2's view (and nothing beyond 4 m of it). +void ideal_camera_without_anyone() { + const auto floor = tilted_floor(10.0f * 3.14159265f / 180.0f, 0, 0.9f); + IdealRun run; + run.step(frame_with({}, floor), 30); + const auto view = run.view(); + require(!run.state.placed && run.state.floor_known && view.rotated, "levelled, not placed"); + require(close_to(view.origin[0], 0) && close_to(view.origin[2], 0) && close_to(view.origin[1], -0.9f + 1.0f, 1e-3f), + "at the v2, the set height above the floor"); + // A point straight ahead of the v2 at 2 m stays in the middle column. + float x = 0, y = 0; + require(sfr::kinect_v1_depth_project(sfr::kinect_depth_view_point(view, {0, 0, 2.0f}), x, y) && close_to(x, 160), + "centred left to right"); +} + +// A standing player (feet on the floor, head 1.7 m up, SpineBase at 0.95 m) +// 2 m in front of a v2 mounted sensor_height above the floor and pitched +// down by pitch, in the v2's own space; the ideal view at 2.5 m and 1.0 m +// must frame the whole body: the head in the top third, the feet in the +// bottom 15%. +struct Framed { + int top = -1, bottom = -1; +}; +Framed frame_a_standing_player(float pitch_degrees, float sensor_height, const std::array& floor_given, + int steps = 30) { + const float pitch = pitch_degrees * 3.14159265f / 180.0f; + // A point of the room (x, height above the floor, distance along the + // floor) in the sensor's space: its axes are x, up (0, cos, sin) and + // forward (0, -sin, cos) seen from the room. + const auto sensor_point = [&](float x, float above_floor, float forward) { + const float y = above_floor - sensor_height; + return std::array{x, y * std::cos(pitch) + forward * std::sin(pitch), + -y * std::sin(pitch) + forward * std::cos(pitch)}; + }; + IdealRun run; + const auto spine = sensor_point(0.1f, 0.95f, 2.0f); + for (int i = 0; i < steps; ++i) run.step(frame_with({{1, spine}}, floor_given)); + Points body; + for (float h = 0.0f; h <= 1.7001f; h += 0.005f) + for (float x = -0.1f; x <= 0.3001f; x += 0.005f) { + const auto p = sensor_point(x, h, 2.0f); + body.add(p[0], p[1], p[2], 0); + } + sfr::KinectImage image; + body.make(image, run.view()); + Framed framed; + for (int row = 0; row < 240; ++row) + for (int column = 0; column < 320; ++column) + if (depth_at(image, uint32_t(column), uint32_t(row)) & 7) { + if (framed.top < 0) framed.top = row; + framed.bottom = row; + } + return framed; +} + +void ideal_frames_a_standing_player() { + for (const float pitch : {0.0f, 10.0f, -10.0f}) { + const float rad = pitch * 3.14159265f / 180.0f; + // The v2's FloorClipPlane: up in its space, its height as W. + const std::array floor{0, std::cos(rad), -std::sin(rad), 1.0f}; + const Framed framed = frame_a_standing_player(pitch, 1.0f, floor); + require(framed.top >= 0 && framed.top < 80, "the head in the top third"); + require(framed.bottom >= 204 && framed.bottom <= 239, "the feet in the bottom 15%, inside the picture"); + // A sensor low on the floor, looking up a little. + const Framed low = frame_a_standing_player(pitch, 0.3f, {0, std::cos(rad), -std::sin(rad), 0.3f}); + require(std::abs(low.top - framed.top) <= 1 && std::abs(low.bottom - framed.bottom) <= 1, + "the camera's height is the floor's, not the sensor's"); + } + // A v2 that knows the floor's direction but not yet its distance (W 0): + // the camera at the sensor's own height, never a metre above it. + const std::array no_height{0, 1, 0, 0}; + const Framed unknown = frame_a_standing_player(0, 1.0f, no_height); + require(unknown.top >= 0 && unknown.top < 80 && unknown.bottom >= 204, "a sensor at 1 m frames the same without W"); + const Framed low_unknown = frame_a_standing_player(0, 0.7f, no_height); + require(low_unknown.top >= 0 && low_unknown.top < 80 && low_unknown.bottom <= 239, + "a lower sensor without W still shows the head and the feet"); + IdealRun run; + run.step(frame_with({{1, {0, -0.05f, 2.0f}}}, no_height), 30); + require(run.state.floor_known && !run.state.height_known && close_to(run.view().origin[1], 0), "levelled, no height yet"); + run.step(frame_with({{1, {0, -0.05f, 2.0f}}}, {0, 1, 0, 0.7f})); + require(run.state.height_known && close_to(run.state.floor[3], 0.7f) && close_to(run.view().origin[1], 0.3f), + "the first distance is taken at once"); + run.step(frame_with({{1, {0, -0.05f, 2.0f}}}, no_height), 30); + require(run.state.height_known && close_to(run.state.floor[3], 0.7f), "a plane without one later leaves it"); +} + +void colour_sampled_per_pixel() { + const std::vector colour{1, 2, 3, 4, 5, 6}; // 3 x 2 + const float positions[] = {0.2f, 0.7f, 2.9f, 1.0f, NAN, 0.0f, 3.0f, 0.0f, -0.1f, 0.5f, 1.5f, 0.5f}; + std::vector out; + sfr::kinect_colour_from_positions(positions, 6, colour.data(), 3, 2, out); + require(out == std::vector{1, 6, 0, 0, 0, 2}, "the nearest pixel, black for none or outside"); +} +} + +int main() { + const std::pair cases[] = { + {"v1 depth back", v1_depth_back_to_the_sensor}, + {"colour per pixel", colour_sampled_per_pixel}, + {"ideal without anyone", ideal_camera_without_anyone}, + {"ideal frames a standing player", ideal_frames_a_standing_player}, + {"choice", choice_is_the_games_settings}, + {"viewer settings", viewer_takes_new_settings}, + {"ideal snapshot", ideal_snapshot_is_the_view}, + {"packing", packing_is_the_v1s}, + {"projection", projection_is_the_nominal_lens}, + {"points land", points_land_where_projected}, + {"z-buffer", nearest_wins}, + {"hole filling", holes_filled_conservatively}, + {"special values", out_of_range_is_no_value}, + {"view transform", view_moves_the_camera}, + {"settings", settings_from_text}, + {"latch holds", latch_holds_still}, + {"latch mean", latch_settles_on_the_mean}, + {"latch height", latch_height_from_the_floor}, + {"fit scale", fit_scale_from_the_lens}, + {"fit whole view", fit_shows_the_whole_v2_view}, + {"fit depth and size", fit_depth_matches_size}, + {"fit range", fit_range_clamps}, + {"fit skeleton and floor", fit_moves_skeleton_and_floor}, + {"fit of one", fit_of_one_is_sensor}, + {"sensor view", sensor_view_stays}, + {"lens fallback", lens_fallback_matches_estimate}, + {"levelling", levelling_takes_out_a_known_tilt}, + {"ideal settings", ideal_settings_from_text}, + {"ideal follow", ideal_follows_slowly}, + {"ideal two players", ideal_two_players}, + {"ideal determinism", ideal_is_deterministic}, + {"ideal without a floor", ideal_without_a_floor}, + {"ideal is a true v1", ideal_is_a_true_v1}, + {"ideal skeleton and floor", ideal_moves_skeleton_and_floor}, + {"viewer keeps the old views", viewer_keeps_the_old_views}, + {"raw runs", raw_runs_round_trip}, + {"raw file", raw_file_round_trip}, + {"raw rays", raw_rays_orientation}, + {"resynth equals online", resynth_equals_online}, + {"skeleton knobs", skeleton_knobs}, + }; + unsigned failures = 0; + for (const auto& [name, test] : cases) { + try { + test(); + } catch (const std::exception& error) { + ++failures; + std::cerr << "FAIL " << name << ": " << error.what() << '\n'; + } + } + if (failures) return 1; + std::cout << "kinect_v2_depth: all passed\n"; + return 0; +} diff --git a/tests/kinect_v2_image_test.cpp b/tests/kinect_v2_image_test.cpp new file mode 100644 index 0000000..31477b5 --- /dev/null +++ b/tests/kinect_v2_image_test.cpp @@ -0,0 +1,363 @@ +#include "kinect_v2_image.h" + +#include +#include +#include +#include +#include +#include + +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +bool close_to(float a, float b, float tolerance = 1e-3f) { return std::fabs(a - b) <= tolerance; } + +uint32_t colour_at(const sfr::KinectImage& image, uint32_t x, uint32_t y) { + const uint8_t* p = &image.pixels[(size_t(y) * image.width + x) * 4]; + return uint32_t(p[0]) | uint32_t(p[1]) << 8 | uint32_t(p[2]) << 16 | uint32_t(p[3]) << 24; +} + +uint16_t depth_at(const sfr::KinectImage& image, uint32_t x, uint32_t y) { + const uint8_t* p = &image.pixels[(size_t(y) * image.width + x) * 2]; + return uint16_t(p[0] | p[1] << 8); +} + +void fits_keep_the_shape() { + // 16:9 colour in 4:3: a third the size, 60-pixel bars above and below. + const sfr::KinectFit colour = sfr::kinect_fit(1920, 1080, 640, 480); + require(close_to(colour.scale, 1.0f / 3.0f, 1e-6f), "colour scale"); + require(close_to(colour.offset_x, 0) && close_to(colour.offset_y, 60), "colour bars above and below"); + // 512x424 depth in 320x240: as tall, with bars at the sides. + const sfr::KinectFit depth = sfr::kinect_fit(512, 424, 320, 240); + require(close_to(depth.scale, 240.0f / 424.0f, 1e-6f), "depth scale"); + require(close_to(depth.offset_y, 0) && close_to(depth.offset_x, (320.0f - 512.0f * 240.0f / 424.0f) / 2), "depth bars at the sides"); + require(depth.offset_x > 15 && depth.offset_x < 15.2f, "about 15 pixels each side"); +} + +using View = sfr::KinectV2View; +using Kind = sfr::KinectImageKind; + +void crops_are_the_common_view() { + // The same formulas as the code, from the lenses' tangent space. + const float colour_half_y = 540.0f / 1060.0f, depth_half_y = 212.0f / 366.0f; // 0.509, 0.579 + const float colour_half_x = 960.0f / 1060.0f, depth_half_x = 256.0f / 366.0f; // 0.906, 0.699 + const float half_y = std::min(colour_half_y, depth_half_y); + const float half_x = std::min(colour_half_x, depth_half_x); + require(close_to(half_x, 0.699f, 1e-3f) && close_to(half_y, 0.509f, 1e-3f), "the intersection of the fields of view"); + const sfr::KinectV2Crop colour = sfr::kinect_v2_crop(Kind::colour, View::common); + require(close_to(colour.x, 960 - half_x * 1060, 0.05f) && close_to(colour.y, 540 - half_y * 1060, 0.05f) && + close_to(colour.width, 2 * half_x * 1060, 0.1f) && close_to(colour.height, 2 * half_y * 1060, 0.1f), + "colour crop from the lens"); + require(colour.x > 218 && colour.x < 219.5f && colour.x + colour.width > 1700 && colour.x + colour.width < 1702 && + close_to(colour.y, 0, 0.1f) && close_to(colour.height, 1080, 0.1f), + "colour crop about x 219..1701 (1482 px), all 1080 rows"); + const sfr::KinectV2Crop depth = sfr::kinect_v2_crop(Kind::depth_and_player, View::common); + require(close_to(depth.x, 256 - half_x * 366, 0.05f) && close_to(depth.y, 212 - half_y * 366, 0.05f) && + close_to(depth.width, 2 * half_x * 366, 0.1f) && close_to(depth.height, 2 * half_y * 366, 0.1f), + "depth crop from the lens"); + require(close_to(depth.x, 0, 0.05f) && close_to(depth.width, 512, 0.1f), "depth keeps its full width"); + require(depth.y > 25.4f && depth.y < 25.8f && depth.y + depth.height > 398.1f && depth.y + depth.height < 398.5f && + depth.height > 372 && depth.height < 374, + "depth crop y about 25.7..398.3 (about 373 rows)"); + // Not 4:3: both crops have the intersection's own shape, about 1.373:1. + const float aspect = 1.373f; + require(close_to(colour.width / colour.height, aspect, 2e-3f) && close_to(depth.width / depth.height, aspect, 2e-3f), + "both crops are about 1.373:1"); + // Both crops look at the same tangent-space box (same angles). + require(close_to(colour.width / 1060.0f, depth.width / 366.0f, 1e-4f) && + close_to(colour.height / 1060.0f, depth.height / 366.0f, 1e-4f), "the same view"); + // Original: the whole image. + const sfr::KinectV2Crop whole = sfr::kinect_v2_crop(Kind::colour, View::original); + require(whole.x == 0 && whole.y == 0 && whole.width == 1920 && whole.height == 1080, "original colour is whole"); + const sfr::KinectV2Crop whole_depth = sfr::kinect_v2_crop(Kind::depth_and_player, View::original); + require(whole_depth.x == 0 && whole_depth.y == 0 && whole_depth.width == 512 && whole_depth.height == 424, + "original depth is whole"); +} + +void preview_images_follow_the_pictures_shape() { + const sfr::KinectImageSize colour = sfr::kinect_v2_preview_size(Kind::colour, View::common); + const sfr::KinectImageSize depth = sfr::kinect_v2_preview_size(Kind::depth_and_player, View::common); + require(colour.width == 640 && colour.height == 466, "common colour 640x466"); + require(depth.width == 512 && depth.height == 373, "common depth 512x373"); + const float colour_aspect = float(colour.width) / float(colour.height); + const float depth_aspect = float(depth.width) / float(depth.height); + require(close_to(colour_aspect, 1.373f, 2e-3f) && close_to(depth_aspect, 1.373f, 2e-3f), "both images about 1.373:1"); + const sfr::KinectImageSize colour_whole = sfr::kinect_v2_preview_size(Kind::colour, View::original); + const sfr::KinectImageSize depth_whole = sfr::kinect_v2_preview_size(Kind::depth_and_player, View::original); + require(colour_whole.width == 640 && colour_whole.height == 360, "original colour 640x360 (16:9)"); + require(depth_whole.width == 512 && depth_whole.height == 424, "original depth 512x424"); + require(close_to(float(colour_whole.width) / float(colour_whole.height), 16.0f / 9.0f, 1e-3f), "colour 16:9"); + require(close_to(float(depth_whole.width) / float(depth_whole.height), 512.0f / 424.0f, 1e-3f), "depth 512:424"); +} + +void pictures_cover_their_whole_image() { + // The image is the crop, so the picture is all of it, in either view. + for (const View view : {View::common, View::original}) + for (const Kind kind : {Kind::colour, Kind::depth_and_player}) { + const sfr::KinectImageSize size = sfr::kinect_v2_preview_size(kind, view); + const sfr::KinectImageRect rect = sfr::kinect_v2_picture_rect(kind, view); + require(rect.x == 0 && rect.y == 0 && close_to(rect.width, float(size.width)) && + close_to(rect.height, float(size.height)), "the rectangle is the whole image"); + // The crop's corners land (to rounding) on the image's corners. + const bool is_colour = kind == Kind::colour; + const sfr::KinectV2Crop crop = sfr::kinect_v2_crop(kind, view); + const auto to_preview = [&](float x, float y) { + return is_colour ? sfr::kinect_v2_colour_to_preview(x, y, view) : sfr::kinect_v2_depth_to_preview(x, y, view); + }; + const sfr::KinectImagePoint top_left = to_preview(crop.x, crop.y); + const sfr::KinectImagePoint bottom_right = to_preview(crop.x + crop.width, crop.y + crop.height); + require(close_to(top_left.x, 0, 0.6f) && close_to(top_left.y, 0, 0.6f) && + close_to(bottom_right.x, rect.width, 0.6f) && close_to(bottom_right.y, rect.height, 0.6f), + "the crop fills the image"); + require(rect.contains(rect.width / 2, rect.height / 2) && rect.contains(0, 0) && rect.contains(1, 1) && + !rect.contains(rect.width, rect.height / 2) && !rect.contains(-1, 1), "no bars"); + } +} + +void yuy2_converts_as_bt601() { + require(sfr::kinect_yuy2_bgrx(16, 128, 128) == 0x000000u, "video black"); + require(sfr::kinect_yuy2_bgrx(235, 128, 128) == 0xFFFFFFu, "video white"); + // Pure red in BT.601 video range is about Y 81, U 90, V 240. + const uint32_t red = sfr::kinect_yuy2_bgrx(81, 90, 240); + require((red >> 16 & 0xFF) > 240 && (red >> 8 & 0xFF) < 10 && (red & 0xFF) < 10, "red"); +} + +std::vector blue_red_frame(uint32_t w, uint32_t h) { + std::vector bgra(size_t(w) * h * 4); + for (uint32_t y = 0; y < h; ++y) + for (uint32_t x = 0; x < w; ++x) { + uint8_t* p = &bgra[(size_t(y) * w + x) * 4]; + p[0] = x < w / 2 ? 255 : 0; + p[2] = x < w / 2 ? 0 : 255; + p[3] = 255; + } + return bgra; +} + +void colour_original_is_the_whole_16_by_9_and_averaged() { + // A BGRA frame whose left half is blue and right half is red, with one + // green pixel in the block the target pixel (0, 0) averages. + const uint32_t w = 1920, h = 1080; + std::vector bgra = blue_red_frame(w, h); + bgra[1] = 255; + sfr::KinectImage image; + image.number = 7; + sfr::kinect_v2_colour_from_bgra(bgra.data(), w, h, View::original, image); + require(image.number == 7, "the number is the caller's"); + require(image.width == 640 && image.height == 360 && image.bytes_per_pixel == 4, "16:9, a third the size"); + require(image.pixels.size() == size_t(640) * 360 * 4, "pixels"); + require(colour_at(image, 10, 0) == 0x0000FFu && colour_at(image, 10, 359) == 0x0000FFu, "no bars above or below"); + require(colour_at(image, 630, 359) == 0xFF0000u, "red on the right"); + // 255 / 9 rounds to 28: the 3x3 block is averaged, not sampled. + require((colour_at(image, 0, 0) >> 8 & 0xFF) == 28, "a 3x3 block averaged"); +} + +void colour_common_is_the_crop_without_bars() { + const uint32_t w = 1920, h = 1080; + std::vector bgra = blue_red_frame(w, h); + bgra[1] = 255; // outside the crop (x < 219): must not show + sfr::KinectImage image; + sfr::kinect_v2_colour_from_bgra(bgra.data(), w, h, View::common, image); + require(image.width == 640 && image.height == 466, "the crop's shape"); + require(colour_at(image, 10, 0) == 0x0000FFu && colour_at(image, 10, 465) == 0x0000FFu, "blue to the top and bottom edge"); + require(colour_at(image, 630, 0) == 0xFF0000u && colour_at(image, 630, 465) == 0xFF0000u, "red to the top and bottom edge"); + require(colour_at(image, 0, 0) == 0x0000FFu, "no bar, no outside pixel"); + // The picture's middle (source x 960) is the image's middle. + require(colour_at(image, 317, 233) == 0x0000FFu && colour_at(image, 323, 233) == 0xFF0000u, "the middle stays the middle"); +} + +void yuy2_frames_use_each_pixels_own_luma() { + // Pixel pairs sharing one U and V: the first pair black then white, + // the others white; each pixel takes its own Y. Small enough to be + // scaled up, so each target pixel shows one source pixel. + const uint32_t w = 6, h = 3; + std::vector yuy2(size_t(w) * h * 2); + for (uint32_t y = 0; y < h; ++y) + for (uint32_t pair = 0; pair < w / 2; ++pair) { + uint8_t* p = &yuy2[(size_t(y) * w + pair * 2) * 2]; + p[0] = pair == 0 ? 16 : 235; // Y0 + p[1] = 128; // U + p[2] = 235; // Y1 + p[3] = 128; // V + } + sfr::KinectImage image; + sfr::kinect_v2_colour_from_yuy2(yuy2.data(), w, h, View::original, image); + require(image.width == 640 && image.height == 360, "always the preview size"); + // 6x3 in 640x360: scaled up, 640 wide and 320 tall, bars of 20. + require(colour_at(image, 0, 19) == 0, "bar above"); + require(colour_at(image, 0, 20) == 0x000000u, "the first pixel is black"); + require(colour_at(image, 639, 20) == 0xFFFFFFu, "the last pixel is white"); + require(colour_at(image, 150, 200) == 0xFFFFFFu, "the second pixel, sharing U and V, is white"); +} + +void depth_carries_millimetres_and_bodies() { + const uint32_t w = 512, h = 424; + std::vector depth(size_t(w) * h, 2000); + std::vector bodies(size_t(w) * h, sfr::kinect_v2_no_body); + // Body 2 in the middle; a far reading beyond 13 bits at the top left. + for (uint32_t y = 200; y < 224; ++y) + for (uint32_t x = 240; x < 272; ++x) bodies[size_t(y) * w + x] = 2; + depth[0] = 9000; + sfr::KinectImage image; + image.number = 3; + sfr::kinect_v2_depth_image(depth.data(), bodies.data(), w, h, View::original, image); + require(image.number == 3, "the number is the caller's"); + require(image.width == 512 && image.height == 424 && image.bytes_per_pixel == 2, "the whole depth image"); + require(depth_at(image, 0, 120) == uint16_t(2000 << 3) && depth_at(image, 511, 120) == uint16_t(2000 << 3), "no bars at the sides"); + require(depth_at(image, 100, 50) == uint16_t(2000 << 3), "millimetres shifted up three, no player"); + require(depth_at(image, 256, 212) == uint16_t(2000 << 3 | 3), "body 2 is player 3"); + require(depth_at(image, 0, 0) == uint16_t(0x1FFF << 3), "far readings clamp to 13 bits"); + // Without the body index, no player at all. + sfr::kinect_v2_depth_image(depth.data(), nullptr, w, h, View::original, image); + require(depth_at(image, 256, 212) == uint16_t(2000 << 3), "no body index, no player"); + // The common view: full width, about 26 rows less above and below, the + // top-left reading (outside the crop) gone. + sfr::kinect_v2_depth_image(depth.data(), bodies.data(), w, h, View::common, image); + require(image.width == 512 && image.height == 373, "the crop's shape"); + require(depth_at(image, 0, 186) == uint16_t(2000 << 3) && depth_at(image, 511, 186) == uint16_t(2000 << 3), + "full width, no bars at the sides"); + require(depth_at(image, 256, 0) == uint16_t(2000 << 3) && depth_at(image, 256, 372) == uint16_t(2000 << 3), "to the top and bottom"); + require(depth_at(image, 0, 0) == uint16_t(2000 << 3), "the far reading is outside the crop"); + require(depth_at(image, 256, 186) == uint16_t(2000 << 3 | 3), "body 2 is still player 3, in the middle"); +} + +void points_move_with_the_images() { + // The principal points are the images' centres, in either view. + for (const View view : {View::common, View::original}) { + const sfr::KinectImageSize depth_size = sfr::kinect_v2_preview_size(Kind::depth_and_player, view); + const sfr::KinectImagePoint centre = sfr::kinect_v2_depth_to_preview(256, 212, view); + require(centre.visible && close_to(centre.x, depth_size.width / 2.0f, 0.01f) && + close_to(centre.y, depth_size.height / 2.0f, 0.01f), "depth centre"); + const sfr::KinectImageSize colour_size = sfr::kinect_v2_preview_size(Kind::colour, view); + const sfr::KinectImagePoint colour_centre = sfr::kinect_v2_colour_to_preview(960, 540, view); + require(colour_centre.visible && close_to(colour_centre.x, colour_size.width / 2.0f, 0.01f) && + close_to(colour_centre.y, colour_size.height / 2.0f, 0.01f), "colour centre"); + } + // Original: the image's corners are the picture's corners. + const sfr::KinectImagePoint corner = sfr::kinect_v2_depth_to_preview(0, 0, View::original); + require(corner.visible && close_to(corner.x, 0, 0.01f) && close_to(corner.y, 0, 0.01f), "depth corner"); + const sfr::KinectImagePoint depth_far = sfr::kinect_v2_depth_to_preview(512, 424, View::original); + require(close_to(depth_far.x, 512, 0.01f) && close_to(depth_far.y, 424, 0.01f), "depth far corner"); + const sfr::KinectImagePoint colour = sfr::kinect_v2_colour_to_preview(1920, 1080, View::original); + require(colour.visible && close_to(colour.x, 640) && close_to(colour.y, 360), "colour corner"); + // Common: the image of the crop, so the image's own corner lies outside. + const sfr::KinectImagePoint image_corner = sfr::kinect_v2_colour_to_preview(0, 0, View::common); + require(image_corner.visible && image_corner.x < 0, "the image's own corner is outside the cropped panel (placed, then clipped)"); + const sfr::KinectImagePoint depth_top = sfr::kinect_v2_depth_to_preview(256, 0, View::common); + require(depth_top.visible && depth_top.y < 0, "the depth rows above the crop are outside"); + // The mapper's answer for a point it cannot place. + require(!sfr::kinect_v2_depth_to_preview(-INFINITY, -INFINITY, View::common).visible, "infinite is not visible"); + require(!sfr::kinect_v2_colour_to_preview(NAN, 3, View::original).visible, "NaN is not visible"); + require(!sfr::kinect_v2_colour_to_preview(1e7f, 3, View::common).visible, "far outside is not drawn"); + // Visible means placed, not inside the image: a point just beyond the + // edge stays placed (and so keeps its bone), for the window to clip. + const sfr::KinectImagePoint beyond = sfr::kinect_v2_colour_to_preview(-100, 540, View::original); + require(beyond.visible && beyond.x < 0, "just outside the image is still placed"); + const sfr::KinectImagePoint beyond_depth = sfr::kinect_v2_depth_to_preview(600, 212, View::original); + require(beyond_depth.visible && beyond_depth.x > 512, "beyond the depth image's right edge is still placed"); +} + +void switching_the_view_changes_pictures_and_points_together() { + // A point near the cropped edge: inside the picture in one view, and + // the picture rectangles differ; each view's point lies in its own + // rectangle exactly when it is in that view's crop. + const sfr::KinectImagePoint in_common = sfr::kinect_v2_colour_to_preview(300, 540, View::common); + const sfr::KinectImagePoint in_original = sfr::kinect_v2_colour_to_preview(300, 540, View::original); + const sfr::KinectImageRect common_rect = sfr::kinect_v2_picture_rect(Kind::colour, View::common); + const sfr::KinectImageRect original_rect = sfr::kinect_v2_picture_rect(Kind::colour, View::original); + require(common_rect.contains(in_common.x, in_common.y) && original_rect.contains(in_original.x, in_original.y), + "inside the crop is inside the picture, either view"); + const sfr::KinectImagePoint cut_common = sfr::kinect_v2_colour_to_preview(100, 540, View::common); + const sfr::KinectImagePoint cut_original = sfr::kinect_v2_colour_to_preview(100, 540, View::original); + require(!common_rect.contains(cut_common.x, cut_common.y) && original_rect.contains(cut_original.x, cut_original.y), + "left of the crop: outside the cropped picture, inside the whole one"); + require(common_rect.height > original_rect.height, "the cropped picture is taller"); + require(in_common.x != in_original.x, "the same sensor point lands differently"); + // The lens fallback lands in the same rectangles. + const sfr::KinectImagePoint ahead_common = sfr::kinect_v2_colour_estimate({0, 0, 2}, View::common); + const sfr::KinectImagePoint ahead_original = sfr::kinect_v2_colour_estimate({0, 0, 2}, View::original); + require(close_to(ahead_common.x, 320, 0.01f) && close_to(ahead_common.y, 233, 0.01f) && + close_to(ahead_original.x, 320, 0.01f) && close_to(ahead_original.y, 180, 0.01f), "straight ahead is the middle"); +} + +void estimates_follow_the_lens() { + // Straight ahead is the image's centre; a metre right at two metres is + // 183 depth pixels right (366 / 2) before fitting. + const sfr::KinectImagePoint ahead = sfr::kinect_v2_depth_estimate({0, 0, 2}, View::common); + require(ahead.visible && close_to(ahead.x, 256, 0.01f) && close_to(ahead.y, 186.5f, 0.01f), "ahead is the centre"); + // Original: 1:1 with the depth image. + const sfr::KinectImagePoint right = sfr::kinect_v2_depth_estimate({1, 0.5f, 2}, View::original); + require(close_to(right.x, 256 + 183, 0.01f) && close_to(right.y, 212 - 91.5f, 0.01f), "the lens"); + // Common: the crop's own scale (image pixels per depth pixel, about 1). + const sfr::KinectV2Crop crop = sfr::kinect_v2_crop(Kind::depth_and_player, View::common); + const sfr::KinectImageRect rect = sfr::kinect_v2_picture_rect(Kind::depth_and_player, View::common); + const float common_scale = rect.width / crop.width; + const sfr::KinectImagePoint right_common = sfr::kinect_v2_depth_estimate({1, 0.5f, 2}, View::common); + require(close_to(right_common.x, 256 + 183 * common_scale, 0.01f) && + close_to(right_common.y, rect.height / 2 - 91.5f * common_scale, 0.01f), "the lens, cropped"); + const sfr::KinectImagePoint up = sfr::kinect_v2_colour_estimate({0, 0.5f, 2.5f}, View::original); + require(close_to(up.x, 320, 0.01f) && close_to(up.y, (540 - 212.0f) / 3.0f, 0.01f), "the colour lens"); + require(!sfr::kinect_v2_depth_estimate({0, 0, 0}, View::common).visible, "no depth, not visible"); + require(!sfr::kinect_v2_colour_estimate({0, 0, 0.05f}, View::original).visible, "too near, not visible"); +} + +void the_mapper_comes_first() { + // A point the mapper placed goes where it says, not where the lens would. + const std::array p{0.3f, 0.2f, 2.0f}; + for (const View view : {View::common, View::original}) { + const float depth_pixel[2] = {100, 50}; + const sfr::KinectImagePoint depth = sfr::kinect_v2_place(Kind::depth_and_player, p, depth_pixel, view); + const sfr::KinectImagePoint depth_expected = sfr::kinect_v2_depth_to_preview(100, 50, view); + require(depth.visible && close_to(depth.x, depth_expected.x) && close_to(depth.y, depth_expected.y), + "a mapped depth point is the mapper's"); + const float colour_pixel[2] = {1500, 300}; + const sfr::KinectImagePoint colour = sfr::kinect_v2_place(Kind::colour, p, colour_pixel, view); + const sfr::KinectImagePoint colour_expected = sfr::kinect_v2_colour_to_preview(1500, 300, view); + require(colour.visible && close_to(colour.x, colour_expected.x) && close_to(colour.y, colour_expected.y), + "a mapped colour point is the mapper's"); + // -infinity (the mapper could not place it) or no mapper: the lens. + const float unplaced[2] = {-INFINITY, -INFINITY}; + const sfr::KinectImagePoint lens = sfr::kinect_v2_depth_estimate(p, view); + const sfr::KinectImagePoint fallback = sfr::kinect_v2_place(Kind::depth_and_player, p, unplaced, view); + require(fallback.visible && close_to(fallback.x, lens.x) && close_to(fallback.y, lens.y), "-inf falls back to the lens"); + const float half[2] = {700, -INFINITY}; + const sfr::KinectImagePoint colour_lens = sfr::kinect_v2_colour_estimate(p, view); + const sfr::KinectImagePoint half_fallback = sfr::kinect_v2_place(Kind::colour, p, half, view); + require(close_to(half_fallback.x, colour_lens.x) && close_to(half_fallback.y, colour_lens.y), + "one infinite coordinate falls back too"); + const sfr::KinectImagePoint none = sfr::kinect_v2_place(Kind::colour, p, nullptr, view); + require(close_to(none.x, colour_lens.x) && close_to(none.y, colour_lens.y), "no mapper, the lens"); + // Behind the sensor neither can place it. + require(!sfr::kinect_v2_place(Kind::depth_and_player, {0, 0, 0}, unplaced, view).visible, "nowhere, not visible"); + } + // Original colour: a third of the sensor's pixels. + const float colour_pixel[2] = {1500, 300}; + const sfr::KinectImagePoint old = sfr::kinect_v2_place(Kind::colour, {0.3f, 0.2f, 2.0f}, colour_pixel, View::original); + require(close_to(old.x, 500) && close_to(old.y, 100), "original colour is the 16:9 image, a third the size"); +} +} + +int main() { + try { + fits_keep_the_shape(); + crops_are_the_common_view(); + preview_images_follow_the_pictures_shape(); + pictures_cover_their_whole_image(); + yuy2_converts_as_bt601(); + colour_original_is_the_whole_16_by_9_and_averaged(); + colour_common_is_the_crop_without_bars(); + yuy2_frames_use_each_pixels_own_luma(); + depth_carries_millimetres_and_bodies(); + points_move_with_the_images(); + switching_the_view_changes_pictures_and_points_together(); + estimates_follow_the_lens(); + the_mapper_comes_first(); + } catch (const std::exception& error) { + std::cerr << "kinect_v2_image_test: " << error.what() << '\n'; + return 1; + } + std::cout << "kinect_v2_image_test passed\n"; + return 0; +} diff --git a/tests/launcher_settings_test.cpp b/tests/launcher_settings_test.cpp index 4644425..4a423e0 100644 --- a/tests/launcher_settings_test.cpp +++ b/tests/launcher_settings_test.cpp @@ -24,6 +24,12 @@ std::string value_of(const sfr::LauncherSettings& settings, const std::string& n throw std::runtime_error("variable missing from the game environment"); } +bool exports(const sfr::LauncherSettings& settings, const std::string& name) { + for (const auto& entry : sfr::game_environment(settings, {})) + if (entry.first == name) return true; + return false; +} + void settings_round_trip() { sfr::LauncherSettings settings; settings.window_width = 1920; @@ -116,20 +122,21 @@ void graphics_backend_settings() { } void kinect_tuning_settings() { - const sfr::LauncherSettings defaults; - require(defaults.kinect_smooth && defaults.kinect_lean == 100 && defaults.kinect_crouch_cm == 15, - "Kinect tuning defaults to the game's own behaviour"); - for (const char* name : {"SFR_KINECT_SMOOTH", "SFR_KINECT_LEAN_SCALE", "SFR_KINECT_CROUCH_DEPTH"}) - require(value_of(defaults, name).empty(), "at the defaults the game decides for itself"); - const auto tuned = sfr::parse_launcher_settings(sfr::format_launcher_settings(sfr::parse_launcher_settings( - "kinect_smooth=0\nkinect_lean=50\nkinect_crouch_cm=20\n"))); - require(!tuned.kinect_smooth && tuned.kinect_lean == 50 && tuned.kinect_crouch_cm == 20, - "Kinect tuning survives a save and a load"); - require(value_of(tuned, "SFR_KINECT_SMOOTH") == "0", "smoothing off reaches the game"); - require(value_of(tuned, "SFR_KINECT_LEAN_SCALE") == "1.75", "half the lean is half of 3.5"); - require(value_of(tuned, "SFR_KINECT_CROUCH_DEPTH") == "0.2", "twenty centimetres are 0.2 metres"); - const auto odd = sfr::parse_launcher_settings("kinect_lean=5\nkinect_crouch_cm=40\n"); - require(odd.kinect_lean == 100 && odd.kinect_crouch_cm == 15, "values outside the sliders are ignored"); + // The v2 smoothing switch, lean sensitivity, crouch depth and the v2 + // depth switch are gone (the game smooths a real Kinect's skeletons once, + // as the title asks): an old settings.ini with them still loads, the old + // keys are ignored, not saved again, and nothing is exported for them. + require(!exports(sfr::LauncherSettings{}, "SFR_KINECT_SMOOTH"), "the launcher never sets the backends' smoothing"); + for (const char* text : {"kinect_smooth=0\ncamera=kinect_v2\n", "kinect_smooth=1\ncamera=kinect_v2\n", + "kinect_v2_depth=1\nkinect_lean=60\nkinect_crouch_cm=20\ncamera=kinect_v2\nkinect_smooth=0\n"}) { + const auto old = sfr::parse_launcher_settings(text); + require(old.camera == "kinect_v2", "old Kinect keys leave the rest alone"); + const std::string saved = sfr::format_launcher_settings(old); + for (const char* key : {"kinect_smooth", "kinect_v2_depth", "kinect_lean", "kinect_crouch_cm"}) + require(saved.find(key) == std::string::npos, "an old Kinect key is not saved again"); + for (const char* name : {"SFR_KINECT_SMOOTH", "SFR_KINECT_V2_DEPTH", "SFR_KINECT_LEAN_SCALE", "SFR_KINECT_CROUCH_DEPTH"}) + require(!exports(old, name), "no removed Kinect switch reaches the game"); + } } void runtime_render_scale() { @@ -545,10 +552,10 @@ void save_directory_settings() { void menu_steady_settings() { const sfr::LauncherSettings defaults; - require(sfr::format_launcher_settings(defaults).find("menu_steady_radius=8\n") != std::string::npos, - "the menu cursor steadying defaults to an 8 mm circle"); - require(value_of(defaults, "SFR_MENU_HAND_STEADY") == "1" && value_of(defaults, "SFR_MENU_STEADY_RADIUS") == "8", - "the default circle reaches the game"); + require(sfr::format_launcher_settings(defaults).find("menu_steady_radius=0\n") != std::string::npos, + "the menu cursor steadying defaults to off"); + require(value_of(defaults, "SFR_MENU_HAND_STEADY") == "0" && value_of(defaults, "SFR_MENU_STEADY_RADIUS").empty(), + "the default (off) reaches the game"); const auto wide = sfr::parse_launcher_settings("menu_steady_radius=15\n"); require(wide.menu_steady_radius == 15 && value_of(wide, "SFR_MENU_STEADY_RADIUS") == "15" && sfr::format_launcher_settings(wide).find("menu_steady_radius=15\n") != std::string::npos, @@ -557,10 +564,62 @@ void menu_steady_settings() { require(value_of(off, "SFR_MENU_HAND_STEADY") == "0" && value_of(off, "SFR_MENU_STEADY_RADIUS").empty(), "0 starts the steadying off"); for (const char* bad : {"31", "-1", "big"}) - require(sfr::parse_launcher_settings(std::string("menu_steady_radius=") + bad + "\n").menu_steady_radius == 8, + require(sfr::parse_launcher_settings(std::string("menu_steady_radius=") + bad + "\n").menu_steady_radius == 0, "an out-of-range circle keeps the default"); } +void kinect_v2_view_settings() { + const sfr::LauncherSettings defaults; + require(defaults.kinect_v2_view == "ideal" && value_of(defaults, "SFR_KINECT_V2_DEPTH_VIEW").empty(), + "the view only reaches the game with a Kinect v2 chosen"); + auto chosen = sfr::parse_launcher_settings("camera=kinect_v2\n"); + require(value_of(chosen, "SFR_KINECT_V2_DEPTH_VIEW") == "ideal" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_DISTANCE") == "2.50" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_HEIGHT") == "1.00" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_FOLLOW") == "2.00" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_DEADBAND") == "0.15" && + value_of(chosen, "SFR_KINECT_V2_SKELETON_DISTANCE").empty(), + "Recompiled is the default and sends its default tuning"); + chosen = sfr::parse_launcher_settings("camera=kinect_v2\nkinect_v2_view=fit\n"); + require(value_of(chosen, "SFR_KINECT_V2_DEPTH_VIEW") == "fit" && value_of(chosen, "SFR_KINECT_V2_DEPTH_DISTANCE").empty(), + "Simulated sends no Recompiled tuning"); + require(sfr::parse_launcher_settings("camera=kinect_v2\nkinect_v2_view=fit\n").kinect_v2_view == "fit" && + sfr::format_launcher_settings(chosen).find("kinect_v2_view=fit\n") != std::string::npos, + "a saved Simulated stays Simulated under the new default"); + chosen = sfr::parse_launcher_settings( + "camera=kinect_v2\nkinect_v2_view=ideal\nkinect_v2_distance=300\nkinect_v2_height=95\n" + "kinect_v2_follow=35\nkinect_v2_deadband=0\n"); + require(value_of(chosen, "SFR_KINECT_V2_DEPTH_VIEW") == "ideal" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_DISTANCE") == "3.00" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_HEIGHT") == "0.95" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_FOLLOW") == "3.50" && + value_of(chosen, "SFR_KINECT_V2_DEPTH_DEADBAND") == "0.00" && + value_of(chosen, "SFR_KINECT_V2_SKELETON_DISTANCE").empty(), + "Recompiled's tuning reaches the game in metres and seconds"); + const std::string saved = sfr::format_launcher_settings(chosen); + require(saved.find("kinect_v2_view=ideal\n") != std::string::npos && + saved.find("kinect_v2_skeleton_distance") == std::string::npos, + "the view and its tuning survive a save"); + require(value_of(sfr::parse_launcher_settings("camera=kinect\nkinect_v2_view=ideal\n"), "SFR_KINECT_V2_DEPTH_VIEW").empty(), + "the Xbox 360 choice sends no v2 view"); + require(sfr::parse_launcher_settings("kinect_v2_view=player\n").kinect_v2_view == "ideal", + "an unknown view keeps the default"); + require(sfr::parse_launcher_settings("kinect_v2_distance=9000\n").kinect_v2_distance == 250, + "an out-of-range distance keeps the default"); + // The preview's choice: the same numbers the game is handed. + const auto choice = sfr::launcher_kinect_depth_choice(chosen); + require(choice.view == sfr::KinectDepthChoiceView::ideal && choice.distance == 3.0f && choice.height == 0.95f && + choice.follow == 3.5f && choice.deadband == 0.0f && choice.skeleton_distance == 0.0f, + "the preview gets Recompiled's tuning in metres and seconds"); + require(sfr::launcher_kinect_depth_choice(sfr::LauncherSettings{}) == sfr::KinectDepthChoice{}, + "the launcher's defaults are the preview's"); + require(sfr::launcher_kinect_depth_choice(sfr::parse_launcher_settings("kinect_v2_view=fit\n")).view == + sfr::KinectDepthChoiceView::fit && + sfr::launcher_kinect_depth_choice(sfr::parse_launcher_settings("kinect_v2_view=sensor\n")).view == + sfr::KinectDepthChoiceView::sensor, + "Simulated and Original"); +} + void motion_trace_settings() { const sfr::LauncherSettings defaults; require(!defaults.motion_trace && sfr::format_launcher_settings(defaults).find("motion_trace=0\n") != std::string::npos, @@ -671,6 +730,27 @@ void malformed_values_keep_defaults() { const sfr::LauncherSettings defaults; require(read.camera == defaults.camera, "an unknown camera choice keeps the default"); require(sfr::parse_launcher_settings("camera=kinect\n").camera == "kinect", "a real Kinect is a camera choice"); + require(sfr::parse_launcher_settings("camera=kinect_v2\n").camera == "kinect_v2", "a Kinect v2 is a camera choice of its own"); + require(sfr::parse_launcher_settings("camera=Kinect_v3\n").camera == defaults.camera, "another Kinect name keeps the default"); + for (const char* choice : {"kinect", "kinect_v2"}) { + sfr::LauncherSettings chosen; + chosen.camera = choice; + require(sfr::parse_launcher_settings(sfr::format_launcher_settings(chosen)).camera == choice, + "a Kinect choice survives saving and reading"); + } + { + sfr::LauncherSettings v1, v2, none; + v1.camera = "kinect"; + v2.camera = "kinect_v2"; + require(value_of(v1, "SFR_CAMERA") == "kinect" && value_of(v1, "SFR_KINECT_MODEL") == "1", + "the Xbox 360 Kinect reaches the game as the Kinect with model 1"); + require(value_of(v2, "SFR_CAMERA") == "kinect" && value_of(v2, "SFR_KINECT_MODEL") == "2", + "the Kinect v2 reaches the game as the Kinect with model 2"); + require(value_of(none, "SFR_KINECT_MODEL").empty(), "no Kinect chosen leaves the model alone"); + none.camera = "motion"; + require(value_of(none, "SFR_CAMERA") == "motion" && value_of(none, "SFR_KINECT_MODEL").empty(), + "the webcam choices do not name a Kinect model"); + } require(sfr::parse_launcher_settings("camera_device=" + std::string(200, 'x') + "\n").camera_device == defaults.camera_device, "a name longer than any camera has keeps the default"); require(read.window_width == defaults.window_width && read.window_height == defaults.window_height, @@ -864,6 +944,7 @@ int main() { camera_debug_settings(); motion_trace_settings(); menu_steady_settings(); + kinect_v2_view_settings(); save_directory_settings(); title_stats_settings(); race_frameskip_settings(); diff --git a/tests/nui_race_hooks_test.cpp b/tests/nui_race_hooks_test.cpp index 4f0f8e1..dbb9dac 100644 --- a/tests/nui_race_hooks_test.cpp +++ b/tests/nui_race_hooks_test.cpp @@ -47,7 +47,6 @@ std::optional second_player_pad() { return harness::second_pad_present ? std::optional(GamepadState{}) : std::nullopt; } GamepadState nui_gamepad() { return harness::input; } -bool nui_sensor_has_depth() { return false; } bool nui_body_from_sensor() { return harness::sensor_body; } uint64_t kinect_frame_generation() { return harness::kinect_sequence; } void enter_function_observed(PPCContext&, const char*, uint32_t) {} @@ -350,8 +349,9 @@ void run() { require(invoke("sub_822CA6B0") == 1 && original_side_calls == 2, "with a sensor the original detectors must run"); - // No depth image reaches the title, so a sensor's lean pair comes from - // the body: upright after the stance calibration, then the torso's roll. + // A real Kinect's race is the title's own depth view and detectors, as on + // the console: nothing is made of the body's pose, so the lean pair the + // title computed and the detectors' answers stand whatever the body does. const auto put = [&](uint32_t offset, float x, float y, float z) { m.store(original + offset, std::bit_cast(x)); m.store(original + offset + 4, std::bit_cast(y)); @@ -361,36 +361,33 @@ void run() { return std::pair{std::bit_cast(m.load(original + 640)), std::bit_cast(m.load(original + 644))}; }; + const std::pair titles{1.25f, 2.5f}; + const auto title_leans = [&] { + m.store(original + 640, std::bit_cast(titles.first)); + m.store(original + 644, std::bit_cast(titles.second)); + }; put(0, 0, 0, 2.2f); // hip centre put(32, 0, 0.5f, 2.2f); // shoulder centre put(224, -0.1f, -0.8f, 2.2f); put(288, 0.1f, -0.8f, 2.2f); + title_leans(); for (int i = 0; i < 60; ++i) { ++kinect_sequence; frame(0); } - require(lean_pair() == std::pair{1.f, 1.f}, "a calibrated upright sensor body leans neither way"); + require(lean_pair() == titles, "the title's lean pair stands for an upright body"); put(32, -0.3f, 0.5f, 2.2f); // shoulders over one side of the hips for (int i = 0; i < 60; ++i) { ++kinect_sequence; frame(0); } - const auto [right, left] = lean_pair(); - require(right > 1.5f && left == 1.f, "a sensor body's roll leans the race one way"); - require(right == 4.5f, "a sensor body's full lean uses the title's whole lean range"); - // The depth view's worker can refill the pair before the race consumer - // reads it through the accessor, which writes the full lean again. + require(lean_pair() == titles, "a body's roll does not write the lean pair"); constexpr uint32_t source = detector + 0x800, object = detector + 0x900; m.store(source, object); m.store(object + 4, original); - m.store(original + 640, std::bit_cast(1.f)); - m.store(original + 644, std::bit_cast(1.f)); { PPCContext ctx; ctx.r3.u64 = source; hooks().at("sub_822C6200")(ctx, m.base()); } - require(lean_pair() == std::pair{4.5f, 1.f}, "the race consumer must read the sensor's full lean"); - put(32, 0.3f, 0.5f, 2.2f); - for (int i = 0; i < 60; ++i) { ++kinect_sequence; frame(0); } - require(lean_pair().first == 1.f && lean_pair().second > 1.5f, "and the other roll the other way"); + require(lean_pair() == titles, "the race consumer reads the title's own lean"); - // The originals cannot see a crouch without the depth view, so the - // skeleton's crouch and the rise out of it answer for them. + // The detectors answer from the title's depth view: a crouch or a rise in + // the skeleton is not an answer. constexpr uint32_t vtable = detector + 0xA00; m.store(object, vtable); m.store(vtable + 4, 0x82918418); @@ -399,53 +396,24 @@ void run() { require(invoke("sub_822C8778", source) == 2, "an upright sensor body does not crouch"); put(0, 0, -0.4f, 2.2f); put(32, 0, 0.1f, 2.2f); - // Feet out of the sensor's view: its guessed ankles follow the hips. put(224, -0.1f, -1.2f, 2.2f); put(288, 0.1f, -1.2f, 2.2f); for (int i = 0; i < 30; ++i) { ++kinect_sequence; frame(0); } m.store(selected + 4, 0); - require(invoke("sub_822C8778", source) == 1 && (primary() & 0x7000) == 0x7000, - "a sensor body's crouch must reach the title"); + require(invoke("sub_822C8778", source) == 2 && (primary() & 0x7000) == 0, + "a skeleton's crouch must not reach the title"); put(0, 0, 0, 2.2f); put(32, 0, 0.5f, 2.2f); - bool jumped = false; - // The motion trace (main sets it) records the jump the fallback gave - // the game's own jump detector, and the bits it left in the result. - std::ostringstream log; - auto* const old = std::cerr.rdbuf(log.rdbuf()); - try { - for (int i = 0; i < 30 && !jumped; ++i) { - ++kinect_sequence; - frame(0); - jumped = invoke("sub_822C9050", source) == 1; - } - } catch (...) { std::cerr.rdbuf(old); throw; } - std::cerr.rdbuf(old); - require(jumped && (primary() & 0x200), "rising out of the crouch must jump"); - const std::string traced = log.str(); - const auto line = traced.find("KINECT_GESTURE address=0x822c9050 original=2 result=1"); - require(line != std::string::npos && traced.find("jump=1", line) != std::string::npos && - traced.find("primary=0x220", line) != std::string::npos, - "the motion trace shows the jump handed to the game's detector and its result bits"); - require(traced.find("KINECT_MOTION jump") != std::string::npos && - traced.find(" generation=", traced.find("KINECT_MOTION jump")) != std::string::npos, - "a detected jump is logged with the sensor frame it came from"); - require(traced.find("CAMERA_RACE_MOTION") != std::string::npos && traced.find(" standing=") != std::string::npos, - "the trace's motion line carries the hips' height and the stance"); + for (int i = 0; i < 30; ++i) { + ++kinect_sequence; + frame(0); + require(invoke("sub_822C9050", source) != 1 && !(primary() & 0x200), "a skeleton's rise must not jump"); + } sensor_body = false; } } int main() { - // The skeleton stand-ins below are the fallback without the sensor's - // depth images, which is what these checks exercise. -#ifdef _WIN32 - _putenv_s("SFR_KINECT_DEPTH", "0"); - _putenv_s("SFR_CAMERA_RACE_TRACE", "1"); -#else - setenv("SFR_KINECT_DEPTH", "0", 1); - setenv("SFR_CAMERA_RACE_TRACE", "1", 1); -#endif try { harness::run(); harness::posture(); diff --git a/tests/nui_race_players_test.cpp b/tests/nui_race_players_test.cpp index aff35c2..ffa3a08 100644 --- a/tests/nui_race_players_test.cpp +++ b/tests/nui_race_players_test.cpp @@ -28,7 +28,7 @@ uint32_t original_overthrow_result = 99; uint32_t original_kick_bits = 0; bool consumer_throws = false; bool camera_active = false; -bool sensor_active = false, sensor_depth = false; +bool sensor_active = false; uint64_t camera_sequence=0; uint64_t camera_now=1000000000; uint32_t consumed_record = 0; @@ -52,7 +52,6 @@ uint64_t camera_pose_generation() { return harness::camera_sequence; } uint64_t camera_motion_clock_ns() { return harness::camera_now; } GamepadState nui_gamepad() { return harness::first; } bool nui_body_from_sensor() { return harness::sensor_active; } -bool nui_sensor_has_depth() { return harness::sensor_depth; } uint64_t kinect_frame_generation() { return harness::camera_sequence; } std::optional second_player_pad() { return harness::second; } void enter_function_observed(PPCContext&, const char*, uint32_t) {} @@ -268,7 +267,7 @@ bool braking(unsigned source) { return invoke("sub_822C9BF0", source) == 1; } void begin() { - sensor_active=false; sensor_depth=false; + sensor_active=false; original_kick_bits=0; mem().store(0x83E516A0,0); camera_active = false; @@ -847,101 +846,35 @@ void sensor_begin() { sensor_pose(original);sensor_pose(other); for(int i=0;i<60;++i){frame();consume(0);consume(1);} } -void sensor_independent() { - sensor_begin(); - require(f(original+640)==1 && f(other+640)==1,"no-depth sensors must default to neutral skeleton lean for both players"); +// A real Kinect's players are read by the title's own depth detectors, as on +// the console: whatever the skeleton does, the lean pair the title computed +// and each detector's own answer stand, for both players. +void sensor_depth_stands() { + sensor_begin();frame(); + for(uint32_t record:{original,other}){store_float(record+640,31);store_float(record+644,47);} + // The accessor alone, within the original consumer scope, retains native depth. + PPCContext ctx;ctx.r3.u64=objects; + {RaceSourceScope scope(sources);hooks().at("sub_82918418")(ctx,mem().base());} + require(f(original+640)==31 && f(original+644)==47,"the title's depth pair must stand"); + // Leaning, crouching and rising skeletons: no stand-in answers for the title. sensor_pose(original,-.5f);sensor_pose(other,.5f); - for(int i=0;i<60;++i){frame();consume(0);consume(1);} - require(f(original+640)>f(original+644) && f(other+640)(race_flag_global,0);frame(); - mem().store(race_flag_global,1);frame();consume(1); - require(invoke("sub_822C8778",1)!=1,"race reentry must clear crouch calibration"); - sensor_pose(other);for(int i=0;i<60;++i){frame();consume(1);} - sensor_pose(other,0,.6f);for(int i=0;i<30;++i){frame();consume(1);} - require(invoke("sub_822C8778",1)==1,"crouch must recover after recalibration"); - for(int i=0;i<40;++i){frame();} // P2 no longer consumed: retire its state. - consume(1); - require(invoke("sub_822C8778",1)!=1,"absent P2 record must not reuse expired calibration"); -} -void sensor_identity_reuse() { - sensor_begin();sensor_pose(other,0,.6f); - for(int i=0;i<30;++i){frame();consume(1);} - require(invoke("sub_822C8778",1)==1,"fixture must begin crouching"); - // Same guest allocation, but the sensor now tracks a different person. - mem().store(other+904,123);consume(1); - require(invoke("sub_822C8778",1)!=1,"new tracking identity must clear old crouch even within one game update"); - sensor_pose(other);for(int i=0;i<60;++i){frame();consume(1);} - sensor_pose(other,0,.6f);for(int i=0;i<30;++i){frame();consume(1);} - require(invoke("sub_822C8778",1)==1,"crouch must recover"); - mem().store(other+900,0);consume(1); - require(invoke("sub_822C8778",1)!=1,"untracked skeleton must not replay cached crouch"); -} - -void sensor_overrides() { - sensor_begin();sensor_depth=true; -#ifdef _WIN32 - _putenv_s("SFR_KINECT_BODY_LEAN","1");_putenv_s("SFR_KINECT_BODY_GESTURES","1"); -#else - setenv("SFR_KINECT_BODY_LEAN","1",1);setenv("SFR_KINECT_BODY_GESTURES","1",1); -#endif - for(int i=0;i<60;++i){frame();consume(0);consume(1);} - sensor_pose(other,0,.6f);for(int i=0;i<30;++i){frame();consume(1);} - const bool forced=f(other+640)>=1 && invoke("sub_822C8778",1)==1; -#ifdef _WIN32 - _putenv_s("SFR_KINECT_BODY_LEAN","0");_putenv_s("SFR_KINECT_BODY_GESTURES","0"); -#else - setenv("SFR_KINECT_BODY_LEAN","0",1);setenv("SFR_KINECT_BODY_GESTURES","0",1); -#endif - sensor_depth=false;frame();consume(1); - const bool disabled=f(other+640)==0 && invoke("sub_822C8778",1)==99; -#ifdef _WIN32 - _putenv_s("SFR_KINECT_BODY_LEAN","");_putenv_s("SFR_KINECT_BODY_GESTURES",""); -#else - unsetenv("SFR_KINECT_BODY_LEAN");unsetenv("SFR_KINECT_BODY_GESTURES"); -#endif - require(forced && disabled,"explicit overrides must take precedence over actual depth capability"); -} - -void sensor_depth_capability() { - sensor_begin();sensor_depth=true;frame(); - store_float(original+640,31);store_float(original+644,47); - // Accessor alone, within the original consumer scope, must retain native depth. - PPCContext ctx;ctx.r3.u64=objects; - {RaceSourceScope scope(sources);hooks().at("sub_82918418")(ctx,mem().base());} - require(f(original+640)==31 && f(original+644)==47,"opened depth stream must retain native depth pair"); - sensor_pose(other,0,.6f); - for(int i=0;i<30;++i){frame();consume(1);} - require(invoke("sub_822C8778",1)==99,"native depth must retain original gesture result"); - sensor_depth=false; - for(int i=0;i<60;++i){frame();consume(0);consume(1);} - require(f(other+640)>=1,"capability change must enable fallback without restart"); } } -int main() { - using namespace harness; +int main() { using namespace harness; sfr::GuestMemory memory; sfr::active_memory = &memory; memory.map(box, 0x10000); @@ -952,7 +885,7 @@ int main() { memory.store(results + 8, 1); unsigned failures = 0; const std::pair cases[] = { - {"sensor independent players", sensor_independent}, {"sensor lifecycle", sensor_lifecycle}, {"sensor identity reuse", sensor_identity_reuse}, {"sensor overrides", sensor_overrides}, {"sensor depth capability", sensor_depth_capability}, + {"sensor depth stands", sensor_depth_stands}, {"two objects per player", baseline}, {"source reuse", source_reuse}, {"record rebind", record_rebind}, {"zero then valid", zero_then_valid}, {"original alias", original_alias}, {"disconnect", disconnect}, diff --git a/tests/nui_transform_smooth_sdk_test.cpp b/tests/nui_transform_smooth_sdk_test.cpp new file mode 100644 index 0000000..0735fad --- /dev/null +++ b/tests/nui_transform_smooth_sdk_test.cpp @@ -0,0 +1,227 @@ +// Our NuiTransformSmooth (nui_transform_smooth.h) against the real one: the +// Kinect for Windows SDK 1.8's Kinect10.dll exports +// NuiTransformSmooth(NUI_SKELETON_FRAME*, const NUI_TRANSFORM_SMOOTH_PARAMETERS*). +// The same synthetic skeleton frames go through both, with the default +// parameters and others, and every joint of every slot must agree within a +// tenth of a millimetre. +// +// No sensor is started: NuiInitialize is never called, only the DLL is +// loaded. When it is not installed, or its NuiTransformSmooth refuses to +// work without an initialized sensor, the test says so and is skipped (exit +// 77); the unit test (nui_transform_smooth_test) checks the algorithm on its +// own either way. +#include "nui_transform_smooth.h" + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +// NuiSkeleton.h (SDK 1.8): the PC's own layout, little-endian. +struct NuiVector4 { float x, y, z, w; }; +struct NuiSkeletonData { + int32_t tracking_state; + DWORD tracking_id, enrollment_index, user_index; + NuiVector4 position; + NuiVector4 positions[20]; + int32_t position_states[20]; + DWORD quality_flags; +}; +struct NuiSkeletonFrame { + LARGE_INTEGER timestamp; + DWORD frame_number, flags; + NuiVector4 floor_clip_plane, normal_to_gravity; + NuiSkeletonData skeletons[6]; +}; +struct NuiTransformSmoothParameters { float smoothing, correction, prediction, jitter_radius, max_deviation_radius; }; +static_assert(sizeof(NuiSkeletonData) == 436 && sizeof(NuiSkeletonFrame) == 2664, "SDK 1.8 layout"); +using NuiTransformSmooth = HRESULT(WINAPI*)(NuiSkeletonFrame*, const NuiTransformSmoothParameters*); +constexpr int skipped = 77; + +struct Body { + uint32_t slot, id; + sfr::SkeletonJoints joints; + std::array states; +}; + +// A frame of the given bodies, in both forms. +void build(const std::vector& bodies, uint32_t number, NuiSkeletonFrame& sdk, + std::array& ours) { + sdk = {}; + ours = {}; + sdk.timestamp.QuadPart = LONGLONG(number) * 33; + sdk.frame_number = number; + sdk.floor_clip_plane = {0.0f, 1.0f, 0.0f, 0.8f}; + sdk.normal_to_gravity = {0.0f, 1.0f, 0.0f, 0.0f}; + for (const Body& body : bodies) { + NuiSkeletonData& data = sdk.skeletons[body.slot]; + data.tracking_state = 2; + data.tracking_id = body.id; + data.position = {body.joints[0][0], body.joints[0][1], body.joints[0][2], 1.0f}; + auto& slot = ours[body.slot]; + slot.tracking_state = 2; + slot.tracking_id = body.id; + for (int j = 0; j < 20; ++j) { + data.positions[j] = {body.joints[j][0], body.joints[j][1], body.joints[j][2], 1.0f}; + data.position_states[j] = int32_t(body.states[j]); + slot.joints[j] = body.joints[j]; + slot.states[j] = body.states[j]; + } + } +} + +struct Run { + const char* name; + bool explicit_parameters; + NuiTransformSmoothParameters parameters; +}; + +// A standing body that sways and reaches, with sensor noise. +Body body_at(uint32_t slot, uint32_t id, uint32_t frame, std::mt19937& random, float x0) { + std::normal_distribution noise(0.0f, 0.006f); + Body body{slot, id, {}, {}}; + const float t = float(frame) / 30.0f; + const float sway = 0.12f * std::sin(t * 1.3f), reach = 0.25f * std::sin(t * 2.7f); + for (int j = 0; j < 20; ++j) { + body.joints[j] = {x0 + sway + 0.05f * float(j % 5) + noise(random), -0.6f + 0.06f * float(j) + noise(random), + 2.4f + (j == 7 || j == 11 ? -reach : 0.0f) + noise(random)}; + body.states[j] = 2; + } + return body; +} + +std::string text(HRESULT value) { + char out[16]; + std::snprintf(out, sizeof out, "0x%08lX", static_cast(value)); + return out; +} +} + +int main() { + HMODULE library = LoadLibraryW(L"Kinect10.dll"); + if (!library) { + std::cout << "SKIPPED: Kinect10.dll (Kinect for Windows SDK/Runtime 1.8) is not installed\n"; + return skipped; + } + const auto smooth = reinterpret_cast(reinterpret_cast(GetProcAddress(library, "NuiTransformSmooth"))); + if (!smooth) { + std::cout << "SKIPPED: Kinect10.dll exports no NuiTransformSmooth\n"; + return skipped; + } + // Clears the SDK's own per-slot history the documented way: a frame with + // no tracked skeleton (our smoother is made anew for each run). + const auto clear_sdk = [&] { + NuiSkeletonFrame empty{}; + return smooth(&empty, nullptr); + }; + { + // Asked both ways (no frame tracked and no parameters; a tracked + // skeleton and the defaults given) before giving up on it. + const HRESULT empty = clear_sdk(); + std::mt19937 random(7); + NuiSkeletonFrame sdk; + std::array slots; + build({body_at(0, 1, 1, random, 0.0f)}, 1, sdk, slots); + const NuiTransformSmoothParameters defaults{0.5f, 0.5f, 0.5f, 0.05f, 0.04f}; + const HRESULT tracked = smooth(&sdk, &defaults); + if (FAILED(empty) && FAILED(tracked)) { + std::cout << "SKIPPED: the SDK's NuiTransformSmooth refuses to work without NuiInitialize (it answered " + << text(empty) << " for an empty frame and " << text(tracked) + << " for a tracked one); no sensor is started here, so nui_transform_smooth_test checks the algorithm alone\n"; + return skipped; + } + } + const std::vector runs = { + {"defaults (no parameters)", false, {}}, + {"defaults, given", true, {0.5f, 0.5f, 0.5f, 0.05f, 0.04f}}, + {"light", true, {0.25f, 0.25f, 0.25f, 0.03f, 0.05f}}, + {"heavy", true, {0.8f, 0.2f, 1.0f, 0.1f, 0.1f}}, + {"no jitter radius", true, {0.5f, 0.5f, 0.5f, 0.0f, 0.04f}}, + {"wide", true, {0.6f, 0.4f, 0.5f, 0.5f, 0.5f}}, + }; + constexpr float tolerance = 1e-4f; // metres + int failures = 0; + for (const Run& run : runs) { + clear_sdk(); + sfr::NuiFrameSmoother ours; + const sfr::NuiSmoothParameters parameters = + run.explicit_parameters ? sfr::NuiSmoothParameters{run.parameters.smoothing, run.parameters.correction, + run.parameters.prediction, run.parameters.jitter_radius, + run.parameters.max_deviation_radius} + : sfr::NuiSmoothParameters{}; + std::mt19937 random(1234); + float worst = 0; + uint32_t worst_frame = 0, compared = 0; + HRESULT failed = S_OK; + for (uint32_t number = 1; number <= 400; ++number) { + std::vector bodies; + // The first player throughout (slot 2), a second one (slot 4) + // who steps out for a while and back in under a new id. + Body first = body_at(2, 101, number, random, -0.3f); + // Some joints inferred, one not tracked, one at the origin (lost). + if (number % 7 == 0) first.states[7] = 1; + if (number % 11 == 0) first.states[15] = 0; + if (number >= 200 && number < 210) first.joints[19] = {0.0f, 0.0f, 0.0f}; + bodies.push_back(first); + if (number < 120 || number >= 160) bodies.push_back(body_at(4, number < 120 ? 202 : 203, number, random, 0.4f)); + NuiSkeletonFrame sdk; + std::array slots; + build(bodies, number, sdk, slots); + const HRESULT result = smooth(&sdk, run.explicit_parameters ? &run.parameters : nullptr); + if (FAILED(result)) { failed = result; break; } + ours.smooth(slots, parameters); + for (int s = 0; s < 6; ++s) { + if (slots[s].tracking_state != 2) continue; + for (int j = 0; j < 20; ++j) { + const NuiVector4& p = sdk.skeletons[s].positions[j]; + const float d = (std::max)({std::fabs(p.x - slots[s].joints[j][0]), std::fabs(p.y - slots[s].joints[j][1]), + std::fabs(p.z - slots[s].joints[j][2])}); + ++compared; + if (d > worst) { worst = d; worst_frame = number; } + } + } + } + const bool ok = failed == S_OK && worst <= tolerance; + std::printf("%-26s %s: %u joints compared, largest difference %.3g m (frame %u)%s\n", run.name, ok ? "agree" : "DIFFER", + compared, double(worst), worst_frame, failed == S_OK ? "" : (" ; the SDK answered " + text(failed)).c_str()); + if (!ok) ++failures; + } + // Informational: a new tracking id in a slot that stayed tracked. Ours + // starts that slot over; does the SDK? + { + clear_sdk(); + sfr::NuiFrameSmoother ours; + std::mt19937 random(99); + float same_history = 0, ours_restart = 0; + for (uint32_t number = 1; number <= 40; ++number) { + NuiSkeletonFrame sdk; + std::array slots; + build({body_at(1, number <= 20 ? 7 : 8, number, random, number <= 20 ? -0.5f : 0.5f)}, number, sdk, slots); + const auto measured = slots[1].joints[3]; + smooth(&sdk, nullptr); + ours.smooth(slots, {}); + if (number == 21) { + same_history = std::fabs(sdk.skeletons[1].positions[3].x - measured[0]); + ours_restart = std::fabs(slots[1].joints[3][0] - measured[0]); + } + } + std::printf("tracking id change in a tracked slot: the SDK's first output is %.3g m from the measurement (0: it starts over), " + "ours %.3g m\n", double(same_history), double(ours_restart)); + } + std::cout << (failures ? "FAILED: the SDK and ours differ\n" : "The SDK's NuiTransformSmooth and ours agree\n"); + return failures ? 1 : 0; +} diff --git a/tests/nui_transform_smooth_test.cpp b/tests/nui_transform_smooth_test.cpp new file mode 100644 index 0000000..52b1c37 --- /dev/null +++ b/tests/nui_transform_smooth_test.cpp @@ -0,0 +1,192 @@ +#include "guest_memory.h" +#include "nui_transform_smooth.h" + +#include +#include +#include +#include + +// NuiTransformSmooth as the title's NUI library has it (82779700), worked +// by hand (nui_transform_smooth.h describes the steps). +namespace { +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +bool near(float a, float b, float tolerance = 1e-6f) { return std::fabs(a - b) <= tolerance; } + +float step(sfr::NuiSmoothJoint& joint, float x, uint32_t state = 2, const sfr::NuiSmoothParameters& p = {}) { + const auto out = sfr::nui_smooth_joint(joint, {x, 1.0f, 2.0f}, state, p); + require(near(out[1], 1.0f) && near(out[2], 2.0f), "an axis that does not move stays put"); + return out[0]; +} + +void the_first_frames() { + sfr::NuiSmoothJoint joint; + require(step(joint, 0.0f) == 0.0f, "the first frame is the measurement"); + // Second: the mean (0.05), trend 0.5 * 0.05; output 0.05 + 0.5 * 0.025. + require(near(step(joint, 0.1f), 0.0625f), "the second frame is the mean plus half the trend"); + // Third: moved 0.05 from the last filtered value, at the jitter radius: + // taken whole; filtered 0.5 * 0.1 + 0.5 * (0.05 + 0.025) = 0.0875, trend + // 0.5 * 0.0375 + 0.5 * 0.025 = 0.03125, output 0.0875 + 0.015625. + require(near(step(joint, 0.1f), 0.103125f), "then Holt's double exponential step"); + sfr::NuiSmoothJoint still; + for (int i = 0; i < 30; ++i) require(near(step(still, 0.2f), 0.2f), "a joint that does not move is left where it is"); +} + +void the_prediction_is_held_back() { + // Two frames at 0, then a 1 m jump: filtered 0.5, trend 0.25, prediction + // 0.625 is 0.375 from the measurement, drawn back to 0.04 (0.96). + sfr::NuiSmoothJoint tracked, inferred; + for (int i = 0; i < 2; ++i) { step(tracked, 0.0f); step(inferred, 0.0f, 1); } + require(near(step(tracked, 1.0f), 0.96f, 1e-5f), "never more than the deviation radius from the measurement"); + require(near(step(inferred, 1.0f, 1), 0.92f, 1e-5f), "an inferred joint has twice the radius"); +} + +void jitter_is_pulled_in() { + // After two frames at 0, a move of 0.02 (inside 0.05): pulled to + // 0.02 * 0.4 = 0.008, filtered 0.004, trend 0.002, output 0.005. + sfr::NuiSmoothJoint joint; + step(joint, 0.0f); + step(joint, 0.0f); + require(near(step(joint, 0.02f), 0.005f), "a small move is damped in proportion to its size"); + // The radius is never under 0.0001 m: a 0.00005 move with a radius of 0 + // is still pulled halfway (0.000025), filtered 0.0000125. + sfr::NuiSmoothParameters none; + none.jitter_radius = 0.0f; + sfr::NuiSmoothJoint small; + step(small, 0.0f, 2, none); + step(small, 0.0f, 2, none); + require(near(step(small, 0.00005f, 2, none), 0.0000125f + 0.5f * 0.00000625f, 1e-9f), "the smallest jitter radius"); +} + +void lost_joints_start_over() { + sfr::NuiSmoothJoint joint; + for (int i = 0; i < 10; ++i) step(joint, 0.3f); + const auto lost = sfr::nui_smooth_joint(joint, {0.0f, 0.0f, 0.0f}, 0, {}); + require(lost[0] == 0.0f && lost[1] == 0.0f && lost[2] == 0.0f, "a joint at the origin passes as it is"); + // It was the first frame of a new history, so the next one is the mean + // with the origin (the library's way, and the SDK sample's), held back + // to the deviation radius from the measurement. + const std::array back{0.9f, 1.0f, 2.0f}; + const auto after = sfr::nui_smooth_joint(joint, back, 2, {}); + const float off = std::sqrt((after[0] - back[0]) * (after[0] - back[0]) + (after[1] - back[1]) * (after[1] - back[1]) + + (after[2] - back[2]) * (after[2] - back[2])); + require(near(off, 0.04f, 1e-5f) && after[0] < back[0], "the frame after a lost one starts over from the origin"); + // A joint the sensor calls not tracked but gives a position is filtered + // like a tracked one (the library looks only at the position). + sfr::NuiSmoothJoint untracked; + step(untracked, 0.0f, 0); + require(near(step(untracked, 0.1f, 0), 0.0625f), "not tracked with a position: filtered"); +} + +sfr::NuiFrameSmoother::Slot slot_at(uint32_t id, float x) { + sfr::NuiFrameSmoother::Slot slot; + slot.tracking_state = 2; + slot.tracking_id = id; + for (uint32_t j = 0; j < sfr::nui_joint_count; ++j) { + slot.joints[j] = {x, 0.1f * float(j), 2.0f}; + slot.states[j] = 2; + } + return slot; +} + +void slots_keep_their_own_history() { + sfr::NuiFrameSmoother smoother; + std::array slots{}; + slots[1] = slot_at(5, 0.0f); + smoother.smooth(slots, {}); + slots[1] = slot_at(5, 0.1f); + smoother.smooth(slots, {}); + require(near(slots[1].joints[4][0], 0.0625f), "a tracked slot is filtered"); + // Another person in the same slot starts from their own measurement. + slots[1] = slot_at(6, 0.7f); + smoother.smooth(slots, {}); + require(slots[1].joints[4][0] == 0.7f, "a new tracking id starts the slot over"); + // A slot that is not tracked is left alone and forgets. + slots[1].tracking_state = 1; + slots[1].joints[4][0] = 0.4f; + smoother.smooth(slots, {}); + require(slots[1].joints[4][0] == 0.4f, "a position-only skeleton is not filtered"); + slots[1] = slot_at(6, 0.2f); + smoother.smooth(slots, {}); + require(slots[1].joints[4][0] == 0.2f, "and its slot starts over when it is tracked again"); +} + +// The title's own big-endian frame. +constexpr uint32_t frame = 0x20000, parameters = 0x21000; +void store_float(sfr::GuestMemory& memory, uint64_t address, float value) { + memory.store(address, std::bit_cast(value)); +} +float load_float(sfr::GuestMemory& memory, uint64_t address) { return std::bit_cast(memory.load(address)); } +void write_frame(sfr::GuestMemory& memory, uint32_t number, float x) { + for (uint32_t offset = 0; offset < sfr::nui_skeleton_frame_size; offset += 4) memory.store(frame + offset, 0); + memory.store(frame + 8, number); + const uint64_t data = frame + sfr::nui_skeleton_data_offset + 3 * sfr::nui_skeleton_data_size; + memory.store(data, 2); + memory.store(data + 4, 9); + store_float(memory, data + 16, x); // the centre + for (uint32_t j = 0; j < sfr::nui_joint_count; ++j) { + store_float(memory, data + 32 + 16 * j, x); + store_float(memory, data + 32 + 16 * j + 4, 1.0f); + store_float(memory, data + 32 + 16 * j + 8, 2.0f); + store_float(memory, data + 32 + 16 * j + 12, 0.5f); + memory.store(data + 352 + 4 * j, 2); + } + // An untracked slot with stale numbers in it. + const uint64_t other = frame + sfr::nui_skeleton_data_offset; + store_float(memory, other + 32, 3.0f); +} + +void the_guest_frame() { + sfr::GuestMemory memory; + memory.map(frame, 0x2000); + sfr::NuiFrameSmoother smoother; + require(sfr::nui_transform_smooth_guest(memory, 0, 0, smoother) == 0x80004003u, "no frame is E_POINTER"); + const uint64_t joint = frame + sfr::nui_skeleton_data_offset + 3 * sfr::nui_skeleton_data_size + 32 + 16 * 11; + write_frame(memory, 1, 0.0f); + require(sfr::nui_transform_smooth_guest(memory, frame, 0, smoother) == 0, "S_OK"); + write_frame(memory, 2, 0.1f); + sfr::nui_transform_smooth_guest(memory, frame, 0, smoother); + require(near(load_float(memory, joint), 0.0625f), "no parameters are the library's defaults"); + require(load_float(memory, joint + 12) == 1.0f, "each joint's w becomes 1"); + require(load_float(memory, frame + sfr::nui_skeleton_data_offset + 3 * sfr::nui_skeleton_data_size + 16) == 0.1f, + "the skeleton's centre is not filtered"); + require(load_float(memory, frame + sfr::nui_skeleton_data_offset + 32) == 3.0f, "an untracked slot is left alone"); + // The skeleton hook handing the same sensor frame over again: the same + // output, not a second step towards the measurement. + write_frame(memory, 2, 0.1f); + sfr::nui_transform_smooth_guest(memory, frame, 0, smoother); + require(near(load_float(memory, joint), 0.0625f), "a repeated frame is not smoothed twice"); + write_frame(memory, 3, 0.1f); + sfr::nui_transform_smooth_guest(memory, frame, 0, smoother); + require(near(load_float(memory, joint), 0.103125f), "the next frame goes on from the first time"); + // Given parameters, big-endian: no smoothing, no correction, no + // prediction is the measurement itself. + sfr::NuiFrameSmoother plain; + for (uint32_t i = 0; i < 5; ++i) store_float(memory, parameters + 4 * i, i < 3 ? 0.0f : 1.0f); + const auto given = sfr::nui_smooth_parameters(memory, parameters); + require(given.smoothing == 0.0f && given.jitter_radius == 1.0f && given.max_deviation_radius == 1.0f, "read big-endian"); + store_float(memory, parameters + 12, 0.0f); + for (uint32_t number = 10; number < 14; ++number) { + write_frame(memory, number, 0.1f * float(number)); + sfr::nui_transform_smooth_guest(memory, frame, parameters, plain); + } + require(near(load_float(memory, joint), 1.3f, 1e-5f), "smoothing 0 follows the measurement"); +} +} + +int main() { + try { + the_first_frames(); + the_prediction_is_held_back(); + jitter_is_pulled_in(); + lost_joints_start_over(); + slots_keep_their_own_history(); + the_guest_frame(); + std::cout << "NuiTransformSmooth checks passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +}