From 210915125af600b034d1024474391b737e84654e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:17:37 +0000 Subject: [PATCH 01/12] Relay proximity voice chat through the game server Players talk over the existing game connection, no separate program or port. Clients send Opus frames unreliably, the server relays them to players in cell range who have voip on and want that speaker's channel (everyone, party or strangers), with a per speaker rate limit. Voice:bVoip in STServer.ini turns it on for a server and is sent to clients in the server settings. The game client keeps its own [Voice] keys in the same file, so saving the server's ini now keeps keys it doesn't own instead of rewriting the file from scratch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- .../console/IniSettingsProvider.cpp | 10 ++ Code/encoding/Messages/ClientMessageFactory.h | 4 +- Code/encoding/Messages/NotifyVoiceData.cpp | 15 +++ Code/encoding/Messages/NotifyVoiceData.h | 23 ++++ Code/encoding/Messages/ServerMessageFactory.h | 3 +- Code/encoding/Messages/VoiceDataRequest.cpp | 13 +++ Code/encoding/Messages/VoiceDataRequest.h | 22 ++++ Code/encoding/Messages/VoiceStateRequest.cpp | 17 +++ Code/encoding/Messages/VoiceStateRequest.h | 31 +++++ Code/encoding/Opcodes.h | 3 + Code/encoding/Structs/ServerSettings.cpp | 4 +- Code/encoding/Structs/ServerSettings.h | 2 + Code/encoding/Structs/VoiceFrame.cpp | 35 ++++++ Code/encoding/Structs/VoiceFrame.h | 21 ++++ Code/server/GameServer.cpp | 3 + Code/server/Services/VoiceService.cpp | 108 ++++++++++++++++++ Code/server/Services/VoiceService.h | 54 +++++++++ Code/server/World.cpp | 2 + Code/tests/encoding.cpp | 82 +++++++++++++ 19 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 Code/encoding/Messages/NotifyVoiceData.cpp create mode 100644 Code/encoding/Messages/NotifyVoiceData.h create mode 100644 Code/encoding/Messages/VoiceDataRequest.cpp create mode 100644 Code/encoding/Messages/VoiceDataRequest.h create mode 100644 Code/encoding/Messages/VoiceStateRequest.cpp create mode 100644 Code/encoding/Messages/VoiceStateRequest.h create mode 100644 Code/encoding/Structs/VoiceFrame.cpp create mode 100644 Code/encoding/Structs/VoiceFrame.h create mode 100644 Code/server/Services/VoiceService.cpp create mode 100644 Code/server/Services/VoiceService.h diff --git a/Code/components/console/IniSettingsProvider.cpp b/Code/components/console/IniSettingsProvider.cpp index 1c148cdcc4..83dea02c2d 100644 --- a/Code/components/console/IniSettingsProvider.cpp +++ b/Code/components/console/IniSettingsProvider.cpp @@ -64,6 +64,16 @@ void SaveSettingsToIni(ConsoleRegistry& aReg, const std::filesystem::path& aPath { CSimpleIni ini; + // Start from the file on disk so keys that aren't settings here survive, the game client keeps its own + // [Voice] keys in the same STServer.ini + std::error_code existsError{}; + if (std::filesystem::exists(aPath, existsError) && !existsError) + { + const auto existing = TiltedPhoques::LoadFile(aPath); + if (ini.LoadData(existing.c_str()) != SI_Error::SI_OK) + ini.Reset(); + } + SI_Error error{SI_Error::SI_OK}; aReg.ForAllSettings( [&](SettingBase* setting) diff --git a/Code/encoding/Messages/ClientMessageFactory.h b/Code/encoding/Messages/ClientMessageFactory.h index 4f94d6d10e..55f1dcda5b 100644 --- a/Code/encoding/Messages/ClientMessageFactory.h +++ b/Code/encoding/Messages/ClientMessageFactory.h @@ -56,6 +56,8 @@ #include #include #include +#include +#include using TiltedPhoques::UniquePtr; @@ -70,7 +72,7 @@ struct ClientMessageFactory PartyChangeLeaderRequest, PartyKickRequest, RequestActorValueChanges, RequestActorMaxValueChanges, EnterExteriorCellRequest, RequestHealthChangeBroadcast, ActivateRequest, LockChangeRequest, AssignObjectsRequest, RequestDeathStateChange, ShiftGridCellRequest, RequestOwnershipTransfer, RequestOwnershipClaim, RequestObjectInventoryChanges, SpellCastRequest, ProjectileLaunchRequest, InterruptCastRequest, AddTargetRequest, ScriptAnimationRequest, DrawWeaponRequest, MountRequest, NewPackageRequest, RequestRespawn, SyncExperienceRequest, RequestEquipmentChanges, SendChatMessageRequest, TeleportCommandRequest, PlayerRespawnRequest, DialogueRequest, SubtitleRequest, PlayerDialogueRequest, PlayerLevelRequest, TeleportRequest, RequestPlayerHealthUpdate, RequestWeatherChange, RequestCurrentWeather, RequestSetWaypoint, - RequestRemoveWaypoint, RemoveSpellRequest, SetTimeCommandRequest, RequestActionEvents>; + RequestRemoveWaypoint, RemoveSpellRequest, SetTimeCommandRequest, RequestActionEvents, VoiceDataRequest, VoiceStateRequest>; return s_visitor(std::forward(func)); } diff --git a/Code/encoding/Messages/NotifyVoiceData.cpp b/Code/encoding/Messages/NotifyVoiceData.cpp new file mode 100644 index 0000000000..ec4e033a0d --- /dev/null +++ b/Code/encoding/Messages/NotifyVoiceData.cpp @@ -0,0 +1,15 @@ +#include + +void NotifyVoiceData::SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept +{ + Serialization::WriteVarInt(aWriter, PlayerId); + Frame.Serialize(aWriter); +} + +void NotifyVoiceData::DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept +{ + ServerMessage::DeserializeRaw(aReader); + + PlayerId = Serialization::ReadVarInt(aReader) & 0xFFFFFFFF; + Frame.Deserialize(aReader); +} diff --git a/Code/encoding/Messages/NotifyVoiceData.h b/Code/encoding/Messages/NotifyVoiceData.h new file mode 100644 index 0000000000..a706ebe000 --- /dev/null +++ b/Code/encoding/Messages/NotifyVoiceData.h @@ -0,0 +1,23 @@ +#pragma once + +#include "Message.h" +#include + +// Another player's voice, relayed by the server +struct NotifyVoiceData final : ServerMessage +{ + static constexpr ServerOpcode Opcode = kNotifyVoiceData; + + NotifyVoiceData() + : ServerMessage(Opcode) + { + } + + void SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept override; + void DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept override; + + bool operator==(const NotifyVoiceData& acRhs) const noexcept { return GetOpcode() == acRhs.GetOpcode() && PlayerId == acRhs.PlayerId && Frame == acRhs.Frame; } + + uint32_t PlayerId{}; + VoiceFrame Frame{}; +}; diff --git a/Code/encoding/Messages/ServerMessageFactory.h b/Code/encoding/Messages/ServerMessageFactory.h index 516878b1d9..5bb30756c4 100644 --- a/Code/encoding/Messages/ServerMessageFactory.h +++ b/Code/encoding/Messages/ServerMessageFactory.h @@ -59,6 +59,7 @@ #include #include #include +#include using TiltedPhoques::UniquePtr; @@ -73,7 +74,7 @@ struct ServerMessageFactory NotifyActorValueChanges, NotifyPartyJoined, NotifyPartyLeft, NotifyActorMaxValueChanges, NotifyHealthChangeBroadcast, NotifyActivate, NotifyLockChange, AssignObjectsResponse, NotifyDeathStateChange, NotifyOwnershipTransfer, NotifyObjectInventoryChanges, NotifySpellCast, NotifyProjectileLaunch, NotifyInterruptCast, NotifyAddTarget, NotifyScriptAnimation, NotifyDrawWeapon, NotifyMount, NotifyNewPackage, NotifyRespawn, NotifySyncExperience, NotifyEquipmentChanges, NotifyChatMessageBroadcast, TeleportCommandResponse, NotifyPlayerRespawn, NotifyDialogue, NotifySubtitle, NotifyPlayerDialogue, NotifyActorTeleport, NotifyPlayerLeft, NotifyPlayerJoined, NotifyDialogue, NotifySubtitle, NotifyPlayerDialogue, NotifyPlayerLevel, NotifyPlayerCellChanged, NotifyTeleport, NotifyPlayerHealthUpdate, NotifySettingsChange, - NotifyWeatherChange, NotifySetWaypoint, NotifyRemoveWaypoint, NotifySetTimeResult, NotifyRemoveSpell, NotifyActionEvents>; + NotifyWeatherChange, NotifySetWaypoint, NotifyRemoveWaypoint, NotifySetTimeResult, NotifyRemoveSpell, NotifyActionEvents, NotifyVoiceData>; return s_visitor(std::forward(func)); } diff --git a/Code/encoding/Messages/VoiceDataRequest.cpp b/Code/encoding/Messages/VoiceDataRequest.cpp new file mode 100644 index 0000000000..24fededade --- /dev/null +++ b/Code/encoding/Messages/VoiceDataRequest.cpp @@ -0,0 +1,13 @@ +#include + +void VoiceDataRequest::SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept +{ + Frame.Serialize(aWriter); +} + +void VoiceDataRequest::DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept +{ + ClientMessage::DeserializeRaw(aReader); + + Frame.Deserialize(aReader); +} diff --git a/Code/encoding/Messages/VoiceDataRequest.h b/Code/encoding/Messages/VoiceDataRequest.h new file mode 100644 index 0000000000..dbbb83a630 --- /dev/null +++ b/Code/encoding/Messages/VoiceDataRequest.h @@ -0,0 +1,22 @@ +#pragma once + +#include "Message.h" +#include + +// The local player's voice, sent unreliably, the server relays it to players in range +struct VoiceDataRequest final : ClientMessage +{ + static constexpr ClientOpcode Opcode = kVoiceDataRequest; + + VoiceDataRequest() + : ClientMessage(Opcode) + { + } + + void SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept override; + void DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept override; + + bool operator==(const VoiceDataRequest& acRhs) const noexcept { return GetOpcode() == acRhs.GetOpcode() && Frame == acRhs.Frame; } + + VoiceFrame Frame{}; +}; diff --git a/Code/encoding/Messages/VoiceStateRequest.cpp b/Code/encoding/Messages/VoiceStateRequest.cpp new file mode 100644 index 0000000000..fcb9e69cf5 --- /dev/null +++ b/Code/encoding/Messages/VoiceStateRequest.cpp @@ -0,0 +1,17 @@ +#include + +void VoiceStateRequest::SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept +{ + Serialization::WriteBool(aWriter, Listening); + aWriter.WriteBits(ListenChannel, 8); +} + +void VoiceStateRequest::DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept +{ + ClientMessage::DeserializeRaw(aReader); + + Listening = Serialization::ReadBool(aReader); + uint64_t channel = 0; + aReader.ReadBits(channel, 8); + ListenChannel = channel < kChannelCount ? static_cast(channel) : kEveryone; +} diff --git a/Code/encoding/Messages/VoiceStateRequest.h b/Code/encoding/Messages/VoiceStateRequest.h new file mode 100644 index 0000000000..3ff6bc4b04 --- /dev/null +++ b/Code/encoding/Messages/VoiceStateRequest.h @@ -0,0 +1,31 @@ +#pragma once + +#include "Message.h" + +// Which voices the local player wants, so the server only relays what will be played +struct VoiceStateRequest final : ClientMessage +{ + static constexpr ClientOpcode Opcode = kVoiceStateRequest; + + enum Channel : uint8_t + { + kEveryone = 0, + kParty, + kStrangers, + kChannelCount + }; + + VoiceStateRequest() + : ClientMessage(Opcode) + { + } + + void SerializeRaw(TiltedPhoques::Buffer::Writer& aWriter) const noexcept override; + void DeserializeRaw(TiltedPhoques::Buffer::Reader& aReader) noexcept override; + + bool operator==(const VoiceStateRequest& acRhs) const noexcept { return GetOpcode() == acRhs.GetOpcode() && Listening == acRhs.Listening && ListenChannel == acRhs.ListenChannel; } + + // False while the player has voip off, nothing is relayed to them + bool Listening{}; + Channel ListenChannel{kEveryone}; +}; diff --git a/Code/encoding/Opcodes.h b/Code/encoding/Opcodes.h index baab493ccb..3de7fc29f3 100644 --- a/Code/encoding/Opcodes.h +++ b/Code/encoding/Opcodes.h @@ -54,6 +54,8 @@ enum ClientOpcode : unsigned char kRequestRemoveWaypoint, kSetTimeCommandRequest, kRequestActionEvents, + kVoiceDataRequest, + kVoiceStateRequest, kClientOpcodeMax }; @@ -114,5 +116,6 @@ enum ServerOpcode : unsigned char kNotifyRemoveWaypoint, kNotifySetTimeResult, kNotifyActionEvents, + kNotifyVoiceData, kServerOpcodeMax }; diff --git a/Code/encoding/Structs/ServerSettings.cpp b/Code/encoding/Structs/ServerSettings.cpp index bdf8f77adc..9cf48ecb24 100644 --- a/Code/encoding/Structs/ServerSettings.cpp +++ b/Code/encoding/Structs/ServerSettings.cpp @@ -5,7 +5,7 @@ using TiltedPhoques::Serialization; bool ServerSettings::operator==(const ServerSettings& acRhs) const noexcept { - return Difficulty == acRhs.Difficulty && GreetingsEnabled == acRhs.GreetingsEnabled && PvpEnabled == acRhs.PvpEnabled && SyncPlayerHomes == acRhs.SyncPlayerHomes && DeathSystemEnabled == acRhs.DeathSystemEnabled && AutoPartyJoin == acRhs.AutoPartyJoin; + return Difficulty == acRhs.Difficulty && GreetingsEnabled == acRhs.GreetingsEnabled && PvpEnabled == acRhs.PvpEnabled && SyncPlayerHomes == acRhs.SyncPlayerHomes && DeathSystemEnabled == acRhs.DeathSystemEnabled && AutoPartyJoin == acRhs.AutoPartyJoin && VoipEnabled == acRhs.VoipEnabled; } bool ServerSettings::operator!=(const ServerSettings& acRhs) const noexcept @@ -22,6 +22,7 @@ void ServerSettings::Serialize(TiltedPhoques::Buffer::Writer& aWriter) const noe Serialization::WriteBool(aWriter, DeathSystemEnabled); Serialization::WriteBool(aWriter, SyncPlayerCalendar); Serialization::WriteBool(aWriter, AutoPartyJoin); + Serialization::WriteBool(aWriter, VoipEnabled); } void ServerSettings::Deserialize(TiltedPhoques::Buffer::Reader& aReader) noexcept @@ -33,4 +34,5 @@ void ServerSettings::Deserialize(TiltedPhoques::Buffer::Reader& aReader) noexcep DeathSystemEnabled = Serialization::ReadBool(aReader); SyncPlayerCalendar = Serialization::ReadBool(aReader); AutoPartyJoin = Serialization::ReadBool(aReader); + VoipEnabled = Serialization::ReadBool(aReader); } diff --git a/Code/encoding/Structs/ServerSettings.h b/Code/encoding/Structs/ServerSettings.h index 02a49302fe..dfe32e5073 100644 --- a/Code/encoding/Structs/ServerSettings.h +++ b/Code/encoding/Structs/ServerSettings.h @@ -17,4 +17,6 @@ struct ServerSettings bool DeathSystemEnabled{}; bool SyncPlayerCalendar{}; bool AutoPartyJoin{}; + // Built-in proximity voice chat, players also need Connect to voip on + bool VoipEnabled{}; }; diff --git a/Code/encoding/Structs/VoiceFrame.cpp b/Code/encoding/Structs/VoiceFrame.cpp new file mode 100644 index 0000000000..75e972fba5 --- /dev/null +++ b/Code/encoding/Structs/VoiceFrame.cpp @@ -0,0 +1,35 @@ +#include "VoiceFrame.h" +#include + +using TiltedPhoques::Serialization; + +void VoiceFrame::Serialize(TiltedPhoques::Buffer::Writer& aWriter) const noexcept +{ + const size_t size = std::min(Data.size(), kMaxSize); + + aWriter.WriteBits(Sequence, 16); + Serialization::WriteVarInt(aWriter, size); + if (size > 0) + aWriter.WriteBytes(Data.data(), size); +} + +bool VoiceFrame::Deserialize(TiltedPhoques::Buffer::Reader& aReader) noexcept +{ + uint64_t sequence = 0; + aReader.ReadBits(sequence, 16); + Sequence = static_cast(sequence); + + const uint64_t size = Serialization::ReadVarInt(aReader); + Data.clear(); + if (size == 0 || size > kMaxSize) + return false; + + Data.resize(static_cast(size)); + if (!aReader.ReadBytes(Data.data(), Data.size())) + { + Data.clear(); + return false; + } + + return true; +} diff --git a/Code/encoding/Structs/VoiceFrame.h b/Code/encoding/Structs/VoiceFrame.h new file mode 100644 index 0000000000..f5201be221 --- /dev/null +++ b/Code/encoding/Structs/VoiceFrame.h @@ -0,0 +1,21 @@ +#pragma once + +using TiltedPhoques::Buffer; + +// One 20 ms Opus packet of a player's voice +struct VoiceFrame +{ + // Opus packets for 20 ms of speech at our bitrate are well under this, anything bigger is dropped + static constexpr size_t kMaxSize = 512; + + bool operator==(const VoiceFrame& acRhs) const noexcept { return Sequence == acRhs.Sequence && Data == acRhs.Data; } + bool operator!=(const VoiceFrame& acRhs) const noexcept { return !this->operator==(acRhs); } + + void Serialize(TiltedPhoques::Buffer::Writer& aWriter) const noexcept; + // False when the frame is empty or too big to be a voice packet + bool Deserialize(TiltedPhoques::Buffer::Reader& aReader) noexcept; + + // Counts up per frame the speaker sends, receivers use it to spot lost and reordered frames + uint16_t Sequence{}; + TiltedPhoques::Vector Data{}; +}; diff --git a/Code/server/GameServer.cpp b/Code/server/GameServer.cpp index de5e917934..3c08255303 100644 --- a/Code/server/GameServer.cpp +++ b/Code/server/GameServer.cpp @@ -45,6 +45,8 @@ Console::Setting bAutoPartyJoin{"Gameplay:bAutoPartyJoin", "Join parties automat Console::Setting bEnableModCheck{"ModPolicy:bEnableModCheck", "Refuse clients whose mods do not match Data/loadorder.txt", false, Console::SettingsFlags::kLocked}; Console::Setting bAllowSKSE{"ModPolicy:bAllowSKSE", "Allow players using SKSE to join. Applies even when bEnableModCheck is off", true, Console::SettingsFlags::kLocked}; Console::Setting bAllowMO2{"ModPolicy:bAllowMO2", "Allow players using Mod Organizer 2 to join. Applies even when bEnableModCheck is off", true, Console::SettingsFlags::kLocked}; +// Voice, the game client reads and writes this section too when the server runs from the game folder +Console::Setting bVoip{"Voice:bVoip", "Built-in proximity voice chat. On a server: lets players talk to players nearby. For a player: the Connect to voip checkbox in Settings", false}; // -- Commands -- Console::Command<> TogglePremium( @@ -141,6 +143,7 @@ ServerSettings GetSettings() settings.DeathSystemEnabled = bEnableDeathSystem; settings.SyncPlayerCalendar = bSyncPlayerCalendar; settings.AutoPartyJoin = bAutoPartyJoin; + settings.VoipEnabled = bVoip; return settings; } diff --git a/Code/server/Services/VoiceService.cpp b/Code/server/Services/VoiceService.cpp new file mode 100644 index 0000000000..20cb7fbd4f --- /dev/null +++ b/Code/server/Services/VoiceService.cpp @@ -0,0 +1,108 @@ +#include + +#include +#include +#include + +#include + +#include +#include +#include + +#include + +extern Console::Setting bVoip; + +namespace +{ +// A speaker sends 50 frames a second, this leaves room for bursts after a stall without letting a client flood +constexpr uint32_t kMaxFramesPerSecond = 75; +} // namespace + +VoiceService::VoiceService(World& aWorld, entt::dispatcher& aDispatcher) noexcept + : m_world(aWorld) +{ + m_voiceDataConnection = aDispatcher.sink>().connect<&VoiceService::OnVoiceDataRequest>(this); + m_voiceStateConnection = aDispatcher.sink>().connect<&VoiceService::OnVoiceStateRequest>(this); + m_playerLeaveConnection = aDispatcher.sink().connect<&VoiceService::OnPlayerLeave>(this); +} + +void VoiceService::OnVoiceDataRequest(const PacketEvent& acMessage) noexcept +{ + if (!bVoip) + return; + + const auto& frame = acMessage.Packet.Frame; + if (frame.Data.empty()) + return; + + Player* const pSpeaker = acMessage.pPlayer; + if (!IsWithinRateLimit(pSpeaker->GetId())) + return; + + // Only players who are in the world can be heard + const auto speakerCharacter = pSpeaker->GetCharacter(); + if (!speakerCharacter) + return; + + const auto* pSpeakerCell = m_world.try_get(*speakerCharacter); + const CellIdComponent& speakerCell = pSpeakerCell ? *pSpeakerCell : pSpeaker->GetCellComponent(); + const auto& speakerParty = pSpeaker->GetParty().JoinedPartyId; + + NotifyVoiceData notify{}; + notify.PlayerId = pSpeaker->GetId(); + notify.Frame = frame; + + for (Player* pListener : m_world.GetPlayerManager()) + { + if (pListener == pSpeaker) + continue; + + const auto listenerItor = m_listeners.find(pListener->GetId()); + if (listenerItor == m_listeners.end() || !listenerItor->second.Listening) + continue; + + const auto& listenerParty = pListener->GetParty().JoinedPartyId; + const bool sameParty = speakerParty.has_value() && listenerParty.has_value() && *speakerParty == *listenerParty; + + const auto channel = listenerItor->second.Channel; + if (channel == VoiceStateRequest::kParty && !sameParty) + continue; + if (channel == VoiceStateRequest::kStrangers && sameParty) + continue; + + if (!speakerCell.IsInRange(pListener->GetCellComponent(), false)) + continue; + + // Late voice is useless, so no resends, and no Nagle delay either + pListener->Send(notify, TiltedPhoques::kUnreliableNoNagle); + } +} + +void VoiceService::OnVoiceStateRequest(const PacketEvent& acMessage) noexcept +{ + auto& listener = m_listeners[acMessage.pPlayer->GetId()]; + listener.Listening = acMessage.Packet.Listening; + listener.Channel = acMessage.Packet.ListenChannel; +} + +void VoiceService::OnPlayerLeave(const PlayerLeaveEvent& acEvent) noexcept +{ + m_listeners.erase(acEvent.pPlayer->GetId()); + m_speakers.erase(acEvent.pPlayer->GetId()); +} + +bool VoiceService::IsWithinRateLimit(uint32_t aPlayerId) noexcept +{ + const auto now = std::chrono::steady_clock::now(); + auto& speaker = m_speakers[aPlayerId]; + + if (now - speaker.WindowStart >= std::chrono::seconds(1)) + { + speaker.WindowStart = now; + speaker.FramesInWindow = 0; + } + + return ++speaker.FramesInWindow <= kMaxFramesPerSecond; +} diff --git a/Code/server/Services/VoiceService.h b/Code/server/Services/VoiceService.h new file mode 100644 index 0000000000..dc00123800 --- /dev/null +++ b/Code/server/Services/VoiceService.h @@ -0,0 +1,54 @@ +#pragma once + +#include + +struct World; +struct Player; +struct PlayerLeaveEvent; +struct VoiceDataRequest; +struct VoiceStateRequest; + +/** + * @brief Relays proximity voice chat between players. + * + * Voice frames are Opus packets the clients encode and decode themselves, the server only decides who gets them: + * players in range of the speaker who have voip on and want that speaker's channel. Distance falloff is done by + * each listener. Enabled with Voice:bVoip in STServer.ini. + */ +struct VoiceService +{ + VoiceService(World& aWorld, entt::dispatcher& aDispatcher) noexcept; + ~VoiceService() noexcept = default; + + TP_NOCOPYMOVE(VoiceService); + +protected: + void OnVoiceDataRequest(const PacketEvent& acMessage) noexcept; + void OnVoiceStateRequest(const PacketEvent& acMessage) noexcept; + void OnPlayerLeave(const PlayerLeaveEvent& acEvent) noexcept; + +private: + struct Listener + { + bool Listening{false}; + uint8_t Channel{0}; + }; + + struct Speaker + { + std::chrono::steady_clock::time_point WindowStart{}; + uint32_t FramesInWindow{0}; + }; + + bool IsWithinRateLimit(uint32_t aPlayerId) noexcept; + + World& m_world; + + // Keyed by player id + Map m_listeners; + Map m_speakers; + + entt::scoped_connection m_voiceDataConnection; + entt::scoped_connection m_voiceStateConnection; + entt::scoped_connection m_playerLeaveConnection; +}; diff --git a/Code/server/World.cpp b/Code/server/World.cpp index 8bfd510499..b0e03da066 100644 --- a/Code/server/World.cpp +++ b/Code/server/World.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -37,6 +38,7 @@ World::World() ctx().emplace(*this, m_dispatcher); ctx().emplace(*this, m_dispatcher); ctx().emplace(*this, m_dispatcher); + ctx().emplace(*this, m_dispatcher); ESLoader::ESLoader loader; // emplace loaded mods into modscomponent. diff --git a/Code/tests/encoding.cpp b/Code/tests/encoding.cpp index defb808418..ac8f7fd049 100644 --- a/Code/tests/encoding.cpp +++ b/Code/tests/encoding.cpp @@ -527,6 +527,88 @@ TEST_CASE("Packets", "[encoding.packets]") REQUIRE(pMessage->GetOpcode() == sendMessage.GetOpcode()); REQUIRE(*CastUnique(std::move(pMessage)) == sendMessage); } + + SECTION("VoiceDataRequest") + { + Buffer buff(1000); + + VoiceDataRequest sendMessage; + sendMessage.Frame.Sequence = 65535; + for (uint32_t i = 0; i < 97; ++i) + sendMessage.Frame.Data.push_back(static_cast(i * 7)); + + Buffer::Writer writer(&buff); + sendMessage.Serialize(writer); + + Buffer::Reader reader(&buff); + + const ClientMessageFactory factory; + auto pMessage = factory.Extract(reader); + + REQUIRE(pMessage); + REQUIRE(pMessage->GetOpcode() == sendMessage.GetOpcode()); + REQUIRE(*CastUnique(std::move(pMessage)) == sendMessage); + } + + SECTION("NotifyVoiceData") + { + Buffer buff(1000); + + NotifyVoiceData sendMessage; + sendMessage.PlayerId = 123456; + sendMessage.Frame.Sequence = 42; + for (uint32_t i = 0; i < 60; ++i) + sendMessage.Frame.Data.push_back(static_cast(255 - i)); + + Buffer::Writer writer(&buff); + sendMessage.Serialize(writer); + + Buffer::Reader reader(&buff); + + const ServerMessageFactory factory; + auto pMessage = factory.Extract(reader); + + REQUIRE(pMessage); + REQUIRE(pMessage->GetOpcode() == sendMessage.GetOpcode()); + REQUIRE(*CastUnique(std::move(pMessage)) == sendMessage); + } + + SECTION("VoiceFrame rejects oversized data") + { + Buffer buff(2000); + + // Written by hand, Serialize itself never writes more than kMaxSize + Buffer::Writer writer(&buff); + writer.WriteBits(7, 16); + Serialization::WriteVarInt(writer, VoiceFrame::kMaxSize + 1); + + Buffer::Reader reader(&buff); + + VoiceFrame frame; + REQUIRE_FALSE(frame.Deserialize(reader)); + REQUIRE(frame.Data.empty()); + } + + SECTION("VoiceStateRequest") + { + Buffer buff(1000); + + VoiceStateRequest sendMessage; + sendMessage.Listening = true; + sendMessage.ListenChannel = VoiceStateRequest::kStrangers; + + Buffer::Writer writer(&buff); + sendMessage.Serialize(writer); + + Buffer::Reader reader(&buff); + + const ClientMessageFactory factory; + auto pMessage = factory.Extract(reader); + + REQUIRE(pMessage); + REQUIRE(pMessage->GetOpcode() == sendMessage.GetOpcode()); + REQUIRE(*CastUnique(std::move(pMessage)) == sendMessage); + } } TEST_CASE("StringCache", "[encoding.string_cache]") From cba4a706f92e5ef9cd9506177361d2c456ecf764 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:17:38 +0000 Subject: [PATCH 02/12] Add built-in proximity voice chat to the client The microphone is captured with miniaudio, gated, Opus encoded and sent to the server. Remote voices are decoded into small jitter buffers and mixed on the voip output device, full volume up to the fade start and silent at twice that distance from the camera, panned left and right. Settings live in [Voice] of Data/SkyrimTogetherReborn/config/STServer.ini: input and output device, bVoip (Connect to voip, the same key a server uses), channel, fade start, mic volume and noise gate. The input device defaults to none, so nothing is captured until a player picks one. The noise gate, falloff, panning and jitter buffer are game independent and unit tested in TPTests. The camera transform offsets were checked in Ghidra on 1.7.104 earlier (see the comment in VoiceService.cpp). New dependencies: libopus 1.5.2 (BSD-3-Clause) and miniaudio 0.11.22 (public domain or MIT-0), Windows client only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- .../client/Services/Generic/OverlayClient.cpp | 34 ++ Code/client/Services/Generic/VoiceAudio.cpp | 518 ++++++++++++++++++ Code/client/Services/Generic/VoiceService.cpp | 464 ++++++++++++++++ Code/client/Services/OverlayClient.h | 1 + Code/client/Services/VoiceAudio.h | 55 ++ Code/client/Services/VoiceService.h | 99 ++++ Code/client/Systems/VoiceDsp.cpp | 117 ++++ Code/client/Systems/VoiceDsp.h | 77 +++ Code/client/World.cpp | 2 + Code/client/xmake.lua | 4 +- Code/tests/voice_dsp.cpp | 96 ++++ Code/tests/xmake.lua | 2 + Code/tp_process/ProcessHandler.cpp | 4 + xmake.lua | 5 +- 14 files changed, 1476 insertions(+), 2 deletions(-) create mode 100644 Code/client/Services/Generic/VoiceAudio.cpp create mode 100644 Code/client/Services/Generic/VoiceService.cpp create mode 100644 Code/client/Services/VoiceAudio.h create mode 100644 Code/client/Services/VoiceService.h create mode 100644 Code/client/Systems/VoiceDsp.cpp create mode 100644 Code/client/Systems/VoiceDsp.h create mode 100644 Code/tests/voice_dsp.cpp diff --git a/Code/client/Services/Generic/OverlayClient.cpp b/Code/client/Services/Generic/OverlayClient.cpp index a0378612fa..9610444371 100644 --- a/Code/client/Services/Generic/OverlayClient.cpp +++ b/Code/client/Services/Generic/OverlayClient.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -82,6 +83,21 @@ bool OverlayClient::OnProcessMessageReceived(CefRefPtr browser, CefR ProcessToggleDebugUI(); else if (eventName == "openPlayGuide") ProcessOpenPlayGuide(); + else if (eventName == "voiceRefresh") + World::Get().GetRunner().Queue([]() { World::Get().ctx().at().RefreshUi(); }); + else if (eventName == "setVoiceSettings") + ProcessSetVoiceSettings(eventArgs); + else if (eventName == "setVoicePlayerVolume") + { + const uint32_t playerId = eventArgs->GetInt(0); + const float volume = static_cast(eventArgs->GetInt(1)) / 100.f; + World::Get().GetRunner().Queue([playerId, volume]() { World::Get().ctx().at().SetPlayerVolume(playerId, volume); }); + } + else if (eventName == "setVoiceMeter") + { + const bool enabled = eventArgs->GetInt(0) != 0; + World::Get().GetRunner().Queue([enabled]() { World::Get().ctx().at().SetMeterEnabled(enabled); }); + } return true; } @@ -157,6 +173,24 @@ void OverlayClient::ProcessOpenPlayGuide() ShellExecuteW(nullptr, L"open", LR"(https://wiki.tiltedphoques.com/tilted-online/general-information/playguide)", nullptr, nullptr, SW_SHOWNORMAL); } +// Numbers come as whole numbers from the UI: percentages for the volumes and units for the fade start +void OverlayClient::ProcessSetVoiceSettings(CefRefPtr aEventArgs) +{ + VoiceService::Settings settings{}; + settings.InputDevice = aEventArgs->GetString(0).ToString().c_str(); + settings.Connect = aEventArgs->GetInt(1) != 0; + settings.OutputDevice = aEventArgs->GetString(2).ToString().c_str(); + + const auto channel = aEventArgs->GetString(3).ToString(); + settings.ListenChannel = channel == "party" ? VoiceService::Channel::kParty : channel == "strangers" ? VoiceService::Channel::kStrangers : VoiceService::Channel::kEveryone; + + settings.FadeStart = static_cast(aEventArgs->GetInt(4)); + settings.MicVolume = static_cast(aEventArgs->GetInt(5)) / 100.f; + settings.NoiseGate = static_cast(aEventArgs->GetInt(6)) / 100.f; + + World::Get().GetRunner().Queue([settings]() { World::Get().ctx().at().ApplySettings(settings); }); +} + void OverlayClient::SetUIVisible(bool aVisible) noexcept { auto pRenderer = GetOverlayRenderHandler(); diff --git a/Code/client/Services/Generic/VoiceAudio.cpp b/Code/client/Services/Generic/VoiceAudio.cpp new file mode 100644 index 0000000000..abc670a19a --- /dev/null +++ b/Code/client/Services/Generic/VoiceAudio.cpp @@ -0,0 +1,518 @@ +#include + +#include +#include + +// Only devices are used, leave the decoders, encoders and engine out +#define MA_NO_DECODING +#define MA_NO_ENCODING +#define MA_NO_ENGINE +#define MA_NO_RESOURCE_MANAGER +#define MA_NO_NODE_GRAPH +#define MA_NO_GENERATION +#define MINIAUDIO_IMPLEMENTATION +#include + +#include + +#include +#include +#include +#include + +namespace +{ +// Speech at this bitrate sounds clear, 50 frames a second come to about 3 KB/s per speaker +constexpr int32_t kOpusBitrate = 24000; +constexpr size_t kMaxQueuedFrames = 50; +// Missing frames up to this many are concealed, a longer gap restarts the voice +constexpr int32_t kMaxConcealedFrames = 3; +constexpr auto kActiveSpeakerTime = std::chrono::milliseconds(300); +constexpr char kDefaultDeviceName[] = "default"; +} // namespace + +struct VoiceAudio::Impl +{ + struct Speaker + { + VoiceDsp::JitterBuffer Buffer; + float LeftGain{0.f}; + float RightGain{0.f}; + }; + + // Main thread only + struct Decoder + { + OpusDecoder* pDecoder{nullptr}; + uint16_t LastSequence{0}; + bool HasSequence{false}; + std::chrono::steady_clock::time_point LastFrameTime{}; + }; + + ~Impl() noexcept + { + StopCapture(); + StopPlayback(); + + for (auto& [id, decoder] : Decoders) + opus_decoder_destroy(decoder.pDecoder); + + if (pEncoder) + opus_encoder_destroy(pEncoder); + + if (HasContext) + ma_context_uninit(&Context); + } + + bool EnsureContext() noexcept + { + if (HasContext) + return true; + + if (ma_context_init(nullptr, 0, nullptr, &Context) != MA_SUCCESS) + { + spdlog::error("Voice chat: no audio backend could be started"); + return false; + } + + HasContext = true; + return true; + } + + bool FindDevice(ma_device_type aType, const String& acName, ma_device_id& aOutId) noexcept + { + ma_device_info* pPlayback = nullptr; + ma_uint32 playbackCount = 0; + ma_device_info* pCapture = nullptr; + ma_uint32 captureCount = 0; + if (ma_context_get_devices(&Context, &pPlayback, &playbackCount, &pCapture, &captureCount) != MA_SUCCESS) + return false; + + ma_device_info* pInfos = aType == ma_device_type_capture ? pCapture : pPlayback; + const ma_uint32 count = aType == ma_device_type_capture ? captureCount : playbackCount; + for (ma_uint32 i = 0; i < count; ++i) + { + if (acName == pInfos[i].name) + { + aOutId = pInfos[i].id; + return true; + } + } + + return false; + } + + void StartCapture(const String& acName) noexcept + { + StopCapture(); + CaptureName = acName; + if (acName.empty()) + return; + + ma_device_id id{}; + const bool useDefault = acName == kDefaultDeviceName; + if (!useDefault && !FindDevice(ma_device_type_capture, acName, id)) + { + spdlog::warn("Voice chat: input device \"{}\" is not connected, nothing is sent", acName); + return; + } + + if (!pEncoder) + { + int error = OPUS_OK; + pEncoder = opus_encoder_create(VoiceDsp::kSampleRate, 1, OPUS_APPLICATION_VOIP, &error); + if (error != OPUS_OK || !pEncoder) + { + spdlog::error("Voice chat: Opus encoder failed to start ({})", error); + pEncoder = nullptr; + return; + } + + opus_encoder_ctl(pEncoder, OPUS_SET_BITRATE(kOpusBitrate)); + opus_encoder_ctl(pEncoder, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE)); + } + + ma_device_config config = ma_device_config_init(ma_device_type_capture); + config.capture.format = ma_format_f32; + config.capture.channels = 1; + config.capture.pDeviceID = useDefault ? nullptr : &id; + config.sampleRate = VoiceDsp::kSampleRate; + config.periodSizeInMilliseconds = 10; + config.dataCallback = &Impl::OnCapture; + config.pUserData = this; + + CaptureFill = 0; + if (ma_device_init(&Context, &config, &CaptureDevice) != MA_SUCCESS) + { + spdlog::error("Voice chat: input device \"{}\" could not be opened", acName); + return; + } + + HasCapture = true; + if (ma_device_start(&CaptureDevice) != MA_SUCCESS) + { + spdlog::error("Voice chat: input device \"{}\" could not be started", acName); + StopCapture(); + return; + } + + spdlog::info("Voice chat: capturing from \"{}\"", CaptureDevice.capture.name); + } + + void StopCapture() noexcept + { + if (HasCapture) + { + ma_device_uninit(&CaptureDevice); + HasCapture = false; + } + + MicLevel = 0.f; + } + + void StartPlayback(const String& acName) noexcept + { + StopPlayback(); + PlaybackName = acName; + + ma_device_id id{}; + bool useDefault = acName.empty() || acName == kDefaultDeviceName; + if (!useDefault && !FindDevice(ma_device_type_playback, acName, id)) + { + spdlog::warn("Voice chat: output device \"{}\" is not connected, using the Windows default", acName); + useDefault = true; + } + + ma_device_config config = ma_device_config_init(ma_device_type_playback); + config.playback.format = ma_format_f32; + config.playback.channels = 2; + config.playback.pDeviceID = useDefault ? nullptr : &id; + config.sampleRate = VoiceDsp::kSampleRate; + config.periodSizeInMilliseconds = 10; + config.dataCallback = &Impl::OnPlayback; + config.pUserData = this; + + if (ma_device_init(&Context, &config, &PlaybackDevice) != MA_SUCCESS) + { + spdlog::error("Voice chat: output device could not be opened"); + return; + } + + HasPlayback = true; + if (ma_device_start(&PlaybackDevice) != MA_SUCCESS) + { + spdlog::error("Voice chat: output device could not be started"); + StopPlayback(); + return; + } + + spdlog::info("Voice chat: playing on \"{}\"", PlaybackDevice.playback.name); + } + + void StopPlayback() noexcept + { + if (HasPlayback) + { + ma_device_uninit(&PlaybackDevice); + HasPlayback = false; + } + } + + // Capture thread + static void OnCapture(ma_device* apDevice, void*, const void* apInput, ma_uint32 aFrameCount) + { + auto* pSelf = static_cast(apDevice->pUserData); + const auto* pInput = static_cast(apInput); + if (!pSelf || !pInput) + return; + + for (ma_uint32 i = 0; i < aFrameCount; ++i) + { + pSelf->CaptureFrame[pSelf->CaptureFill++] = pInput[i]; + if (pSelf->CaptureFill == VoiceDsp::kFrameSamples) + { + pSelf->CaptureFill = 0; + pSelf->ProcessCapturedFrame(); + } + } + } + + // Capture thread + void ProcessCapturedFrame() noexcept + { + const float volume = MicVolume.load(std::memory_order_relaxed); + for (float& sample : CaptureFrame) + sample = std::clamp(sample * volume, -1.f, 1.f); + + const float peak = VoiceDsp::Peak(CaptureFrame.data(), CaptureFrame.size()); + MicLevel.store(peak, std::memory_order_relaxed); + + const bool open = Gate.Process(peak, NoiseGate.load(std::memory_order_relaxed)); + if (!open || !Transmitting.load(std::memory_order_relaxed) || !pEncoder) + return; + + uint8_t packet[VoiceFrame::kMaxSize]; + const opus_int32 size = opus_encode_float(pEncoder, CaptureFrame.data(), VoiceDsp::kFrameSamples, packet, sizeof(packet)); + if (size <= 0) + return; + + VoiceFrame frame{}; + frame.Sequence = NextSequence++; + frame.Data.assign(packet, packet + size); + + std::scoped_lock lock(OutgoingMutex); + if (Outgoing.size() >= kMaxQueuedFrames) + Outgoing.erase(Outgoing.begin()); + Outgoing.push_back(std::move(frame)); + } + + // Playback thread + static void OnPlayback(ma_device* apDevice, void* apOutput, const void*, ma_uint32 aFrameCount) + { + auto* pSelf = static_cast(apDevice->pUserData); + auto* pOutput = static_cast(apOutput); + if (!pOutput) + return; + + std::memset(pOutput, 0, sizeof(float) * 2 * aFrameCount); + if (!pSelf) + return; + + { + std::scoped_lock lock(pSelf->SpeakersMutex); + // Map values are only writable through the iterator + for (auto itor = pSelf->Speakers.begin(); itor != pSelf->Speakers.end(); ++itor) + { + auto& speaker = itor.value(); + speaker.Buffer.MixInto(pOutput, aFrameCount, speaker.LeftGain, speaker.RightGain); + } + } + + for (ma_uint32 i = 0; i < aFrameCount * 2; ++i) + pOutput[i] = std::clamp(pOutput[i], -1.f, 1.f); + } + + ma_context Context{}; + bool HasContext{false}; + + ma_device CaptureDevice{}; + bool HasCapture{false}; + String CaptureName{}; + ma_device PlaybackDevice{}; + bool HasPlayback{false}; + String PlaybackName{}; + bool Running{false}; + + // Capture thread state + OpusEncoder* pEncoder{nullptr}; + std::array CaptureFrame{}; + uint32_t CaptureFill{0}; + VoiceDsp::NoiseGate Gate{}; + uint16_t NextSequence{0}; + + std::atomic MicVolume{1.f}; + std::atomic NoiseGate{0.1f}; + std::atomic MicLevel{0.f}; + std::atomic Transmitting{false}; + + std::mutex OutgoingMutex; + Vector Outgoing; + + std::mutex SpeakersMutex; + Map Speakers; + + Map Decoders; +}; + +VoiceAudio::VoiceAudio() noexcept + : m_pImpl(MakeUnique()) +{ +} + +VoiceAudio::~VoiceAudio() noexcept = default; + +bool VoiceAudio::ListDevices(DeviceLists& aOutLists) noexcept +{ + aOutLists.Inputs.clear(); + aOutLists.Outputs.clear(); + + if (!m_pImpl->EnsureContext()) + return false; + + ma_device_info* pPlayback = nullptr; + ma_uint32 playbackCount = 0; + ma_device_info* pCapture = nullptr; + ma_uint32 captureCount = 0; + if (ma_context_get_devices(&m_pImpl->Context, &pPlayback, &playbackCount, &pCapture, &captureCount) != MA_SUCCESS) + return false; + + for (ma_uint32 i = 0; i < captureCount; ++i) + aOutLists.Inputs.push_back(pCapture[i].name); + for (ma_uint32 i = 0; i < playbackCount; ++i) + aOutLists.Outputs.push_back(pPlayback[i].name); + + return true; +} + +void VoiceAudio::Start(const String& acInputDevice, const String& acOutputDevice) noexcept +{ + if (!m_pImpl->EnsureContext()) + return; + + if (!m_pImpl->Running || acInputDevice != m_pImpl->CaptureName) + m_pImpl->StartCapture(acInputDevice); + if (!m_pImpl->Running || acOutputDevice != m_pImpl->PlaybackName) + m_pImpl->StartPlayback(acOutputDevice); + + m_pImpl->Running = true; +} + +void VoiceAudio::Stop() noexcept +{ + m_pImpl->StopCapture(); + m_pImpl->StopPlayback(); + m_pImpl->Running = false; + + { + std::scoped_lock lock(m_pImpl->OutgoingMutex); + m_pImpl->Outgoing.clear(); + } + + RemoveAllSpeakers(); +} + +bool VoiceAudio::IsRunning() const noexcept +{ + return m_pImpl->Running; +} + +void VoiceAudio::SetCaptureSettings(float aMicVolume, float aNoiseGate) noexcept +{ + m_pImpl->MicVolume.store(std::clamp(aMicVolume, 0.f, 1.f), std::memory_order_relaxed); + m_pImpl->NoiseGate.store(std::clamp(aNoiseGate, 0.f, 1.f), std::memory_order_relaxed); +} + +void VoiceAudio::SetTransmitting(bool aTransmitting) noexcept +{ + m_pImpl->Transmitting.store(aTransmitting, std::memory_order_relaxed); +} + +float VoiceAudio::GetMicLevel() const noexcept +{ + return m_pImpl->HasCapture ? m_pImpl->MicLevel.load(std::memory_order_relaxed) : 0.f; +} + +bool VoiceAudio::PopEncodedFrame(VoiceFrame& aOutFrame) noexcept +{ + std::scoped_lock lock(m_pImpl->OutgoingMutex); + if (m_pImpl->Outgoing.empty()) + return false; + + aOutFrame = std::move(m_pImpl->Outgoing.front()); + m_pImpl->Outgoing.erase(m_pImpl->Outgoing.begin()); + return true; +} + +void VoiceAudio::PushRemoteFrame(uint32_t aPlayerId, const VoiceFrame& acFrame) noexcept +{ + if (!m_pImpl->Running || acFrame.Data.empty()) + return; + + auto& decoder = m_pImpl->Decoders[aPlayerId]; + if (!decoder.pDecoder) + { + int error = OPUS_OK; + decoder.pDecoder = opus_decoder_create(VoiceDsp::kSampleRate, 1, &error); + if (error != OPUS_OK || !decoder.pDecoder) + { + spdlog::error("Voice chat: Opus decoder failed to start ({})", error); + m_pImpl->Decoders.erase(aPlayerId); + return; + } + } + + // Late and duplicate frames are dropped, a few missing ones are concealed by Opus + int32_t concealed = 0; + if (decoder.HasSequence) + { + const int32_t delta = VoiceDsp::SequenceDelta(decoder.LastSequence, acFrame.Sequence); + if (delta <= 0) + return; + + concealed = std::min(delta - 1, kMaxConcealedFrames); + } + + decoder.LastSequence = acFrame.Sequence; + decoder.HasSequence = true; + decoder.LastFrameTime = std::chrono::steady_clock::now(); + + std::array pcm{}; + size_t samples = 0; + + for (int32_t i = 0; i < concealed; ++i) + { + const int decoded = opus_decode_float(decoder.pDecoder, nullptr, 0, pcm.data() + samples, VoiceDsp::kFrameSamples, 0); + if (decoded > 0) + samples += static_cast(decoded); + } + + const int decoded = opus_decode_float(decoder.pDecoder, acFrame.Data.data(), static_cast(acFrame.Data.size()), pcm.data() + samples, VoiceDsp::kFrameSamples, 0); + if (decoded > 0) + samples += static_cast(decoded); + + if (samples == 0) + return; + + std::scoped_lock lock(m_pImpl->SpeakersMutex); + m_pImpl->Speakers[aPlayerId].Buffer.Push(pcm.data(), samples); +} + +void VoiceAudio::SetSpeakerGains(uint32_t aPlayerId, float aLeft, float aRight) noexcept +{ + std::scoped_lock lock(m_pImpl->SpeakersMutex); + const auto itor = m_pImpl->Speakers.find(aPlayerId); + if (itor == m_pImpl->Speakers.end()) + return; + + itor.value().LeftGain = aLeft; + itor.value().RightGain = aRight; +} + +void VoiceAudio::RemoveSpeaker(uint32_t aPlayerId) noexcept +{ + { + std::scoped_lock lock(m_pImpl->SpeakersMutex); + m_pImpl->Speakers.erase(aPlayerId); + } + + const auto itor = m_pImpl->Decoders.find(aPlayerId); + if (itor != m_pImpl->Decoders.end()) + { + opus_decoder_destroy(itor->second.pDecoder); + m_pImpl->Decoders.erase(itor); + } +} + +void VoiceAudio::RemoveAllSpeakers() noexcept +{ + { + std::scoped_lock lock(m_pImpl->SpeakersMutex); + m_pImpl->Speakers.clear(); + } + + for (auto& [id, decoder] : m_pImpl->Decoders) + opus_decoder_destroy(decoder.pDecoder); + m_pImpl->Decoders.clear(); +} + +void VoiceAudio::GetActiveSpeakers(Vector& aOutPlayerIds) const noexcept +{ + aOutPlayerIds.clear(); + + const auto now = std::chrono::steady_clock::now(); + for (const auto& [id, decoder] : m_pImpl->Decoders) + { + if (now - decoder.LastFrameTime < kActiveSpeakerTime) + aOutPlayerIds.push_back(id); + } +} diff --git a/Code/client/Services/Generic/VoiceService.cpp b/Code/client/Services/Generic/VoiceService.cpp new file mode 100644 index 0000000000..a10d6a4cae --- /dev/null +++ b/Code/client/Services/Generic/VoiceService.cpp @@ -0,0 +1,464 @@ +#include + +#include +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace +{ +// Same file the dedicated server uses when it runs from the game folder, so bVoip is one switch for both +constexpr char kSettingsPath[] = "Data/SkyrimTogetherReborn/config/STServer.ini"; +constexpr char kSection[] = "Voice"; + +constexpr float kMaxFadeStart = 100000.f; +// How often talking icons and the mic meter are pushed to the UI +constexpr double kUiInterval = 0.066; +// Voices are panned this much at most, fully one sided sounds odd on headphones +constexpr float kMaxPan = 0.7f; + +// NiAVObject::world (NiTransform), checked in Ghidra on 1.7.104: NiAVObject::UpdateWorldData (0x140eebf40) writes +// world = parent world * local to 0x7C..0xAF, row major rotate at 0x7C, translate at 0xA0. The camera position getter +// (ID 50785) reads cameraRoot (TESCamera +0x20) world translate at 0xA0, and the engine's audio listener update +// (ID 40449) uses a camera node's world rotate Y column as forward. +constexpr size_t kNiAVObjectWorldRotateOffset = 0x7C; +constexpr size_t kNiAVObjectWorldTranslateOffset = 0xA0; +static_assert(kNiAVObjectWorldTranslateOffset + sizeof(float) * 3 <= sizeof(NiAVObject)); +static_assert(offsetof(TESCamera, cameraNode) == 0x20); + +struct Listener +{ + NiPoint3 Position{}; + // Horizontal, unit length, east/north + float RightX{1.f}; + float RightY{0.f}; +}; + +// Voices are heard from the camera, so free look, camera mods and ragdolls don't matter +bool GetListener(Listener& aOutListener) noexcept +{ + PlayerCharacter* pPlayer = PlayerCharacter::Get(); + if (!pPlayer) + return false; + + aOutListener.Position = pPlayer->position; + // Skyrim heading is clockwise from north, so facing is (sin, cos) and right is (cos, -sin) + aOutListener.RightX = std::cos(pPlayer->rotation.z); + aOutListener.RightY = -std::sin(pPlayer->rotation.z); + + PlayerCamera* pCamera = PlayerCamera::Get(); + // PlayerCamera redeclares the TESCamera members at the wrong offsets, go through the base on purpose + const NiNode* pCameraRoot = pCamera ? static_cast(pCamera)->cameraNode : nullptr; + if (!pCameraRoot) + return true; + + const auto* pBytes = reinterpret_cast(pCameraRoot); + float rotate[3][3]; + float translate[3]; + std::memcpy(rotate, pBytes + kNiAVObjectWorldRotateOffset, sizeof(rotate)); + std::memcpy(translate, pBytes + kNiAVObjectWorldTranslateOffset, sizeof(translate)); + + if (!std::isfinite(translate[0]) || !std::isfinite(translate[1]) || !std::isfinite(translate[2])) + return true; + + aOutListener.Position = glm::vec3(translate[0], translate[1], translate[2]); + + const float frontX = rotate[0][1]; + const float frontY = rotate[1][1]; + const float length = std::sqrt(frontX * frontX + frontY * frontY); + // Looking straight up or down, keep the character's facing + if (std::isfinite(length) && length > 0.01f) + { + aOutListener.RightX = frontY / length; + aOutListener.RightY = -frontX / length; + } + + return true; +} + +const char* ChannelToString(VoiceService::Channel aChannel) noexcept +{ + switch (aChannel) + { + case VoiceService::Channel::kParty: return "party"; + case VoiceService::Channel::kStrangers: return "strangers"; + default: return "everyone"; + } +} + +VoiceService::Channel ChannelFromString(const std::string_view acValue) noexcept +{ + if (acValue == "party") + return VoiceService::Channel::kParty; + if (acValue == "strangers") + return VoiceService::Channel::kStrangers; + return VoiceService::Channel::kEveryone; +} + +float Clamp01(double aValue, float aFallback) noexcept +{ + return std::isfinite(aValue) ? std::clamp(static_cast(aValue), 0.f, 1.f) : aFallback; +} + +std::filesystem::path GetSettingsPath() noexcept +{ + std::error_code error{}; + return std::filesystem::current_path(error) / kSettingsPath; +} +} // namespace + +VoiceService::VoiceService(World& aWorld, entt::dispatcher& aDispatcher, TransportService& aTransport) noexcept + : m_world(aWorld) + , m_transport(aTransport) +{ + LoadSettings(); + m_audio.SetCaptureSettings(m_settings.MicVolume, m_settings.NoiseGate); + + m_updateConnection = aDispatcher.sink().connect<&VoiceService::OnUpdate>(this); + m_connectedConnection = aDispatcher.sink().connect<&VoiceService::OnConnected>(this); + m_disconnectedConnection = aDispatcher.sink().connect<&VoiceService::OnDisconnected>(this); + m_voiceDataConnection = aDispatcher.sink().connect<&VoiceService::OnVoiceData>(this); + m_playerLeftConnection = aDispatcher.sink().connect<&VoiceService::OnPlayerLeft>(this); +} + +VoiceService::~VoiceService() noexcept +{ + m_audio.Stop(); +} + +void VoiceService::RefreshUi() noexcept +{ + // The ini may have been edited by hand or by the server since the menu was last open + LoadSettings(); + m_audio.SetCaptureSettings(m_settings.MicVolume, m_settings.NoiseGate); + + auto* pOverlay = m_world.GetOverlayService().GetOverlayApp(); + if (pOverlay) + { + VoiceAudio::DeviceLists devices{}; + m_audio.ListDevices(devices); + + auto pInputs = CefListValue::Create(); + for (size_t i = 0; i < devices.Inputs.size(); ++i) + pInputs->SetString(i, devices.Inputs[i].c_str()); + + auto pOutputs = CefListValue::Create(); + for (size_t i = 0; i < devices.Outputs.size(); ++i) + pOutputs->SetString(i, devices.Outputs[i].c_str()); + + auto pArguments = CefListValue::Create(); + pArguments->SetList(0, pInputs); + pArguments->SetList(1, pOutputs); + pOverlay->ExecuteAsync("voiceDevices", pArguments); + } + + SendSettingsToUi(); + UpdateAudioState(); +} + +void VoiceService::ApplySettings(const Settings& acSettings) noexcept +{ + m_settings = acSettings; + m_settings.FadeStart = std::isfinite(m_settings.FadeStart) ? std::clamp(m_settings.FadeStart, 0.f, kMaxFadeStart) : 2500.f; + m_settings.MicVolume = Clamp01(m_settings.MicVolume, 1.f); + m_settings.NoiseGate = Clamp01(m_settings.NoiseGate, 0.1f); + + m_audio.SetCaptureSettings(m_settings.MicVolume, m_settings.NoiseGate); + SaveSettings(); + UpdateAudioState(); +} + +void VoiceService::SetPlayerVolume(uint32_t aPlayerId, float aVolume) noexcept +{ + m_playerVolumes[aPlayerId] = Clamp01(aVolume, 1.f); +} + +void VoiceService::SetMeterEnabled(bool aEnabled) noexcept +{ + m_meterEnabled = aEnabled; + UpdateAudioState(); +} + +void VoiceService::OnUpdate(const UpdateEvent& acEvent) noexcept +{ + UpdateAudioState(); + + if (m_audio.IsRunning()) + { + SendOutgoingFrames(); + UpdateSpeakerGains(); + } + + UpdateUi(acEvent.Delta); +} + +void VoiceService::OnConnected(const ConnectedEvent&) noexcept +{ + // A new server knows nothing about us yet + m_listeningSent = false; +} + +void VoiceService::OnDisconnected(const DisconnectedEvent&) noexcept +{ + m_listeningSent = false; + m_playerVolumes.clear(); + UpdateAudioState(); +} + +void VoiceService::OnVoiceData(const NotifyVoiceData& acMessage) noexcept +{ + if (!IsVoiceWanted()) + return; + + m_audio.PushRemoteFrame(acMessage.PlayerId, acMessage.Frame); +} + +void VoiceService::OnPlayerLeft(const NotifyPlayerLeft& acMessage) noexcept +{ + m_audio.RemoveSpeaker(acMessage.PlayerId); + m_playerVolumes.erase(acMessage.PlayerId); +} + +bool VoiceService::IsVoiceWanted() const noexcept +{ + return m_settings.Connect && m_transport.IsOnline() && m_world.GetServerSettings().VoipEnabled; +} + +void VoiceService::UpdateAudioState() noexcept +{ + const bool wanted = IsVoiceWanted(); + // The mic popup's meter works with voip off too, it only reads the microphone locally + const bool metering = m_meterEnabled && !m_settings.InputDevice.empty(); + + if (wanted || metering) + m_audio.Start(m_settings.InputDevice, m_settings.OutputDevice); + else if (m_audio.IsRunning()) + m_audio.Stop(); + + m_audio.SetTransmitting(wanted && !m_settings.InputDevice.empty()); + + if (!wanted && m_audio.IsRunning()) + m_audio.RemoveAllSpeakers(); + + SendListeningState(); +} + +void VoiceService::SendListeningState() noexcept +{ + if (!m_transport.IsOnline()) + return; + + const bool listening = IsVoiceWanted(); + if (m_listeningSent && m_sentListening == listening && m_sentChannel == m_settings.ListenChannel) + return; + + VoiceStateRequest request{}; + request.Listening = listening; + request.ListenChannel = static_cast(m_settings.ListenChannel); + if (!m_transport.Send(request)) + return; + + m_listeningSent = true; + m_sentListening = listening; + m_sentChannel = m_settings.ListenChannel; +} + +void VoiceService::SendOutgoingFrames() noexcept +{ + VoiceDataRequest request{}; + while (m_audio.PopEncodedFrame(request.Frame)) + { + if (!IsVoiceWanted()) + continue; + + // Late voice is useless, so no resends, and no Nagle delay either + m_transport.Send(request, TiltedPhoques::kUnreliableNoNagle); + } +} + +void VoiceService::UpdateSpeakerGains() noexcept +{ + Vector speakers{}; + m_audio.GetActiveSpeakers(speakers); + if (speakers.empty()) + return; + + Listener listener{}; + const bool hasListener = GetListener(listener); + + const auto& partyMembers = m_world.GetPartyService().GetPartyMembers(); + auto view = m_world.view(); + + for (const uint32_t playerId : speakers) + { + float left = 0.f; + float right = 0.f; + + const bool inParty = std::find(partyMembers.begin(), partyMembers.end(), playerId) != partyMembers.end(); + const bool channelAllows = m_settings.ListenChannel == Channel::kEveryone || (m_settings.ListenChannel == Channel::kParty && inParty) || + (m_settings.ListenChannel == Channel::kStrangers && !inParty); + + const auto volumeItor = m_playerVolumes.find(playerId); + const float volume = volumeItor != m_playerVolumes.end() ? volumeItor->second : 1.f; + + // A player whose character isn't loaded here is too far away to be heard + const auto entityItor = std::find_if(view.begin(), view.end(), [&view, playerId](auto aEntity) { return view.get(aEntity).Id == playerId; }); + Actor* pActor = entityItor != view.end() ? Cast(TESForm::GetById(view.get(*entityItor).Id)) : nullptr; + + if (hasListener && channelAllows && volume > 0.f && pActor) + { + const float dx = pActor->position.x - listener.Position.x; + const float dy = pActor->position.y - listener.Position.y; + const float dz = pActor->position.z - listener.Position.z; + const float distance = std::sqrt(dx * dx + dy * dy + dz * dz); + + const float gain = volume * VoiceDsp::DistanceGain(distance, m_settings.FadeStart); + + const float horizontal = std::sqrt(dx * dx + dy * dy); + const float pan = horizontal > 1.f ? (dx * listener.RightX + dy * listener.RightY) / horizontal * kMaxPan : 0.f; + + VoiceDsp::PanGains(pan, left, right); + left *= gain; + right *= gain; + } + + m_audio.SetSpeakerGains(playerId, left, right); + } +} + +void VoiceService::UpdateUi(double aDelta) noexcept +{ + m_uiTimer += aDelta; + if (m_uiTimer < kUiInterval) + return; + m_uiTimer = 0.0; + + auto* pOverlay = m_world.GetOverlayService().GetOverlayApp(); + if (!pOverlay) + return; + + // Players whose voice is getting through their noise gate right now + Vector talking{}; + if (IsVoiceWanted()) + m_audio.GetActiveSpeakers(talking); + std::sort(talking.begin(), talking.end()); + + if (talking != m_lastTalking) + { + m_lastTalking = talking; + + auto pIds = CefListValue::Create(); + for (size_t i = 0; i < talking.size(); ++i) + pIds->SetInt(i, static_cast(talking[i])); + + auto pArguments = CefListValue::Create(); + pArguments->SetList(0, pIds); + pOverlay->ExecuteAsync("voiceTalking", pArguments); + } + + if (m_meterEnabled) + { + auto pArguments = CefListValue::Create(); + pArguments->SetInt(0, static_cast(std::lround(m_audio.GetMicLevel() * 100.f))); + pOverlay->ExecuteAsync("voiceMicLevel", pArguments); + } +} + +void VoiceService::LoadSettings() noexcept +{ + Settings settings{}; + + const auto path = GetSettingsPath(); + std::error_code error{}; + if (std::filesystem::exists(path, error) && !error) + { + CSimpleIni ini(true); + if (ini.LoadFile(path.c_str()) == SI_Error::SI_OK) + { + settings.InputDevice = ini.GetValue(kSection, "sInputDevice", ""); + settings.Connect = ini.GetBoolValue(kSection, "bVoip", false); + settings.OutputDevice = ini.GetValue(kSection, "sOutputDevice", ""); + settings.ListenChannel = ChannelFromString(ini.GetValue(kSection, "sVoipChannel", "everyone")); + settings.FadeStart = static_cast(ini.GetDoubleValue(kSection, "fVoipFadeStart", 2500.0)); + settings.MicVolume = static_cast(ini.GetDoubleValue(kSection, "fMicVolume", 1.0)); + settings.NoiseGate = static_cast(ini.GetDoubleValue(kSection, "fNoiseGate", 0.1)); + } + else + { + spdlog::warn("Voice chat: {} could not be read, using the defaults", kSettingsPath); + } + } + + settings.FadeStart = std::isfinite(settings.FadeStart) ? std::clamp(settings.FadeStart, 0.f, kMaxFadeStart) : 2500.f; + settings.MicVolume = Clamp01(settings.MicVolume, 1.f); + settings.NoiseGate = Clamp01(settings.NoiseGate, 0.1f); + + m_settings = settings; +} + +void VoiceService::SaveSettings() const noexcept +{ + const auto path = GetSettingsPath(); + + std::error_code error{}; + std::filesystem::create_directories(path.parent_path(), error); + + // Load first so the server's own settings in the file are kept + CSimpleIni ini(true); + if (std::filesystem::exists(path, error) && !error && ini.LoadFile(path.c_str()) != SI_Error::SI_OK) + { + spdlog::error("Voice chat: {} could not be read, voice settings are not saved", kSettingsPath); + return; + } + + ini.SetValue(kSection, "sInputDevice", m_settings.InputDevice.c_str(), "; Player: microphone for voice chat. Empty sends nothing, default follows the Windows default device"); + ini.SetBoolValue(kSection, "bVoip", m_settings.Connect, + "; Built-in proximity voice chat. On a server: lets players talk to players nearby. For a player: the Connect to voip checkbox in Settings"); + ini.SetValue(kSection, "sOutputDevice", m_settings.OutputDevice.c_str(), "; Player: where voices play. Empty uses the Windows default device, like the game"); + ini.SetValue(kSection, "sVoipChannel", ChannelToString(m_settings.ListenChannel), "; Player: who you hear, everyone, party or strangers"); + ini.SetDoubleValue(kSection, "fVoipFadeStart", m_settings.FadeStart, "; Player: voices are full volume up to this distance in game units, silent at twice it"); + ini.SetDoubleValue(kSection, "fMicVolume", m_settings.MicVolume, "; Player: microphone volume, 0 to 1"); + ini.SetDoubleValue(kSection, "fNoiseGate", m_settings.NoiseGate, "; Player: microphone level below which nothing is sent, 0 to 1 like the Windows mic test bar"); + + if (ini.SaveFile(path.c_str(), false) != SI_Error::SI_OK) + spdlog::error("Voice chat: could not write {}", kSettingsPath); +} + +void VoiceService::SendSettingsToUi() const noexcept +{ + auto* pOverlay = m_world.GetOverlayService().GetOverlayApp(); + if (!pOverlay) + return; + + auto pArguments = CefListValue::Create(); + pArguments->SetString(0, m_settings.InputDevice.c_str()); + pArguments->SetInt(1, m_settings.Connect ? 1 : 0); + pArguments->SetString(2, m_settings.OutputDevice.c_str()); + pArguments->SetString(3, ChannelToString(m_settings.ListenChannel)); + pArguments->SetInt(4, static_cast(std::lround(m_settings.FadeStart))); + pArguments->SetInt(5, static_cast(std::lround(m_settings.MicVolume * 100.f))); + pArguments->SetInt(6, static_cast(std::lround(m_settings.NoiseGate * 100.f))); + pOverlay->ExecuteAsync("voiceSettings", pArguments); +} diff --git a/Code/client/Services/OverlayClient.h b/Code/client/Services/OverlayClient.h index 0c71ac87da..0e19336156 100644 --- a/Code/client/Services/OverlayClient.h +++ b/Code/client/Services/OverlayClient.h @@ -30,6 +30,7 @@ struct OverlayClient : TiltedPhoques::OverlayClient void ProcessTeleportMessage(CefRefPtr aEventArgs); void ProcessToggleDebugUI(); void ProcessOpenPlayGuide(); + void ProcessSetVoiceSettings(CefRefPtr aEventArgs); void SetUIVisible(bool aVisible) noexcept; TransportService& m_transport; diff --git a/Code/client/Services/VoiceAudio.h b/Code/client/Services/VoiceAudio.h new file mode 100644 index 0000000000..5cd46f4f7e --- /dev/null +++ b/Code/client/Services/VoiceAudio.h @@ -0,0 +1,55 @@ +#pragma once + +#include + +/** + * @brief The audio side of voice chat: microphone capture with the noise gate and Opus encoding, and decoding and + * mixing of remote voices into an output device. Uses miniaudio for the devices and libopus for the codec. + * + * Capture and playback run on miniaudio's device threads. Everything else is called from the game's main thread, + * the two sides only share small locked queues and atomics. + */ +struct VoiceAudio +{ + struct DeviceLists + { + Vector Inputs; + Vector Outputs; + }; + + VoiceAudio() noexcept; + ~VoiceAudio() noexcept; + + TP_NOCOPYMOVE(VoiceAudio); + + // Device names as Windows shows them, for the Settings menu + bool ListDevices(DeviceLists& aOutLists) noexcept; + + // An empty input name captures nothing, "default" follows the Windows default device. An empty output name + // plays on the Windows default device, which is the one the game uses too. Restarts devices that changed. + void Start(const String& acInputDevice, const String& acOutputDevice) noexcept; + void Stop() noexcept; + [[nodiscard]] bool IsRunning() const noexcept; + + // Volume 0 to 1 applied before the gate and the meter, gate threshold 0 to 1 of peak level + void SetCaptureSettings(float aMicVolume, float aNoiseGate) noexcept; + // While false the microphone is still read for the meter, but nothing is encoded or sent + void SetTransmitting(bool aTransmitting) noexcept; + // Peak level of the last microphone frame after volume, 0 to 1, for the Settings meter + [[nodiscard]] float GetMicLevel() const noexcept; + // Encoded frames waiting to be sent, oldest first + bool PopEncodedFrame(VoiceFrame& aOutFrame) noexcept; + + // Decodes a remote player's frame into their jitter buffer + void PushRemoteFrame(uint32_t aPlayerId, const VoiceFrame& acFrame) noexcept; + // Stereo gains for a remote player, 0 mutes them without dropping their audio + void SetSpeakerGains(uint32_t aPlayerId, float aLeft, float aRight) noexcept; + void RemoveSpeaker(uint32_t aPlayerId) noexcept; + void RemoveAllSpeakers() noexcept; + // Players whose voice arrived in the last few hundred ms, whether or not they can be heard + void GetActiveSpeakers(Vector& aOutPlayerIds) const noexcept; + +private: + struct Impl; + UniquePtr m_pImpl; +}; diff --git a/Code/client/Services/VoiceService.h b/Code/client/Services/VoiceService.h new file mode 100644 index 0000000000..faaaf035e0 --- /dev/null +++ b/Code/client/Services/VoiceService.h @@ -0,0 +1,99 @@ +#pragma once + +#include + +struct World; +struct TransportService; +struct UpdateEvent; +struct ConnectedEvent; +struct DisconnectedEvent; +struct NotifyVoiceData; +struct NotifyPlayerLeft; + +/** + * @brief Built-in proximity voice chat. + * + * The microphone is captured, gated and Opus encoded by VoiceAudio, sent to the server over the game connection + * and relayed to players nearby. Remote voices get louder or quieter with distance from the camera: full volume up to + * the fade start, silent at twice that. Nothing else is needed, no separate program and no extra port. + * + * Voice runs while the server's Voice:bVoip is on and the player ticked Connect to voip, which is the same key in + * the player's own Data/SkyrimTogetherReborn/config/STServer.ini. The other [Voice] keys there are the player's + * Settings menu choices. Per player volumes live in the UI, which pushes them here by player id. + */ +struct VoiceService +{ + enum class Channel : uint8_t + { + kEveryone = 0, + kParty, + kStrangers + }; + + struct Settings + { + // Empty captures nothing, "default" follows the Windows default microphone + String InputDevice{}; + bool Connect{false}; + // Empty plays on the Windows default device, the one the game uses + String OutputDevice{}; + Channel ListenChannel{Channel::kEveryone}; + float FadeStart{2500.f}; + float MicVolume{1.f}; + float NoiseGate{0.1f}; + }; + + VoiceService(World& aWorld, entt::dispatcher& aDispatcher, TransportService& aTransport) noexcept; + ~VoiceService() noexcept; + + TP_NOCOPYMOVE(VoiceService); + + // Called from the UI, on the main thread + void RefreshUi() noexcept; + void ApplySettings(const Settings& acSettings) noexcept; + void SetPlayerVolume(uint32_t aPlayerId, float aVolume) noexcept; + void SetMeterEnabled(bool aEnabled) noexcept; + + [[nodiscard]] const Settings& GetSettings() const noexcept { return m_settings; } + +protected: + void OnUpdate(const UpdateEvent& acEvent) noexcept; + void OnConnected(const ConnectedEvent& acEvent) noexcept; + void OnDisconnected(const DisconnectedEvent& acEvent) noexcept; + void OnVoiceData(const NotifyVoiceData& acMessage) noexcept; + void OnPlayerLeft(const NotifyPlayerLeft& acMessage) noexcept; + +private: + [[nodiscard]] bool IsVoiceWanted() const noexcept; + void UpdateAudioState() noexcept; + void SendListeningState() noexcept; + void SendOutgoingFrames() noexcept; + void UpdateSpeakerGains() noexcept; + void UpdateUi(double aDelta) noexcept; + + void LoadSettings() noexcept; + void SaveSettings() const noexcept; + void SendSettingsToUi() const noexcept; + + World& m_world; + TransportService& m_transport; + VoiceAudio m_audio; + + Settings m_settings{}; + Map m_playerVolumes; + + // What the server was last told, so it's only sent on change + bool m_listeningSent{false}; + bool m_sentListening{false}; + Channel m_sentChannel{Channel::kEveryone}; + + bool m_meterEnabled{false}; + double m_uiTimer{0.0}; + Vector m_lastTalking; + + entt::scoped_connection m_updateConnection; + entt::scoped_connection m_connectedConnection; + entt::scoped_connection m_disconnectedConnection; + entt::scoped_connection m_voiceDataConnection; + entt::scoped_connection m_playerLeftConnection; +}; diff --git a/Code/client/Systems/VoiceDsp.cpp b/Code/client/Systems/VoiceDsp.cpp new file mode 100644 index 0000000000..f138b3d956 --- /dev/null +++ b/Code/client/Systems/VoiceDsp.cpp @@ -0,0 +1,117 @@ +#include "VoiceDsp.h" + +#include +#include + +namespace VoiceDsp +{ +float Peak(const float* apSamples, size_t aCount) noexcept +{ + float peak = 0.f; + for (size_t i = 0; i < aCount; ++i) + peak = std::max(peak, std::fabs(apSamples[i])); + + return std::min(peak, 1.f); +} + +float DistanceGain(float aDistance, float aFadeStart) noexcept +{ + if (!std::isfinite(aDistance) || !std::isfinite(aFadeStart) || aFadeStart <= 0.f) + return 0.f; + + if (aDistance <= aFadeStart) + return 1.f; + + const float silentAt = aFadeStart * 2.f; + if (aDistance >= silentAt) + return 0.f; + + return (silentAt - aDistance) / aFadeStart; +} + +void PanGains(float aPan, float& aOutLeft, float& aOutRight) noexcept +{ + constexpr float kQuarterPi = 0.78539816f; + + const float pan = std::isfinite(aPan) ? std::clamp(aPan, -1.f, 1.f) : 0.f; + const float angle = (pan + 1.f) * kQuarterPi; + + aOutLeft = std::cos(angle); + aOutRight = std::sin(angle); +} + +int32_t SequenceDelta(uint16_t aLast, uint16_t aNow) noexcept +{ + return static_cast(static_cast(aNow - aLast)); +} + +bool NoiseGate::Process(float aPeak, float aThreshold) noexcept +{ + if (aPeak > 0.f && aPeak >= aThreshold) + m_holdLeft = kHoldFrames; + else if (m_holdLeft > 0) + --m_holdLeft; + + return m_holdLeft > 0; +} + +JitterBuffer::JitterBuffer() + : m_samples(kCapacity, 0.f) +{ +} + +void JitterBuffer::Push(const float* apSamples, size_t aCount) noexcept +{ + for (size_t i = 0; i < aCount; ++i) + { + if (m_size == kCapacity) + { + m_read = (m_read + 1) % kCapacity; + --m_size; + } + + m_samples[(m_read + m_size) % kCapacity] = apSamples[i]; + ++m_size; + } + + // Too far behind, skip ahead to the start threshold + if (m_size > kMaxSamples) + { + const size_t drop = m_size - kStartSamples; + m_read = (m_read + drop) % kCapacity; + m_size -= drop; + } + + if (!m_playing && m_size >= kStartSamples) + m_playing = true; +} + +size_t JitterBuffer::MixInto(float* apStereoOut, size_t aFrames, float aLeftGain, float aRightGain) noexcept +{ + if (!m_playing) + return 0; + + const size_t count = std::min(aFrames, m_size); + for (size_t i = 0; i < count; ++i) + { + const float sample = m_samples[(m_read + i) % kCapacity]; + apStereoOut[i * 2] += sample * aLeftGain; + apStereoOut[i * 2 + 1] += sample * aRightGain; + } + + m_read = (m_read + count) % kCapacity; + m_size -= count; + + if (m_size == 0) + m_playing = false; + + return count; +} + +void JitterBuffer::Clear() noexcept +{ + m_read = 0; + m_size = 0; + m_playing = false; +} +} // namespace VoiceDsp diff --git a/Code/client/Systems/VoiceDsp.h b/Code/client/Systems/VoiceDsp.h new file mode 100644 index 0000000000..8a0ddcf3e5 --- /dev/null +++ b/Code/client/Systems/VoiceDsp.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +/** + * @brief Game independent pieces of voice chat: the noise gate, distance falloff, panning and the per speaker + * jitter buffer. Kept apart from the audio devices and the game so they can be unit tested. + * + * Audio is 48 kHz, mono per speaker, in 20 ms frames of 960 samples. + */ +namespace VoiceDsp +{ +constexpr uint32_t kSampleRate = 48000; +constexpr uint32_t kFrameSamples = 960; + +// Peak of a frame, 0 to 1. This is what the Windows "Test your microphone" bar shows. +float Peak(const float* apSamples, size_t aCount) noexcept; + +// Full volume up to aFadeStart, then linear down to silent at twice that distance +float DistanceGain(float aDistance, float aFadeStart) noexcept; + +// aPan is -1 (all left) to 1 (all right), constant power so a voice moving around doesn't change loudness +void PanGains(float aPan, float& aOutLeft, float& aOutRight) noexcept; + +// Frames between the last sequence and this one, wrapping at 65536. 1 is the next frame, 0 or less is a duplicate or late. +int32_t SequenceDelta(uint16_t aLast, uint16_t aNow) noexcept; + +/** + * @brief Opens on a frame whose peak reaches the threshold, and stays open a moment after so word endings and short + * pauses aren't cut off. A threshold of 0 is always open. + */ +struct NoiseGate +{ + // 300 ms + static constexpr uint32_t kHoldFrames = 15; + + // Returns whether this frame should be sent + bool Process(float aPeak, float aThreshold) noexcept; + + [[nodiscard]] bool IsOpen() const noexcept { return m_holdLeft > 0; } + +private: + uint32_t m_holdLeft{0}; +}; + +/** + * @brief Decoded audio of one remote speaker, waiting to be played. + * + * Playback starts once kStartSamples are buffered so small network jitter doesn't cause gaps. When the buffer + * runs dry the speaker goes quiet and buffers up again. If it grows past kMaxSamples, e.g. after our game froze, + * the oldest audio is dropped so the voice doesn't lag behind. + */ +struct JitterBuffer +{ + static constexpr size_t kStartSamples = kFrameSamples * 3; // 60 ms + static constexpr size_t kMaxSamples = kFrameSamples * 10; // 200 ms + static constexpr size_t kCapacity = kFrameSamples * 16; + + JitterBuffer(); + + void Push(const float* apSamples, size_t aCount) noexcept; + // Adds up to aFrames samples into the interleaved stereo output, returns how many were mixed + size_t MixInto(float* apStereoOut, size_t aFrames, float aLeftGain, float aRightGain) noexcept; + void Clear() noexcept; + + [[nodiscard]] size_t GetBuffered() const noexcept { return m_size; } + [[nodiscard]] bool IsPlaying() const noexcept { return m_playing; } + +private: + std::vector m_samples; + size_t m_read{0}; + size_t m_size{0}; + bool m_playing{false}; +}; +} // namespace VoiceDsp diff --git a/Code/client/World.cpp b/Code/client/World.cpp index b0596604b5..ec19a112af 100644 --- a/Code/client/World.cpp +++ b/Code/client/World.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,7 @@ World::World() ctx().emplace(*this, m_transport, m_dispatcher); ctx().emplace(*this, m_transport, m_dispatcher); ctx().emplace(*this, m_dispatcher, m_transport); + ctx().emplace(*this, m_dispatcher, m_transport); BehaviorVar::Get()->Init(); } diff --git a/Code/client/xmake.lua b/Code/client/xmake.lua index ec4be6bc32..2855c5810a 100644 --- a/Code/client/xmake.lua +++ b/Code/client/xmake.lua @@ -59,7 +59,9 @@ target(name) "entt", "glm", "mem", - "xbyak") + "xbyak", + "libopus", + "miniaudio") add_syslinks( "version", diff --git a/Code/tests/voice_dsp.cpp b/Code/tests/voice_dsp.cpp new file mode 100644 index 0000000000..a32346f915 --- /dev/null +++ b/Code/tests/voice_dsp.cpp @@ -0,0 +1,96 @@ +#include + +#include "../client/Systems/VoiceDsp.h" + +#include + +using namespace VoiceDsp; + +TEST_CASE("Voice distance falloff", "[voice.dsp]") +{ + REQUIRE(DistanceGain(0.f, 2500.f) == 1.f); + REQUIRE(DistanceGain(2500.f, 2500.f) == 1.f); + REQUIRE(DistanceGain(3750.f, 2500.f) == Approx(0.5f)); + REQUIRE(DistanceGain(5000.f, 2500.f) == 0.f); + REQUIRE(DistanceGain(9000.f, 2500.f) == 0.f); + // A fade start of 0 would divide by zero, it means nobody is heard + REQUIRE(DistanceGain(10.f, 0.f) == 0.f); +} + +TEST_CASE("Voice panning keeps loudness", "[voice.dsp]") +{ + float left = 0.f, right = 0.f; + + PanGains(0.f, left, right); + REQUIRE(left == Approx(right)); + REQUIRE(left * left + right * right == Approx(1.f)); + + PanGains(-1.f, left, right); + REQUIRE(left == Approx(1.f)); + REQUIRE(right == Approx(0.f).margin(1e-6)); + + PanGains(5.f, left, right); + REQUIRE(right == Approx(1.f)); +} + +TEST_CASE("Voice sequence numbers wrap", "[voice.dsp]") +{ + REQUIRE(SequenceDelta(10, 11) == 1); + REQUIRE(SequenceDelta(10, 13) == 3); + REQUIRE(SequenceDelta(65535, 0) == 1); + REQUIRE(SequenceDelta(65534, 2) == 4); + REQUIRE(SequenceDelta(11, 10) == -1); + REQUIRE(SequenceDelta(10, 10) == 0); +} + +TEST_CASE("Voice noise gate", "[voice.dsp]") +{ + NoiseGate gate; + + // Below the threshold stays closed + REQUIRE_FALSE(gate.Process(0.05f, 0.1f)); + + // Opens on a loud frame and holds through a pause + REQUIRE(gate.Process(0.3f, 0.1f)); + for (uint32_t i = 1; i < NoiseGate::kHoldFrames; ++i) + REQUIRE(gate.Process(0.01f, 0.1f)); + REQUIRE_FALSE(gate.Process(0.01f, 0.1f)); + + // A threshold of 0 sends anything but digital silence + NoiseGate openGate; + REQUIRE(openGate.Process(0.001f, 0.f)); + + REQUIRE(Peak(std::vector{0.1f, -0.4f, 0.2f}.data(), 3) == Approx(0.4f)); +} + +TEST_CASE("Voice jitter buffer", "[voice.dsp]") +{ + JitterBuffer buffer; + std::vector frame(kFrameSamples, 0.5f); + std::vector out(kFrameSamples * 2, 0.f); + + // Waits for 60 ms before playing + buffer.Push(frame.data(), frame.size()); + buffer.Push(frame.data(), frame.size()); + REQUIRE_FALSE(buffer.IsPlaying()); + REQUIRE(buffer.MixInto(out.data(), kFrameSamples, 1.f, 1.f) == 0); + + buffer.Push(frame.data(), frame.size()); + REQUIRE(buffer.IsPlaying()); + + REQUIRE(buffer.MixInto(out.data(), kFrameSamples, 1.f, 0.5f) == kFrameSamples); + REQUIRE(out[0] == Approx(0.5f)); + REQUIRE(out[1] == Approx(0.25f)); + + // Runs dry and stops until it has buffered again + buffer.MixInto(out.data(), kFrameSamples, 1.f, 1.f); + buffer.MixInto(out.data(), kFrameSamples, 1.f, 1.f); + REQUIRE_FALSE(buffer.IsPlaying()); + REQUIRE(buffer.GetBuffered() == 0); + + // Falls too far behind and drops the oldest audio + for (int i = 0; i < 12; ++i) + buffer.Push(frame.data(), frame.size()); + REQUIRE(buffer.GetBuffered() <= JitterBuffer::kMaxSamples); + REQUIRE(buffer.IsPlaying()); +} diff --git a/Code/tests/xmake.lua b/Code/tests/xmake.lua index 19abce5747..89f4d28342 100644 --- a/Code/tests/xmake.lua +++ b/Code/tests/xmake.lua @@ -8,6 +8,8 @@ target("TPTests") add_files("*.cpp") -- Has no game dependencies, so the client's remote playback timing can be tested here add_files("../client/Systems/PlaybackClock.cpp") + -- Voice chat's noise gate, falloff and jitter buffer, also game independent + add_files("../client/Systems/VoiceDsp.cpp") add_deps("SkyrimEncoding") add_packages( "tiltedcore", diff --git a/Code/tp_process/ProcessHandler.cpp b/Code/tp_process/ProcessHandler.cpp index 9c67fd1ec4..b59b3d05a1 100644 --- a/Code/tp_process/ProcessHandler.cpp +++ b/Code/tp_process/ProcessHandler.cpp @@ -28,6 +28,10 @@ void ProcessHandler::OnContextCreated(CefRefPtr browser, CefRefPtrSetValue("teleportToPlayer", CefV8Value::CreateFunction("teleportToPlayer", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); m_pCoreObject->SetValue("toggleDebugUI", CefV8Value::CreateFunction("toggleDebugUI", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); m_pCoreObject->SetValue("openPlayGuide", CefV8Value::CreateFunction("openPlayGuide", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); + m_pCoreObject->SetValue("voiceRefresh", CefV8Value::CreateFunction("voiceRefresh", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); + m_pCoreObject->SetValue("setVoiceSettings", CefV8Value::CreateFunction("setVoiceSettings", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); + m_pCoreObject->SetValue("setVoicePlayerVolume", CefV8Value::CreateFunction("setVoicePlayerVolume", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); + m_pCoreObject->SetValue("setVoiceMeter", CefV8Value::CreateFunction("setVoiceMeter", m_pOverlayHandler), V8_PROPERTY_ATTRIBUTE_NONE); } void ProcessHandler::OnContextReleased(CefRefPtr browser, CefRefPtr frame, CefRefPtr context) diff --git a/xmake.lua b/xmake.lua index 31ea6831cf..d6d8720508 100644 --- a/xmake.lua +++ b/xmake.lua @@ -58,7 +58,10 @@ add_requires( if is_plat("windows") then add_requires( "discord 3.2.1", - "imgui v1.89.7" + "imgui v1.89.7", + -- built-in voice chat: Opus codec (BSD-3-Clause) and audio devices (miniaudio, MIT-0 or public domain) + "libopus 1.5.2", + "miniaudio 0.11.22" ) end From f152e957b650af83051b9baa84165418528607e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:17:38 +0000 Subject: [PATCH 03/12] Add voice chat controls to Settings and the player manager Settings gets a Voice chat section: input device (None by default), Connect to voip, output device (the game's device by default), channel and fade start. The Player List and Party Menu tabs get a Voip column: the header mic opens your microphone volume and a noise gate with a live level meter, each player's speaker opens their volume. Speakers show one wave when the player is quiet and two while their voice gets through, party members in the party colour. Per player volumes are kept in the UI store as playerUi_voiceVolumes, by username, so they carry across servers and sessions. The F2 menu itself is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- Code/skyrim_ui/src/app/app.module.ts | 4 + .../party-menu/party-menu.component.html | 8 + .../party-menu/party-menu.component.scss | 5 + .../player-list/player-list.component.html | 8 + .../player-list/player-list.component.scss | 5 + .../src/app/components/root/root.component.ts | 3 + .../settings/settings.component.html | 107 +++++++++ .../settings/settings.component.scss | 6 + .../components/settings/settings.component.ts | 32 +++ .../components/voip/voip-mic.component.html | 84 +++++++ .../app/components/voip/voip-mic.component.ts | 64 ++++++ .../voip/voip-speaker.component.html | 55 +++++ .../components/voip/voip-speaker.component.ts | 99 ++++++++ .../app/components/voip/voip.component.scss | 96 ++++++++ .../src/app/mock/skyrimtogether.mock.ts | 83 +++++++ .../src/app/services/voip.service.ts | 211 ++++++++++++++++++ Code/skyrim_ui/src/assets/i18n/en.json | 30 +++ Code/skyrim_ui/src/styles.scss | 1 + Code/skyrim_ui/src/typings.d.ts | 69 ++++++ 19 files changed, 970 insertions(+) create mode 100644 Code/skyrim_ui/src/app/components/voip/voip-mic.component.html create mode 100644 Code/skyrim_ui/src/app/components/voip/voip-mic.component.ts create mode 100644 Code/skyrim_ui/src/app/components/voip/voip-speaker.component.html create mode 100644 Code/skyrim_ui/src/app/components/voip/voip-speaker.component.ts create mode 100644 Code/skyrim_ui/src/app/components/voip/voip.component.scss create mode 100644 Code/skyrim_ui/src/app/services/voip.service.ts diff --git a/Code/skyrim_ui/src/app/app.module.ts b/Code/skyrim_ui/src/app/app.module.ts index e27772139e..49c1822a69 100644 --- a/Code/skyrim_ui/src/app/app.module.ts +++ b/Code/skyrim_ui/src/app/app.module.ts @@ -6,6 +6,8 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms'; import { BrowserModule } from '@angular/platform-browser'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { FontAwesomeModule } from '@fortawesome/angular-fontawesome'; +import { VoipMicComponent } from './components/voip/voip-mic.component'; +import { VoipSpeakerComponent } from './components/voip/voip-speaker.component'; import { environment } from '../environments/environment'; import { ActionButtonsComponent } from './components/action-buttons/action-buttons.component'; import { ChatComponent } from './components/chat/chat.component'; @@ -71,6 +73,8 @@ import { TranslocoRootModule } from './transloco-root.module'; ServerListComponent, SettingsComponent, ToggleComponent, + VoipMicComponent, + VoipSpeakerComponent, WindowComponent, CheckboxDirective, diff --git a/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.html b/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.html index dda94c2de4..0e88ab8113 100644 --- a/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.html +++ b/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.html @@ -69,6 +69,7 @@ {{ 'COMPONENT.PARTY_MENU.TABLE_HEADERS.LEVEL' | transloco }} + {{ 'COMPONENT.PARTY_MENU.TABLE_HEADERS.NAME' | transloco }} {{ 'COMPONENT.PARTY_MENU.TABLE_HEADERS.LOCATION' | transloco }} @@ -79,6 +80,13 @@ {{ player.level }} + + + {{ player.name }} {{ player.cellName }} diff --git a/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.scss b/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.scss index 2d3ad298ef..c823855d90 100644 --- a/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.scss +++ b/Code/skyrim_ui/src/app/components/party-menu/party-menu.component.scss @@ -81,3 +81,8 @@ hr { } } } + +.voip-col { + white-space: nowrap; + text-align: center; +} diff --git a/Code/skyrim_ui/src/app/components/player-list/player-list.component.html b/Code/skyrim_ui/src/app/components/player-list/player-list.component.html index 712f2c59a5..df3df7e427 100644 --- a/Code/skyrim_ui/src/app/components/player-list/player-list.component.html +++ b/Code/skyrim_ui/src/app/components/player-list/player-list.component.html @@ -11,6 +11,7 @@ {{ 'COMPONENT.PARTY_LIST.TABLE_HEADERS.LEVEL' | transloco }} + {{ 'COMPONENT.PARTY_LIST.TABLE_HEADERS.NAME' | transloco }} @@ -18,6 +19,13 @@ {{ player.level }} + + + {{ player.name }} + + + +
+
+ {{ 'COMPONENT.VOIP.MIC_VOLUME' | transloco }} + {{ settings.micVolume }}% +
+ + +
+ +
+ {{ 'COMPONENT.VOIP.NOISE_GATE' | transloco }} + {{ settings.noiseGate }}% +
+ + +
+ {{ + (settings.inputDevice + ? 'COMPONENT.VOIP.NOISE_GATE_HINT' + : 'COMPONENT.VOIP.NO_DEVICE_HINT' + ) | transloco + }} +
+
+ {{ 'COMPONENT.VOIP.NO_DEVICE' | transloco }} +
+
+ {{ 'COMPONENT.VOIP.SAVED_UNTIL_CONNECTED' | transloco }} +
+
+ diff --git a/Code/skyrim_ui/src/app/components/voip/voip-mic.component.ts b/Code/skyrim_ui/src/app/components/voip/voip-mic.component.ts new file mode 100644 index 0000000000..b3250dfb34 --- /dev/null +++ b/Code/skyrim_ui/src/app/components/voip/voip-mic.component.ts @@ -0,0 +1,64 @@ +import { ChangeDetectionStrategy, Component, OnDestroy } from '@angular/core'; +import { + faMicrophone, + faMicrophoneSlash, +} from '@fortawesome/free-solid-svg-icons'; +import { map } from 'rxjs'; +import { VoipService } from '../../services/voip.service'; + +/** "Voip" column header with the local microphone button and its popup: mic volume and noise gate. */ +@Component({ + selector: 'app-voip-mic', + templateUrl: './voip-mic.component.html', + styleUrls: ['./voip.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class VoipMicComponent implements OnDestroy { + readonly faMicrophone = faMicrophone; + readonly faMicrophoneSlash = faMicrophoneSlash; + + isOpen = false; + + readonly settings$ = this.voipService.settings; + readonly micLevel$ = this.voipService.micLevel; + readonly state$ = this.voipService.settings.pipe( + map(settings => { + if (!settings.connect || !settings.inputDevice) { + return 'off'; + } + return settings.micVolume === 0 ? 'muted' : 'on'; + }), + ); + + constructor(private readonly voipService: VoipService) {} + + toggle(): void { + this.setOpen(!this.isOpen); + } + + close(): void { + this.setOpen(false); + } + + setMicVolume(value: number): void { + this.voipService.update({ micVolume: Number(value) }); + } + + setNoiseGate(value: number): void { + this.voipService.update({ noiseGate: Number(value) }); + } + + ngOnDestroy(): void { + if (this.isOpen) { + this.voipService.setMeter(false); + } + } + + private setOpen(open: boolean): void { + if (open === this.isOpen) { + return; + } + this.isOpen = open; + this.voipService.setMeter(open); + } +} diff --git a/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.html b/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.html new file mode 100644 index 0000000000..7b4b109a63 --- /dev/null +++ b/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.html @@ -0,0 +1,55 @@ + + + + +
+
+ {{ + 'COMPONENT.VOIP.PLAYER_VOLUME' | transloco : { name: name$ | async } + }} + {{ volume$ | async }}% +
+ +
+ {{ + (settings.connect + ? 'COMPONENT.VOIP.MUTES_PLAYER' + : 'COMPONENT.VOIP.SAVED_UNTIL_CONNECTED' + ) | transloco + }} +
+
+ + diff --git a/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.ts b/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.ts new file mode 100644 index 0000000000..dd7fffb4ad --- /dev/null +++ b/Code/skyrim_ui/src/app/components/voip/voip-speaker.component.ts @@ -0,0 +1,99 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { + faVolumeHigh, + faVolumeLow, + faVolumeXmark, +} from '@fortawesome/free-solid-svg-icons'; +import { BehaviorSubject, combineLatest, map } from 'rxjs'; +import { VoipChannel, VoipService } from '../../services/voip.service'; + +type SpeakerState = 'off' | 'muted' | 'quiet' | 'talking'; + +/** A player's speaker icon in the Voip column, and the popup with their volume. */ +@Component({ + selector: 'app-voip-speaker', + templateUrl: './voip-speaker.component.html', + styleUrls: ['./voip.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class VoipSpeakerComponent { + readonly faVolumeHigh = faVolumeHigh; + readonly faVolumeLow = faVolumeLow; + readonly faVolumeXmark = faVolumeXmark; + + private readonly player$ = new BehaviorSubject<{ + id: number; + name: string; + isMember: boolean; + }>({ + id: 0, + name: '', + isMember: false, + }); + + @Input() set playerId(value: number) { + this.player$.next({ ...this.player$.value, id: value }); + } + @Input() set playerName(value: string) { + this.player$.next({ ...this.player$.value, name: value }); + } + @Input() set isMember(value: boolean) { + this.player$.next({ ...this.player$.value, isMember: !!value }); + } + + isOpen = false; + + readonly settings$ = this.voipService.settings; + readonly volume$ = combineLatest([ + this.player$, + this.voipService.volumes, + ]).pipe(map(([player]) => this.voipService.getVolume(player.name))); + readonly state$ = combineLatest([ + this.player$, + this.voipService.settings, + this.voipService.volumes, + this.voipService.talking, + ]).pipe( + map(([player, settings, , talking]): SpeakerState => { + const heard = + settings.channel === VoipChannel.EVERYONE || + (settings.channel === VoipChannel.PARTY && player.isMember) || + (settings.channel === VoipChannel.STRANGERS && !player.isMember); + if (!settings.connect || !heard) { + return 'off'; + } + if (this.voipService.getVolume(player.name) === 0) { + return 'muted'; + } + return talking.has(player.id) ? 'talking' : 'quiet'; + }), + ); + readonly isMember$ = this.player$.pipe(map(player => player.isMember)); + readonly name$ = this.player$.pipe(map(player => player.name)); + + constructor(private readonly voipService: VoipService) {} + + icon(state: SpeakerState) { + switch (state) { + case 'talking': + return this.faVolumeHigh; + case 'quiet': + return this.faVolumeLow; + default: + return this.faVolumeXmark; + } + } + + toggle(): void { + this.isOpen = !this.isOpen; + } + + close(): void { + this.isOpen = false; + } + + setVolume(value: number): void { + const player = this.player$.value; + this.voipService.setVolume(player.id, player.name, Number(value)); + } +} diff --git a/Code/skyrim_ui/src/app/components/voip/voip.component.scss b/Code/skyrim_ui/src/app/components/voip/voip.component.scss new file mode 100644 index 0000000000..ecf9e5453b --- /dev/null +++ b/Code/skyrim_ui/src/app/components/voip/voip.component.scss @@ -0,0 +1,96 @@ +// Shared by the Voip column's mic header and speaker buttons. The popups render in the CDK overlay container, +// outside the component, so their styles are global through ::ng-deep. + +.voip-head { + display: inline-flex; + align-items: center; + gap: 0.25rem; +} + +.voip-icon-button { + background: none; + border: 0; + padding: 0.1rem 0.3rem; + cursor: pointer; + color: #e8e4dc; + line-height: 1; + + &.off { + color: #8a857c; + } + + &.muted { + color: #d9534f; + } + + // Same colour as the party chat + &.party { + color: #9affe6; + } +} + +::ng-deep .voip-popup { + display: grid; + gap: 0.35rem; + width: 16rem; + margin-top: 0.4rem; + padding: 0.6rem 0.75rem; + background: #151515; + border: 1px solid grey; + box-shadow: 0 6px 24px rgba(0, 0, 0, 0.6); + color: #e8e4dc; + font-size: 0.9rem; + + .row { + display: flex; + justify-content: space-between; + color: #a9a49b; + + b { + color: #e8e4dc; + font-variant-numeric: tabular-nums; + } + } + + input[type='range'] { + width: 100%; + } + + .voip-separator { + border-top: 1px solid #3a3833; + margin: 0.25rem 0 0.1rem; + } + + .meter { + position: relative; + height: 0.5rem; + background: #0d0d0c; + border: 1px solid #3a3833; + + .level { + position: absolute; + top: 0; + bottom: 0; + left: 0; + background: #6f6a60; + transition: width 60ms linear; + + &.open { + background: #8fd18a; + } + } + + .gate { + position: absolute; + top: -3px; + bottom: -3px; + width: 2px; + background: #e8e4dc; + } + } + + .hint { + font-size: 0.8rem; + color: #a9a49b; + } +} diff --git a/Code/skyrim_ui/src/app/mock/skyrimtogether.mock.ts b/Code/skyrim_ui/src/app/mock/skyrimtogether.mock.ts index 9a2ec37900..519f7cbf77 100644 --- a/Code/skyrim_ui/src/app/mock/skyrimtogether.mock.ts +++ b/Code/skyrim_ui/src/app/mock/skyrimtogether.mock.ts @@ -29,6 +29,17 @@ export class SkyrimtogetherMock extends EventEmitter implements SkyrimTogether { private showEvents = true; private localPlayerId: number; public readonly players$ = playerStore.pipe(selectAllEntities()); + private voiceSettings: [ + string, + number, + string, + string, + number, + number, + number, + ] = ['', 0, '', 'everyone', 2500, 100, 10]; + private voiceMeterTimer: ReturnType | undefined; + private voiceTalkingTimer: ReturnType | undefined; connect(host: string, port: number, password: string): void { if (!this.connected) { @@ -310,6 +321,78 @@ export class SkyrimtogetherMock extends EventEmitter implements SkyrimTogether { ); } + voiceRefresh(): void { + this.emit( + 'voiceDevices', + [ + 'Microphone (Realtek High Definition Audio)', + 'Headset Microphone (HyperX Cloud II)', + ], + [ + 'Speakers (Realtek High Definition Audio)', + 'Headphones (HyperX Cloud II)', + ], + ); + this.emit('voiceSettings', ...this.voiceSettings); + } + + setVoiceSettings( + inputDevice: string, + connect: number, + outputDevice: string, + channel: string, + fadeStart: number, + micVolume: number, + noiseGate: number, + ): void { + this.voiceSettings = [ + inputDevice, + connect, + outputDevice, + channel, + fadeStart, + micVolume, + noiseGate, + ]; + this.emit('voiceSettings', ...this.voiceSettings); + + // Pretend some connected players talk now and then + clearInterval(this.voiceTalkingTimer); + this.voiceTalkingTimer = undefined; + if (connect) { + let tick = 0; + this.voiceTalkingTimer = setInterval(() => { + tick++; + const ids = playerStore + .query(getAllEntities()) + .filter((p, i) => (tick + i * 3) % 7 < 4) + .map(p => p.id); + this.emit('voiceTalking', ids); + }, 500); + } else { + this.emit('voiceTalking', []); + } + } + + setVoicePlayerVolume(playerId: number, volume: number): void {} + + setVoiceMeter(enabled: number): void { + clearInterval(this.voiceMeterTimer); + this.voiceMeterTimer = undefined; + if (enabled && this.voiceSettings[0] !== '') { + let tick = 0; + this.voiceMeterTimer = setInterval(() => { + tick++; + const speaking = tick % 40 < 22; + const raw = speaking ? 25 + Math.random() * 45 : 2 + Math.random() * 7; + this.emit( + 'voiceMicLevel', + Math.round((raw * this.voiceSettings[5]) / 100), + ); + }, 90); + } + } + openPlayGuide(): void { window.open( 'https://wiki.tiltedphoques.com/tilted-online/general-information/playguide', diff --git a/Code/skyrim_ui/src/app/services/voip.service.ts b/Code/skyrim_ui/src/app/services/voip.service.ts new file mode 100644 index 0000000000..32b8d09d30 --- /dev/null +++ b/Code/skyrim_ui/src/app/services/voip.service.ts @@ -0,0 +1,211 @@ +import { Injectable, NgZone } from '@angular/core'; +import { BehaviorSubject } from 'rxjs'; +import { PlayerListService } from './player-list.service'; +import { StoreService } from './store.service'; + +export enum VoipChannel { + EVERYONE = 'everyone', + PARTY = 'party', + STRANGERS = 'strangers', +} + +/** Voice chat settings, saved by the game in the player's STServer.ini. */ +export interface VoipSettings { + /** Empty sends nothing, 'default' follows the Windows default device. */ + inputDevice: string; + connect: boolean; + /** Empty plays on the game's (the Windows default) device. */ + outputDevice: string; + channel: VoipChannel; + /** Full volume up to this distance in game units, silent at twice it. */ + fadeStart: number; + /** 0 to 100 */ + micVolume: number; + /** 0 to 100, like the Windows "Test your microphone" bar. */ + noiseGate: number; +} + +export const defaultVoipSettings: VoipSettings = { + inputDevice: '', + connect: false, + outputDevice: '', + channel: VoipChannel.EVERYONE, + fadeStart: 2500, + micVolume: 100, + noiseGate: 10, +}; + +/** Per player voice volumes by username, 0 to 1, kept across servers and sessions like the name overrides. */ +const volumesStoreKey = 'playerUi_voiceVolumes'; +const defaultPlayerVolume = 100; + +@Injectable({ + providedIn: 'root', +}) +export class VoipService { + public readonly settings = new BehaviorSubject( + defaultVoipSettings, + ); + public readonly inputDevices = new BehaviorSubject([]); + public readonly outputDevices = new BehaviorSubject([]); + /** Players whose voice is getting through their noise gate right now. */ + public readonly talking = new BehaviorSubject>(new Set()); + /** Local microphone level, 0 to 100, while the meter is on. */ + public readonly micLevel = new BehaviorSubject(0); + /** Per player volumes by username, 0 to 100. */ + public readonly volumes: BehaviorSubject>; + + constructor( + private readonly zone: NgZone, + private readonly storeService: StoreService, + private readonly playerListService: PlayerListService, + ) { + this.volumes = new BehaviorSubject(this.loadVolumes()); + + skyrimtogether.on('voiceSettings', this.onSettings.bind(this)); + skyrimtogether.on('voiceDevices', this.onDevices.bind(this)); + skyrimtogether.on('voiceTalking', this.onTalking.bind(this)); + skyrimtogether.on('voiceMicLevel', this.onMicLevel.bind(this)); + + // The game forgets volumes when it disconnects and only knows players by id, so they're sent again whenever + // the player list changes + this.playerListService.playerList.subscribe(() => this.sendVolumes()); + + this.refresh(); + } + + /** Asks the game for the saved settings and the device lists. */ + public refresh(): void { + skyrimtogether.voiceRefresh(); + } + + public update(changes: Partial): void { + const settings = { ...this.settings.value, ...changes }; + settings.fadeStart = Math.max( + 0, + Math.min(100000, Math.round(settings.fadeStart || 0)), + ); + settings.micVolume = this.clampPercent(settings.micVolume); + settings.noiseGate = this.clampPercent(settings.noiseGate); + this.settings.next(settings); + + skyrimtogether.setVoiceSettings( + settings.inputDevice, + settings.connect ? 1 : 0, + settings.outputDevice, + settings.channel, + settings.fadeStart, + settings.micVolume, + settings.noiseGate, + ); + } + + public getVolume(name: string): number { + return this.volumes.value[name] ?? defaultPlayerVolume; + } + + public setVolume(playerId: number, name: string, volume: number): void { + const clamped = this.clampPercent(volume); + const volumes = { ...this.volumes.value }; + if (clamped === defaultPlayerVolume) { + delete volumes[name]; + } else { + volumes[name] = clamped; + } + + this.volumes.next(volumes); + this.saveVolumes(volumes); + skyrimtogether.setVoicePlayerVolume(playerId, clamped); + } + + public setMeter(enabled: boolean): void { + if (!enabled) { + this.micLevel.next(0); + } + skyrimtogether.setVoiceMeter(enabled ? 1 : 0); + } + + private sendVolumes(): void { + const players = this.playerListService.playerList.value?.players ?? []; + for (const player of players) { + skyrimtogether.setVoicePlayerVolume( + player.id, + this.getVolume(player.name), + ); + } + } + + private onSettings( + inputDevice: string, + connect: number, + outputDevice: string, + channel: string, + fadeStart: number, + micVolume: number, + noiseGate: number, + ): void { + const knownChannel = Object.values(VoipChannel).includes( + channel as VoipChannel, + ) + ? (channel as VoipChannel) + : VoipChannel.EVERYONE; + + this.zone.run(() => { + this.settings.next({ + inputDevice, + connect: !!connect, + outputDevice, + channel: knownChannel, + fadeStart, + micVolume, + noiseGate, + }); + }); + } + + private onDevices(inputs: string[], outputs: string[]): void { + this.zone.run(() => { + this.inputDevices.next(inputs ?? []); + this.outputDevices.next(outputs ?? []); + }); + } + + private onTalking(playerIds: number[]): void { + this.zone.run(() => { + this.talking.next(new Set(playerIds ?? [])); + }); + } + + private onMicLevel(level: number): void { + this.zone.run(() => { + this.micLevel.next(this.clampPercent(level)); + }); + } + + private loadVolumes(): Record { + try { + const stored = JSON.parse(this.storeService.get(volumesStoreKey, '{}')); + const volumes: Record = {}; + for (const [name, value] of Object.entries(stored ?? {})) { + if (typeof value === 'number' && isFinite(value)) { + volumes[name] = this.clampPercent(value * 100); + } + } + return volumes; + } catch { + return {}; + } + } + + private saveVolumes(volumes: Record): void { + const stored: Record = {}; + for (const [name, value] of Object.entries(volumes)) { + stored[name] = value / 100; + } + this.storeService.set(volumesStoreKey, JSON.stringify(stored)); + } + + private clampPercent(value: number): number { + return Math.max(0, Math.min(100, Math.round(Number(value) || 0))); + } +} diff --git a/Code/skyrim_ui/src/assets/i18n/en.json b/Code/skyrim_ui/src/assets/i18n/en.json index 1f03d698cd..2fabc42af8 100644 --- a/Code/skyrim_ui/src/assets/i18n/en.json +++ b/Code/skyrim_ui/src/assets/i18n/en.json @@ -117,6 +117,25 @@ "AUDIO": "Audio", "MUTE_UI": "Mute User Interface", "UI_VOLUME": "User Interface Volume", + "VOICE": { + "HEADER": "Voice chat", + "INPUT_DEVICE": "Voip input device", + "INPUT_HINT": "None sends nothing, not even to test", + "INPUT_NONE": "None", + "INPUT_DEFAULT": "Windows default", + "CONNECT": "Connect to voip", + "CONNECT_HINT": "Hear and talk to nearby players on servers that allow it", + "OUTPUT_DEVICE": "Voip output device", + "OUTPUT_HINT": "Default follows the game's sound device", + "OUTPUT_GAME": "Game audio device (same as the game)", + "CHANNEL": "Voip channel", + "CHANNEL_HINT": "Party: only your party. Strangers: everyone but your party, e.g. when your party is on Discord", + "CHANNEL_EVERYONE": "Everyone", + "CHANNEL_PARTY": "Party", + "CHANNEL_STRANGERS": "Strangers", + "FADE_START": "Voip fade start", + "FADE_HINT": "Full volume up to this distance, silent at {{ silent }} units" + }, "PARTY": "Party", "SHOW_PARTY": "Show Party", "AUTO_HIDE_PARTY": "Auto Hide Party", @@ -145,6 +164,17 @@ "XL": "Extra Large" }, "BACK": "Back" + }, + "VOIP": { + "HEADER": "Voip", + "MIC_VOLUME": "Your microphone volume", + "NOISE_GATE": "Noise gate", + "NOISE_GATE_HINT": "Sound below the line is not sent.", + "NO_DEVICE_HINT": "Pick an input device in Settings to see your level.", + "NO_DEVICE": "No input device picked in Settings", + "SAVED_UNTIL_CONNECTED": "Saved, applies when Connected to Voip", + "PLAYER_VOLUME": "{{ name }} volume", + "MUTES_PLAYER": "0 mutes this player" } }, "SERVICE": { diff --git a/Code/skyrim_ui/src/styles.scss b/Code/skyrim_ui/src/styles.scss index 6948fd0368..ed93d672bd 100644 --- a/Code/skyrim_ui/src/styles.scss +++ b/Code/skyrim_ui/src/styles.scss @@ -1,3 +1,4 @@ +@import '@angular/cdk/overlay-prebuilt.css'; @import 'functions/mod'; @import 'functions/rem'; diff --git a/Code/skyrim_ui/src/typings.d.ts b/Code/skyrim_ui/src/typings.d.ts index be77792364..6299a0911e 100644 --- a/Code/skyrim_ui/src/typings.d.ts +++ b/Code/skyrim_ui/src/typings.d.ts @@ -96,6 +96,29 @@ declare namespace SkyrimTogetherTypes { type PartyInviteReceivedCallback = (inviterId: number) => void; type QuestUpdatedCallback = () => void; + + /** Voice chat settings, as saved in the player's STServer.ini. Volumes are 0 to 100. */ + type VoiceSettingsCallback = ( + inputDevice: string, + connect: number, + outputDevice: string, + channel: string, + fadeStart: number, + micVolume: number, + noiseGate: number, + ) => void; + + /** Audio devices Windows knows about, by name. */ + type VoiceDevicesCallback = ( + inputs: Array, + outputs: Array, + ) => void; + + /** Players whose voice is getting through their noise gate right now. */ + type VoiceTalkingCallback = (playerIds: Array) => void; + + /** The local microphone's peak level, 0 to 100, while the meter is on. */ + type VoiceMicLevelCallback = (level: number) => void; } /** Global Skyrim: Together object. */ @@ -449,4 +472,50 @@ interface SkyrimTogether { * Open the play guide on the wiki in the default browser. */ openPlayGuide(): void; + + /** Add listener to voice chat settings. */ + on( + event: 'voiceSettings', + callback: SkyrimTogetherTypes.VoiceSettingsCallback, + ): void; + + /** Add listener to the audio device lists. */ + on( + event: 'voiceDevices', + callback: SkyrimTogetherTypes.VoiceDevicesCallback, + ): void; + + /** Add listener to who is talking. */ + on( + event: 'voiceTalking', + callback: SkyrimTogetherTypes.VoiceTalkingCallback, + ): void; + + /** Add listener to the local microphone level. */ + on( + event: 'voiceMicLevel', + callback: SkyrimTogetherTypes.VoiceMicLevelCallback, + ): void; + + /** Re-read the voice settings and audio devices, they arrive as voiceSettings and voiceDevices. */ + voiceRefresh(): void; + + /** + * Save and apply voice chat settings. Numbers are whole numbers: volumes 0 to 100, the fade start in game units. + */ + setVoiceSettings( + inputDevice: string, + connect: number, + outputDevice: string, + channel: string, + fadeStart: number, + micVolume: number, + noiseGate: number, + ): void; + + /** Volume of one player's voice, 0 to 100. */ + setVoicePlayerVolume(playerId: number, volume: number): void; + + /** Turn the microphone meter on (1) or off (0). */ + setVoiceMeter(enabled: number): void; } From d9a702bdf61e5612a5dfa585616e35f1604e7ab5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:26:35 +0000 Subject: [PATCH 04/12] Ship the Opus licence notice with the client Opus is BSD-3-Clause, which asks binary redistributions to include its copyright notice and licence. THIRD_PARTY_NOTICES.txt is copied next to SkyrimTogether.exe on install, so it ends up in the release zip. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- Code/client/xmake.lua | 2 ++ THIRD_PARTY_NOTICES.txt | 62 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 THIRD_PARTY_NOTICES.txt diff --git a/Code/client/xmake.lua b/Code/client/xmake.lua index 2855c5810a..6147dc8df7 100644 --- a/Code/client/xmake.lua +++ b/Code/client/xmake.lua @@ -26,6 +26,8 @@ target(name) local uidir = path.join(target:scriptdir(), "..", "skyrim_ui", "src") os.cp(path.join(uidir, "assets", "images", "cursor.dds"), path.join(target:installdir(), "bin", "assets", "images", "cursor.dds")) os.cp(path.join(uidir, "assets", "images", "cursor.png"), path.join(target:installdir(), "bin", "assets", "images", "cursor.png")) + -- licence notices of bundled libraries (Opus asks for its notice to ship with the binaries) + os.cp(path.join(target:scriptdir(), "..", "..", "THIRD_PARTY_NOTICES.txt"), path.join(target:installdir(), "bin", "THIRD_PARTY_NOTICES.txt")) os.rm(path.join(target:installdir(), "bin", "**Tests.exe")) end) diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt new file mode 100644 index 0000000000..5d88b16cbf --- /dev/null +++ b/THIRD_PARTY_NOTICES.txt @@ -0,0 +1,62 @@ +Third-party notices for Skyrim Together +======================================= + +The game client includes the software below. Their licences ask for these +notices to ship with the binaries. + +------------------------------------------------------------------------------- +Opus 1.5.2 (https://opus-codec.org), voice chat codec, BSD-3-Clause +------------------------------------------------------------------------------- + +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, + Jean-Marc Valin, Timothy B. Terriberry, + CSIRO, Gregory Maxwell, Mark Borgerding, + Erik de Castro Lopo, Mozilla, Amazon + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses which are +specified at: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ + +------------------------------------------------------------------------------- +miniaudio 0.11.22 (https://miniaud.io), audio devices for voice chat +------------------------------------------------------------------------------- + +Copyright David Reid. Used under its public domain dedication (Unlicense), +alternatively MIT No Attribution. Neither requires this notice, it is listed +for completeness. From 7e51214f50f5bd349424de603a294bb44a763b92 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 19:43:39 +0000 Subject: [PATCH 05/12] Qualify MakeUnique with TiltedPhoques namespace in VoiceAudio Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- Code/client/Services/Generic/VoiceAudio.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/client/Services/Generic/VoiceAudio.cpp b/Code/client/Services/Generic/VoiceAudio.cpp index abc670a19a..004607cc46 100644 --- a/Code/client/Services/Generic/VoiceAudio.cpp +++ b/Code/client/Services/Generic/VoiceAudio.cpp @@ -325,7 +325,7 @@ struct VoiceAudio::Impl }; VoiceAudio::VoiceAudio() noexcept - : m_pImpl(MakeUnique()) + : m_pImpl(TiltedPhoques::MakeUnique()) { } From 4daf20758e0b1550011d720143a0402b99ddcc30 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 22:33:15 +0000 Subject: [PATCH 06/12] Read the camera root by offset so voice builds on upstream dev too Upstream dev renamed TESCamera::cameraNode (NiNode*) to cameraRoot (NiPointer). Both are a single pointer at 0x20, so read it by offset and the voice listener compiles against either header. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- Code/client/Services/Generic/VoiceService.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Code/client/Services/Generic/VoiceService.cpp b/Code/client/Services/Generic/VoiceService.cpp index a10d6a4cae..f4655601ec 100644 --- a/Code/client/Services/Generic/VoiceService.cpp +++ b/Code/client/Services/Generic/VoiceService.cpp @@ -46,7 +46,10 @@ constexpr float kMaxPan = 0.7f; constexpr size_t kNiAVObjectWorldRotateOffset = 0x7C; constexpr size_t kNiAVObjectWorldTranslateOffset = 0xA0; static_assert(kNiAVObjectWorldTranslateOffset + sizeof(float) * 3 <= sizeof(NiAVObject)); -static_assert(offsetof(TESCamera, cameraNode) == 0x20); +// TESCamera's camera root node pointer. Read by offset because the member is named cameraNode (NiNode*) on our +// master and cameraRoot (NiPointer) on upstream dev, both a single pointer at 0x20. +constexpr size_t kTESCameraRootOffset = 0x20; +static_assert(kTESCameraRootOffset + sizeof(NiNode*) <= sizeof(TESCamera)); struct Listener { @@ -70,7 +73,9 @@ bool GetListener(Listener& aOutListener) noexcept PlayerCamera* pCamera = PlayerCamera::Get(); // PlayerCamera redeclares the TESCamera members at the wrong offsets, go through the base on purpose - const NiNode* pCameraRoot = pCamera ? static_cast(pCamera)->cameraNode : nullptr; + const NiNode* pCameraRoot = nullptr; + if (pCamera) + std::memcpy(&pCameraRoot, reinterpret_cast(static_cast(pCamera)) + kTESCameraRootOffset, sizeof(pCameraRoot)); if (!pCameraRoot) return true; From f7c0e75079d75ee4edca3a82c6af9c82f081d478 Mon Sep 17 00:00:00 2001 From: absol89 Date: Tue, 3 Nov 2026 13:33:36 +0100 Subject: [PATCH 07/12] Add a voice master volume slider up to 500 percent Default received voice playback to 250 percent and save the setting in STServer.ini. Multiply per-player and distance gains without changing microphone capture, and retain the final output clamp. Validation: client release build and production UI build passed; TPTests passed 134 assertions in 12 cases. In-game listening and persistence checks remain. --- .../client/Services/Generic/OverlayClient.cpp | 2 ++ Code/client/Services/Generic/VoiceService.cpp | 7 +++++- Code/client/Services/VoiceService.h | 1 + Code/client/Systems/VoiceDsp.cpp | 5 ++++ Code/client/Systems/VoiceDsp.h | 3 +++ .../settings/settings.component.html | 15 ++++++++++++ .../src/app/mock/skyrimtogether.mock.ts | 5 +++- .../src/app/services/voip.service.ts | 9 +++++++ Code/skyrim_ui/src/assets/i18n/en.json | 1 + Code/skyrim_ui/src/typings.d.ts | 2 ++ Code/tests/voice_dsp.cpp | 24 +++++++++++++++++++ 11 files changed, 72 insertions(+), 2 deletions(-) diff --git a/Code/client/Services/Generic/OverlayClient.cpp b/Code/client/Services/Generic/OverlayClient.cpp index 9610444371..55afd2365e 100644 --- a/Code/client/Services/Generic/OverlayClient.cpp +++ b/Code/client/Services/Generic/OverlayClient.cpp @@ -187,6 +187,8 @@ void OverlayClient::ProcessSetVoiceSettings(CefRefPtr aEventArgs) settings.FadeStart = static_cast(aEventArgs->GetInt(4)); settings.MicVolume = static_cast(aEventArgs->GetInt(5)) / 100.f; settings.NoiseGate = static_cast(aEventArgs->GetInt(6)) / 100.f; + if (aEventArgs->GetSize() > 7) + settings.MasterVolume = static_cast(aEventArgs->GetInt(7)) / 100.f; World::Get().GetRunner().Queue([settings]() { World::Get().ctx().at().ApplySettings(settings); }); } diff --git a/Code/client/Services/Generic/VoiceService.cpp b/Code/client/Services/Generic/VoiceService.cpp index f4655601ec..72b7c97c29 100644 --- a/Code/client/Services/Generic/VoiceService.cpp +++ b/Code/client/Services/Generic/VoiceService.cpp @@ -189,6 +189,7 @@ void VoiceService::ApplySettings(const Settings& acSettings) noexcept m_settings.FadeStart = std::isfinite(m_settings.FadeStart) ? std::clamp(m_settings.FadeStart, 0.f, kMaxFadeStart) : 2500.f; m_settings.MicVolume = Clamp01(m_settings.MicVolume, 1.f); m_settings.NoiseGate = Clamp01(m_settings.NoiseGate, 0.1f); + m_settings.MasterVolume = VoiceDsp::ClampMasterVolume(m_settings.MasterVolume); m_audio.SetCaptureSettings(m_settings.MicVolume, m_settings.NoiseGate); SaveSettings(); @@ -339,7 +340,7 @@ void VoiceService::UpdateSpeakerGains() noexcept const float dz = pActor->position.z - listener.Position.z; const float distance = std::sqrt(dx * dx + dy * dy + dz * dz); - const float gain = volume * VoiceDsp::DistanceGain(distance, m_settings.FadeStart); + const float gain = m_settings.MasterVolume * volume * VoiceDsp::DistanceGain(distance, m_settings.FadeStart); const float horizontal = std::sqrt(dx * dx + dy * dy); const float pan = horizontal > 1.f ? (dx * listener.RightX + dy * listener.RightY) / horizontal * kMaxPan : 0.f; @@ -409,6 +410,7 @@ void VoiceService::LoadSettings() noexcept settings.FadeStart = static_cast(ini.GetDoubleValue(kSection, "fVoipFadeStart", 2500.0)); settings.MicVolume = static_cast(ini.GetDoubleValue(kSection, "fMicVolume", 1.0)); settings.NoiseGate = static_cast(ini.GetDoubleValue(kSection, "fNoiseGate", 0.1)); + settings.MasterVolume = static_cast(ini.GetDoubleValue(kSection, "fVoipMasterVolume", 2.5)); } else { @@ -419,6 +421,7 @@ void VoiceService::LoadSettings() noexcept settings.FadeStart = std::isfinite(settings.FadeStart) ? std::clamp(settings.FadeStart, 0.f, kMaxFadeStart) : 2500.f; settings.MicVolume = Clamp01(settings.MicVolume, 1.f); settings.NoiseGate = Clamp01(settings.NoiseGate, 0.1f); + settings.MasterVolume = VoiceDsp::ClampMasterVolume(settings.MasterVolume); m_settings = settings; } @@ -444,6 +447,7 @@ void VoiceService::SaveSettings() const noexcept ini.SetValue(kSection, "sOutputDevice", m_settings.OutputDevice.c_str(), "; Player: where voices play. Empty uses the Windows default device, like the game"); ini.SetValue(kSection, "sVoipChannel", ChannelToString(m_settings.ListenChannel), "; Player: who you hear, everyone, party or strangers"); ini.SetDoubleValue(kSection, "fVoipFadeStart", m_settings.FadeStart, "; Player: voices are full volume up to this distance in game units, silent at twice it"); + ini.SetDoubleValue(kSection, "fVoipMasterVolume", m_settings.MasterVolume, "; Player: all received voices, 0 to 5 (0-500%), default 2.5 (250%)"); ini.SetDoubleValue(kSection, "fMicVolume", m_settings.MicVolume, "; Player: microphone volume, 0 to 1"); ini.SetDoubleValue(kSection, "fNoiseGate", m_settings.NoiseGate, "; Player: microphone level below which nothing is sent, 0 to 1 like the Windows mic test bar"); @@ -465,5 +469,6 @@ void VoiceService::SendSettingsToUi() const noexcept pArguments->SetInt(4, static_cast(std::lround(m_settings.FadeStart))); pArguments->SetInt(5, static_cast(std::lround(m_settings.MicVolume * 100.f))); pArguments->SetInt(6, static_cast(std::lround(m_settings.NoiseGate * 100.f))); + pArguments->SetInt(7, static_cast(std::lround(m_settings.MasterVolume * 100.f))); pOverlay->ExecuteAsync("voiceSettings", pArguments); } diff --git a/Code/client/Services/VoiceService.h b/Code/client/Services/VoiceService.h index faaaf035e0..53922d55d4 100644 --- a/Code/client/Services/VoiceService.h +++ b/Code/client/Services/VoiceService.h @@ -41,6 +41,7 @@ struct VoiceService float FadeStart{2500.f}; float MicVolume{1.f}; float NoiseGate{0.1f}; + float MasterVolume{2.5f}; }; VoiceService(World& aWorld, entt::dispatcher& aDispatcher, TransportService& aTransport) noexcept; diff --git a/Code/client/Systems/VoiceDsp.cpp b/Code/client/Systems/VoiceDsp.cpp index f138b3d956..423b7901da 100644 --- a/Code/client/Systems/VoiceDsp.cpp +++ b/Code/client/Systems/VoiceDsp.cpp @@ -5,6 +5,11 @@ namespace VoiceDsp { +float ClampMasterVolume(float aVolume) noexcept +{ + return std::isfinite(aVolume) ? std::clamp(aVolume, 0.f, 5.f) : 2.5f; +} + float Peak(const float* apSamples, size_t aCount) noexcept { float peak = 0.f; diff --git a/Code/client/Systems/VoiceDsp.h b/Code/client/Systems/VoiceDsp.h index 8a0ddcf3e5..661fb67fdb 100644 --- a/Code/client/Systems/VoiceDsp.h +++ b/Code/client/Systems/VoiceDsp.h @@ -15,6 +15,9 @@ namespace VoiceDsp constexpr uint32_t kSampleRate = 48000; constexpr uint32_t kFrameSamples = 960; +// Playback multiplier: 0 mutes, 1 is unchanged, 5 is the maximum. Invalid values use 250%. +float ClampMasterVolume(float aVolume) noexcept; + // Peak of a frame, 0 to 1. This is what the Windows "Test your microphone" bar shows. float Peak(const float* apSamples, size_t aCount) noexcept; diff --git a/Code/skyrim_ui/src/app/components/settings/settings.component.html b/Code/skyrim_ui/src/app/components/settings/settings.component.html index 4e40e7aa16..b290e047c5 100644 --- a/Code/skyrim_ui/src/app/components/settings/settings.component.html +++ b/Code/skyrim_ui/src/app/components/settings/settings.component.html @@ -108,6 +108,21 @@

{{ 'COMPONENT.SETTINGS.VOICE.HEADER' | transloco }}

> +
+ + +
| undefined; private voiceTalkingTimer: ReturnType | undefined; diff --git a/Code/skyrim_ui/src/app/services/voip.service.ts b/Code/skyrim_ui/src/app/services/voip.service.ts index 426d905283..12dfd8ef09 100644 --- a/Code/skyrim_ui/src/app/services/voip.service.ts +++ b/Code/skyrim_ui/src/app/services/voip.service.ts @@ -17,7 +17,7 @@ export interface VoipSettings { /** Empty plays on the game's (the Windows default) device. */ outputDevice: string; channel: VoipChannel; - /** Full volume up to this distance in game units, silent at twice it. */ + /** Full volume up to this distance in game units, then quieter like real sound, silent at four times it. */ fadeStart: number; /** 0 to 100 */ micVolume: number; @@ -32,7 +32,7 @@ export const defaultVoipSettings: VoipSettings = { connect: false, outputDevice: '', channel: VoipChannel.EVERYONE, - fadeStart: 2500, + fadeStart: 2000, micVolume: 100, noiseGate: 5, masterVolume: 100, diff --git a/Code/skyrim_ui/src/assets/i18n/en.json b/Code/skyrim_ui/src/assets/i18n/en.json index 963f9266fe..8f613cd957 100644 --- a/Code/skyrim_ui/src/assets/i18n/en.json +++ b/Code/skyrim_ui/src/assets/i18n/en.json @@ -135,7 +135,7 @@ "CHANNEL_PARTY": "Party", "CHANNEL_STRANGERS": "Strangers", "FADE_START": "Voip fade start", - "FADE_HINT": "Full volume up to this distance, silent at {{ silent }} units" + "FADE_HINT": "Full volume up to this distance, then fades like real sound, silent at {{ silent }} units" }, "PARTY": "Party", "SHOW_PARTY": "Show Party", diff --git a/Code/tests/voice_dsp.cpp b/Code/tests/voice_dsp.cpp index 33104994e2..3943428f7b 100644 --- a/Code/tests/voice_dsp.cpp +++ b/Code/tests/voice_dsp.cpp @@ -9,11 +9,25 @@ using namespace VoiceDsp; TEST_CASE("Voice distance falloff", "[voice.dsp]") { - REQUIRE(DistanceGain(0.f, 2500.f) == 1.f); - REQUIRE(DistanceGain(2500.f, 2500.f) == 1.f); - REQUIRE(DistanceGain(3750.f, 2500.f) == Approx(0.5f)); - REQUIRE(DistanceGain(5000.f, 2500.f) == 0.f); - REQUIRE(DistanceGain(9000.f, 2500.f) == 0.f); + REQUIRE(DistanceGain(0.f, 2000.f) == 1.f); + REQUIRE(DistanceGain(2000.f, 2000.f) == 1.f); + // -6 dB per doubling: half the amplitude at twice the fade start, a third at three times before the taper + REQUIRE(DistanceGain(3000.f, 2000.f) == Approx(2.f / 3.f)); + REQUIRE(DistanceGain(4000.f, 2000.f) == Approx(0.5f)); + // Halfway through the taper the inverse law (1/3) is halved + REQUIRE(DistanceGain(6000.f, 2000.f) == Approx(1.f / 6.f)); + REQUIRE(DistanceGain(8000.f, 2000.f) == 0.f); + REQUIRE(DistanceGain(9000.f, 2000.f) == 0.f); + + // Never gets louder further away + float previous = 1.f; + for (float distance = 0.f; distance <= 8400.f; distance += 50.f) + { + const float gain = DistanceGain(distance, 2000.f); + REQUIRE(gain <= previous); + previous = gain; + } + // A fade start of 0 would divide by zero, it means nobody is heard REQUIRE(DistanceGain(10.f, 0.f) == 0.f); } From 2f51981258b96fe9e9cf8ac047ed19a72c25b54a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 14:52:21 +0000 Subject: [PATCH 12/12] Keep the voice fade hint to the two distances The settings hint names where voices start to fade and where they go silent, without describing the curve. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018fvQxRbmmR1Uv7KRvfWVgW --- Code/client/Services/Generic/VoiceService.cpp | 2 +- Code/skyrim_ui/src/assets/i18n/en.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/client/Services/Generic/VoiceService.cpp b/Code/client/Services/Generic/VoiceService.cpp index 7b6584dcc4..ae79b6d793 100644 --- a/Code/client/Services/Generic/VoiceService.cpp +++ b/Code/client/Services/Generic/VoiceService.cpp @@ -455,7 +455,7 @@ void VoiceService::SaveSettings() const noexcept "; Built-in proximity voice chat. On a server: lets players talk to players nearby. For a player: the Connect to voip checkbox in Settings"); ini.SetValue(kSection, "sOutputDevice", m_settings.OutputDevice.c_str(), "; Player: where voices play. Empty uses the Windows default device, like the game"); ini.SetValue(kSection, "sVoipChannel", ChannelToString(m_settings.ListenChannel), "; Player: who you hear, everyone, party or strangers"); - ini.SetDoubleValue(kSection, "fVoipFadeStart", m_settings.FadeStart, "; Player: voices are full volume up to this distance in game units, then -6 dB per doubling, silent at 4 times it"); + ini.SetDoubleValue(kSection, "fVoipFadeStart", m_settings.FadeStart, "; Player: voices are full volume up to this distance in game units, silent at 4 times it"); ini.SetDoubleValue(kSection, "fVoipMasterVolume", m_settings.MasterVolume, "; Player: all received voices, 0 to 5 (0-500%), default 1 (100%)"); ini.SetDoubleValue(kSection, "fMicVolume", m_settings.MicVolume, "; Player: microphone volume, 0 to 1"); ini.SetDoubleValue(kSection, "fNoiseGate", m_settings.NoiseGate, "; Player: microphone level below which nothing is sent, 0 to 1 like the Windows mic test bar"); diff --git a/Code/skyrim_ui/src/assets/i18n/en.json b/Code/skyrim_ui/src/assets/i18n/en.json index 8f613cd957..963f9266fe 100644 --- a/Code/skyrim_ui/src/assets/i18n/en.json +++ b/Code/skyrim_ui/src/assets/i18n/en.json @@ -135,7 +135,7 @@ "CHANNEL_PARTY": "Party", "CHANNEL_STRANGERS": "Strangers", "FADE_START": "Voip fade start", - "FADE_HINT": "Full volume up to this distance, then fades like real sound, silent at {{ silent }} units" + "FADE_HINT": "Full volume up to this distance, silent at {{ silent }} units" }, "PARTY": "Party", "SHOW_PARTY": "Show Party",