diff --git a/.gitignore b/.gitignore index 086bba3..d889b6a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,6 @@ Docs/security-backlog.md deploy/.env deploy/models/* !deploy/models/.gitkeep +# The Google Home HomeGraph service-account key lands here. It is a live Google credential. +deploy/secrets/ diff --git a/App/serval_app/lib/data/live_repository.dart b/App/serval_app/lib/data/live_repository.dart index 2eb174a..e43808f 100644 --- a/App/serval_app/lib/data/live_repository.dart +++ b/App/serval_app/lib/data/live_repository.dart @@ -6,12 +6,14 @@ import 'package:shared_preferences/shared_preferences.dart'; import '../models/activity.dart'; import '../models/alert.dart'; import '../models/camera.dart'; +import '../models/cast_target.dart'; import '../models/clip_selection.dart'; import '../models/config_backup.dart'; import '../models/conversation.dart'; import '../models/saved_clip.dart'; import '../media/media_saver.dart'; import '../media/media_sharer.dart'; +import '../models/google_home.dart'; import '../models/ptz.dart'; import '../models/push.dart'; import '../models/server_settings.dart'; @@ -2163,6 +2165,16 @@ class LiveServalRepository implements ServalRepository { ); } + @override + Future googleHomeStatus() => _api.googleHomeStatus(); + + @override + Future> googleHomeLinks() => _api.googleHomeLinks(); + + @override + Future unlinkGoogleHome(String agentUserId) => + _api.unlinkGoogleHome(agentUserId); + @override Future pushConfig() => _api.pushConfig(); @@ -2565,6 +2577,31 @@ class LiveServalRepository implements ServalRepository { @override Future deleteClip(String id) => _api.deleteClip(id); + @override + Future castReceiverAppId() => _api.castReceiverAppId(); + + @override + Future castVodUrl( + String cameraId, { + required DateTime from, + required DateTime to, + required DateTime at, + }) async { + final token = await _auth.mintStreamToken(); + if (token == null) return null; + + return _api.castVodUrl( + cameraId, + from: from, + to: to, + at: at, + streamToken: token, + ); + } + + @override + Future castTarget(String cameraId) => _api.castTarget(cameraId); + @override Future savedClipUrl(String id) async => _api.savedClipUrl(id, streamToken: await _auth.mintStreamToken()); diff --git a/App/serval_app/lib/data/sample_repository.dart b/App/serval_app/lib/data/sample_repository.dart index c9f7724..d8357e5 100644 --- a/App/serval_app/lib/data/sample_repository.dart +++ b/App/serval_app/lib/data/sample_repository.dart @@ -4,11 +4,13 @@ import 'package:flutter/painting.dart'; import '../models/activity.dart'; import '../models/alert.dart'; import '../models/camera.dart'; +import '../models/cast_target.dart'; import '../models/clip_selection.dart'; import '../models/saved_clip.dart'; import '../models/config_backup.dart'; import '../models/conversation.dart'; import '../media/media_saver.dart'; +import '../models/google_home.dart'; import '../models/ptz.dart'; import '../models/push.dart'; import '../models/server_settings.dart'; @@ -249,6 +251,33 @@ class SampleServalRepository implements ServalRepository { List? rules, }) async {} + /// **Switched off, which is the state nearly every deployment is in** — and the more useful one + /// to draw, because it is the state the card exists to explain. Turning it on needs a public + /// HTTPS endpoint and a Nest Hub on the LAN; a sample that pretended to have both would show the + /// one layout an operator is least likely to see. + /// + /// The sentence is the Server's own wording, copied rather than invented: the App renders + /// whatever reason it is handed, so the sample has to prove that path rather than a local map. + @override + Future googleHomeStatus() async => const GoogleHomeStatus( + effective: false, + blocker: 'disabled', + reason: + 'Serval:GoogleHome:Enabled is false, so the Google Home routes are closed. ' + 'This is the default. See Docs/google-home.md for what it needs before turning it on.', + publicBaseUrl: null, + homeGraphKeyConfigured: false, + castReceiverConfigured: false, + ); + + /// Nothing linked, matching the status above — a linked account on a disabled integration would + /// be a state the Server cannot produce. + @override + Future> googleHomeLinks() async => const []; + + @override + Future unlinkGoogleHome(String agentUserId) async {} + /// A deployment with notifications on and a key that looks like a real one. The sample never /// reaches a browser's push machinery — `PushClient` is the stub under `flutter test` — so this /// only has to be well-formed enough for the screen to draw its enabled state. @@ -467,6 +496,22 @@ class SampleServalRepository implements ServalRepository { @override Future deleteClip(String id) async {} + /// No server, so no receiver to launch and nothing for one to fetch — the button stays absent in + /// samples, the same posture [canStreamLive] takes on the wall. + @override + Future castReceiverAppId() async => null; + + @override + Future castVodUrl( + String cameraId, { + required DateTime from, + required DateTime to, + required DateTime at, + }) async => null; + + @override + Future castTarget(String cameraId) async => null; + /// Null, so the sample clip screens draw their placeholder rather than reaching for a Server — /// the same posture [canStreamLive] takes on the wall. @override diff --git a/App/serval_app/lib/data/serval_api.dart b/App/serval_app/lib/data/serval_api.dart index 25a4583..b433a5d 100644 --- a/App/serval_app/lib/data/serval_api.dart +++ b/App/serval_app/lib/data/serval_api.dart @@ -4,8 +4,10 @@ import 'package:http/http.dart' as http; import '../models/alert.dart'; import '../models/camera.dart'; +import '../models/cast_target.dart'; import '../models/clip_selection.dart'; import '../models/config_backup.dart'; +import '../models/google_home.dart'; import '../models/ptz.dart'; import '../models/push.dart'; import '../models/saved_clip.dart'; @@ -325,6 +327,32 @@ class ServalApi { PushDevice.fromJson(device as Map), ]; + // --------------------------------------------------------------- google home + // + // Three reads and one action, and deliberately no writes: every Serval:GoogleHome:* key is + // environment-only, so there is nothing here for a form to submit. See Docs/google-home.md. + + Future googleHomeStatus() async => + GoogleHomeStatus.fromJson( + await _getJson('/api/google/status') as Map, + ); + + Future> googleHomeLinks() async => [ + for (final link in await _getJson('/api/google/links') as List) + GoogleHomeLink.fromJson(link as Map), + ]; + + /// Unlinks the Google account, revoking every credential issued to it. Google keeps showing the + /// cameras until it next calls and is refused. + Future unlinkGoogleHome(String agentUserId) async { + final response = await _client.delete( + config.resolve('/api/google/links/$agentUserId'), + ); + if (response.statusCode != 204) { + throw ServalApiException(response.statusCode, _errorMessage(response)); + } + } + /// Registers this browser, or refreshes what the Server holds for it. /// /// Called on every launch, not only when somebody turns notifications on: the row is keyed by a @@ -722,6 +750,35 @@ class ServalApi { await _getJson('/api/clips/$id/status') as Map, ); + /// The Cast application this deployment casts with, or null where none is registered. + /// + /// Read before anything is cast, because Google's sender SDK finds no devices until it knows + /// which application to look for. 404 is the ordinary answer on a deployment that has not set one + /// up, so it is not a failure. + Future castReceiverAppId() async { + final response = await _client.get(config.resolve('/api/cast/receiver')); + if (response.statusCode == 404 || response.statusCode == 503) return null; + + final json = _decode(response) as Map; + final id = json['receiverAppId']?.toString(); + return (id == null || id.isEmpty) ? null : id; + } + + /// Which Cast receiver to launch for a camera, and what to hand it. + /// + /// Null where this deployment cannot cast — no Cast application registered (501), or the + /// configuration the receiver is served behind is incomplete (503). Both are ordinary states + /// rather than faults, and the button is simply absent in each, so neither throws. + Future castTarget(String cameraId) async { + final response = await _client.post( + config.resolve('/api/cameras/$cameraId/cast'), + ); + + if (response.statusCode == 501 || response.statusCode == 503) return null; + + return CastTarget.fromJson(_decode(response) as Map); + } + Future renameClip(String id, String name) async { final response = await _client.patch( config.resolve('/api/clips/$id'), @@ -871,6 +928,25 @@ class ServalApi { 'to': to.toUtc().toIso8601String(), }); + /// The same window, re-encoded for a television. + /// + /// A separate route rather than a flag on [vodUrl], because the two want opposite things: this + /// player wants the recording untouched, and a Cast device cannot decode it at all — every camera + /// here records above the 1080p a Cast device will take. [streamToken] rides in the URL because a + /// Cast receiver cannot set a header, and the segments inherit it. + Uri castVodUrl( + String cameraId, { + required DateTime from, + required DateTime to, + required DateTime at, + required String streamToken, + }) => config.resolve('/api/cameras/$cameraId/cast.m3u8', { + 'from': from.toUtc().toIso8601String(), + 'to': to.toUtc().toIso8601String(), + 'at': at.toUtc().toIso8601String(), + 'stream_token': streamToken, + }); + /// How far into the playlist the instant it was asked for actually sits. /// /// Segments are four seconds and a window can be asked for at any instant inside one, so the diff --git a/App/serval_app/lib/data/serval_repository.dart b/App/serval_app/lib/data/serval_repository.dart index fd6a976..fe57ba5 100644 --- a/App/serval_app/lib/data/serval_repository.dart +++ b/App/serval_app/lib/data/serval_repository.dart @@ -3,10 +3,12 @@ import 'package:flutter/foundation.dart'; import '../models/activity.dart'; import '../models/alert.dart'; import '../models/camera.dart'; +import '../models/cast_target.dart'; import '../models/clip_selection.dart'; import '../models/config_backup.dart'; import '../models/conversation.dart'; import '../media/media_saver.dart'; +import '../models/google_home.dart'; import '../models/ptz.dart'; import '../models/push.dart'; import '../models/saved_clip.dart'; @@ -419,6 +421,22 @@ abstract interface class ServalRepository { // only: which alerts reach this person is [notificationPreferences] above, because that belongs // to the account and this belongs to one of their browsers. + // ----------------------------------------------------------- google home + // + // Read-only, on purpose. Every Serval:GoogleHome:* key is environment-only — see + // Docs/google-home.md — so what the App offers is diagnosis, not configuration: which one + // condition is unmet, and whether an account is linked. Unlinking is the single exception, + // because it is an act rather than a setting. + + /// Whether the Google Home integration is live, and the sentence naming what is stopping it. + Future googleHomeStatus(); + + /// The linked Google account. At most one, and usually none. + Future> googleHomeLinks(); + + /// Unlinks, revoking every credential issued to Google. + Future unlinkGoogleHome(String agentUserId); + /// The deployment's VAPID public key and whether notifications are switched on at all. Future pushConfig(); @@ -576,6 +594,29 @@ abstract interface class ServalRepository { Future deleteClip(String id); + /// The Cast application this deployment casts with, or null where none is registered — which is + /// also what a deployment with no Server answers. + Future castReceiverAppId(); + + /// A VOD playlist a Cast device can fetch on its own, for a past window. + /// + /// Separate from [vodUrlFor] because the credential differs, not the playlist: a Cast receiver + /// cannot set an `Authorization` header, so the token has to ride in the URL — and the segments + /// inherit it, since the receiver resolves their names against this URL. Null where there is no + /// Server to mint one against. + Future castVodUrl( + String cameraId, { + required DateTime from, + required DateTime to, + required DateTime at, + }); + + /// Which Cast receiver to launch for a camera, and what to hand it. + /// + /// Null where this deployment cannot cast: no Cast application registered, or no Server at all. + /// That is the ordinary case rather than a failure — the button is absent, not broken. + Future castTarget(String cameraId); + /// The clip's video, ready for a player or a download. Null where there is no Server. Future savedClipUrl(String id); diff --git a/App/serval_app/lib/main.dart b/App/serval_app/lib/main.dart index 4ded22f..340e026 100644 --- a/App/serval_app/lib/main.dart +++ b/App/serval_app/lib/main.dart @@ -12,6 +12,7 @@ import 'data/sample_repository.dart'; import 'data/serval_api.dart'; import 'data/serval_config.dart'; import 'data/serval_repository.dart'; +import 'platform/frame_watchdog.dart'; import 'playback/vod_player.dart'; import 'push/push_client.dart'; import 'router/serval_router.dart'; @@ -121,6 +122,12 @@ class _ServalMaterialAppState extends State<_ServalMaterialApp> { // // A no-op off the web and on a browser with no push, so there is no platform branch here. PushClient.onNavigate(_router.go); + + // The other half of a tap landing where it was sent. Routing it is only half the job: a + // browser that has come back from the background unable to paint holds the screen it was on + // however correct the route underneath it is. See [watchFrames], including why the route it + // recovers to has to come from the router rather than from the address bar. + watchFrames(() => _router.routeInformationProvider.value.uri.toString()); } @override diff --git a/App/serval_app/lib/models/cast_target.dart b/App/serval_app/lib/models/cast_target.dart new file mode 100644 index 0000000..59695f8 --- /dev/null +++ b/App/serval_app/lib/models/cast_target.dart @@ -0,0 +1,112 @@ +import 'timeline.dart'; + +/// Where to cast one camera: which receiver application to launch, and what to hand it. +/// +/// Both halves come from the server rather than being built here. The application id is registered +/// by the operator against their own deployment, so it is not something the App could know; and the +/// URL carries a camera-scoped ticket the server mints, which is what lets the receiver — running +/// on a television, with no Serval session — reach the camera at all. +class CastTarget { + const CastTarget({required this.receiverAppId, required this.contentUrl}); + + factory CastTarget.fromJson(Map json) => CastTarget( + receiverAppId: json['receiverAppId']?.toString() ?? '', + contentUrl: Uri.parse(json['contentUrl']?.toString() ?? ''), + ); + + /// The Cast application the sender launches. Serval's own receiver, not Google's default one: + /// the default can only play [contentUrl] as HLS, several seconds behind, where this one + /// negotiates WebRTC first and is live. + final String receiverAppId; + + /// A live HLS playlist for the camera. What the receiver actually plays if its peer connection + /// does not come up — so it is the fallback rather than the plan, and it is a real URL either + /// way. + final Uri contentUrl; + + bool get usable => receiverAppId.isNotEmpty && contentUrl.hasScheme; +} + +/// The stretch of recording a television is currently playing. +/// +/// Held while a recording is cast so that scrubbing in the App can be mirrored there. What it +/// decides is which of two very different things a scrub means — see [covers]. +class CastWindow { + const CastWindow({required this.from, required this.to}); + + /// Where the cast started, and how far the playlist reaches. [to] is the moment the cast was + /// begun, not now: the playlist was built then and does not grow. + final DateTime from; + final DateTime to; + + /// Whether [at] is inside the media the television already has. + /// + /// **Inside is a seek; outside is a whole new cast.** The playlist covers this window and nothing + /// else, so scrubbing back to before it began — or forward past the moment it was started — asks + /// for footage that was never sent, and no seek can reach it. Inclusive at both ends, because the + /// first and last instants are in the playlist. + bool covers(DateTime at) => !at.isBefore(from) && !at.isAfter(to); + + /// How far into the cast [at] sits, which is what a seek is measured in. + Duration offsetOf(DateTime at) => at.difference(from); + + /// The most footage one cast covers. + /// + /// The window wants to be the whole visible timeline, and at the shorter spans it is. The two + /// longest are not free to send: a day is around 21,600 recorded segments, which is a playlist of + /// several thousand lines for a television to parse and hold. Six hours covers every span up to + /// its own, and a scrub past it on a wider one costs a re-cast rather than being refused. + static const maxSpan = Duration(hours: 6); + + /// The window to cast when the playhead is at [at] and the scrubber is showing [timeline]. + /// + /// **Wide on purpose.** Casting only from the playhead made every move of it a fresh cast, and a + /// fresh cast is a second or two of black screen. Sending the whole visible timeline instead + /// means a click anywhere on the bar is already inside what the television has, so it is a seek. + /// + /// Centred on [at] when the timeline is wider than [maxSpan], because scrubbing goes both ways. + factory CastWindow.around(DateTime at, TimelineWindow timeline) { + final span = timeline.to.difference(timeline.from); + + var from = timeline.from; + var to = timeline.to; + + if (span.isNegative || span > maxSpan) { + final half = maxSpan ~/ 2; + from = at.subtract(half); + to = at.add(half); + + // Slid back inside the timeline rather than truncated, so a playhead near either edge still + // gets the full span instead of half of one. + if (from.isBefore(timeline.from)) { + from = timeline.from; + to = from.add(maxSpan); + } + if (to.isAfter(timeline.to)) { + to = timeline.to; + from = to.subtract(maxSpan); + if (from.isBefore(timeline.from)) from = timeline.from; + } + } + + return CastWindow(from: _firstFootageFrom(from, timeline), to: to); + } + + /// Where the recording the television is sent actually begins. + /// + /// **This is what a seek is measured from, so it has to be the footage and not the window.** The + /// cast playlist's clock starts at its first segment, and a window that opens on a stretch with + /// nothing recorded in it has its first segment wherever recording resumed — an hour later, on a + /// camera that was off overnight. Measuring seeks from the window's own left edge would then miss + /// by that whole hour. Gaps *inside* the window need no such treatment: the playlist spans them + /// at wall-clock length, so everything after one still lines up. + static DateTime _firstFootageFrom(DateTime from, TimelineWindow timeline) { + for (final span in timeline.coverage) { + if (span.to.isAfter(from)) { + return span.from.isAfter(from) ? span.from : from; + } + } + + return from; + } +} diff --git a/App/serval_app/lib/models/google_home.dart b/App/serval_app/lib/models/google_home.dart new file mode 100644 index 0000000..e7e5283 --- /dev/null +++ b/App/serval_app/lib/models/google_home.dart @@ -0,0 +1,104 @@ +/// Whether the Google Home integration is live, and what is stopping it. +/// +/// **Read-only, and there is nothing to write.** Every `Serval:GoogleHome:*` key is +/// environment-only — two are secrets, and two more decide where an anonymous endpoint sends +/// credentials — so the App neither renders a form for them nor holds any of them. What it can +/// usefully do is say which single condition is unmet, which is the thing an operator cannot get +/// from a 503. See `Docs/google-home.md`. +class GoogleHomeStatus { + const GoogleHomeStatus({ + required this.effective, + required this.blocker, + required this.reason, + required this.publicBaseUrl, + required this.homeGraphKeyConfigured, + required this.castReceiverConfigured, + }); + + /// Reads defensively, and the reason is a scar. `blocker` first shipped as an unattributed C# + /// enum, which System.Text.Json writes as a *number* — so this parsed `1` as a String, threw, + /// and the screen drew nothing at all. The Server now sends a name and a test pins it, but a + /// cast that can take a whole feature off the screen is not worth keeping for tidiness: every + /// field below tolerates the wrong shape rather than throwing. + factory GoogleHomeStatus.fromJson(Map json) => + GoogleHomeStatus( + effective: json['effective'] == true, + blocker: json['blocker']?.toString() ?? 'Disabled', + reason: json['reason']?.toString(), + publicBaseUrl: json['publicBaseUrl']?.toString(), + homeGraphKeyConfigured: json['homeGraphKeyConfigured'] == true, + castReceiverConfigured: json['castReceiverConfigured'] == true, + ); + + /// Whether a request to any Google Home route would be served rather than answered 503. + final bool effective; + + /// The Server's name for the first unmet condition, or `None`. Not shown; [reason] is. + final String blocker; + + /// The Server's own sentence naming the fix. Null when [effective]. + /// + /// Rendered verbatim rather than mapped to text here, the same contract the settings page has + /// with the catalogue: the Server owns the wording, so a condition added there needs no App + /// release to explain itself. + final String? reason; + + /// Echoed back so the value can be checked against what was pasted into the Google console. + final String? publicBaseUrl; + + /// Whether a HomeGraph key path is set. Not a requirement — without one the integration works + /// and Google simply does not hear about a renamed camera until someone re-links. + final bool homeGraphKeyConfigured; + + /// Whether a Cast application is registered to play streams with Serval's own receiver. + /// + /// Not a requirement. Without one there is simply no Cast button, which is worth reporting + /// because nothing else says so: Google will not put a camera on a television by voice whatever + /// is configured here, so the button is the only route to one and its absence looks like a bug. + final bool castReceiverConfigured; + + /// The deployment has not turned this on — which for almost every deployment is the permanent + /// state, since it needs a public HTTPS address and a Nest Hub. + /// + /// **The App draws nothing at all in this case.** The card's whole value is naming the one + /// remaining unmet condition while somebody is part-way through setting it up; when the switch + /// itself is off there is no diagnosis to offer, and a permanently inert card on the status page + /// of every deployment that will never use this is clutter for a feature nobody asked for. + /// + /// Matched on [blocker] rather than on [reason], because the blocker is a stable machine-readable + /// name and the reason is prose the Server owns and may reword. + bool get switchedOff => blocker.toLowerCase() == 'disabled'; +} + +/// A linked Google account. There is at most one. +class GoogleHomeLink { + const GoogleHomeLink({ + required this.agentUserId, + required this.linkedAt, + required this.lastFulfillmentAt, + required this.lastSyncAt, + }); + + factory GoogleHomeLink.fromJson(Map json) => GoogleHomeLink( + agentUserId: json['agentUserId']?.toString() ?? '', + linkedAt: DateTime.tryParse(json['linkedAt']?.toString() ?? '')?.toLocal(), + lastFulfillmentAt: DateTime.tryParse( + json['lastFulfillmentAt']?.toString() ?? '', + )?.toLocal(), + lastSyncAt: DateTime.tryParse( + json['lastSyncAt']?.toString() ?? '', + )?.toLocal(), + ); + + /// A generated id, not a username — it is what Google is told and sends back. + final String agentUserId; + + final DateTime? linkedAt; + + /// The last time Google actually called. This is the field worth reading: it distinguishes a + /// link that works from one that was made and then quietly stopped being used. + final DateTime? lastFulfillmentAt; + + /// The last successful `requestSync`. Null when there is no HomeGraph key. + final DateTime? lastSyncAt; +} diff --git a/App/serval_app/lib/platform/cast_sender.dart b/App/serval_app/lib/platform/cast_sender.dart new file mode 100644 index 0000000..873c696 --- /dev/null +++ b/App/serval_app/lib/platform/cast_sender.dart @@ -0,0 +1,81 @@ +// Google Cast, behind the same conditional-import shape `frame_watchdog.dart` and +// `push_client.dart` use. `dart.library.io` is true under the VM — including `flutter test` — so +// the stub is what the test binary compiles, and no widget test ever reaches for a Chromecast. +import 'cast_sender_stub.dart' + if (dart.library.js_interop) 'cast_sender_web.dart' + if (dart.library.io) 'cast_sender_stub.dart' + as platform; + +/// Sending a camera to a Chromecast or a Google TV, from the browser, without Google's cloud in +/// the middle. +/// +/// **Why this exists beside the Google Home integration.** Because that integration cannot do it. +/// Google routes camera streams to Nest displays and to the Home app and refuses televisions — it +/// never even calls Serval — and the same refusal happens for other vendors' certified +/// integrations, so it is not something a Serval change or a certification would unlock. This path +/// talks to the Cast device directly and skips the Assistant entirely. +/// +/// **What the Cast device plays is WebRTC**, the same sub-second stream the App shows. It launches +/// Serval's own receiver application, which negotiates a peer connection back to the server; media +/// then flows straight from go2rtc to the television over the LAN. The URL handed over is a live +/// HLS playlist, and that is what the receiver plays if the peer connection does not come up — so +/// there is always a picture, just occasionally a delayed one. +/// +/// **It needs a receiver to launch.** The application id is registered by the operator against +/// their own server and served by [ServalRepository.castTarget]; where none is registered there is +/// nothing to cast to and [available] stays false. +/// +/// **Web only, and Chrome only at that.** Google's sender SDK runs on Chrome and Chromium-based +/// browsers on desktop and Android; it is absent on Safari, on Firefox, and on iOS altogether, and +/// it needs the page itself served over HTTPS. [available] is what the UI asks, so the button +/// simply is not there rather than being there and failing. +abstract final class CastSender { + /// Loads Google's sender SDK and starts looking for [appId]. + /// + /// The application id is required rather than supplied at launch: the SDK discovers only devices + /// that can run a *named* application, so nothing is ever found until it has one — and with + /// nothing found there is no button to launch anything from. Safe to call more than once, and a + /// no-op anywhere the SDK cannot run. + static Future initialise(String appId) => platform.initialise(appId); + + /// Whether a Cast device has been found and can be cast to *right now*. + /// + /// A stream rather than a getter because discovery is asynchronous and ongoing: a TV that is + /// switched on after the page loads should make the button appear, and one that goes away should + /// take it back. + static Stream get available => platform.available; + + /// Whether a session is currently playing, so the button can offer to stop it. + static Stream get casting => platform.casting; + + /// Asks the viewer to pick a receiver, then plays [url] on it through receiver [appId]. + /// + /// [title] is what the receiver shows on screen while it connects. [live] says whether [url] is + /// the live camera or a recording; a recording is cast as buffered media, which is what gives the + /// television a duration and working transport controls. Returns the error Google reported, or + /// null on success — including the viewer simply dismissing the picker, which is not an error + /// worth showing. + /// [startAt] is how far into a recording to open, and is ignored when [live]. + /// + /// Sent because the playlist covers the whole visible timeline rather than the playhead, so + /// without it a cast begins hours before whatever is being watched. The playlist says the same + /// thing in an `EXT-X-START` tag and the receiver ignores it — that tag needs HLS version 6, and + /// a playlist of MPEG-TS segments is version 3. + static Future cast( + String appId, + Uri url, { + required String title, + required bool live, + Duration startAt = Duration.zero, + }) => platform.cast(appId, url, title: title, live: live, startAt: startAt); + + /// Moves the television to [position] into what it is already playing. + /// + /// A seek rather than a fresh cast, so scrubbing here lands there immediately instead of + /// restarting the receiver's media. Only meaningful while a recording is playing, and silently + /// does nothing otherwise. + static Future seek(Duration position) => platform.seek(position); + + /// Ends the session and returns the receiver to its idle screen. + static Future stop() => platform.stop(); +} diff --git a/App/serval_app/lib/platform/cast_sender_stub.dart b/App/serval_app/lib/platform/cast_sender_stub.dart new file mode 100644 index 0000000..e2b89a8 --- /dev/null +++ b/App/serval_app/lib/platform/cast_sender_stub.dart @@ -0,0 +1,21 @@ +/// Everywhere that is not a browser: no Cast SDK, so no receivers and no button. +/// +/// Empty streams rather than `Stream.value(false)` — the UI derives the button's presence from +/// what these produce, and a stream that never produces is exactly "there is nothing here". +Future initialise(String appId) async {} + +Stream get available => const Stream.empty(); + +Stream get casting => const Stream.empty(); + +Future cast( + String appId, + Uri url, { + required String title, + required bool live, + Duration startAt = Duration.zero, +}) async => 'Casting is only available in the browser.'; + +Future seek(Duration position) async {} + +Future stop() async {} diff --git a/App/serval_app/lib/platform/cast_sender_web.dart b/App/serval_app/lib/platform/cast_sender_web.dart new file mode 100644 index 0000000..d0d1e34 --- /dev/null +++ b/App/serval_app/lib/platform/cast_sender_web.dart @@ -0,0 +1,89 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; + +/// The browser half of [CastSender], talking to `web/cast.js`. +/// +/// **This polls JavaScript rather than taking callbacks from it, deliberately.** A callback from +/// JavaScript into Dart is the direction that kept breaking: dart2js binds arguments and checks +/// types *before* the body runs, so an SDK that calls back with one argument where two are +/// documented throws inside Google's own code, with no Dart frame in the stack to find it by. A +/// poll costs one boolean read a second and cannot fail that way. +const _pollInterval = Duration(seconds: 1); + +/// How long to watch the error slot after a launch, and how often. +/// +/// Not a single wait, because the two failures arrive at different times: a synchronous refusal — +/// no SDK, an application id the device cannot run — is recorded immediately, while a load that +/// the receiver drops is only given up on after the sender has retried it. Reading once at two +/// seconds caught the first and silently missed the second. +const _errorWindow = Duration(seconds: 7); +const _errorPoll = Duration(milliseconds: 500); + +@JS() +extension type _ServalCast._(JSObject _) implements JSObject { + external void initialise(String appId); + external bool available(); + external bool casting(); + external String takeError(); + external void start( + String appId, + String url, + String title, + bool live, + double startSeconds, + ); + external void seek(double seconds); + external void stop(); +} + +_ServalCast? get _cast => + globalContext.getProperty<_ServalCast?>('servalCast'.toJS); + +Future initialise(String appId) async => _cast?.initialise(appId); + +/// Polled rather than pushed, and distinct so each has its own subscription in the screen. +Stream get available => _poll(() => _cast?.available() ?? false); + +Stream get casting => _poll(() => _cast?.casting() ?? false); + +Stream _poll(bool Function() read) => + Stream.periodic(_pollInterval, (_) => read()).distinct(); + +Future cast( + String appId, + Uri url, { + required String title, + required bool live, + Duration startAt = Duration.zero, +}) async { + final api = _cast; + if (api == null) return 'Casting is not available in this browser.'; + + api.start( + appId, + url.toString(), + title, + live, + startAt.inMilliseconds / 1000.0, + ); + + // Nothing calls back from JavaScript — see above — so the error slot is watched rather than + // awaited. Returns the moment something is recorded, and null if nothing is by the end of the + // window, which is the ordinary case: the picture is on the television and there is nothing to + // say about it. + final deadline = DateTime.now().add(_errorWindow); + while (DateTime.now().isBefore(deadline)) { + await Future.delayed(_errorPoll); + + final error = api.takeError(); + if (error.isNotEmpty) return error; + } + + return null; +} + +Future seek(Duration position) async => + _cast?.seek(position.inMilliseconds / 1000.0); + +Future stop() async => _cast?.stop(); diff --git a/App/serval_app/lib/platform/frame_watchdog.dart b/App/serval_app/lib/platform/frame_watchdog.dart new file mode 100644 index 0000000..178ad8d --- /dev/null +++ b/App/serval_app/lib/platform/frame_watchdog.dart @@ -0,0 +1,45 @@ +// A browser that stops drawing the App, behind the same conditional-import shape +// `secure_context.dart` and `push_client.dart` use. `dart.library.io` is true under the VM — +// including `flutter test` — so the stub is what the test binary compiles, and no widget test ever +// reloads a page. +import 'frame_watchdog_stub.dart' + if (dart.library.js_interop) 'frame_watchdog_web.dart' + if (dart.library.io) 'frame_watchdog_stub.dart' + as platform; + +/// Watches for the App coming back from the background unable to paint, and recovers it. +/// +/// Flutter's frame pipeline on web is latched in two places, and both latches are cleared only +/// from inside the `requestAnimationFrame` callback that a scheduled frame is waiting on — +/// `FrameService._isFrameScheduled` in the engine, `SchedulerBinding._hasScheduledFrame` in the +/// framework. Neither is cleared by anything else, and both are consulted before a new frame is +/// asked for: `scheduleFrame()` returns early while a frame is believed to be outstanding. +/// +/// The web engine's own note on the first of the two says it plainly — *"if this value is stuck in +/// `true` state, there will be no way to schedule new frames and the app will freeze"*. +/// +/// A phone puts that state within easy reach. This App is animating whenever it is in front of +/// somebody — six live tiles and a feed — so there is almost always a frame in flight at the moment +/// it is sent to the background. `requestAnimationFrame` does not run for a hidden page, and a +/// backgrounded PWA that Android freezes or whose renderer is torn down and restored can come back +/// without ever delivering that callback. The latches then never clear, `scheduleFrame` is a +/// permanent no-op, and nothing else in the framework recovers it. +/// +/// What that looks like is *not* a blank screen: the canvas keeps the last frame it painted, so the +/// App sits there showing the screen it was on, ignoring every tap. Everything that is not painting +/// carries on — timers, sockets, and the routing a tapped notification does — which is why a tap +/// that lands here appears to have opened nothing. +/// +/// This does not try to unwedge the pipeline, because the engine's latch is not reachable from +/// here. It reloads, which is the same thing the only available workaround does — closing the App +/// and opening it again — minus the person having to know that. +/// +/// [route] is where to reload *to*, and it must be the router's own answer rather than the address +/// bar's. `Router` reports a navigation to the browser from a post-frame callback, so an App that +/// cannot paint never writes the new address — a tapped notification routed while wedged leaves +/// `location` still naming the screen it was already on, and reloading that would land back on +/// exactly the screen this is trying to get somebody off. `GoRouter.go` sets the route information +/// provider's value synchronously, which is why that is the thing to ask. +/// +/// A no-op off the web, so there is no platform branch at the call site. +void watchFrames(String Function() route) => platform.watchFrames(route); diff --git a/App/serval_app/lib/platform/frame_watchdog_stub.dart b/App/serval_app/lib/platform/frame_watchdog_stub.dart new file mode 100644 index 0000000..c4e6f1d --- /dev/null +++ b/App/serval_app/lib/platform/frame_watchdog_stub.dart @@ -0,0 +1,7 @@ +/// The conditional import's default branch — neither `dart.library.js_interop` nor a browser. +/// Reached by `flutter test`, which runs on the VM, and by the Linux build. +/// +/// There is nothing to watch off the web. The wedge this recovers from is browser frame scheduling +/// specifically — a `requestAnimationFrame` that a hidden page never receives — and a native +/// embedder drives its frames from a vsync signal that a backgrounded app is simply not sent. +void watchFrames(String Function() route) {} diff --git a/App/serval_app/lib/platform/frame_watchdog_web.dart b/App/serval_app/lib/platform/frame_watchdog_web.dart new file mode 100644 index 0000000..4326888 --- /dev/null +++ b/App/serval_app/lib/platform/frame_watchdog_web.dart @@ -0,0 +1,111 @@ +import 'dart:async'; +import 'dart:js_interop'; + +import 'package:flutter/scheduler.dart'; +import 'package:web/web.dart' as web; + +/// Browser animation frames the probe will sit through before deciding nothing is being painted. +/// +/// Deliberately a count of frames rather than a stopwatch, because the two failures it has to tell +/// apart share a thread. A main thread merely busy — a burst of snapshots decoded on resume, a +/// WebRTC session renegotiating — delays *these* callbacks exactly as much as it delays Flutter's, +/// so waiting a fixed wall-clock second would report a slow phone as a broken one. A page that is +/// genuinely animating and has painted none of thirty frames is not busy. +const _frames = 30; + +/// How long one animation frame is waited for before the probe gives up on the browser. +/// +/// A page can stop animating for reasons that are nobody's fault — it went back to the background +/// mid-probe, or it never really came forward. That is not the fault this recovers from, and +/// reloading somebody's App on the strength of it would be worse than the fault. Abandoning also +/// releases [_probing], so the next time the page is shown it is asked again. +const _stall = Duration(seconds: 5); + +/// Whether a probe is already running. +/// +/// `visibilitychange` can fire repeatedly while one is in flight — a phone answering a call, a +/// notification shade pulled down and let go — and each of those would otherwise start a second +/// probe racing the first to reload the page. +bool _probing = false; + +void watchFrames(String Function() route) { + // The DOM event rather than `AppLifecycleListener`, and that is the point: this asks whether + // Flutter's own machinery is still running, so it cannot be scheduled by that machinery. The + // lifecycle listener is delivered through the engine and the framework binding, which is the + // half under suspicion. + web.document.addEventListener( + 'visibilitychange', + (web.Event _) { + if (web.document.visibilityState == 'visible') { + unawaited(_probe(route)); + } + }.toJS, + ); +} + +/// Asks the framework for a frame and watches whether one arrives. +Future _probe(String Function() route) async { + if (_probing) return; + _probing = true; + + try { + var painted = false; + + // A post-frame callback rather than anything that inspects the scheduler's own flags: it is + // set at the end of `handleDrawFrame`, so it is evidence a frame was actually produced rather + // than evidence one was asked for — and asking is precisely what is believed to have already + // happened. `ensureVisualUpdate` is what makes an idle App produce one; a frame with nothing + // dirty still runs the callback. + SchedulerBinding.instance + ..addPostFrameCallback((_) => painted = true) + ..ensureVisualUpdate(); + + for (var frame = 0; frame < _frames; frame++) { + if (!await _animationFrame()) return; + if (painted) return; + } + + // Thirty frames delivered to this file and none to Flutter. The pipeline is latched shut and + // there is no way to open it from here. + _reloadTo(route()); + } finally { + _probing = false; + } +} + +/// Loads the App again at [target]. +/// +/// `replace` rather than `assign` where the address has to change: the entry being left is a +/// broken copy of this same App, and leaving it on the history stack would put the back button +/// one press away from returning to it. +void _reloadTo(String target) { + final here = '${web.window.location.pathname}${web.window.location.search}'; + if (target == here) { + web.window.location.reload(); + return; + } + + web.window.location.replace(target); +} + +/// One animation frame. False if the browser stopped delivering them instead. +/// +/// Ordering matters and is in our favour: rAF callbacks run in the order they were registered, and +/// anything Flutter scheduled — including the frame [_probe] just asked for — was registered before +/// this one. So on a healthy page the frame is fully painted, post-frame callback and all, before +/// this future completes. +Future _animationFrame() { + final frame = Completer(); + + web.window.requestAnimationFrame( + (JSNumber _) { + if (!frame.isCompleted) frame.complete(true); + }.toJS, + ); + + Timer(_stall, () { + if (!frame.isCompleted) frame.complete(false); + }); + + return frame.future; +} diff --git a/App/serval_app/lib/screens/camera/chrome.dart b/App/serval_app/lib/screens/camera/chrome.dart index 9b5a158..fbb4f97 100644 --- a/App/serval_app/lib/screens/camera/chrome.dart +++ b/App/serval_app/lib/screens/camera/chrome.dart @@ -10,6 +10,9 @@ class _TopBar extends StatelessWidget { this.snapshotJob, this.clipJob, this.choosingClip = false, + this.castState = CastState.unavailable, + this.onCast, + this.castProblem, }); final Camera camera; @@ -20,6 +23,13 @@ class _TopBar extends StatelessWidget { final _SaveJob? snapshotJob; final _SaveJob? clipJob; + /// Whether a Cast device is reachable, and whether one is already playing this. Absent is the + /// ordinary case — no Chromecast on the network, or a browser without the Cast SDK at all — and + /// the button is not rendered then rather than rendered dead. + final CastState castState; + final VoidCallback? onCast; + final String? castProblem; + /// The screen is a trimmer. The bar says so, and everything that would take you off it stops /// working — a gear pressed mid-trim would lose a range that took a minute to set. final bool choosingClip; @@ -78,8 +88,26 @@ class _TopBar extends StatelessWidget { // much. Expanded also ellipsises a long failure — the Server's own sentence — rather than // pushing the buttons off the bar. Expanded( - child: _SaveStatus(snapshot: snapshotJob, clip: clipJob), + child: _SaveStatus( + snapshot: snapshotJob, + clip: clipJob, + castProblem: castProblem, + ), ), + // Only where there is something to cast to. A disabled button would raise the question + // of what is wrong, and on most machines nothing is — there is simply no television. + if (castState != CastState.unavailable && !choosingClip) ...[ + NocturneButton( + label: castState == CastState.casting ? 'Stop casting' : 'Cast', + // The same icon the phone layouts put in the corner of the picture, and the one a cast + // control is recognised by. Filled while a session runs. + icon: castState == CastState.casting + ? PhosphorIconsFill.screencast + : PhosphorIconsRegular.screencast, + onPressed: onCast, + ), + const SizedBox(width: 8), + ], NocturneButton( label: switch (snapshotJob) { _SaveWorking() => 'Saving…', diff --git a/App/serval_app/lib/screens/camera/overlays.dart b/App/serval_app/lib/screens/camera/overlays.dart index 58c9ab9..f2f02cf 100644 --- a/App/serval_app/lib/screens/camera/overlays.dart +++ b/App/serval_app/lib/screens/camera/overlays.dart @@ -197,6 +197,8 @@ class _PictureBand extends StatelessWidget { required this.theirAudio, required this.micStage, required this.onExpand, + this.castState = CastState.unavailable, + this.onCast, this.clip, this.onPlayClip, }); @@ -220,6 +222,11 @@ class _PictureBand extends StatelessWidget { final VoidCallback onExpand; + /// See [_CastOverlayButton]. Drawn in the free corner: the pills have the top left and the + /// full-screen control the bottom right. + final CastState castState; + final VoidCallback? onCast; + /// The range being trimmed, when the screen is a trimmer. See [_VideoStage.clip]. final ClipSelection? clip; @@ -291,6 +298,12 @@ class _PictureBand extends StatelessWidget { onPressed: onExpand, ), ), + if (castState != CastState.unavailable) + Positioned( + right: 12, + top: 12, + child: _CastOverlayButton(state: castState, onCast: onCast), + ), ], ], ), @@ -483,6 +496,36 @@ class _RoundOverlayButton extends StatelessWidget { ); } +/// *Cast*, in the corner of the picture, on every layout that has no room for a bar. +/// +/// The same control in the same place whether the picture is a band or the whole screen, because +/// on a phone it is the same gesture — and the corner of the video is where a cast icon is looked +/// for. Filled while a session is running, which is the platform's own convention for connected and +/// the one [_ActionRow] already uses for a live toggle. +/// +/// Absent, never disabled: [CastState.unavailable] means no receiver was found, or a browser with +/// no Cast support at all, and on most machines nothing is wrong — there is simply no television. +class _CastOverlayButton extends StatelessWidget { + const _CastOverlayButton({required this.state, required this.onCast}); + + final CastState state; + final VoidCallback? onCast; + + @override + Widget build(BuildContext context) { + final casting = state == CastState.casting; + + return _RoundOverlayButton( + icon: casting + ? PhosphorIconsFill.screencast + : PhosphorIconsRegular.screencast, + tooltip: casting ? 'Stop casting' : 'Cast to a television', + active: casting, + onPressed: onCast, + ); + } +} + /// The four things the desktop floats over the video, as a row beneath it. /// /// *Audio* is a toggle and reads as one — lit and underlined while their sound is coming through. diff --git a/App/serval_app/lib/screens/camera/save.dart b/App/serval_app/lib/screens/camera/save.dart index 5ecde3a..78710d9 100644 --- a/App/serval_app/lib/screens/camera/save.dart +++ b/App/serval_app/lib/screens/camera/save.dart @@ -58,13 +58,32 @@ String _megabytes(int bytes) => /// A pill and a sentence rather than a `SnackBar`: Material's is a filled surface carrying a /// ripple, which is the flood Nocturne forbids — and it would cover the video it is reporting on. class _SaveStatus extends StatelessWidget { - const _SaveStatus({this.snapshot, this.clip}); + const _SaveStatus({this.snapshot, this.clip, this.castProblem}); final _SaveJob? snapshot; final _SaveJob? clip; + /// Why the last cast attempt failed, if one did. Shares this line because it is the same + /// question — what happened to the thing I just pressed — and a second status line would + /// compete with this one for the same space. + final String? castProblem; + @override Widget build(BuildContext context) { + // A cast failure outranks a finished save: it is the newer news, and the save's own outcome + // has already been read by the time somebody presses Cast. + if (castProblem case final problem?) { + return Text( + problem, + overflow: TextOverflow.ellipsis, + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + color: Serval.alert, + ), + ); + } + final job = clip is _SaveWorking || clip is _SaveFailed ? clip : (snapshot ?? clip); diff --git a/App/serval_app/lib/screens/camera_screen.dart b/App/serval_app/lib/screens/camera_screen.dart index 7ac4b0b..dbb63ca 100644 --- a/App/serval_app/lib/screens/camera_screen.dart +++ b/App/serval_app/lib/screens/camera_screen.dart @@ -19,9 +19,11 @@ import '../data/time_labels.dart'; import '../models/activity.dart'; import '../media/media_saver.dart'; import '../models/camera.dart'; +import '../models/cast_target.dart'; import '../models/clip_selection.dart'; import '../models/ptz.dart'; import '../models/saved_clip.dart'; +import '../platform/cast_sender.dart'; import '../platform/secure_context.dart'; import '../models/timeline.dart'; import '../playback/microphone_gate.dart'; @@ -88,9 +90,37 @@ class CameraScreen extends ConsumerStatefulWidget { ConsumerState createState() => _CameraScreenState(); } +/// What the Cast button should say, if anything. +enum CastState { + /// No receiver found, or a browser with no Cast support at all. The button is not shown. + unavailable, + + /// A receiver is reachable and idle. + ready, + + /// A receiver is playing this camera. + casting, +} + class _CameraScreenState extends ConsumerState { late final ServalRepository _repository = ref.read(repositoryProvider); + /// Whether a Cast receiver is reachable, and whether one is playing. The button's state is + /// derived from the pair rather than assigned, because the two arrive on separate streams and + /// assigning from either one loses what the other said — which is how "Stop casting" ended up + /// with no way back to "Cast". + bool _castReceiver = false; + bool _castPlaying = false; + + CastState get _castState => !_castReceiver + ? CastState.unavailable + : (_castPlaying ? CastState.casting : CastState.ready); + + /// Why the last cast attempt failed. Cleared when another is started. + String? _castProblem; + StreamSubscription? _castAvailable; + StreamSubscription? _castSession; + // An hour, not the design's twelve: the thing you have come to find is nearly always in the // last few minutes, and at 12 h that is a few pixels of track. Widened in [initState] when // arriving from a feed row older than that, since a track that does not reach the instant asked @@ -228,8 +258,177 @@ class _CameraScreenState extends ConsumerState { // condition the Server guards the route with and it is known synchronously — waiting for the // probe would cost a rebuild to learn something a fixed camera already answers. if (widget.camera.ptzConfigured) unawaited(_readZoomPosition()); + + // Loads Google's sender SDK and starts discovery. A no-op off the web, on a browser with no + // Cast support, and on a deployment with no receiver registered — in each of which the two + // streams never produce and the button never appears. + // + // The application id has to be fetched first: the SDK discovers only devices that can run a + // named application, so passing it at launch time instead would mean no discovery, no + // receiver found, and no button to launch from. + unawaited(_startCastDiscovery()); + + _castAvailable = CastSender.available.listen((available) { + if (!mounted) return; + setState(() => _castReceiver = available); + }); + + _castSession = CastSender.casting.listen((casting) { + if (!mounted) return; + setState(() => _castPlaying = casting); + }); + } + + /// Asks the Server which receiver to look for, and starts discovery if there is one. + /// + /// Silent on failure of every kind. No receiver registered, no Server, an older Server without + /// the route — all mean the same thing to the screen, which is that there is nothing to cast to, + /// and none of them is worth a message on a live view. + Future _startCastDiscovery() async { + try { + final appId = await _repository.castReceiverAppId(); + if (appId != null) await CastSender.initialise(appId); + } on Object { + // Nothing to cast to. See above. + } } + /// Sends this camera to a Cast device, or stops one already playing. + /// + /// The receiver launched is Serval's own, so what plays is the same WebRTC stream on screen here + /// rather than a delayed recording — see [CastSender]. The server decides both which receiver to + /// launch and what to hand it, because the application id belongs to the deployment and the URL + /// carries a credential only the server can mint. + Future _onCast() async { + if (_castState == CastState.casting) { + setState(() { + _castProblem = null; + _castWindow = null; + }); + await CastSender.stop(); + return; + } + + // Whatever is on screen here is what goes to the television. Scrubbed back into the recording, + // that is the recording from where you are; otherwise it is the live camera. Casting live from + // a screen showing an hour ago would be the surprising reading of one button. + await _startCast(_replay.replaying ? _replay.playhead.value : null); + } + + /// The window a recording is being cast over, or null when the television has the live camera. + /// + /// Held so that scrubbing here can be mirrored there: a target inside this window is a seek in + /// what is already playing, and one outside it needs a fresh cast. See [_mirrorCastSeek]. + CastWindow? _castWindow; + + /// Starts casting, opening at [at] for a recording or null for the live camera. + /// + /// **A recording is cast over the whole visible timeline, not from the playhead.** The window is + /// the scrubber's own range, and [at] only says where inside it to start playing. That is what + /// makes a click on the bar a seek in media the television already has rather than a fresh cast: + /// anywhere the bar can be clicked is, by construction, inside the window. Re-casting is what + /// takes seconds and what leaves a screen black while it happens, so the fewer the better. + Future _startCast(DateTime? at) async { + setState(() => _castProblem = null); + + final target = await _repository.castTarget(widget.camera.id); + if (!mounted) return; + + if (target == null || !target.usable) { + _showCastProblem( + 'This server has no Cast receiver set up — see Docs/google-home.md.', + ); + return; + } + + final timeline = _repository.timelineFor(widget.camera.id, _range); + final window = at == null ? null : CastWindow.around(at, timeline); + final from = window?.from; + final to = window?.to ?? timeline.to; + + final url = from == null + ? target.contentUrl + : await _repository.castVodUrl( + widget.camera.id, + from: from, + to: to, + at: at!, + ); + if (!mounted) return; + + if (url == null) { + _showCastProblem('This recording cannot be cast.'); + return; + } + + setState(() => _castWindow = window); + + final failure = await CastSender.cast( + target.receiverAppId, + url, + title: widget.camera.name, + live: from == null, + + // Where the playhead is, not where the window starts. The two are the same thing only when + // casting from the very left of the scrubber, and every other time the difference is how far + // back the television would otherwise open. + startAt: window == null || at == null + ? Duration.zero + : window.offsetOf(at), + ); + if (!mounted || failure == null) return; + + _showCastProblem(failure); + } + + /// Every deliberate move of the playhead, in one place. + /// + /// The screen seeks and the television follows — a scrub here that left a cast showing somewhere + /// else would be the obvious bug. Continuous scrubbing does not come through here: only the + /// committed seek does, so the television is told once rather than on every frame of a drag. + Future _seekTo(DateTime at, TimelineWindow timeline) async { + await _replay.seekTo(at, timeline); + await _mirrorCastSeek(at); + } + + /// Back to the live camera, here and on the television. + Future _backToLive() async { + await _replay.backToLive(); + await _mirrorCastSeek(null); + } + + /// Takes the television to wherever this screen just went. + /// + /// **Two outcomes, and the difference is what is already on the television.** The cast playlist + /// covers one window — where it was started from, up to then — so a target inside it is a seek in + /// media the receiver already has, which lands immediately. A target *outside* it is not in that + /// playlist at all: scrubbing back before the cast began, or forward past the moment it started, + /// asks for footage never sent. That needs a fresh cast over a new window, which costs the second + /// or two of a reload, and is the only thing that can work. + /// + /// Live is the same problem in the other direction: a television showing the live camera has no + /// timeline to seek in, so scrubbing here restarts it as a recording. + Future _mirrorCastSeek(DateTime? at) async { + if (_castState != CastState.casting) return; + + // Back to live, and the television is already there. + if (at == null) { + if (_castWindow != null) await _startCast(null); + return; + } + + final window = _castWindow; + if (window != null && window.covers(at)) { + await CastSender.seek(window.offsetOf(at)); + return; + } + + await _startCast(at); + } + + void _showCastProblem(String message) => + setState(() => _castProblem = message); + /// Replaces the reckoned position with the camera's own, where it has one. /// /// Silent on null: a camera that does not report a position is the fallback case, not an error, @@ -285,7 +484,7 @@ class _CameraScreenState extends ConsumerState { // Exactly [at]. Whoever asked for this screen decided what instant it should open on — a // feed row backs its own off by a few seconds because a detection is stamped part-way into // what caused it, and a tile handed over from the wall means the frame it was showing. - if (mounted) unawaited(_replay.seekTo(at, timeline)); + if (mounted) unawaited(_seekTo(at, timeline)); }); } @@ -301,7 +500,7 @@ class _CameraScreenState extends ConsumerState { void _openRow(DateTime? at, TimelineWindow timeline) { if (_tray.detent == SheetDetent.raised) _tray.goTo(SheetDetent.resting); - unawaited(at == null ? _replay.backToLive() : _replay.seekTo(at, timeline)); + unawaited(at == null ? _backToLive() : _seekTo(at, timeline)); } @override @@ -317,6 +516,8 @@ class _CameraScreenState extends ConsumerState { _clearSaved?.cancel(); // The save itself is the Server's and carries on regardless — this only stops asking about it. _clipPoll?.cancel(); + _castAvailable?.cancel(); + _castSession?.cancel(); _replay.dispose(); _liveVideoSize.dispose(); _pictureZoom.dispose(); @@ -895,6 +1096,9 @@ class _CameraScreenState extends ConsumerState { choosingClip: _clipMode != null, onSnapshot: _onSnapshot, onSaveClip: () => _onSaveClip(timeline), + castState: _castState, + onCast: _onCast, + castProblem: _castProblem, ), Expanded( child: Row( @@ -978,9 +1182,9 @@ class _CameraScreenState extends ConsumerState { setState(() => _range = r), live: !replaying, playhead: _replay.playhead, - onSeek: (at) => _replay.seekTo(at, timeline), + onSeek: (at) => _seekTo(at, timeline), onScrub: (at) => _replay.scrubTo(at, timeline), - onBackToLive: _replay.backToLive, + onBackToLive: _backToLive, // The same control the wall has, from the same constants: a rate // that meant something different depending on which screen you // were on would be worse than not offering one. @@ -1114,6 +1318,8 @@ class _CameraScreenState extends ConsumerState { theirAudio: _theirAudio, micStage: _micStage, onExpand: () => setState(() => _expanded = true), + castState: _castState, + onCast: _onCast, clip: _clipMode?.selection, onPlayClip: _clipMode == null ? null @@ -1358,9 +1564,9 @@ class _CameraScreenState extends ConsumerState { onRangeChanged: (r) => setState(() => _range = r), live: !replaying, playhead: _replay.playhead, - onSeek: (at) => _replay.seekTo(at, window), + onSeek: (at) => _seekTo(at, window), onScrub: (at) => _replay.scrubTo(at, window), - onBackToLive: _replay.backToLive, + onBackToLive: _backToLive, transport: !replaying ? null : ReplayTransport( @@ -1603,16 +1809,28 @@ class _CameraScreenState extends ConsumerState { // The top-right corner, which is where a full-screen video is left. Not the bottom-right // one it was entered from: down there is the control row. - if (collapsible) - Positioned( - right: 16, - top: 14, - child: _RoundOverlayButton( - icon: PhosphorIconsRegular.cornersIn, - tooltip: 'Leave full screen', - onPressed: () => setState(() => _expanded = false), - ), + // + // Cast sits beside it rather than replacing it, in the same corner it occupies on the + // band — full screen is the mode somebody is most likely to be in when they want the + // picture on a television, so leaving the mode to reach the button would be backwards. + Positioned( + right: 16, + top: 14, + child: Row( + children: [ + if (_castState != CastState.unavailable) ...[ + _CastOverlayButton(state: _castState, onCast: _onCast), + const SizedBox(width: 8), + ], + if (collapsible) + _RoundOverlayButton( + icon: PhosphorIconsRegular.cornersIn, + tooltip: 'Leave full screen', + onPressed: () => setState(() => _expanded = false), + ), + ], ), + ), // Where pan, tilt and zoom belong: full size, over the video, with the whole picture // underneath to aim by. diff --git a/App/serval_app/lib/screens/mask_editor_screen.dart b/App/serval_app/lib/screens/mask_editor_screen.dart index bb130e8..d3e3e56 100644 --- a/App/serval_app/lib/screens/mask_editor_screen.dart +++ b/App/serval_app/lib/screens/mask_editor_screen.dart @@ -231,13 +231,13 @@ class _MaskEditorScreenState extends ConsumerState { }, child: Actions( actions: { - _UndoPointIntent: CallbackAction<_UndoPointIntent>( + _UndoPointIntent: _CanvasAction<_UndoPointIntent>( onInvoke: (_) { _undoPoint(); return null; }, ), - _AbandonDraftIntent: CallbackAction<_AbandonDraftIntent>( + _AbandonDraftIntent: _CanvasAction<_AbandonDraftIntent>( onInvoke: (_) { _abandonDraft(); return null; @@ -655,6 +655,30 @@ class _AbandonDraftIntent extends Intent { const _AbandonDraftIntent(); } +/// A canvas rule that stands down while the caret is in a field. +/// +/// The rules hang over the whole screen because the whole screen is the drawing surface — but the +/// inspector has a name box in it, and Backspace there is a character, not a point. `Shortcuts` +/// asks the nearest enclosing `Actions` first and reaches the editing shortcuts the framework +/// installs above the app *only* when the action it finds is disabled: an action that merely +/// declines to do anything still eats the key. So being disabled is the whole of it. +class _CanvasAction extends CallbackAction { + _CanvasAction({required super.onInvoke}); + + @override + bool isEnabled(T intent) => !_caretIsInAField; +} + +/// Whether the keyboard is currently for typing. +/// +/// [EditableText] builds its focus node inside itself, so a focused node with one above it is a +/// field with the caret — every box in the app is an [EditableText], including the class chips'. +bool get _caretIsInAField { + final focused = FocusManager.instance.primaryFocus?.context; + return focused != null && + focused.findAncestorWidgetOfExactType() != null; +} + class _BackToCamera extends StatelessWidget { const _BackToCamera({required this.name, required this.onTap}); diff --git a/App/serval_app/lib/screens/server_screen.dart b/App/serval_app/lib/screens/server_screen.dart index 5ea54d5..43c35e6 100644 --- a/App/serval_app/lib/screens/server_screen.dart +++ b/App/serval_app/lib/screens/server_screen.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:convert'; import 'package:file_picker/file_picker.dart'; @@ -10,6 +11,7 @@ import '../data/providers.dart'; import '../data/serval_api.dart' show ServalApiException; import '../data/serval_repository.dart'; import '../models/config_backup.dart'; +import '../models/google_home.dart'; import '../models/system_stats.dart'; import '../models/vitals_history.dart'; import '../theme/app_theme.dart'; @@ -17,6 +19,7 @@ import '../theme/nocturne.dart'; import '../theme/serval_tokens.dart'; import '../widgets/compact_app_bar.dart'; import '../widgets/config_backup_section.dart'; +import '../widgets/google_home_section.dart'; import '../widgets/nocturne_button.dart'; import '../widgets/nocturne_dialog.dart'; import '../widgets/nocturne_field.dart'; @@ -59,6 +62,75 @@ class _ServerScreenState extends ConsumerState { String? _status; String? _error; + /// Read once when the page opens rather than on the vitals sweep: none of this moves on its own + /// — the configuration is environment-only and a link is made from the Google Home app, not from + /// here — so polling it every five seconds would be two requests a minute answering the same way + /// forever. + GoogleHomeStatus? _google; + List _googleLinks = const []; + String? _googleError; + + /// The two cases where the section is meant to be absent rather than present and complaining: + /// an account that may not read the status (a Viewer, 403), and a deployment that has not + /// switched the integration on at all. + bool _googleHidden = false; + + @override + void initState() { + super.initState(); + unawaited(_loadGoogleHome()); + } + + Future _loadGoogleHome() async { + try { + final status = await _repository.googleHomeStatus(); + // Only worth asking who has linked once the integration is actually serving; a closed one + // has nothing to list and the route would answer for an empty collection. + final links = status.effective + ? await _repository.googleHomeLinks() + : const []; + if (!mounted) return; + setState(() { + // A deployment that has not switched this on gets no card — see + // GoogleHomeStatus.switchedOff. + _googleHidden = status.switchedOff; + _google = status; + _googleLinks = links; + _googleError = null; + }); + } on ServalApiException catch (error) { + if (!mounted) return; + // A Viewer gets 403, and the section simply does not appear for them — they have nothing to + // do about it and it is not their page. Every other status is reported. + setState(() { + _googleHidden = error.statusCode == 403; + _googleError = error.statusCode == 403 ? null : error.message; + }); + } catch (error) { + // Deliberately broad, and it is the fix for a real failure rather than defensive habit: the + // first build of this screen caught only ServalApiException, a decoding error escaped into an + // unawaited future, and the section vanished from the page with nothing logged anywhere. A + // fault that removes a feature from the UI is the worst shape a fault can take, because it + // is indistinguishable from the feature not existing. + if (!mounted) return; + setState( + () => _googleError = 'Could not read the Google Home status: $error', + ); + } + } + + Future _unlinkGoogleHome(GoogleHomeLink link) async { + setState(() => _googleError = null); + try { + await _repository.unlinkGoogleHome(link.agentUserId); + } on ServalApiException catch (error) { + if (!mounted) return; + setState(() => _googleError = error.message); + return; + } + await _loadGoogleHome(); + } + @override Widget build(BuildContext context) { // Both actions are Admin-only on the Server. Hiding them from a Viewer rather than letting the @@ -81,6 +153,11 @@ class _ServerScreenState extends ConsumerState { configBusy: _busy, configStatus: _status, configError: _error, + googleHome: _googleHidden ? null : _google, + googleHomeHidden: _googleHidden, + googleHomeLinks: _googleLinks, + onUnlinkGoogleHome: canBackUp ? _unlinkGoogleHome : null, + googleHomeError: _googleError, ), ), ); @@ -217,6 +294,11 @@ class ServerScreenBody extends StatelessWidget { this.configBusy, this.configStatus, this.configError, + this.googleHome, + this.googleHomeHidden = false, + this.googleHomeLinks = const [], + this.onUnlinkGoogleHome, + this.googleHomeError, }); final SystemStats? stats; @@ -242,6 +324,26 @@ class ServerScreenBody extends StatelessWidget { final String? configStatus; final String? configError; + /// Whether the Google Home integration is live, and what is stopping it. **Null drops the whole + /// section** — which is what the page draws before the read lands, and what a Viewer gets, since + /// `GET /api/google/status` is Admin-only and the 403 is swallowed rather than reported. + /// + /// Unlike the backup section above, this is *not* dropped for the sample repository: the sample + /// answers the way nearly every real deployment does — switched off — and that is the state the + /// section exists to explain. + final GoogleHomeStatus? googleHome; + + /// True only for an account that may not read the status at all. It is what separates "this is + /// not your page" — draw nothing — from "the read failed", which must always draw something. + final bool googleHomeHidden; + + final List googleHomeLinks; + + /// Null draws the section without its one action, which is what a Viewer sees. + final void Function(GoogleHomeLink link)? onUnlinkGoogleHome; + + final String? googleHomeError; + /// Below this the two columns will not both hold their content, so the page becomes one column. /// The meters' column is fixed at its design width and the volume needs a comparable share. static const _twoColumnWidth = 780.0; @@ -295,6 +397,21 @@ class ServerScreenBody extends StatelessWidget { // button inside a stack of readings. Last, after everything the page // reports, it reads as a footer of actions — which suits one carrying a // warning. + // Above the backup section, because it reports rather than acts and this + // page reads reports first. It has one button, which is why it is not + // further up: the actions belong together at the foot. + if (!googleHomeHidden && + (googleHome != null || googleHomeError != null)) ...[ + const SizedBox(height: 22), + const SettingsDivider(), + const SizedBox(height: 22), + GoogleHomeSection( + status: googleHome, + links: googleHomeLinks, + onUnlink: onUnlinkGoogleHome, + error: googleHomeError, + ), + ], if (onBackup != null && onRestore != null) ...[ const SizedBox(height: 22), const SettingsDivider(), diff --git a/App/serval_app/lib/widgets/google_home_section.dart b/App/serval_app/lib/widgets/google_home_section.dart new file mode 100644 index 0000000..8c45bb1 --- /dev/null +++ b/App/serval_app/lib/widgets/google_home_section.dart @@ -0,0 +1,295 @@ +import 'package:flutter/widgets.dart'; + +import '../data/time_labels.dart'; +import '../models/google_home.dart'; +import '../theme/app_theme.dart'; +import '../theme/nocturne.dart'; +import '../theme/serval_tokens.dart'; +import 'nocturne_button.dart'; +import 'nocturne_field.dart'; + +/// Whether the cameras are reachable from Google Home, and if not, the one reason why. +/// +/// **Nothing here is a setting, which is why it belongs on this page.** Every +/// `Serval:GoogleHome:*` key is environment-only — two are secrets, and two more decide where an +/// anonymous endpoint sends credentials Google issued — so there is no form to render and no value +/// to submit. What is worth showing is the thing an operator cannot get any other way: six +/// conditions have to hold for the integration to answer anything but a 503, and knowing *which* +/// one is unmet is the difference between a minute and an afternoon. +/// +/// **The reason sentence comes from the Server and is rendered verbatim.** The same contract the +/// settings page has with the catalogue: the Server owns the wording, so a condition added there +/// explains itself without an App release. Mapping [GoogleHomeStatus.blocker] to local text here +/// would put the two out of step on the first change. +/// +/// Prop-driven like everything under `widgets/`, so every state can be pumped into a widget test. +class GoogleHomeSection extends StatelessWidget { + const GoogleHomeSection({ + super.key, + required this.status, + required this.links, + this.onUnlink, + this.busy = false, + this.error, + }); + + /// Null when the status could not be read at all. The section still draws — with [error] — so a + /// failure reads as a failure rather than as the feature not existing. + final GoogleHomeStatus? status; + + /// At most one, and usually none. + final List links; + + /// Null hides the action — a Viewer reaching this page reads it and does not act on it. + final void Function(GoogleHomeLink link)? onUnlink; + + final bool busy; + + final String? error; + + @override + Widget build(BuildContext context) => SettingsSection( + title: 'Google Home', + blurb: Text( + 'Cameras on a Nest Hub or a Chromecast, by voice. Configured entirely in the ' + 'deployment’s environment — there is nothing to change here.', + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + height: 1.45, + color: Nocturne.mix(Nocturne.text, 50), + ), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + _StateLine( + effective: status?.effective ?? false, + linked: links.isNotEmpty, + unknown: status == null, + ), + + // The Server's own sentence. Not an error strip: for the overwhelmingly common case — + // the integration is simply switched off — nothing is wrong, and painting that in the + // alert colour would make a default look like a fault. + if (status?.reason case final reason?) ...[ + const SizedBox(height: 12), + Text( + reason, + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + height: 1.45, + color: Nocturne.mix(Nocturne.text, 55), + ), + ), + ], + + if (status?.effective ?? false) ...[ + const SizedBox(height: 14), + _Fact( + label: 'Public address', + // Only ever set when effective, since the gate requires it — but rendered as absent + // rather than as an empty string if it somehow is not. + value: status?.publicBaseUrl ?? '—', + ), + _Fact( + label: 'Camera changes', + value: (status?.homeGraphKeyConfigured ?? false) + ? 'Pushed to Google automatically' + : 'Not pushed — re-link, or say “sync my devices”, after adding a camera', + ), + // Both answers are a working deployment, so this says what you get rather than what is + // missing. Unset is not a degraded television, it is no Cast button at all — Google will + // not send a camera to a television by voice however this is configured. + _Fact( + label: 'On a television', + value: (status?.castReceiverConfigured ?? false) + ? 'Cast from a camera screen — live, or a recording from where you are' + : 'No Cast receiver registered, so the app offers no Cast button', + ), + ], + + if (links.isEmpty && (status?.effective ?? false)) ...[ + const SizedBox(height: 12), + Text( + 'No Google account has linked yet. Add it from the Google Home app: ' + 'Add → Works with Google.', + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + height: 1.45, + color: Nocturne.mix(Nocturne.text, 45), + ), + ), + ], + + for (final link in links) ...[ + const SizedBox(height: 14), + _LinkRow( + link: link, + onUnlink: busy || onUnlink == null ? null : () => onUnlink!(link), + ), + ], + + if (error case final message?) ...[ + const SizedBox(height: 12), + Text( + message, + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + height: 1.4, + color: Serval.alertText, + ), + ), + ], + ], + ), + ); +} + +/// One line saying what state the integration is in, in the words an operator would use. +class _StateLine extends StatelessWidget { + const _StateLine({ + required this.effective, + required this.linked, + this.unknown = false, + }); + + final bool effective; + final bool linked; + + /// The status could not be read. Distinct from "not active": one is a deployment that has not + /// turned this on, the other is a question we failed to get an answer to. + final bool unknown; + + @override + Widget build(BuildContext context) { + // Four states. "On but nobody has linked" is a real and common step in the middle of the setup + // runbook, and collapsing it into "on" would leave somebody waiting for cameras that are never + // going to appear; "could not be read" must not read as "off", which would send them looking + // at configuration that is fine. + final (String label, Color colour) = switch ((unknown, effective, linked)) { + (true, _, _) => ('Status unavailable', Serval.alertText), + (false, false, _) => ('Not active', Nocturne.mix(Nocturne.text, 40)), + (false, true, false) => ( + 'Ready — no account linked', + Nocturne.mix(Nocturne.text, 60), + ), + (false, true, true) => ('Active', Serval.healthyText), + }; + + return Row( + children: [ + Container( + width: 8, + height: 8, + decoration: BoxDecoration(color: colour, shape: BoxShape.circle), + ), + const SizedBox(width: 9), + Text( + label, + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 13, + fontWeight: FontWeight.w600, + color: colour, + ), + ), + ], + ); + } +} + +class _Fact extends StatelessWidget { + const _Fact({required this.label, required this.value}); + + final String label; + final String value; + + @override + Widget build(BuildContext context) => Padding( + padding: const EdgeInsets.only(bottom: 8), + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + SizedBox( + width: 132, + child: Text( + label, + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + color: Nocturne.mix(Nocturne.text, 40), + ), + ), + ), + Expanded( + child: Text( + value, + style: monoStyle( + fontSize: 11.5, + color: Nocturne.mix(Nocturne.text, 60), + ), + ), + ), + ], + ), + ); +} + +class _LinkRow extends StatelessWidget { + const _LinkRow({required this.link, this.onUnlink}); + + final GoogleHomeLink link; + final VoidCallback? onUnlink; + + @override + Widget build(BuildContext context) => Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'Linked ${link.linkedAt == null ? '—' : _whenLabel(link.linkedAt!)}', + style: TextStyle( + fontFamily: Nocturne.fontBody, + fontSize: 12.5, + color: Nocturne.mix(Nocturne.text, 55), + ), + ), + const SizedBox(height: 4), + // The field worth reading. A link that was made and then never used again looks + // identical to a working one from everything else on this card, and this is what + // separates them. + Text( + link.lastFulfillmentAt == null + ? 'Google has not called since linking' + : 'Google last called ${_whenLabel(link.lastFulfillmentAt!)}', + style: monoStyle( + fontSize: 11.5, + color: Nocturne.mix(Nocturne.text, 40), + ), + ), + ], + ), + ), + if (onUnlink != null) ...[ + const SizedBox(width: 12), + NocturneButton(label: 'Unlink', onPressed: onUnlink), + ], + ], + ); +} + +/// `8:12 am` today, `3 Aug` before that — the same rule the notifications screen uses, because a +/// clock time is only a useful answer for the day you are on. +String _whenLabel(DateTime at) { + final now = DateTime.now(); + final today = + at.year == now.year && at.month == now.month && at.day == now.day; + return today ? clockLabel(at) : dayLabel(at); +} diff --git a/App/serval_app/test/cast_window_test.dart b/App/serval_app/test/cast_window_test.dart new file mode 100644 index 0000000..02d8088 --- /dev/null +++ b/App/serval_app/test/cast_window_test.dart @@ -0,0 +1,113 @@ +// Which stretch of recording a cast covers, and where its clock starts. +// +// Both answers matter for the same reason and it is not obvious from either name: a seek on the +// television is sent as an offset in seconds, computed here as `at - window.from`. If the window +// is narrower than the scrubber, a click on the bar lands outside it and costs a re-cast — a +// second or two of black screen. If `window.from` is not where the footage actually begins, every +// seek in the session misses by the difference. +import 'package:flutter_test/flutter_test.dart'; +import 'package:serval_app/models/cast_target.dart'; +import 'package:serval_app/models/timeline.dart'; + +/// A timeline with continuous footage across the whole of it, which is the ordinary case. +TimelineWindow _covered(DateTime from, DateTime to) => + TimelineWindow(from: from, to: to, coverage: [CoverageSpan(from, to)]); + +void main() { + final noon = DateTime(2026, 8, 21, 12); + + group('the window covers what the scrubber shows', () { + test('a short range is cast whole, whatever the playhead is doing', () { + final timeline = _covered(noon, noon.add(const Duration(hours: 1))); + + final window = CastWindow.around( + noon.add(const Duration(minutes: 5)), + timeline, + ); + + expect(window.from, noon); + expect(window.to, timeline.to); + + // The point of all of it: the far end of the bar is a seek, not a new cast. + expect(window.covers(timeline.to), isTrue); + expect(window.offsetOf(timeline.to), const Duration(hours: 1)); + }); + + test('a day is cut to six hours around the playhead', () { + final timeline = _covered(noon, noon.add(const Duration(hours: 24))); + final at = noon.add(const Duration(hours: 12)); + + final window = CastWindow.around(at, timeline); + + expect(window.to.difference(window.from), CastWindow.maxSpan); + expect(window.from, at.subtract(const Duration(hours: 3))); + expect(window.to, at.add(const Duration(hours: 3))); + }); + + // Half a window is what centring naively would give, and it would halve the reach of every + // seek for a viewer watching the beginning or the end of a long day — which is most of them. + test('a playhead at the edge still gets a full span', () { + final timeline = _covered(noon, noon.add(const Duration(hours: 24))); + + final atStart = CastWindow.around(noon, timeline); + expect(atStart.from, noon); + expect(atStart.to.difference(atStart.from), CastWindow.maxSpan); + + final atEnd = CastWindow.around(timeline.to, timeline); + expect(atEnd.to, timeline.to); + expect(atEnd.to.difference(atEnd.from), CastWindow.maxSpan); + }); + }); + + group('the clock starts at the footage', () { + /// **The failure this guards.** The cast playlist's zero is its first segment. Open a window on + /// a camera that was switched off until 3 am and the first segment is at 3 am — so a seek + /// measured from midnight would be sent three hours short, every time, for the whole session. + test('a window that opens on a gap starts where recording resumed', () { + final resumed = noon.add(const Duration(hours: 2)); + final timeline = TimelineWindow( + from: noon, + to: noon.add(const Duration(hours: 4)), + coverage: [CoverageSpan(resumed, noon.add(const Duration(hours: 4)))], + ); + + final window = CastWindow.around(resumed, timeline); + + expect(window.from, resumed); + expect(window.offsetOf(resumed), Duration.zero); + }); + + // A gap in the middle is spanned by the playlist at wall-clock length, so it needs no + // correction — and applying one would break the far commoner case of footage either side. + test('a gap in the middle does not move the start', () { + final timeline = TimelineWindow( + from: noon, + to: noon.add(const Duration(hours: 4)), + coverage: [ + CoverageSpan(noon, noon.add(const Duration(hours: 1))), + CoverageSpan( + noon.add(const Duration(hours: 3)), + noon.add(const Duration(hours: 4)), + ), + ], + ); + + final window = CastWindow.around(noon, timeline); + + expect(window.from, noon); + }); + + // Coverage the App has not fetched yet is not evidence of a gap. Casting from the window's own + // edge is the same thing every version before this did, and it is right whenever there is + // footage there. + test('an unknown coverage leaves the window alone', () { + final timeline = TimelineWindow( + from: noon, + to: noon.add(const Duration(hours: 1)), + loading: true, + ); + + expect(CastWindow.around(noon, timeline).from, noon); + }); + }); +} diff --git a/App/serval_app/test/golden_capture_test.dart b/App/serval_app/test/golden_capture_test.dart index 278639a..a7bc662 100644 --- a/App/serval_app/test/golden_capture_test.dart +++ b/App/serval_app/test/golden_capture_test.dart @@ -17,6 +17,7 @@ import 'package:serval_app/data/serval_config.dart'; import 'package:serval_app/main.dart'; import 'package:serval_app/models/activity.dart'; import 'package:serval_app/models/camera.dart'; +import 'package:serval_app/models/google_home.dart'; import 'package:serval_app/models/timeline.dart'; import 'package:serval_app/push/push_client.dart'; import 'package:serval_app/theme/app_theme.dart'; @@ -25,6 +26,7 @@ import 'package:serval_app/widgets/activity_filter_panel.dart'; import 'package:serval_app/widgets/activity_sheet.dart'; import 'package:serval_app/widgets/camera_tile.dart'; import 'package:serval_app/widgets/config_backup_section.dart'; +import 'package:serval_app/widgets/google_home_section.dart'; import 'package:serval_app/widgets/timeline_scrubber.dart'; /// Renders the screens at the design's 1440x900 with the real vendored @@ -336,6 +338,11 @@ void main() { // repository, whose `canSaveMedia` is false, so both callbacks are null and `ServerScreenBody` // drops the section. This golden is therefore the page as the design drew it, and 2d below is // where the section gets its own picture. + // + // No Google Home section either, for a related reason: the sample answers `googleHomeStatus` + // the way nearly every real deployment does — switched off — and a deployment that never + // turned the integration on gets no card, because there is nothing to diagnose. 2e below + // carries the states where it does appear. await expectLater( find.byType(ServalApp), matchesGoldenFile('goldens/server.png'), @@ -383,6 +390,79 @@ void main() { ); }); + /// The Google Home section in the two states the page-level golden cannot reach: live with an + /// account linked, and live with nobody linked yet. + /// + /// The disabled state is in 2c already, on the sample repository — which is deliberately the + /// common one, since turning this on needs a public HTTPS endpoint most deployments do not have. + testWidgets('2e — google home', (tester) async { + await tester.pumpWidget( + MaterialApp( + debugShowCheckedModeBanner: false, + theme: buildServalTheme(), + home: Scaffold( + backgroundColor: Serval.panel, + body: Align( + alignment: Alignment.topLeft, + child: SizedBox( + width: 780, + child: Padding( + padding: const EdgeInsets.all(24), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + GoogleHomeSection( + status: const GoogleHomeStatus( + effective: true, + blocker: 'None', + reason: null, + publicBaseUrl: 'https://serval.example.com', + homeGraphKeyConfigured: true, + castReceiverConfigured: true, + ), + links: [ + GoogleHomeLink( + agentUserId: '0f8fad5bd9cb469fa16570867728950e', + // Fixed rather than relative to now, like the notification device dates + // and for the same reason: a date derived from now is a different string + // every day the goldens are captured. + linkedAt: DateTime(2026, 8, 3, 9, 40), + lastFulfillmentAt: DateTime(2026, 8, 8, 14, 3), + lastSyncAt: DateTime(2026, 8, 8, 14, 3), + ), + ], + onUnlink: (_) {}, + ), + const SizedBox(height: 26), + // Live, but nobody has linked yet — the middle of the setup runbook, and the + // state somebody waiting for cameras to appear is actually in. + const GoogleHomeSection( + status: GoogleHomeStatus( + effective: true, + blocker: 'None', + reason: null, + publicBaseUrl: 'https://serval.example.com', + homeGraphKeyConfigured: false, + castReceiverConfigured: false, + ), + links: [], + ), + ], + ), + ), + ), + ), + ), + ), + ); + await tester.pumpAndSettle(); + + await expectLater( + find.byType(Column).first, + matchesGoldenFile('goldens/server-google-home.png'), + ); + }); + testWidgets('3a — sign in', (tester) async { // Built through the real `ServalApp` rather than by pumping `LoginScreen` under a hand-rolled // `MaterialApp`: the login screen sits outside any `Scaffold`, so it depends on the app's own diff --git a/App/serval_app/test/goldens/server-google-home.png b/App/serval_app/test/goldens/server-google-home.png new file mode 100644 index 0000000..d2452b9 Binary files /dev/null and b/App/serval_app/test/goldens/server-google-home.png differ diff --git a/App/serval_app/test/google_home_test.dart b/App/serval_app/test/google_home_test.dart new file mode 100644 index 0000000..e009e76 --- /dev/null +++ b/App/serval_app/test/google_home_test.dart @@ -0,0 +1,371 @@ +// What the App makes of the Google Home status, and what the section draws from it. +// +// The section is the only place an operator finds out *why* the integration is not working, so +// the tests here are mostly about it saying the right thing in each of the three states rather +// than about layout — the goldens hold the layout. +// +// Nothing here writes configuration, because nothing can: every Serval:GoogleHome:* key is +// environment-only. See Docs/google-home.md. +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:serval_app/models/google_home.dart'; +import 'package:serval_app/theme/app_theme.dart'; +import 'package:serval_app/widgets/google_home_section.dart'; + +void main() { + group('status', () { + test('reads a live integration', () { + final status = GoogleHomeStatus.fromJson({ + 'effective': true, + 'blocker': 'None', + 'reason': null, + 'publicBaseUrl': 'https://serval.example.com', + 'homeGraphKeyConfigured': true, + }); + + expect(status.effective, isTrue); + expect(status.reason, isNull); + expect(status.publicBaseUrl, 'https://serval.example.com'); + expect(status.homeGraphKeyConfigured, isTrue); + }); + + // A Server older or newer than this build, or one that answered oddly. Absent means off, + // which is the safe reading: claiming the integration is live when the payload did not say so + // would send somebody looking for a networking fault that does not exist. + test('an empty payload reads as switched off', () { + final status = GoogleHomeStatus.fromJson(const {}); + + expect(status.effective, isFalse); + expect(status.homeGraphKeyConfigured, isFalse); + expect(status.reason, isNull); + }); + + /// **The bug this file exists for.** `blocker` first shipped as an unattributed C# enum, which + /// System.Text.Json writes as its ordinal — so the App received `"blocker": 1`, the cast to + /// String threw, the exception escaped into an unawaited future, and the whole card silently + /// vanished from the Server status page. It looked exactly like the feature had never been + /// built. + /// + /// The Server now sends a name and a test pins that. This is the second line of defence: no + /// payload shape may take the card off the screen. + test('an integer blocker does not throw', () { + final status = GoogleHomeStatus.fromJson(const { + 'effective': false, + 'blocker': 1, + 'reason': 'Serval:GoogleHome:Enabled is false.', + 'publicBaseUrl': null, + 'homeGraphKeyConfigured': false, + }); + + expect(status.effective, isFalse); + expect(status.blocker, '1'); + expect(status.reason, 'Serval:GoogleHome:Enabled is false.'); + }); + + /// Nothing in the payload may be load-bearing enough to throw on. A Server one version ahead + /// or behind must degrade, not blank the page. + test('wrong types anywhere do not throw', () { + final status = GoogleHomeStatus.fromJson(const { + 'effective': 'yes', + 'blocker': ['odd'], + 'reason': 42, + 'publicBaseUrl': 7, + 'homeGraphKeyConfigured': 1, + }); + + // A non-bool is not true — the safe reading, since claiming the integration is live when the + // payload did not say so sends somebody hunting a networking fault that does not exist. + expect(status.effective, isFalse); + expect(status.homeGraphKeyConfigured, isFalse); + expect(status.reason, '42'); + }); + + /// A deployment that never turned this on gets no card at all — the state almost every + /// deployment is permanently in, and one where there is nothing to diagnose. + test('a switched-off deployment is recognised', () { + expect( + GoogleHomeStatus.fromJson(const { + 'effective': false, + 'blocker': 'disabled', + }).switchedOff, + isTrue, + ); + + // Case-insensitive: the Server sends camelCase, but nothing should hinge on that holding. + expect( + GoogleHomeStatus.fromJson(const {'blocker': 'Disabled'}).switchedOff, + isTrue, + ); + + // Every other blocker means somebody is part-way through setting it up, which is exactly + // when the card has something worth saying. + for (final blocker in const [ + 'none', + 'webRtcDisabled', + 'publicBaseUrlInvalid', + 'projectIdMissing', + 'clientIdMissing', + 'clientSecretMissing', + ]) { + expect( + GoogleHomeStatus.fromJson({'blocker': blocker}).switchedOff, + isFalse, + reason: blocker, + ); + } + }); + + test('a link tolerates wrong types too', () { + final link = GoogleHomeLink.fromJson(const { + 'agentUserId': 12345, + 'linkedAt': 0, + 'lastFulfillmentAt': null, + 'lastSyncAt': null, + }); + + expect(link.agentUserId, '12345'); + expect(link.linkedAt, isNull); + }); + + test('a link keeps the times it was given, and null where it has none', () { + final link = GoogleHomeLink.fromJson({ + 'agentUserId': '0f8fad5b', + 'linkedAt': '2026-08-03T09:40:00Z', + 'lastFulfillmentAt': null, + 'lastSyncAt': null, + }); + + expect(link.agentUserId, '0f8fad5b'); + expect(link.linkedAt, isNotNull); + // Never called since linking — the state the card exists to distinguish from a working + // link, and one a default-to-now would erase. + expect(link.lastFulfillmentAt, isNull); + expect(link.lastSyncAt, isNull); + }); + }); + + group('section', () { + Future pump( + WidgetTester tester, { + required GoogleHomeStatus? status, + List links = const [], + void Function(GoogleHomeLink)? onUnlink, + String? error, + }) => tester.pumpWidget( + MaterialApp( + theme: buildServalTheme(), + home: Scaffold( + body: SingleChildScrollView( + child: GoogleHomeSection( + status: status, + links: links, + onUnlink: onUnlink, + error: error, + ), + ), + ), + ), + ); + + const off = GoogleHomeStatus( + effective: false, + blocker: 'ClientIdMissing', + reason: + 'Serval:GoogleHome:ClientId is not set, so account linking rejects every request.', + publicBaseUrl: null, + homeGraphKeyConfigured: false, + castReceiverConfigured: false, + ); + + const live = GoogleHomeStatus( + effective: true, + blocker: 'None', + reason: null, + publicBaseUrl: 'https://serval.example.com', + homeGraphKeyConfigured: true, + castReceiverConfigured: true, + ); + + /// The whole point of the card: the Server's own sentence, rendered as given. Mapping the + /// blocker to local text here would put the App a release behind the Server on the first + /// condition anyone adds. + testWidgets('shows the Server’s reason verbatim', (tester) async { + await pump(tester, status: off); + + expect(find.textContaining('Serval:GoogleHome:ClientId'), findsOneWidget); + expect(find.text('Not active'), findsOneWidget); + }); + + // Not a fault, so it must not read as one — the overwhelming majority of deployments are here + // and nothing is wrong with them. + testWidgets('a closed integration shows no address and no link prompt', ( + tester, + ) async { + await pump(tester, status: off); + + expect(find.textContaining('Public address'), findsNothing); + expect(find.textContaining('No Google account has linked'), findsNothing); + }); + + /// Three states, not two. "On but nobody has linked" is a real step in the middle of the + /// runbook, and collapsing it into "on" leaves somebody waiting for cameras that will never + /// arrive. + testWidgets('live with nobody linked says so', (tester) async { + await pump(tester, status: live); + + expect(find.text('Ready — no account linked'), findsOneWidget); + expect( + find.textContaining('No Google account has linked'), + findsOneWidget, + ); + expect(find.text('https://serval.example.com'), findsOneWidget); + }); + + testWidgets('a linked account reports when Google last called', ( + tester, + ) async { + await pump( + tester, + status: live, + links: [ + GoogleHomeLink( + agentUserId: 'agent-1', + linkedAt: DateTime(2026, 8, 3), + lastFulfillmentAt: null, + lastSyncAt: null, + ), + ], + ); + + expect(find.text('Active'), findsOneWidget); + expect(find.text('Google has not called since linking'), findsOneWidget); + }); + + /// A Viewer reads this page and does not act on it, so the action is absent rather than + /// present-and-inert — there is no 403 to explain if the button was never offered. + testWidgets('no unlink handler means no unlink button', (tester) async { + await pump( + tester, + status: live, + links: [ + GoogleHomeLink( + agentUserId: 'agent-1', + linkedAt: DateTime(2026, 8, 3), + lastFulfillmentAt: DateTime(2026, 8, 8), + lastSyncAt: null, + ), + ], + ); + + expect(find.text('Unlink'), findsNothing); + }); + + testWidgets('unlinking reports the account it was asked about', ( + tester, + ) async { + GoogleHomeLink? unlinked; + + await pump( + tester, + status: live, + links: [ + GoogleHomeLink( + agentUserId: 'agent-1', + linkedAt: DateTime(2026, 8, 3), + lastFulfillmentAt: DateTime(2026, 8, 8), + lastSyncAt: null, + ), + ], + onUnlink: (link) => unlinked = link, + ); + + await tester.tap(find.text('Unlink')); + await tester.pump(); + + expect(unlinked?.agentUserId, 'agent-1'); + }); + + testWidgets('an error is shown alongside the state, not instead of it', ( + tester, + ) async { + await pump( + tester, + status: live, + error: 'The server refused the request.', + ); + + expect(find.text('The server refused the request.'), findsOneWidget); + expect(find.text('Ready — no account linked'), findsOneWidget); + }); + + /// With no status at all the card still draws, and says the status is unavailable rather than + /// disappearing or claiming the integration is off. Those are different situations: one is a + /// deployment that has not turned this on, the other is a question that failed — and reading + /// the second as the first sends somebody to check configuration that is fine. + testWidgets('a failed read still draws the card', (tester) async { + await pump( + tester, + status: null, + error: 'Could not read the Google Home status.', + ); + + expect(find.text('Google Home'), findsOneWidget); + expect(find.text('Status unavailable'), findsOneWidget); + expect( + find.text('Could not read the Google Home status.'), + findsOneWidget, + ); + expect(find.text('Not active'), findsNothing); + }); + + /// Without a HomeGraph key the integration works and the device list goes stale — a real + /// difference an operator has to be told about, since the symptom appears days later as a + /// renamed camera Google never heard about. + testWidgets('says when camera changes will not reach Google', ( + tester, + ) async { + await pump( + tester, + status: const GoogleHomeStatus( + effective: true, + blocker: 'None', + reason: null, + publicBaseUrl: 'https://serval.example.com', + homeGraphKeyConfigured: false, + castReceiverConfigured: false, + ), + ); + + expect(find.textContaining('Not pushed'), findsOneWidget); + }); + + /// The receiver row says what the operator gets, in both directions. + /// + /// Unset is a working deployment, so the row cannot read as a fault — but it does have to say + /// there is no Cast button, because nothing else does and its absence otherwise looks like one. + /// Google will not put a camera on a television by voice whatever is configured here, so the + /// button is the only route to one. + testWidgets('says whether a television can be cast to at all', ( + tester, + ) async { + await pump(tester, status: live); + expect(find.textContaining('Cast from a camera screen'), findsOneWidget); + + await pump( + tester, + status: const GoogleHomeStatus( + effective: true, + blocker: 'None', + reason: null, + publicBaseUrl: 'https://serval.example.com', + homeGraphKeyConfigured: true, + castReceiverConfigured: false, + ), + ); + expect( + find.textContaining('No Cast receiver registered'), + findsOneWidget, + ); + }); + }); +} diff --git a/App/serval_app/test/mask_editor_test.dart b/App/serval_app/test/mask_editor_test.dart index 56149bc..2f99277 100644 --- a/App/serval_app/test/mask_editor_test.dart +++ b/App/serval_app/test/mask_editor_test.dart @@ -8,6 +8,7 @@ import 'package:serval_app/data/sample_repository.dart'; import 'package:serval_app/screens/mask_editor_screen.dart'; import 'package:serval_app/theme/app_theme.dart'; import 'package:serval_app/widgets/mask_canvas.dart'; +import 'package:serval_app/widgets/nocturne_field.dart'; /// Design 9b — drawing a mask, and the rules that make a polygon finishable. /// @@ -148,6 +149,59 @@ void main() { expect(find.text('Unnamed area'), findsNothing); }); + /// The canvas rules are hung over the whole screen, and the name box is on that screen — so a + /// keystroke that means "the last point" out on the frame has to mean "the last character" once + /// the caret is in a field, or a mask cannot be renamed without retyping the name from scratch. + group('with the caret in the name box', () { + Finder nameField() => find.descendant( + of: find.byWidgetPredicate( + (widget) => widget is NocturneField && widget.label == 'Name', + ), + matching: find.byType(EditableText), + ); + + testWidgets('Backspace deletes a character, not a point', (tester) async { + await tester.pumpWidget(harness()); + await tester.pumpAndSettle(); + + await click(tester, 0.2, 0.2); + await click(tester, 0.6, 0.2); + await click(tester, 0.6, 0.6); + await click(tester, 0.2, 0.2); + + await tester.enterText(nameField(), 'drivw'); + await tester.pumpAndSettle(); + + await tester.sendKeyEvent(LogicalKeyboardKey.backspace); + await tester.pumpAndSettle(); + + expect(tester.widget(nameField()).controller.text, 'driv'); + }); + + testWidgets('Esc does not abandon the shape being drawn', (tester) async { + await tester.pumpWidget(harness()); + await tester.pumpAndSettle(); + + // A mask to select, so the inspector has a name box in it, and then a second shape in hand. + await click(tester, 0.2, 0.2); + await click(tester, 0.6, 0.2); + await click(tester, 0.6, 0.6); + await click(tester, 0.2, 0.2); + + await click(tester, 0.3, 0.7); + await click(tester, 0.5, 0.7); + expect(find.text('Undo point'), findsOneWidget); + + await tester.tap(nameField()); + await tester.pumpAndSettle(); + + await tester.sendKeyEvent(LogicalKeyboardKey.escape); + await tester.pumpAndSettle(); + + expect(find.text('Undo point'), findsOneWidget); + }); + }); + testWidgets('a drawn mask can be saved', (tester) async { await tester.pumpWidget(harness()); await tester.pumpAndSettle(); diff --git a/App/serval_app/web/cast.js b/App/serval_app/web/cast.js new file mode 100644 index 0000000..df2aa1e --- /dev/null +++ b/App/serval_app/web/cast.js @@ -0,0 +1,339 @@ +// Google Cast sender, kept in JavaScript rather than Dart interop on purpose. +// +// A callback from JavaScript into Dart is the direction that kept breaking: dart2js binds +// arguments and checks types *before* the body runs, so an SDK that calls back with one argument +// where two are documented throws inside Google's own code, with no Dart frame in the stack. So +// nothing here calls into Dart. It publishes plain state and Dart polls it — see cast_sender_web. +// +// What gets cast is Serval's own receiver application, not Google's default one. That receiver +// opens a WebRTC connection back to this server and plays the URL it is handed only if that fails, +// so a television gets the same sub-second picture the App shows. The default receiver could only +// ever play the URL, several seconds behind, which is what this used to do. +// +// The application id is per-deployment — the operator registers their own against their own +// server's receiver URL — so it arrives from the server at start() time rather than being a +// constant here. +(function () { + 'use strict'; + + var SDK = 'https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1'; + var READY_TIMEOUT_MS = 20000; + var POLL_MS = 250; + + // How long to give a cold-launched receiver before loading again. Long enough for a device to + // fetch and start the page, short enough that a viewer does not give up and press it themselves — + // which is what they have been doing. + var RETRY_AFTER_MS = 4000; + + var ready = false; + var receiverAvailable = false; + var lastError = ''; + var initialised = false; + var currentAppId = null; + + // The application the SDK has been told to look for, as opposed to the one it has been + // configured with. Held separately because the two are set at different times: this arrives from + // the server as soon as a camera screen opens, and is applied once the SDK finishes loading. + var wantedAppId = null; + + function log(message) { + if (window.__servalCastDebug) console.log('[cast] ' + message); + } + + /** + * Starts the framework for one application id. + * + * Re-initialising with a different id is supported because the id is server-supplied and can + * change under us (an operator registering a receiver, or clearing one). The SDK tolerates + * setOptions being called again; what it does not tolerate is being asked to launch an id it was + * not initialised with. + */ + function configure(appId) { + if (!window.chrome || !chrome.cast || !chrome.cast.isAvailable) return false; + if (currentAppId === appId) return true; + + cast.framework.CastContext.getInstance().setOptions({ + receiverApplicationId: appId, + + // Leave a session running when the tab goes away. Casting a camera to a television is a + // "put it on that screen and walk off" action, and tearing it down on navigation would make + // the button useless for the one thing it is for. + autoJoinPolicy: chrome.cast.AutoJoinPolicy.ORIGIN_SCOPED + }); + + currentAppId = appId; + watchReceivers(); + return true; + } + + function watchReceivers() { + var context = cast.framework.CastContext.getInstance(); + + // CastState is on cast.framework, not chrome.cast — the two namespaces both exist and only one + // has it, so getting this wrong throws inside the discovery callback and the button never + // appears. The string is the enum's own value, kept as a fallback so a namespace that moves + // again degrades to a working comparison rather than to no casting at all. + var noDevices = + (cast.framework.CastState && cast.framework.CastState.NO_DEVICES_AVAILABLE) + || 'NO_DEVICES_AVAILABLE'; + + function refresh() { + receiverAvailable = context.getCastState() !== noDevices; + log('cast state: ' + context.getCastState()); + } + + context.addEventListener( + cast.framework.CastContextEventType.CAST_STATE_CHANGED, refresh); + refresh(); + } + + /** + * Loads Google's sender SDK. + * + * Both the callback and a poll, because Chrome injects its own sender script into a tab that has + * cast before — so the API can already be initialised, and the callback already dispatched, + * before this file ever runs. Waiting only on the callback means the button never appears in + * exactly the tabs most likely to want it. + */ + function loadSdk() { + if (initialised) return; + initialised = true; + + window.__onGCastApiAvailable = function (available) { + ready = !!available; + log('api available: ' + available); + }; + + var script = document.createElement('script'); + script.src = SDK; + document.head.appendChild(script); + + var waited = 0; + var poll = setInterval(function () { + waited += POLL_MS; + if (window.chrome && chrome.cast && chrome.cast.isAvailable) { + ready = true; + clearInterval(poll); + log('api ready after ' + waited + 'ms'); + + // Discovery cannot start until the SDK knows which application to look for, so applying + // this is what makes a receiver findable at all — and therefore what makes the button + // appear. Doing it only at launch time was a deadlock: no discovery, no button, no launch. + if (wantedAppId) configure(wantedAppId); + } else if (waited >= READY_TIMEOUT_MS) { + clearInterval(poll); + log('api never became available'); + } + }, POLL_MS); + } + + function session() { + if (!ready || !window.cast || !cast.framework) return null; + return cast.framework.CastContext.getInstance().getCurrentSession(); + } + + /** + * Describes what to play. Shared by both launch paths, which differ only in whether a session + * has to be asked for first. + * + * The URL is a live HLS playlist, which is what the receiver falls back to — it negotiates WebRTC + * off the same URL first, so this describes the fallback rather than what will actually play. + */ + function mediaInfo(url, title, live) { + var info = new chrome.cast.media.MediaInfo(url, 'application/vnd.apple.mpegurl'); + + // A recording is BUFFERED, which is what gives the television a duration, a scrub bar and + // working transport controls. Marking one LIVE would take all three away, and marking a live + // stream BUFFERED would have it seek to a beginning that does not exist. + info.streamType = live + ? chrome.cast.media.StreamType.LIVE + : chrome.cast.media.StreamType.BUFFERED; + + // The live fallback is fMP4, straight off the recorder. A recording is MPEG-TS, because it is + // transcoded and TS segments carry their own parameter sets. Say which: the media player + // library assumes transport stream, and an fMP4 stream that does not declare itself is parsed, + // fetched in full, and rendered as nothing at all, with no error on either side. + var formats = chrome.cast.media.HlsVideoSegmentFormat; + info.hlsVideoSegmentFormat = live + ? ((formats && formats.FMP4) || 'fmp4') + : ((formats && formats.MPEG2_TS) || 'mpeg2_ts'); + + if (live) { + info.hlsSegmentFormat = + (chrome.cast.media.HlsSegmentFormat && chrome.cast.media.HlsSegmentFormat.FMP4) || 'fmp4'; + } + + info.metadata = new chrome.cast.media.GenericMediaMetadata(); + info.metadata.title = title; + + return info; + } + + /** + * Loads the media, and loads it again if the first one goes nowhere. + * + * A cold launch is a race the sender loses: the session is reported established before the + * receiver page has finished starting, so the first load can arrive at a receiver that has not + * registered its message interceptor yet and is dropped — the television then sits on the + * receiver's own screen forever, and a second press works because the receiver is by then + * already running. Retrying is safe: the receiver treats a second load as a fresh camera. + * + * Both failure shapes are covered because they are not the same. A dropped load may reject, and + * it may equally just never settle, so there is a timer as well — whichever fires first retries, + * and `settled` keeps that to exactly one extra attempt. + */ + function load(active, info, title, startSeconds) { + var settled = false; + var retried = false; + + function request() { + var req = new chrome.cast.media.LoadRequest(info); + + // Where in the recording to open. + // + // The playlist carries an EXT-X-START for the same instant and the receiver ignores it — + // that tag arrived in HLS version 6 and this playlist is version 3, which it has to be for + // MPEG-TS segments with no EXT-X-MAP. Saying it in the load request is what actually works, + // and it is the sender that knows: the window is far wider than the playhead, so without it + // a cast opens hours before whatever is being watched. + if (startSeconds > 0) req.currentTime = startSeconds; + + return req; + } + + function attempt() { + active.loadMedia(request()).then(function () { + settled = true; + log('loaded ' + title); + }, function (err) { + if (settled) return; + if (retried) { + settled = true; + lastError = 'The Cast device refused the stream (' + err + ').'; + return; + } + retry('it was refused (' + err + ')'); + }); + } + + function retry(why) { + if (settled || retried) return; + retried = true; + log('retrying the load: ' + why); + attempt(); + } + + attempt(); + setTimeout(function () { retry('the receiver did not answer in time'); }, RETRY_AFTER_MS); + } + + window.servalCast = { + /** + * Loads the sender SDK and starts looking for `appId`. + * + * Called when a camera screen opens, with the application this deployment registered. Safe to + * call repeatedly and with a different id — the SDK is loaded once, and the id is applied as + * soon as it is ready. + */ + initialise: function (appId) { + wantedAppId = appId || null; + loadSdk(); + if (ready && wantedAppId) configure(wantedAppId); + }, + + /** Whether a receiver is reachable AND this deployment has a receiver to launch. */ + available: function () { + return ready && receiverAvailable && currentAppId !== null; + }, + + casting: function () { + return session() !== null; + }, + + /** The last failure, cleared by reading it — Dart shows it once and moves on. */ + takeError: function () { + var error = lastError; + lastError = ''; + return error; + }, + + /** + * Casts one camera. `appId` is which receiver to launch, `url` what to hand it, `live` + * whether that URL is the live camera or a recording — the two want different stream types — + * and `startSeconds` how far into a recording to open, which is ignored when live. + */ + start: function (appId, url, title, live, startSeconds) { + lastError = ''; + + if (!ready) { + lastError = 'Casting is not available in this browser.'; + return; + } + + if (!configure(appId)) { + lastError = 'Casting is not available in this browser.'; + return; + } + + // Already casting: load into the session that exists rather than asking for one. + // + // requestSession() is how a viewer *chooses* a device, and it puts Cast's own dialog on the + // screen to do it. Calling it while a session is running therefore interrupts somebody who + // has already chosen — which is what scrubbing outside the cast window was doing, popping the + // device and volume dialog onto the phone in the middle of a seek. + var existing = session(); + if (existing) { + load(existing, mediaInfo(url, title, live), title, startSeconds || 0); + return; + } + + cast.framework.CastContext.getInstance().requestSession().then(function () { + var active = session(); + if (!active) { + lastError = 'No Cast device was chosen.'; + return; + } + + load(active, mediaInfo(url, title, live), title, startSeconds || 0); + }, function (err) { + // Cancelling the device picker arrives here too, and is not worth reporting. + if (err !== 'cancel') lastError = 'Could not start casting (' + err + ').'; + }); + }, + + /** + * Moves the television to `seconds` into whatever it is already playing. + * + * Used when somebody scrubs the timeline here while a recording is on screen there. A seek + * rather than a fresh load because a load restarts the receiver's media — several seconds of + * black — where this is immediate, and because the playlist already covers the whole window. + * Silent when nothing is playing: the caller cannot know that without asking, and a scrub is + * not the moment to report it. + */ + seek: function (seconds) { + var active = session(); + if (!active) return; + + var media = active.getMediaSession(); + if (!media) return; + + var request = new chrome.cast.media.SeekRequest(); + request.currentTime = seconds; + + // Keep playing across the jump — the default resumes whatever state it was in, and a scrub + // landing on a paused television reads as a seek that did not work. + request.resumeState = chrome.cast.media.ResumeState.PLAYBACK_START; + + media.seek(request, function () { + log('sought to ' + seconds.toFixed(1) + 's'); + }, function (err) { + lastError = 'The Cast device would not seek (' + err + ').'; + }); + }, + + stop: function () { + var active = session(); + if (active) active.endSession(true); + } + }; +})(); diff --git a/App/serval_app/web/index.html b/App/serval_app/web/index.html index 2fa842a..617836a 100644 --- a/App/serval_app/web/index.html +++ b/App/serval_app/web/index.html @@ -61,6 +61,10 @@ dist build ends with (the .map is not vendored, so it only 404s). --> + + + + + + + + + + + + + + +
+ + + + + diff --git a/Server/Serval.Server/GoogleHome/SdpSummary.cs b/Server/Serval.Server/GoogleHome/SdpSummary.cs new file mode 100644 index 0000000..f66b3c9 --- /dev/null +++ b/Server/Serval.Server/GoogleHome/SdpSummary.cs @@ -0,0 +1,103 @@ +using System.Text; + +namespace Serval.Server.GoogleHome; + +/// +/// A one-line description of an SDP, for the log. +/// +/// Why this exists rather than logging the SDP itself. An offer is several kilobytes +/// and unreadable at a glance, but the handful of facts that decide whether media will actually +/// flow are all in there: which media sections survived negotiation, what codecs they carry, and — +/// the question this was written to answer — what kind of ICE candidates the far end is +/// offering. Host candidates mean the peer is on the same network; server-reflexive or relay +/// candidates mean it is somewhere else entirely and reaching it depends on NAT or a TURN server. +/// The full SDP is still logged at Debug for when the summary is not enough. +/// +/// Deliberately tolerant: this parses attacker-adjacent text from another party's WebRTC +/// stack purely to write a log line, so a shape it does not recognise must produce a worse summary +/// rather than an exception on a live stream request. +/// +internal static class SdpSummary +{ + public static string Describe(string? sdp) + { + if (string.IsNullOrWhiteSpace(sdp)) + { + return "(empty)"; + } + + var media = new List(); + var candidates = new List(); + var payloads = new Dictionary(StringComparer.Ordinal); + string? setup = null; + bool bundle = false; + + foreach (string raw in sdp.Split('\n')) + { + string line = raw.Trim(); + + if (line.StartsWith("m=", StringComparison.Ordinal)) + { + // "m=video 9 UDP/TLS/RTP/SAVPF 96 97" — a port of 0 means the section was rejected, + // which is the quiet way a negotiation ends up with no media at all. + string[] parts = line[2..].Split(' '); + if (parts.Length >= 2) + { + media.Add(parts[1] == "0" ? $"{parts[0]}:REJECTED" : parts[0]); + } + } + else if (line.StartsWith("a=rtpmap:", StringComparison.Ordinal)) + { + string[] parts = line[9..].Split(' ', 2); + if (parts.Length == 2) + { + payloads[parts[0]] = parts[1].Split('/')[0]; + } + } + else if (line.StartsWith("a=candidate:", StringComparison.Ordinal)) + { + // "a=candidate:1 1 udp 2130706431 192.168.1.20 8555 typ host" + string[] f = line[12..].Split(' '); + int typ = Array.IndexOf(f, "typ"); + if (typ >= 0 && typ + 1 < f.Length && f.Length > 5) + { + candidates.Add($"{f[typ + 1]} {f[4]}:{f[5]}/{f[2]}"); + } + } + else if (line.StartsWith("a=setup:", StringComparison.Ordinal)) + { + setup = line[8..]; + } + else if (line.StartsWith("a=group:BUNDLE", StringComparison.Ordinal)) + { + bundle = true; + } + } + + var summary = new StringBuilder(); + summary.Append(media.Count == 0 ? "no media" : string.Join('+', media)); + + if (payloads.Count > 0) + { + summary.Append(" codecs=").Append(string.Join(',', payloads.Values.Distinct(StringComparer.OrdinalIgnoreCase))); + } + + summary.Append(" candidates="); + summary.Append(candidates.Count == 0 + // Not a fault on its own: trickle-ICE offers arrive with none, and the far end sends + // them separately. It is a fault here, because this signaling contract is one exchange + // with nowhere for a later candidate to arrive. + ? "none" + : string.Join(" | ", candidates)); + + if (setup is not null) + { + summary.Append(" setup=").Append(setup); + } + + summary.Append(bundle ? " bundle" : " no-bundle"); + summary.Append(" bytes=").Append(sdp.Length); + + return summary.ToString(); + } +} diff --git a/Server/Serval.Server/GoogleHome/SmartHomeContracts.cs b/Server/Serval.Server/GoogleHome/SmartHomeContracts.cs new file mode 100644 index 0000000..dbd0998 --- /dev/null +++ b/Server/Serval.Server/GoogleHome/SmartHomeContracts.cs @@ -0,0 +1,260 @@ +using System.Text.Json.Serialization; + +namespace Serval.Server.GoogleHome; + +/// +/// The fulfillment wire format. +/// +/// Every field name is spelled out, and it has to be. Google's intent payloads are +/// camelCase, which happens to match this server's default — but the action.devices.* +/// vocabulary is not derivable from a C# name, and a property renamed during a refactor would +/// silently stop matching. Naming them here makes the contract explicit rather than incidental. +/// +public sealed record SmartHomeRequest( + [property: JsonPropertyName("requestId")] string RequestId, + [property: JsonPropertyName("inputs")] IReadOnlyList Inputs); + +/// +/// One of action.devices.SYNC, .QUERY, .EXECUTE, .DISCONNECT. +/// +/// +/// Shaped differently per intent, so it stays a and each +/// handler reads what it needs. Modelling four unrelated shapes as one type would only move the +/// branching somewhere less honest. +/// +public sealed record SmartHomeInput( + [property: JsonPropertyName("intent")] string Intent, + [property: JsonPropertyName("payload")] System.Text.Json.JsonElement Payload); + +/// The envelope every intent answers in. is echoed unchanged. +public sealed record SmartHomeResponse( + [property: JsonPropertyName("requestId")] string RequestId, + [property: JsonPropertyName("payload")] object Payload); + +/// +/// SYNC's answer: who the user is to us, and every device they have. +/// +public sealed record SyncPayload( + [property: JsonPropertyName("agentUserId")] string AgentUserId, + [property: JsonPropertyName("devices")] IReadOnlyList Devices); + +/// +/// One camera as Google models it. +/// +/// is true only where Report State can actually be delivered, +/// which means a HomeGraph key is loaded — see CameraDeviceMapper.ToDevice. +/// is omitted rather than sent empty — an empty string is a room named "" in +/// the Google Home app, which the user then has to clean up by hand. +/// +public sealed record SyncDevice( + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("traits")] IReadOnlyList Traits, + [property: JsonPropertyName("name")] SyncDeviceName Name, + [property: JsonPropertyName("willReportState")] bool WillReportState, + [property: JsonPropertyName("attributes")] CameraStreamAttributes Attributes, + [property: JsonPropertyName("deviceInfo")] SyncDeviceInfo DeviceInfo, + [property: JsonPropertyName("roomHint")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? RoomHint); + +public sealed record SyncDeviceName( + [property: JsonPropertyName("name")] string Name); + +public sealed record SyncDeviceInfo( + [property: JsonPropertyName("manufacturer")] string Manufacturer, + [property: JsonPropertyName("model")] string Model, + [property: JsonPropertyName("swVersion")] string SwVersion); + +/// +/// What the camera can do, declared at SYNC so Google knows what to ask for later. +/// +/// webrtc first, then hls for a camera that is recorded — see +/// CameraDeviceMapper.ProtocolsFor. The order is the preference: WebRTC is sub-second and +/// keeps the media on the LAN between the display and go2rtc, while HLS has the Cast device fetch +/// the playlist and every segment through the operator's public origin. HLS is offered anyway +/// because a Cast Web Receiver speaks nothing else, so a camera advertising WebRTC alone cannot be +/// shown on a TV at all. +/// +public sealed record CameraStreamAttributes( + [property: JsonPropertyName("cameraStreamSupportedProtocols")] + IReadOnlyList SupportedProtocols, + [property: JsonPropertyName("cameraStreamNeedAuthToken")] bool NeedAuthToken); + +/// QUERY's answer, keyed by device id. +public sealed record QueryPayload( + [property: JsonPropertyName("devices")] IReadOnlyDictionary Devices); + +/// "SUCCESS" or "ERROR". +/// +/// Whether the camera is producing frames — reachability, and Google's own concept. Deliberately +/// not driven by : a camera switched off in the Home app is still +/// perfectly reachable, and reporting it offline is what greys out the control that would switch it +/// back on. Omitted on an error. +/// +/// +/// Whether Serval will offer this camera to Google, as set from the Home app. Omitted on an error, +/// and for a device that does not carry the trait. +/// +/// Google's vocabulary, e.g. "deviceNotFound". Omitted on success. +public sealed record QueryDeviceState( + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("online")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + bool? Online, + [property: JsonPropertyName("on")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + bool? On, + [property: JsonPropertyName("errorCode")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? ErrorCode); + +/// The states an OnOff execution answers with. +public sealed record OnOffState([property: JsonPropertyName("on")] bool On); + +/// EXECUTE's answer: one entry per group of device ids that share an outcome. +public sealed record ExecutePayload( + [property: JsonPropertyName("commands")] IReadOnlyList Commands); + +public sealed record ExecuteCommandResult( + [property: JsonPropertyName("ids")] IReadOnlyList Ids, + [property: JsonPropertyName("status")] string Status, + // Typed as object because the shape depends on the command: GetCameraStream answers with a + // CameraStreamState and OnOff with an OnOffState. Google returns both in this one slot. + [property: JsonPropertyName("states")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + object? States, + [property: JsonPropertyName("errorCode")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? ErrorCode, + + /// + /// Present when the command needs a second factor. It is what turns the Assistant's reply from + /// "sorry, something went wrong" into "what's your PIN?", so its absence is not a small error — + /// it is the difference between a challenge and a failure. + /// + [property: JsonPropertyName("challengeNeeded")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + ChallengeNeeded? Challenge = null); + +/// "pinNeeded" or "ackNeeded". +public sealed record ChallengeNeeded( + [property: JsonPropertyName("type")] string Type); + +/// +/// What Google needs to open the stream. Which fields are populated depends on +/// : WebRTC uses and HLS uses +/// , and each omits the other's — one record for both because Google +/// returns them in the same states slot. +/// +/// is deliberately never populated: leaving it out is what makes Google +/// generate the SDP offer, and go2rtc answers offers rather than making them. +/// is a JSON-encoded string rather than an array — Google's +/// choice, not ours — and is omitted when unconfigured, which is the normal case for a Hub and +/// go2rtc on one LAN. +/// +/// is sent for both, and is the same camera-scoped ticket that the +/// URL already carries. It is redundant on the HLS path — the Cast receiver that fetches those +/// segments cannot set a header at all, which is why cameraStreamNeedAuthToken is false — +/// and harmless to include. +/// +public sealed record CameraStreamState( + [property: JsonPropertyName("cameraStreamProtocol")] string Protocol, + [property: JsonPropertyName("cameraStreamSignalingUrl")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? SignalingUrl, + [property: JsonPropertyName("cameraStreamAccessUrl")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? AccessUrl, + [property: JsonPropertyName("cameraStreamAuthToken")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? AuthToken, + [property: JsonPropertyName("cameraStreamIceServers")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? IceServers, + [property: JsonPropertyName("cameraStreamOffer")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? Offer, + + /// + /// The Cast application to open the stream with, in place of Google's own receiver. + /// + /// Only meaningful alongside a non-WebRTC protocol: Google's words are "Cast receiver ID + /// to process the camera stream when the StreamToChromecast parameter is true; default + /// receiver will be used if not provided", and for webrtc it plays with its own player + /// and never consults this. That asymmetry is the whole reason this field is worth having — + /// WebRTC reaches only Nest displays and Chromecast with Google TV, so on any other Cast + /// device the receiver named here is the only thing that can open a WebRTC connection. + /// + /// Null unless Serval:GoogleHome:CastReceiverAppId is set, which is the default. + /// Then Google uses its own receiver and plays as ordinary HLS. + /// + [property: JsonPropertyName("cameraStreamReceiverAppId")] + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + string? ReceiverAppId = null); + +/// +/// The signaling request Google POSTs to cameraStreamSignalingUrl, and the answer it +/// expects back. +/// +/// "offer", "answer", or "end". +/// The camera, as named in SYNC. Cross-checked against the ticket. +/// The session description. Empty for "end". +public sealed record SignalingRequest( + [property: JsonPropertyName("action")] string Action, + [property: JsonPropertyName("deviceId")] string? DeviceId, + [property: JsonPropertyName("sdp")] string? Sdp); + +/// must be the literal "answer"; Google checks it. +public sealed record SignalingResponse( + [property: JsonPropertyName("action")] string Action, + [property: JsonPropertyName("sdp")] string Sdp); + +/// The names Google uses, in one place rather than as literals across three files. +public static class SmartHome +{ + public const string SyncIntent = "action.devices.SYNC"; + public const string QueryIntent = "action.devices.QUERY"; + public const string ExecuteIntent = "action.devices.EXECUTE"; + public const string DisconnectIntent = "action.devices.DISCONNECT"; + + public const string CameraType = "action.devices.types.CAMERA"; + public const string CameraStreamTrait = "action.devices.traits.CameraStream"; + public const string GetCameraStreamCommand = "action.devices.commands.GetCameraStream"; + + /// + /// Declared so the Google Home app has something to switch. It governs whether Serval offers + /// Google a stream and nothing else — the camera keeps recording either way. See + /// . + /// + public const string OnOffTrait = "action.devices.traits.OnOff"; + + public const string OnOffCommand = "action.devices.commands.OnOff"; + + public const string WebRtcProtocol = "webrtc"; + + /// + /// The fallback protocol, and the only one a Cast Web Receiver speaks. A Google TV does not + /// ask for WebRTC: casting launches a receiver on the device, and that receiver plays + /// hls/dash/smooth_stream/progressive_mp4 and nothing else. + /// + public const string HlsProtocol = "hls"; + + public const string Success = "SUCCESS"; + public const string Error = "ERROR"; + + /// + /// Asks the Assistant to collect a PIN and send the command again. Google requires this for + /// OnOff on a camera — see 's VerificationPin. + /// + public const string ChallengeNeeded = "challengeNeeded"; + + /// The PIN was wrong. A distinct code, so the Assistant re-prompts rather than gives up. + public const string ChallengeFailedPinNeeded = "challengeFailedPinNeeded"; + + public const string PinNeeded = "pinNeeded"; + + public const string DeviceNotFound = "deviceNotFound"; + public const string DeviceOffline = "deviceOffline"; + public const string FunctionNotSupported = "functionNotSupported"; +} diff --git a/Server/Serval.Server/GoogleHome/SmartHomeFulfillment.cs b/Server/Serval.Server/GoogleHome/SmartHomeFulfillment.cs new file mode 100644 index 0000000..e991ffd --- /dev/null +++ b/Server/Serval.Server/GoogleHome/SmartHomeFulfillment.cs @@ -0,0 +1,674 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using Serval.Server.Cameras; +using Serval.Server.Configuration; +using Serval.Server.Snapshots; + +namespace Serval.Server.GoogleHome; + +/// +/// The four intents Google sends, and what each answers. +/// +/// SYNC lists the cameras. QUERY says whether each is up. EXECUTE hands +/// back a signaling URL for one. DISCONNECT is the user unlinking from the Google Home app, +/// and must take everything away. +/// +/// The device set is throughout, which is the go2rtc +/// sync worker's own predicate — see there for why that is not merely tidy. +/// +public sealed class SmartHomeFulfillment +{ + private readonly CameraRepository _cameras; + private readonly SnapshotBroadcaster _snapshots; + private readonly GoogleOAuthStore _store; + private readonly CameraStreamTicketService _tickets; + private readonly GoogleHomeGate _gate; + private readonly HomeGraphKeyStore _homeGraphKeys; + private readonly GoogleCameraSwitchStore _switches; + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + + public SmartHomeFulfillment( + CameraRepository cameras, + SnapshotBroadcaster snapshots, + GoogleOAuthStore store, + CameraStreamTicketService tickets, + GoogleHomeGate gate, + HomeGraphKeyStore homeGraphKeys, + GoogleCameraSwitchStore switches, + IOptionsMonitor options, + ILogger logger) + { + _cameras = cameras; + _snapshots = snapshots; + _store = store; + _tickets = tickets; + _gate = gate; + _homeGraphKeys = homeGraphKeys; + _switches = switches; + _options = options; + _logger = logger; + } + + /// + /// Whether the Home app's switch is offered, which is to say whether it can be protected. + /// + /// Google requires a pinNeeded challenge for OnOff on a camera. With no PIN + /// configured there is nothing to challenge with, so the trait is not declared, no switch is + /// accepted, and — importantly — any switch already stored is ignored. That last part is what + /// stops a camera being stranded off by a row written while a PIN was configured and left + /// behind when it was removed. + /// + private bool Switchable => + !string.IsNullOrEmpty(_options.CurrentValue.GoogleHome.VerificationPin); + + /// + /// The Cast application to open an HLS stream with, or null to leave Google on its own + /// receiver. Null is the default and a working deployment; see + /// . + /// + private string? ReceiverAppId => _gate.CastReceiverAppId; + + /// + /// Dispatches one request. Google sends a single input in practice but the contract is a list, + /// so the first is taken and any others ignored rather than guessed at. + /// + public async Task HandleAsync( + SmartHomeRequest request, string agentUserId, CancellationToken ct) + { + SmartHomeInput? input = request.Inputs?.FirstOrDefault(); + + // Every intent, named. Without this the only trace a request left was a Mongo timestamp, + // which says one arrived and nothing about which or what it asked for. + _logger.LogInformation("Google Home {Intent}.", input?.Intent ?? "(no intent)"); + + object payload = input?.Intent switch + { + SmartHome.SyncIntent => await SyncAsync(agentUserId, ct), + SmartHome.QueryIntent => await QueryAsync(input.Payload, ct), + SmartHome.ExecuteIntent => await ExecuteAsync(input.Payload, ct), + SmartHome.DisconnectIntent => await DisconnectAsync(agentUserId, ct), + + // Google's own vocabulary for "I do not know what you asked me". Answering with an + // empty success instead would have it believe the account has no cameras. + _ => new { errorCode = "notSupported" }, + }; + + return new SmartHomeResponse(request.RequestId, payload); + } + + private async Task SyncAsync(string agentUserId, CancellationToken ct) + { + List cameras = await _cameras.ListAsync(ct); + + // Whether the key *loaded*, not whether a path is configured: a path pointing at an + // unreadable file would otherwise have us promise reports nothing can send. + bool reportsState = _homeGraphKeys.Key is not null; + + SyncDevice[] devices = + [ + .. CameraDeviceMapper.Eligible(cameras) + .Select(camera => CameraDeviceMapper.ToDevice(camera, reportsState, Switchable)), + ]; + + _logger.LogInformation( + "Google Home SYNC: offering {Count} of {Total} cameras.", devices.Length, cameras.Count); + + return new SyncPayload(agentUserId, devices); + } + + private async Task QueryAsync(JsonElement payload, CancellationToken ct) + { + List cameras = await _cameras.ListAsync(ct); + + HashSet eligible = [.. CameraDeviceMapper.Eligible(cameras).Select(c => c.Id)]; + HashSet off = Switchable ? await _switches.OffAsync(ct) : []; + DateTimeOffset now = DateTimeOffset.UtcNow; + + var states = new Dictionary(StringComparer.Ordinal); + + foreach (string id in DeviceIds(payload)) + { + states[id] = eligible.Contains(id) + ? new QueryDeviceState( + SmartHome.Success, + + // Reachability, unaffected by the switch: a camera switched off in the Home app + // is still answering, and saying otherwise is what would grey out the control + // that switches it back on. + CameraDeviceMapper.IsOnline(_snapshots.Latest(id), now), + On: Switchable ? !off.Contains(id) : null, + ErrorCode: null) + + // Disabled, deleted, or a file camera. Google keeps a device it was told about + // until the next SYNC, so this is a normal answer rather than an exceptional one. + : new QueryDeviceState( + SmartHome.Error, Online: null, On: null, SmartHome.DeviceNotFound); + } + + return new QueryPayload(states); + } + + /// + /// Answers the one command this integration supports, per device. + /// + /// Success here means "a ticket was minted", not "the stream works". Nothing is + /// asked of go2rtc — the camera could be unreachable and this still returns SUCCESS, after + /// which the Assistant says "showing the front door" and the display fails to connect. That is + /// inherent to the protocol: the media negotiation happens after this response. A pre-flight + /// check would cost an RTSP connection on every voice command to buy a better error message on + /// a rare one. + /// + private async Task ExecuteAsync(JsonElement payload, CancellationToken ct) + { + List cameras = await _cameras.ListAsync(ct); + Dictionary eligible = CameraDeviceMapper.Eligible(cameras) + .ToDictionary(c => c.Id, StringComparer.Ordinal); + + Uri? publicBase = _gate.PublicBaseUri; + string? iceServers = _gate.IceServersJson; + + var results = new List(); + + // What the requesting surface says it can play. This is the field that decides whether a + // phone, a display or a TV gets a stream at all, and it is the only place the surface's + // own capabilities are visible to us — Google does not say which device is asking. + _logger.LogInformation( + "Google Home EXECUTE requested protocols: {Protocols} (to a Cast device: {Chromecast}).", + RequestedProtocols(payload), + StreamToChromecast(payload)); + + // The Home app's switch. Handled first so that switching a camera off and asking for its + // stream in one payload cannot answer with a stream it has just withdrawn. + foreach ((string id, bool on, string? pin) in OnOffTargets(payload)) + { + if (!eligible.ContainsKey(id)) + { + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.DeviceNotFound)); + continue; + } + + if (!Switchable) + { + // No PIN configured, so the trait was never declared. Reachable only if Google is + // holding an older SYNC, and refusing is the safe answer either way. + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.FunctionNotSupported)); + continue; + } + + // The second factor, and only in the direction that needs one. + // + // Switching a security camera *off* is the sensitive act — a voice carries through an + // open window, and out of a television — and it is what Google singles out. Switching + // one back on restores the safe state, so challenging it buys nothing and costs + // something real: a camera left off with the way back gated behind a prompt the Home + // app may never offer is a camera nobody can recover without touching the database. + if (NeedsChallenge(on)) + { + if (pin is null) + { + results.Add(new ExecuteCommandResult( + [id], + SmartHome.Error, + States: null, + SmartHome.ChallengeNeeded, + new ChallengeNeeded(SmartHome.PinNeeded))); + continue; + } + + if (!PinMatches(pin)) + { + // A distinct code from challengeNeeded, so the Assistant says the PIN was wrong + // and asks again rather than reporting a failure the speaker cannot act on. + _logger.LogWarning( + "Google Home: switching camera {CameraId} off was refused — wrong PIN.", id); + + results.Add(new ExecuteCommandResult( + [id], + SmartHome.Error, + States: null, + SmartHome.ChallengeFailedPinNeeded, + new ChallengeNeeded(SmartHome.PinNeeded))); + continue; + } + } + + await _switches.SetAsync(id, on, ct); + + // A session already running would otherwise keep playing after the switch was thrown, + // which reads as the switch not working. Dropping the tickets stops the next segment or + // renegotiation; nothing in Serval's own pipeline is touched. + if (!on) + { + _tickets.RevokeForCamera(id); + } + + _logger.LogInformation( + "Google Home: camera {CameraId} switched {State} from the Home app. Recording and " + + "the Serval app are unaffected.", id, on ? "on" : "off"); + + results.Add(new ExecuteCommandResult( + [id], SmartHome.Success, new OnOffState(on), ErrorCode: null)); + } + + HashSet switchedOff = Switchable ? await _switches.OffAsync(ct) : []; + + foreach ((string id, bool webRtcWanted, bool hlsWanted) in ExecuteTargets(payload)) + { + if (!eligible.TryGetValue(id, out Camera? camera)) + { + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.DeviceNotFound)); + continue; + } + + if (switchedOff.Contains(id)) + { + // Switched off in the Home app. deviceOffline rather than functionNotSupported: + // the camera can do this, it is just not offering it right now, and Google words + // that to the user as unavailable rather than unsupported. + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.DeviceOffline)); + continue; + } + + if (publicBase is null) + { + // Unreachable behind the gate, which already requires a valid https base URL. Kept + // as a real branch rather than a null-forgiving operator so a future change to the + // gate cannot turn this into a NullReferenceException on a live voice command. + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.DeviceOffline)); + continue; + } + + // WebRTC first whenever the receiver will take it: sub-second, media stays on the LAN, + // and no dependence on the camera being recorded. Google offers it only from a Nest + // display or a Chromecast with Google TV; every other Cast device asks for HLS, and + // that branch is answered only for a camera being recorded, because those segments are + // what it plays. + if (webRtcWanted) + { + string ticket = _tickets.Mint(id); + + results.Add(new ExecuteCommandResult( + [id], + SmartHome.Success, + new CameraStreamState( + Protocol: SmartHome.WebRtcProtocol, + SignalingUrl: SignalingUrl(publicBase, ticket), + AccessUrl: null, + AuthToken: ticket, + IceServers: iceServers, + + // Never set: omitting it is what makes Google generate the offer, which is + // the direction go2rtc can answer in. + Offer: null), + ErrorCode: null)); + continue; + } + + if (hlsWanted && CameraDeviceMapper.CanPlayHls(camera)) + { + string ticket = _tickets.MintForPlayback(id); + + results.Add(new ExecuteCommandResult( + [id], + SmartHome.Success, + new CameraStreamState( + Protocol: SmartHome.HlsProtocol, + SignalingUrl: null, + AccessUrl: HlsUrl(publicBase, id, ticket), + AuthToken: ticket, + + // Both are read by Google's own player, which is not the one that runs when + // a receiver app is named below. Serval's receiver is handed its ICE + // configuration in the page instead — see CameraStreamReceiver. + IceServers: null, + Offer: null, + + // Turns this from "play a playlist" into "run Serval's receiver, which will + // try WebRTC and use the playlist only if that fails". Null unless the + // operator registered a Cast application, which is the default. + ReceiverAppId: ReceiverAppId), + ErrorCode: null)); + continue; + } + + // A receiver that can take neither — or one asking for HLS on a camera that is not + // being recorded, which has no segments to serve. + results.Add(new ExecuteCommandResult( + [id], SmartHome.Error, States: null, SmartHome.FunctionNotSupported)); + } + + return new ExecutePayload(results); + } + + /// + /// The user unlinked in the Google Home app. Everything issued under the link goes, so a + /// request arriving afterwards finds no token and is refused. + /// + private async Task DisconnectAsync(string agentUserId, CancellationToken ct) + { + await _store.UnlinkAsync(agentUserId, ct); + + // The switches were set from the Home app, so they belong to the link that is going away. + // Leaving them would have a camera silently unavailable to whoever links next, with the + // reason recorded nowhere they would think to look. + await _switches.ClearAsync(ct); + + _logger.LogInformation("Google Home DISCONNECT: link {AgentUserId} removed.", agentUserId); + + // An empty object is the whole contract. + return new { }; + } + + /// + /// EXECUTE's payload, flattened to (device id, requested on state) for OnOff commands. + /// + /// A separate walk from rather than one pass returning a + /// tagged union: the two commands share no parameters and no outcome, and reading each on its + /// own terms is what keeps either one comprehensible. + /// + internal static IEnumerable<(string Id, bool On, string? Pin)> OnOffTargets(JsonElement payload) + { + if (!payload.TryGetProperty("commands", out JsonElement commands) + || commands.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (JsonElement command in commands.EnumerateArray()) + { + if (RequestedOnState(command) is not bool on) + { + continue; + } + + string? pin = ChallengePin(command); + + if (!command.TryGetProperty("devices", out JsonElement devices) + || devices.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (JsonElement device in devices.EnumerateArray()) + { + if (device.TryGetProperty("id", out JsonElement id) + && id.GetString() is { Length: > 0 } value) + { + yield return (value, on, pin); + } + } + } + } + + /// + /// The PIN Google collected for a command, or null if it sent none. + /// + /// Google puts it on the execution as challenge.pin, alongside the params rather + /// than inside them. Null means "not asked yet" and is the normal first pass — every switch + /// arrives twice, once without a challenge and once with. + /// + private static string? ChallengePin(JsonElement command) + { + if (!command.TryGetProperty("execution", out JsonElement executions) + || executions.ValueKind != JsonValueKind.Array) + { + return null; + } + + foreach (JsonElement execution in executions.EnumerateArray()) + { + if (execution.TryGetProperty("command", out JsonElement name) + && name.GetString() == SmartHome.OnOffCommand + && execution.TryGetProperty("challenge", out JsonElement challenge) + && challenge.TryGetProperty("pin", out JsonElement pin) + && pin.GetString() is { Length: > 0 } value) + { + return value; + } + } + + return null; + } + + /// + /// Whether switching a camera to needs the PIN. + /// + /// Only switching off does. That is what Google requires a pinNeeded + /// challenge for, and it is the half that matters: a voice within earshot disabling a security + /// camera. Turning one back on returns it to the safe state, and gating that as well leaves a + /// camera stranded off whenever the Home app does not present the prompt. + /// + internal static bool NeedsChallenge(bool on) => !on; + + /// + /// Whether a supplied PIN is the configured one, compared in constant time — the discipline + /// TelemetryEndpoints uses for its API key, and for the same reason: a comparison that + /// returns early leaks the PIN one character at a time to anything that can time it. + /// + private bool PinMatches(string supplied) => + CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(supplied), + Encoding.UTF8.GetBytes(_options.CurrentValue.GoogleHome.VerificationPin)); + + /// + /// The on parameter of a command's OnOff execution, or null if it carries none. + /// Null rather than false, so a malformed command is ignored instead of silently switching + /// every camera it names off. + /// + private static bool? RequestedOnState(JsonElement command) + { + if (!command.TryGetProperty("execution", out JsonElement executions) + || executions.ValueKind != JsonValueKind.Array) + { + return null; + } + + foreach (JsonElement execution in executions.EnumerateArray()) + { + if (execution.TryGetProperty("command", out JsonElement name) + && name.GetString() == SmartHome.OnOffCommand + && execution.TryGetProperty("params", out JsonElement parameters) + && parameters.TryGetProperty("on", out JsonElement on) + && on.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + return on.GetBoolean(); + } + } + + return null; + } + + /// + /// Google's StreamToChromecast parameter, for the log only. + /// + /// Nothing routes on it — SupportedStreamProtocols is the field that actually says + /// what the far end can play, and acting on both would mean two sources of truth for one + /// decision. It is logged because it is the only thing in the request that distinguishes a TV + /// asking from a phone asking, and "which surface asked" is otherwise invisible to us. + /// + internal static string StreamToChromecast(JsonElement payload) + { + if (payload.TryGetProperty("commands", out JsonElement commands) + && commands.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement command in commands.EnumerateArray()) + { + if (!command.TryGetProperty("execution", out JsonElement executions) + || executions.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (JsonElement execution in executions.EnumerateArray()) + { + if (execution.TryGetProperty("params", out JsonElement p) + && p.TryGetProperty("StreamToChromecast", out JsonElement cast) + && cast.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + return cast.GetBoolean() ? "yes" : "no"; + } + } + } + } + + return "(not stated)"; + } + + /// + /// Every protocol named anywhere in an EXECUTE payload, for the log. Diagnostic only — the + /// decision is made per command in . + /// + internal static string RequestedProtocols(JsonElement payload) + { + var found = new List(); + + if (payload.TryGetProperty("commands", out JsonElement commands) + && commands.ValueKind == JsonValueKind.Array) + { + foreach (JsonElement command in commands.EnumerateArray()) + { + if (!command.TryGetProperty("execution", out JsonElement executions) + || executions.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (JsonElement execution in executions.EnumerateArray()) + { + if (execution.TryGetProperty("params", out JsonElement p) + && p.TryGetProperty("SupportedStreamProtocols", out JsonElement protocols) + && protocols.ValueKind == JsonValueKind.Array) + { + found.AddRange(protocols.EnumerateArray() + .Select(x => x.GetString()) + .Where(x => !string.IsNullOrEmpty(x))!); + } + } + } + } + + return found.Count == 0 ? "(none stated)" : string.Join(",", found.Distinct(StringComparer.Ordinal)); + } + + internal static string SignalingUrl(Uri publicBase, string ticket) => + new Uri(publicBase, "/api/google/camerastream/signal").AbsoluteUri + + "?t=" + Uri.EscapeDataString(ticket); + + /// + /// The playlist URL handed to a Cast receiver. + /// + /// The camera is in the path rather than carried only by the ticket, because the receiver + /// resolves the segment names in the playlist relative to this URL — so the path is + /// what puts them in the right camera's directory. The ticket is still what authorises it, and + /// the handler refuses a ticket whose camera does not match this path. + /// + internal static string HlsUrl(Uri publicBase, string cameraId, string ticket) => + new Uri(publicBase, $"/api/google/camerastream/hls/{Uri.EscapeDataString(cameraId)}/index.m3u8").AbsoluteUri + + "?t=" + Uri.EscapeDataString(ticket); + + /// QUERY's payload: {"devices":[{"id":"front-door"}]}. + internal static IEnumerable DeviceIds(JsonElement payload) + { + if (!payload.TryGetProperty("devices", out JsonElement devices) + || devices.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (JsonElement device in devices.EnumerateArray()) + { + if (device.TryGetProperty("id", out JsonElement id) + && id.GetString() is { Length: > 0 } value) + { + yield return value; + } + } + } + + /// + /// EXECUTE's payload, flattened to (device id, protocols the receiver will take) rows. + /// + /// Google nests it three deep — commands[].devices[] against + /// commands[].execution[] — because one command may address several devices and carry + /// several executions. Only GetCameraStream means anything here, and its + /// SupportedStreamProtocols is what says what the receiver on the other end can play. + /// It is the receiver's own answer and the only one worth having: what a surface supports is + /// not something to infer from what kind of device we imagine is asking. + /// + internal static IEnumerable<(string Id, bool WebRtc, bool Hls)> ExecuteTargets(JsonElement payload) + { + if (!payload.TryGetProperty("commands", out JsonElement commands) + || commands.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (JsonElement command in commands.EnumerateArray()) + { + HashSet wanted = SupportedProtocols(command); + bool webRtc = wanted.Contains(SmartHome.WebRtcProtocol); + bool hls = wanted.Contains(SmartHome.HlsProtocol); + + if (!command.TryGetProperty("devices", out JsonElement devices) + || devices.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (JsonElement device in devices.EnumerateArray()) + { + if (device.TryGetProperty("id", out JsonElement id) + && id.GetString() is { Length: > 0 } value) + { + yield return (value, webRtc, hls); + } + } + } + } + + /// Every protocol one command's GetCameraStream executions say they can play. + private static HashSet SupportedProtocols(JsonElement command) + { + var wanted = new HashSet(StringComparer.Ordinal); + + if (!command.TryGetProperty("execution", out JsonElement executions) + || executions.ValueKind != JsonValueKind.Array) + { + return wanted; + } + + foreach (JsonElement execution in executions.EnumerateArray()) + { + if (!execution.TryGetProperty("command", out JsonElement name) + || name.GetString() != SmartHome.GetCameraStreamCommand) + { + continue; + } + + if (!execution.TryGetProperty("params", out JsonElement parameters) + || !parameters.TryGetProperty("SupportedStreamProtocols", out JsonElement protocols) + || protocols.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (JsonElement protocol in protocols.EnumerateArray()) + { + if (protocol.GetString() is { Length: > 0 } value) + { + wanted.Add(value); + } + } + } + + return wanted; + } +} diff --git a/Server/Serval.Server/Ingest/Go2RtcClient.cs b/Server/Serval.Server/Ingest/Go2RtcClient.cs index 7999229..a60cb64 100644 --- a/Server/Serval.Server/Ingest/Go2RtcClient.cs +++ b/Server/Serval.Server/Ingest/Go2RtcClient.cs @@ -15,8 +15,15 @@ public interface IGo2RtcClient /// Names of every stream go2rtc currently has configured. Task> ListStreamNamesAsync(CancellationToken cancellationToken); - /// Adds (or replaces) a stream named sourced from . - Task PutStreamAsync(string name, string src, CancellationToken cancellationToken); + /// + /// Adds (or replaces) a stream named drawing on . + /// + /// A go2rtc stream is a set of sources, not one: it negotiates per consumer across all of + /// them, so a track a consumer asks for can come from a source that exists only to supply it. + /// Later sources may name this stream, which is how one is built on top of another without + /// opening a second session to the camera. + /// + Task PutStreamAsync(string name, IReadOnlyList sources, CancellationToken cancellationToken); /// Removes the stream named . Task DeleteStreamAsync(string name, CancellationToken cancellationToken); @@ -31,9 +38,9 @@ public interface IGo2RtcClient /// /// A thin typed over go2rtc's HTTP API. go2rtc's stream API has one -/// quirk worth pinning down: PUT identifies the stream by name and takes the source -/// as src, but DELETE identifies the stream by src. We map name to the -/// camera id throughout, so callers never see this. +/// quirk worth pinning down: PUT identifies the stream by name and takes its sources +/// as repeated src parameters, but DELETE identifies the stream by src. We map +/// name to the camera id throughout, so callers never see this. /// public sealed class Go2RtcClient : IGo2RtcClient { @@ -62,9 +69,13 @@ public async Task> ListStreamNamesAsync(CancellationToken c return names; } - public async Task PutStreamAsync(string name, string src, CancellationToken cancellationToken) + public async Task PutStreamAsync(string name, IReadOnlyList sources, CancellationToken cancellationToken) { - string url = $"api/streams?name={Uri.EscapeDataString(name)}&src={Uri.EscapeDataString(src)}"; + // `src` is read as a repeated parameter, not a single value, and the whole set replaces + // whatever the stream had. Order is preserved and matters: a source naming this stream has + // to come after the one that supplies it. + string src = string.Join('&', sources.Select(s => $"src={Uri.EscapeDataString(s)}")); + string url = $"api/streams?name={Uri.EscapeDataString(name)}&{src}"; using HttpResponseMessage response = await _http.PutAsync(url, content: null, cancellationToken); response.EnsureSuccessStatusCode(); } diff --git a/Server/Serval.Server/Ingest/Go2RtcSyncWorker.cs b/Server/Serval.Server/Ingest/Go2RtcSyncWorker.cs index 59f7f50..5756207 100644 --- a/Server/Serval.Server/Ingest/Go2RtcSyncWorker.cs +++ b/Server/Serval.Server/Ingest/Go2RtcSyncWorker.cs @@ -25,10 +25,10 @@ public sealed class Go2RtcSyncWorker : PeriodicWorker private readonly IOptionsMonitor _options; private readonly ILogger _logger; - // The source string we last registered per camera id. Lets reconcile notice when a camera's - // source changes (a new live-stream URL, or talk-back toggled, which flips the backchannel - // suffix) and re-register it — go2rtc only tells us stream names exist, not their sources. - private readonly Dictionary _registered = new(StringComparer.Ordinal); + // The sources we last registered per camera id. Lets reconcile notice when a camera's sources + // change (a new live-stream URL, or talk-back toggled, which flips the backchannel suffix) and + // re-register it — go2rtc only tells us stream names exist, not what they draw on. + private readonly Dictionary> _registered = new(StringComparer.Ordinal); /// Whether the last tick found WebRTC off, so the log records the change, not the state. private bool _idle; @@ -65,6 +65,7 @@ protected override async Task TickAsync(CancellationToken stoppingToken) } List cameras = await _cameras.ListAsync(stoppingToken); + await ReconcileAsync(cameras, _go2rtc, _registered, _logger, stoppingToken); } @@ -96,36 +97,37 @@ private bool Idle() /// /// Brings go2rtc's stream set in line with the desired set for one snapshot of the registry: - /// register (or re-register) a stream for every eligible camera whose source go2rtc is missing + /// register (or re-register) a stream for every eligible camera whose sources go2rtc is missing /// or has stale, and delete every stream we no longer want. - /// is the caller-owned memory of what source each stream was last given, so a changed source is + /// is the caller-owned memory of what sources each stream was last given, so a change is /// re-pushed. Static and pure over its inputs, the client, and that map — so tests drive it /// directly with a fake client and a camera list, no worker instance and no Mongo. /// internal static async Task ReconcileAsync( IReadOnlyList cameras, IGo2RtcClient go2rtc, - IDictionary registered, + IDictionary> registered, ILogger logger, CancellationToken cancellationToken) { var desired = cameras .Where(IsWebRtcEligible) - .ToDictionary(c => c.Id, SourceFor, StringComparer.Ordinal); + .ToDictionary(c => c.Id, SourcesFor, StringComparer.Ordinal); IReadOnlySet existing = await go2rtc.ListStreamNamesAsync(cancellationToken); - // Register a stream when go2rtc doesn't have it, or when its source changed since we last - // pushed it (new live-stream URL, or talk-back toggled → the backchannel suffix flipped). - // PUT is a replace, so re-pushing a changed source is all it takes; a restart re-adds it. - foreach ((string id, string src) in desired) + // Register a stream when go2rtc doesn't have it, or when its sources changed since we last + // pushed them (new live-stream URL, or talk-back toggled → the backchannel suffix flipped). + // PUT is a replace, so re-pushing is all it takes; a restart re-adds it. + foreach ((string id, IReadOnlyList sources) in desired) { bool missing = !existing.Contains(id); - bool drifted = !registered.TryGetValue(id, out string? last) || last != src; + bool drifted = !registered.TryGetValue(id, out IReadOnlyList? last) + || !last.SequenceEqual(sources, StringComparer.Ordinal); if (missing || drifted) { - await go2rtc.PutStreamAsync(id, src, cancellationToken); - registered[id] = src; + await go2rtc.PutStreamAsync(id, sources, cancellationToken); + registered[id] = sources; logger.LogInformation("Registered go2rtc stream for camera {CameraId}.", id); } } @@ -145,6 +147,42 @@ internal static async Task ReconcileAsync( } } + /// + /// The sources a camera's go2rtc stream draws on: the camera itself, and a rendering of its + /// audio into every codec a WebRTC consumer may ask for. + /// + /// Why the second one exists. Cameras send AAC, which WebRTC cannot carry. A + /// consumer that negotiates an audio m-line against a passthrough source therefore gets one + /// that is answered and then never filled — and a player waiting on a track that never arrives + /// shows nothing, while the video beside it flows perfectly. Nothing on either side reports a + /// fault. Rendering the audio into codecs WebRTC does carry is what lets go2rtc answer that + /// m-line honestly. + /// + /// Why all three, and not Opus alone. go2rtc will answer an m-line with any codec + /// it believes it can produce, and it believes that of G.711 whether or not a source actually + /// supplies it. So a consumer offering opus,PCMU,PCMA — which the Google Home app on a + /// phone does — could be answered PCMU and then handed silence, reproducing the exact + /// fault above on a surface where Opus alone looked sufficient. It is per-negotiation, which is + /// why it presents as a camera that worked and then stopped. With all three offered, go2rtc + /// picks Opus and can honour the other two if it ever does not. + /// + /// It costs nothing until something asks for it. go2rtc negotiates per consumer + /// across all of a stream's sources and starts each one only when a track it supplies is + /// actually wanted, so a viewer who takes video alone never launches it. The source names this + /// stream rather than the camera, so it draws on the session go2rtc already holds — pointing it + /// at the camera instead opens a second RTSP session, which cameras that cap concurrent + /// sessions refuse, and the picture cuts out on a cycle as the transcode is relaunched. + /// + /// Talk-back is unaffected: the camera's backchannel belongs to the first source, which + /// still holds the RTSP session itself, and go2rtc's rule that only one source may claim a + /// backchannel is satisfied because this one never opens a camera session at all. + /// + /// A camera with no audio at all simply fails this source, leaving the m-line unfilled — + /// exactly what a lone passthrough source does today, so there is nothing to guard against. + /// + internal static IReadOnlyList SourcesFor(Camera camera) => + [SourceFor(camera), $"ffmpeg:{camera.Id}#audio=opus#audio=pcmu#audio=pcma"]; + /// /// The go2rtc source for a camera: its live stream's URL, with #backchannel=0 appended /// unless talk-back is enabled. go2rtc probes the backchannel by default and that probe breaks @@ -168,8 +206,15 @@ internal static string SourceFor(Camera camera) /// Live role, which is assigned explicitly to a stream and resolves to nothing else — /// so this is independent of whether the camera records at all. Test file cameras are skipped: /// go2rtc has nothing to pull. + /// + /// Internal because the Google Home integration must ask the same question. The + /// set of cameras it offers Google has to be exactly the set registered here — a camera Google + /// knows about but go2rtc has no stream for is not a degraded experience, it is a stream + /// request that succeeds and then never connects, with the failure landing on a display in + /// somebody's kitchen. Two filters that merely agree today would drift; there is one, and + /// GoogleHomeSyncTests drives a single camera list through both to prove it. /// - private static bool IsWebRtcEligible(Camera camera) => + internal static bool IsWebRtcEligible(Camera camera) => camera.Enabled && camera.LiveStream is { Url: var url } && !string.IsNullOrWhiteSpace(url) diff --git a/Server/Serval.Server/Media/CastTranscoder.cs b/Server/Serval.Server/Media/CastTranscoder.cs new file mode 100644 index 0000000..ab07ebc --- /dev/null +++ b/Server/Serval.Server/Media/CastTranscoder.cs @@ -0,0 +1,424 @@ +using System.Diagnostics; +using System.Globalization; +using Microsoft.Extensions.Options; +using Serval.Server.Configuration; +using Serval.Server.Recordings; + +namespace Serval.Server.Media; + +/// +/// Turns one recorded segment into something a television can decode, on demand. +/// +/// Why this exists. A Cast device decodes H.264 to Level 4.2 — 1080p — and records +/// here are the camera's main stream copied untouched: 4K HEVC on one, 2560x1440 on another, +/// 1920x2560 on a doorbell. Measured on the real hardware, a Cast device fetches such a playlist +/// and every segment in it, all 200, and renders nothing at all. Casting a recording therefore +/// means re-encoding it, and the only question was where. +/// +/// Per request, not per session. Each request transcodes one batch of recorded +/// segments, so the playlist stays an ordinary VOD playlist with real durations: seeking and +/// scrubbing work without any of this knowing they happened, there is no ffmpeg lifecycle tied to a +/// viewer who may have walked away, and nothing is encoded that nobody watches. +/// +/// A batch rather than a segment, because a run costs more than the work. Launching +/// ffmpeg and initialising VAAPI is most of what a four-second segment took. Batching +/// of them pays that once instead of once each, and +/// everything inside a batch is a single continuous encode rather than several independent ones — +/// fewer joins, and fewer places for a decoder to object. +/// +/// MPEG-TS out, not fMP4. A TS segment carries its own parameter sets, so independent +/// ffmpeg runs need not agree on an initialisation segment — which they cannot be relied on to do. +/// It is also Cast's native HLS format, and fMP4 is the one this deployment has twice watched a +/// television fetch in full and not draw. +/// +public sealed class CastTranscoder +{ + /// + /// The height to encode to when the receiver has not said what its screen will take. 1080p is + /// the floor every Cast device decodes, and the safe answer when nothing better is known. + /// + public const int DefaultHeight = 1080; + + /// + /// The ceiling, whatever a receiver claims. A device reporting something absurd would otherwise + /// have this encoding at that size, which is expensive and pointless in equal measure. + /// + private const int MaxHeight = 2160; + + private const int MinHeight = 360; + + /// + /// How many segments may be transcoded at once. + /// + /// A bound rather than a queue depth: seeking makes a player abandon what it asked for + /// and ask for somewhere else, and without a ceiling a few scrubs would leave a handful of + /// ffmpeg processes competing for one GPU and finishing none of them in time. Two, because this + /// server's first job is recording and the GPU is shared with it. + /// + private static readonly SemaphoreSlim Slots = new(2, 2); + + private readonly IngestOptions _options; + private readonly string _mediaRoot; + private readonly ILogger _logger; + + public CastTranscoder(IOptions options, ILogger logger) + { + _options = options.Value.Ingest; + _mediaRoot = options.Value.Media.Root; + _logger = logger; + } + + /// + /// The init a segment cannot be decoded without. + /// + /// Derived from the name rather than looked up, because the naming is the mechanism: a + /// session stamps every file it writes with its own start time, which is what ties + /// seg-<stamp>-NNNNN.m4s to init-<stamp>.mp4. See + /// FfmpegStreamSession. Null for anything not shaped like a segment, which the caller + /// treats as not found. + /// + internal static string? InitFor(string segmentName) + { + if (!segmentName.StartsWith("seg-", StringComparison.Ordinal)) + { + return null; + } + + int lastDash = segmentName.LastIndexOf('-'); + return lastDash <= 4 ? null : $"init-{segmentName[4..lastDash]}.mp4"; + } + + /// + /// Writes one batch to as MPEG-TS, re-encoded to fit. + /// + /// is how many consecutive recorded segments the batch covers, as the + /// playlist worked out — they share an init and are fed to one ffmpeg in order. + /// is where the batch sits in that playlist, which is what + /// pins its timestamps so that a seek means the same thing to both ends. + /// is how long the playlist said that slot is, which the + /// encode is trimmed to so that it cannot run into the next one. + /// + public async Task WriteSegmentAsync( + string cameraId, + string segmentName, + int count, + double offsetSeconds, + double? durationSeconds, + int? maxHeight, + Stream destination, + CancellationToken cancellationToken) + { + string? init = InitFor(segmentName); + if (init is null) + { + throw new InvalidOperationException($"'{segmentName}' is not a recorded segment name."); + } + + string cameraDir = Path.Combine(_mediaRoot, cameraId); + string initPath = Path.Combine(cameraDir, init); + + IReadOnlyList paths = BatchPaths(cameraDir, segmentName, count); + + if (paths.Count == 0 || !File.Exists(initPath)) + { + throw new FileNotFoundException($"No recorded segment {segmentName} for camera {cameraId}."); + } + + await Slots.WaitAsync(cancellationToken); + + try + { + await RunAsync( + cameraId, initPath, paths, offsetSeconds, durationSeconds, Clamp(maxHeight), + destination, cancellationToken); + } + finally + { + Slots.Release(); + } + } + + /// + /// The files in a batch, in order, stopping at the first one that is not there. + /// + /// Stopping rather than failing: retention runs while somebody is watching, and a batch + /// whose tail has been deleted is still playable up to the gap. The alternative is a segment + /// that 404s in the middle of a recording somebody is part-way through. + /// + internal static IReadOnlyList BatchPaths(string cameraDir, string firstSegment, int count) + { + int lastDash = firstSegment.LastIndexOf('-'); + if (lastDash < 0 || !int.TryParse(firstSegment[(lastDash + 1)..], out int first)) + { + return []; + } + + string prefix = firstSegment[..(lastDash + 1)]; + int width = firstSegment.Length - lastDash - 1; + + var paths = new List(); + + for (int i = 0; i < Math.Max(1, count); i++) + { + string name = prefix + (first + i).ToString(CultureInfo.InvariantCulture).PadLeft(width, '0'); + string path = Path.Combine(cameraDir, $"{name}.m4s"); + + if (!File.Exists(path)) + { + break; + } + + paths.Add(path); + } + + return paths; + } + + /// + /// The height to actually encode to: what the receiver asked for, inside what is sensible. + /// Absent means the receiver could not say, which is the 1080p floor rather than a failure. + /// + internal static int Clamp(int? maxHeight) => + maxHeight is not int h ? DefaultHeight : Math.Clamp(h, MinHeight, MaxHeight); + + /// + /// Bitrate for a height, in the shape a Cast device is happy with. + /// + /// Scaled with the pixels rather than fixed, because the ceiling is now the receiver's to + /// choose: 4M is generous at 1080p and visibly poor at 2160p, and encoding a 4K screen's worth + /// of detail at a 1080p bitrate throws away most of what asking the device bought. + /// + internal static string Bitrate(int height) => height switch + { + >= 2160 => "16M", + >= 1440 => "8M", + >= 1080 => "5M", + >= 720 => "3M", + _ => "1500k", + }; + + private async Task RunAsync( + string cameraId, + string initPath, + IReadOnlyList segmentPaths, + double offsetSeconds, + double? durationSeconds, + int height, + Stream destination, + CancellationToken cancellationToken) + { + var startInfo = new ProcessStartInfo(_options.FfmpegPath) + { + RedirectStandardInput = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + foreach (string arg in Arguments(offsetSeconds, durationSeconds, height)) + { + startInfo.ArgumentList.Add(arg); + } + + using Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Could not start ffmpeg to transcode a segment."); + + Task feed = FeedAsync(process, initPath, segmentPaths, cancellationToken); + Task drain = DrainStderrAsync(process, cameraId, cancellationToken); + + try + { + long written = await CopyCountingAsync( + process.StandardOutput.BaseStream, destination, cancellationToken); + await feed; + await process.WaitForExitAsync(cancellationToken); + + // The response has already been sent with a 200 in front of it — it is streamed, so + // there is no moment at which a failed run could still be answered with an error. What + // the player receives instead is a short or empty segment, which it reports as a decode + // failure and stops on. That failure is only diagnosable from this side, and only if + // somebody wrote it down. + if (process.ExitCode != 0 || written == 0) + { + _logger.LogWarning( + "ffmpeg produced {Bytes} bytes and exited {ExitCode} transcoding {Segment} " + + "(+{Count}) for camera {CameraId}; the player has already been sent them.", + written, process.ExitCode, Path.GetFileName(segmentPaths[0]), + segmentPaths.Count - 1, cameraId); + } + } + finally + { + if (!process.HasExited) + { + try { process.Kill(entireProcessTree: true); } catch { /* already gone */ } + } + + // The viewer seeking away closes the response mid-copy, which leaves the feed writing + // into a pipe whose process has just been killed. Somebody has to observe that. + try { await feed; } catch { /* the request was abandoned */ } + try { await drain; } catch { /* not interesting */ } + } + } + + /// + /// The whole command line, as one list so the hardware and software paths are read side by + /// side rather than assembled from fragments. + /// + /// With a render node, decode, scale and encode all stay on the GPU — the frames never + /// come back to the CPU, which is what makes 4K HEVC affordable on an N100 that is also running + /// detection. Without one it is the same shape in software, which will not keep up with 4K but + /// is the honest fallback for a deployment that has no GPU to give. + /// + internal IReadOnlyList Arguments(double offsetSeconds, double? durationSeconds, int height) + { + string offset = offsetSeconds.ToString("0.###", CultureInfo.InvariantCulture); + string device = _options.HwAccelDevice ?? ""; + + // Width is the 16:9 partner of the height, and only a bound: force_original_aspect_ratio + // fits the source inside the box rather than filling it, which is what keeps a portrait + // doorbell portrait — 1920x2560 becomes 810x1080 instead of a squeezed landscape. + int width = height * 16 / 9; + string scale = $"w={width}:h={height}:force_original_aspect_ratio=decrease"; + + List args = ["-nostdin", "-hide_banner", "-loglevel", "warning"]; + + if (!string.IsNullOrWhiteSpace(device)) + { + args.AddRange([ + "-hwaccel", "vaapi", + "-hwaccel_device", device.Trim(), + "-hwaccel_output_format", "vaapi", + "-i", "pipe:0", + "-vf", $"scale_vaapi={scale}", + "-c:v", "h264_vaapi", + ]); + } + else + { + args.AddRange([ + "-i", "pipe:0", + + // force_divisible_by, because an odd dimension is not encodable as yuv420p and a + // portrait source scaled to fit will land on one. + "-vf", $"scale={scale}:force_divisible_by=2", + "-c:v", "libx264", + "-preset", "veryfast", + ]); + } + + args.AddRange([ + "-b:v", Bitrate(height), + + // Audio is normalised, not passed through at the camera's own shape. These cameras + // record AAC at 16 kHz mono, and asking the encoder for a television's bitrate at that + // rate overruns what a frame can hold — ffmpeg says so on every segment and clamps. + // Resampling to 48 kHz stereo first is what makes the bitrate valid, and 16 kHz mono is + // an odd thing to hand a Cast device besides: audio is what has broken every playback + // path in this integration so far, and none of them said so. + "-af", "aresample=async=1", + "-ar", "48000", + "-ac", "2", + "-c:a", "aac", + "-b:a", "128k", + + // Where this batch sits in its playlist. ffmpeg normalises timestamps per invocation, + // so without it every batch claims to begin at the same instant; with it, playlist time + // and media time are the same and a seek lands where it was aimed. + // + // The alternative — keeping the recording's own timestamps — reads better and does not + // work: the recorder restarts every few minutes, and each restart begins afresh, so any + // window of length spans several and its timeline jumps backwards partway through. + "-output_ts_offset", offset, + + // The muxer's own head start, removed. Left at its default the stream begins 1.4 + // seconds after the offset it was given, which is 1.4 seconds this batch is not where + // the playlist says it is. + "-muxdelay", "0", + "-muxpreload", "0", + ]); + + // Trimmed to the slot the playlist declared for it. + // + // **This is what a Cast device was stopping on.** Each batch is an independent encode + // positioned absolutely, and its natural length is a frame or two more than the wall-clock + // spacing the playlist measured — so the next batch's first packet carried a timestamp + // *earlier* than this one's last, at every single join. Measured on the real recordings: 60 + // ms of video and 120 ms of audio, enough for the decoder to call the stream corrupt and + // give up around thirty seconds in. Trimming turns that overlap into a gap of a few + // milliseconds, which players simply skip. + if (durationSeconds is double seconds and > 0) + { + args.AddRange(["-t", seconds.ToString("0.###", CultureInfo.InvariantCulture)]); + } + + args.AddRange(["-f", "mpegts", "pipe:1"]); + + return args; + } + + /// Writes the init and then the batch into ffmpeg's stdin — the bytes a decoder needs, + /// in the order it needs them. One init, because a batch never spans a session. + private static async Task FeedAsync( + Process process, + string initPath, + IReadOnlyList segmentPaths, + CancellationToken cancellationToken) + { + try + { + await CopyAsync(initPath, process.StandardInput.BaseStream, cancellationToken); + + foreach (string path in segmentPaths) + { + await CopyAsync(path, process.StandardInput.BaseStream, cancellationToken); + } + } + finally + { + // ffmpeg will not finish its output until it has seen the end of its input. + try { process.StandardInput.BaseStream.Close(); } catch { /* already gone */ } + } + } + + /// + /// , but says how much it copied. + /// The size of what ffmpeg produced is the difference between a run that worked and one that + /// fell over part-way, and nothing else on this path can tell them apart. + /// + private static async Task CopyCountingAsync( + Stream source, Stream destination, CancellationToken cancellationToken) + { + byte[] buffer = new byte[64 * 1024]; + long total = 0; + + while (true) + { + int read = await source.ReadAsync(buffer, cancellationToken); + if (read == 0) + { + return total; + } + + await destination.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + total += read; + } + } + + private static async Task CopyAsync(string path, Stream destination, CancellationToken cancellationToken) + { + await using FileStream source = File.OpenRead(path); + await source.CopyToAsync(destination, cancellationToken); + } + + private async Task DrainStderrAsync(Process process, string cameraId, CancellationToken cancellationToken) + { + string errors = await process.StandardError.ReadToEndAsync(cancellationToken); + if (!string.IsNullOrWhiteSpace(errors)) + { + _logger.LogWarning( + "ffmpeg transcoding a cast segment for camera {CameraId}: {Errors}", + cameraId, errors.Trim()); + } + } +} diff --git a/Server/Serval.Server/Media/MediaEndpoints.cs b/Server/Serval.Server/Media/MediaEndpoints.cs index fbaa65c..9c76b0f 100644 --- a/Server/Serval.Server/Media/MediaEndpoints.cs +++ b/Server/Serval.Server/Media/MediaEndpoints.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Options; using Serval.Server.Cameras; using Serval.Server.Configuration; +using Serval.Server.Ingest; using Serval.Server.Recordings; using Serval.Server.Snapshots; @@ -206,10 +207,59 @@ IResult Segment(string id, string file, string extension, IOptions - /// Only the two shapes ffmpeg writes — init-<stamp> and - /// seg-<stamp>-<n> — and no path separators to escape the directory. + /// Only the shapes ffmpeg writes — init-<stamp>, seg-<stamp>-<n>, + /// and the preview ring's preview-init-<stamp> and + /// preview-<stamp>-<n> — and no path separators to escape the directory. + /// + /// The ring's prefix belongs here because the Google Home playback route serves that + /// playlist: without it the playlist is fetched happily and every segment in it 404s, which + /// reads on a television as a stream that connected and then showed nothing. /// - private static bool IsSafeSegmentName(string file) => - (file.StartsWith("init-", StringComparison.Ordinal) || file.StartsWith("seg-", StringComparison.Ordinal)) + internal static bool IsSafeSegmentName(string file) => + (file.StartsWith("init-", StringComparison.Ordinal) + || file.StartsWith("seg-", StringComparison.Ordinal) + || file.StartsWith(PreviewRing.FilePrefix, StringComparison.Ordinal)) && file.All(c => char.IsAsciiLetterOrDigit(c) || c is '-'); + + /// + /// The live HLS a Cast device is given: the preview ring, not the recording. + /// + /// Why not the recording. The recording is the camera's main stream copied + /// untouched — 4K HEVC on one of these cameras, 7680x2160 on another, 1920x2560 on the + /// doorbell. A Cast receiver decodes H.264 to Level 4.2, which is 1080p; every one of those is + /// beyond it. The receiver fetches the playlist, fetches the segments, decodes nothing, and + /// sits on the title card with no error to report — which is exactly what it did. + /// + /// The preview ring is the detect stream copied instead: 640x360 H.264 with AAC, + /// already written, already a rolling window with its own init, and already sized for exactly + /// this. It costs nothing extra because it is being written whether anybody casts or not — see + /// for why it exists. + /// + /// Null when there is no ring. A camera whose detect stream is its recorded + /// stream writes none, because the recording already is those bytes — the caller falls back to + /// the recording index there, and on such a camera the recorded stream is the small one + /// anyway. + /// + internal static string? PreviewPlaylist(string mediaRoot, string cameraId, string? streamToken) + { + string path = Path.Combine(mediaRoot, cameraId, PreviewRing.PlaylistName); + if (!File.Exists(path)) + { + return null; + } + + try + { + // ffmpeg rewrites this file in place, so a read can catch it mid-write. A torn playlist + // is not worth serving; the player asks again within a segment's time either way. + string playlist = File.ReadAllText(path); + return playlist.Contains("#EXTM3U", StringComparison.Ordinal) + ? HlsPlaylist.WithStreamToken(playlist, streamToken) + : null; + } + catch (IOException) + { + return null; + } + } } diff --git a/Server/Serval.Server/Program.cs b/Server/Serval.Server/Program.cs index c06aa8a..f564303 100644 --- a/Server/Serval.Server/Program.cs +++ b/Server/Serval.Server/Program.cs @@ -14,10 +14,12 @@ using Serval.Server.Auth; using Serval.Server.Backup; using Serval.Server.Cameras; +using Serval.Server.Cast; using Serval.Server.Clips; using Serval.Server.Configuration; using Serval.Server.Dashboard; using Serval.Server.Events; +using Serval.Server.GoogleHome; using Serval.Server.Ingest; using Serval.Server.Live; using Serval.Server.Media; @@ -150,6 +152,32 @@ + "Output:ApiKey. Nothing else is affected — a Server with no modules needs no key."); } +// Named once at boot, and specifically. Six conditions have to hold for the Google Home routes to +// answer anything but 503, and an operator who has just filled in five of them needs to be told +// which one is left — not that "Google Home is off", which is true of every failure and points at +// none of them. Silent while the feature is simply switched off, which is the ordinary state. +if (serverOptions.GoogleHome.Enabled) +{ + GoogleHomeStatus googleHome = GoogleHomeGate.Evaluate(serverOptions); + if (!googleHome.Effective) + { + startupWarnings.Add( + $"Google Home is switched on but not serving: {googleHome.Reason}"); + } + else if (!googleHome.HomeGraphKeyConfigured) + { + // Not a failure. The integration works without it; what is lost is worth naming because + // the symptom appears days later, as a camera that Google never hears about. + startupWarnings.Add( + "Google Home is live without a HomeGraph key (Serval:GoogleHome:HomeGraphKeyPath is " + + "unset), so Google will not learn about camera changes on its own. Re-link in the " + + "Google Home app, or say \"Hey Google, sync my devices\", after adding or renaming " + + "a camera. Camera online/offline is also not reported, so SYNC declares " + + "willReportState: false and Google's Test Suite will skip these devices. Streaming " + + "is unaffected."); + } +} + IngestOptions ingest = serverOptions.Ingest; // What this host's ffmpeg can actually encode, read once. Fatal if it cannot be read at all: a @@ -209,6 +237,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); // Saved clips. Registered unconditionally — the summary worker resolves the vision model through // the provider and does nothing when there is none, so a server with no AI still keeps clips. @@ -236,9 +265,11 @@ // nothing, and Serval:Push:Enabled is checked at enqueue rather than here so turning notifications // on and off does not need a restart. // -// The HTTP client is the only one here pointed at the public internet — Google, Mozilla and Apple's -// push services, whose addresses come from the browsers themselves, so it has no BaseAddress. What -// crosses that boundary is ciphertext the relay cannot read; see WebPushCrypto. +// The HTTP client is pointed at the public internet — Google, Mozilla and Apple's push services, +// whose addresses come from the browsers themselves, so it has no BaseAddress. What crosses that +// boundary is ciphertext the relay cannot read; see WebPushCrypto. The only other client here that +// leaves the LAN is HomeGraphClient, registered below and used only when a Google Home integration +// has been configured. builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddHttpClient(http => http.Timeout = TimeSpan.FromSeconds(15)); @@ -246,6 +277,25 @@ builder.Services.AddSingleton(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); +// Registered unconditionally, like the notifier above and for the same reason: the gate reads +// Serval:GoogleHome:* through IOptionsMonitor at request time, so switching the integration on and +// off is a configuration change rather than a restart. +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +// The second HTTP client in this process pointed at the public internet, after Web Push. What +// crosses is an agent user id and nothing else — no camera name, no image, no telemetry. Absent a +// HomeGraph key the client is registered and never used, which is the ordinary deployment. +builder.Services.AddSingleton(); +builder.Services.AddHttpClient(http => http.Timeout = TimeSpan.FromSeconds(15)); +builder.Services.AddHostedService(); +builder.Services.AddHostedService(); + // Where each camera's rolling detect-stream buffer is. In memory rather than in Mongo — see the // type — so it is a singleton shared by the ingest session that fills it and the worker that cuts // clips out of it. @@ -504,6 +554,9 @@ void ConfigureJwtBearer(JwtBearerOptions options, bool requireStreamScope) // explicitly below and is visible at the one place it takes effect. const string AppCorsPolicy = "app"; +/// The Google Home camera-stream signalling route, which a gstatic.com page calls directly. +const string GoogleSignalingCorsPolicy = "google-signaling"; + builder.Services.AddCors(options => options.AddPolicy(AppCorsPolicy, policy => { string[] origins = [.. serverOptions.Cors.AllowedOrigins]; @@ -533,6 +586,34 @@ void ConfigureJwtBearer(JwtBearerOptions options, bool requireStreamScope) "X-Serval-Clip-Truncated"); })); +// A second, narrow policy for the one route a *Google-hosted page* calls in a browser. +// +// Google's CameraStream signalling is not the server-to-server exchange it looks like: the player +// runs in a web view served from gstatic.com and fetches the signalling URL itself, so the browser +// applies CORS to the answer. Google's documentation asks for that exact origin, and the exactness +// is load-bearing — a credentialed fetch refuses a wildcard and requires the literal origin echoed +// back with Allow-Credentials. The app policy above deliberately never allows credentials, so +// leaving this route on it means the browser silently discards a perfectly good SDP answer: the +// request arrives, the server answers 200, setRemoteDescription is never called, no ICE is ever +// started, and every side of the system reports success while the picture never appears. +// +// Its own policy rather than a widened app policy, so that tightening Serval:Cors:AllowedOrigins — +// which every publicly reachable deployment should do — cannot break Google Home. +builder.Services.AddCors(options => options.AddPolicy(GoogleSignalingCorsPolicy, policy => policy + + // gstatic.com is where Google serves the player that does the signaling, and it is the one + // origin this route exists for. It cannot be a wildcard: the fetch is credentialed, and a + // credentialed fetch refuses "*" — which fails as an answer the browser silently discards, + // with a 200 in every log on this side. + // + // The App's own configured origins are allowed alongside it so that this route is reachable + // from the Serval UI too, on the same terms as every other API route. Without them a browser + // on the operator's own origin is refused by a policy meant to constrain Google. + .WithOrigins([.. serverOptions.Cors.AllowedOrigins, "https://www.gstatic.com"]) + .WithMethods("GET", "POST", "OPTIONS") + .WithHeaders("content-type", "authorization") + .AllowCredentials())); + builder.Services.AddOpenApi(options => { // The document is titled after the product rather than the assembly, since it is what the @@ -653,12 +734,14 @@ await CameraRegistrySweep.RunAsync( app.MapLiveEventsEndpoint(); app.MapAudioLevelsEndpoint(); app.MapWebRtcEndpoints(); +app.MapCastEndpoints(); app.MapPtzEndpoints(); app.MapOnvifEndpoints(); app.MapSystemEndpoints(); app.MapSettingsEndpoints(); app.MapPreferencesEndpoints(); app.MapPushEndpoints(); +app.MapGoogleHomeEndpoints(); app.MapConfigBackupEndpoints(); // The App's web build, when wwwroot holds one (see Dockerfile) — every API route lives under diff --git a/Server/Serval.Server/README.md b/Server/Serval.Server/README.md index 6e17736..005353f 100644 --- a/Server/Serval.Server/README.md +++ b/Server/Serval.Server/README.md @@ -116,6 +116,13 @@ Prefs GET /api/preferences the signed-in account's own s omitted property is left alone. No id in the route: it is always your own. Telemetry POST /api/cameras/{id}/telemetry module → server (X-Api-Key) +Google GET /api/google/status is the integration live, and if not, why (Admin) + GET /api/google/links the linked Google account, if any (Admin) + DELETE /api/google/links/{agentUserId} unlink, revoking every credential (Admin) + GET /api/google/oauth/authorize account linking starts here (public) + POST /api/google/oauth/token code and refresh grants (public) + POST /api/google/fulfillment SYNC · QUERY · EXECUTE · DISCONNECT (public) + POST /api/google/camerastream/signal WebRTC signaling for one camera (public) AI serve GET /api/cameras/{id}/utterances?from&to&limit GET /api/cameras/{id}/scenes?from&to&limit GET /api/cameras/{id}/detections?from&to&limit @@ -128,6 +135,11 @@ AI serve GET /api/cameras/{id}/utterances?from&to&limit **Dashboard frames** are binary — `[uint32 cameraId length][cameraId UTF-8][JPEG]` — to skip the ~33% base64 tax of images-in-JSON. **Live events** are JSON text — `{ camera_id, type, document }`. +**The four public `/api/google/*` routes are the only ones on this server meant to be reachable +from the internet**, and each authenticates itself rather than relying on a session — Google's +servers have none. They all answer 503 until the integration is configured, which is the default. +See [Docs/google-home.md](../../Docs/google-home.md). + `GET /scalar/v1` serves a [Scalar](https://scalar.com) API reference over the generated OpenAPI document, with a "Test Request" button that calls the live server. Unlike the usual ASP.NET template it is **not** gated on `Development` — camera CRUD needs a GUI and the deployed container diff --git a/Server/Serval.Server/Recordings/HlsPlaylist.cs b/Server/Serval.Server/Recordings/HlsPlaylist.cs index e8129d6..85f73a2 100644 --- a/Server/Serval.Server/Recordings/HlsPlaylist.cs +++ b/Server/Serval.Server/Recordings/HlsPlaylist.cs @@ -35,6 +35,161 @@ public static class HlsPlaylist /// Null when the caller authenticated with a header (curl, desktop debugging), which it can /// equally set on the segment requests. /// + /// + /// The same window, as MPEG-TS segments a Cast device can actually decode. + /// + /// Why a second builder rather than a parameter. Three things differ and each one + /// is load-bearing. There is no EXT-X-MAP, because a TS segment carries its own + /// parameter sets — which is the whole reason for TS here: every segment is transcoded by a + /// separate ffmpeg, and independent runs cannot be relied on to emit byte-identical + /// initialisation. The version drops to 3, since EXT-X-MAP is what required 7. And each + /// URI carries the segment's offset into the window. + /// + /// Every batch is pinned to where the playlist says it is, by the o= it + /// carries and the -output_ts_offset the transcoder passes on. So playlist time and media + /// time are the same thing, which is what makes a seek mean anything: a player told to go to + /// twelve minutes lands twelve minutes in. + /// + /// Keeping the recording's own timestamps instead does not work here, and it is + /// worth saying why since it looks tidier. The recorder restarts every few minutes — seven + /// sessions in an hour on this deployment — and each restart begins its timestamps afresh. A + /// window of any length therefore spans several, and its media timeline jumps backwards partway + /// through: seeks land nowhere, and playback ends early. Normalising removes that, and a + /// discontinuity tag becomes unnecessary along with it, because every batch is re-encoded to + /// identical parameters and joins the previous one seamlessly. + /// + /// Segment names are relative — cast/<name>.ts — so they resolve against this + /// playlist's own URL, and the token rides on each because RFC 3986 drops the playlist's query + /// when it does. + /// + public static string BuildCastVod( + IReadOnlyList segments, + DateTimeOffset? from = null, + string? streamToken = null, + int? maxHeight = null) + { + var sb = new StringBuilder(); + + string token = string.IsNullOrEmpty(streamToken) + ? string.Empty + : $"&stream_token={Uri.EscapeDataString(streamToken)}"; + + // Carried per segment rather than held server-side, because it belongs to the screen and + // not to the window: the receiver asks its own device what it will decode and puts the + // answer on the playlist URL, so the same recording cast to a 4K television and to a 1080p + // one is encoded differently, with nothing to remember between them. + string height = maxHeight is int h ? $"&h={h.ToString(CultureInfo.InvariantCulture)}" : string.Empty; + + int target = segments.Count == 0 + ? 1 + : Math.Max(1, (int)Math.Ceiling(segments.Max(s => s.DurationSeconds))); + + sb.Append("#EXTM3U\n"); + sb.Append("#EXT-X-VERSION:3\n"); + sb.Append(CultureInfo.InvariantCulture, $"#EXT-X-TARGETDURATION:{target}\n"); + sb.Append("#EXT-X-PLAYLIST-TYPE:VOD\n"); + sb.Append("#EXT-X-MEDIA-SEQUENCE:0\n"); + + if (from is { } start && segments.Count > 0) + { + double offset = (start - segments[0].StartedAt).TotalSeconds; + if (offset > 0.05) + { + sb.Append(CultureInfo.InvariantCulture, + $"#EXT-X-START:TIME-OFFSET={offset.ToString("0.###", CultureInfo.InvariantCulture)},PRECISE=YES\n"); + } + } + + List> batches = [.. Batches(segments)]; + double elapsed = 0; + + for (int i = 0; i < batches.Count; i++) + { + IReadOnlyList batch = batches[i]; + + // How long this batch occupies the timeline: the distance to where the next one starts, + // which is what the recorder's own clock says. Its segments' declared durations run + // fractionally short of that, and using them accumulated a drift of seconds over a long + // window. The last batch has no successor to measure against and falls back to them. + // + // Sent to the transcoder as well as declared here, because the two have to agree + // exactly. A batch encoded to its own natural length overruns this slot by a frame or + // two, which puts the next batch's first packet *before* the last packet of this one — + // a timestamp running backwards mid-stream, which a Cast device reports as a decode + // failure and stops on. Measured at every join: 60 ms of video and 120 ms of audio. + double duration = i + 1 < batches.Count + ? (batches[i + 1][0].StartedAt - batch[0].StartedAt).TotalSeconds + : batch.Sum(s => s.DurationSeconds); + + sb.Append(CultureInfo.InvariantCulture, + $"#EXTINF:{duration.ToString("0.######", CultureInfo.InvariantCulture)},\n"); + sb.Append(CultureInfo.InvariantCulture, + $"cast/{CastSegmentName(batch[0].FileName)}.ts" + + $"?n={batch.Count}&o={elapsed.ToString("0.###", CultureInfo.InvariantCulture)}" + + $"&d={duration.ToString("0.###", CultureInfo.InvariantCulture)}{height}{token}\n"); + + elapsed += duration; + } + + sb.Append("#EXT-X-ENDLIST\n"); + return sb.ToString(); + } + + /// + /// How many recorded segments one transcoded segment covers. + /// + /// Each is a separate ffmpeg run, and a run costs a process launch and a VAAPI + /// initialisation whatever it then does — most of the time a four-second segment took. Batching + /// pays that once per batch instead of once per segment, and everything inside a batch is one + /// continuous encode rather than several independent ones, which is fewer joins for a decoder + /// to object to. + /// + /// Four, not more: a batch is also the seek granularity and the smallest thing that can + /// be transcoded ahead, so a large one makes scrubbing coarse and wastes work whenever somebody + /// moves. + /// + public const int CastBatchSegments = 4; + + /// + /// Consecutive runs of at most segments sharing one init. + /// + /// A batch never spans a session restart: the segments in one are concatenated and fed to + /// a single decoder, and across a restart they are not decodable together at all. + /// + internal static IEnumerable> Batches( + IReadOnlyList segments) + { + var batch = new List(); + + foreach (RecordingSegment segment in segments) + { + bool sameRun = batch.Count > 0 + && string.Equals(batch[0].InitFileName, segment.InitFileName, StringComparison.Ordinal); + + if (batch.Count > 0 && (!sameRun || batch.Count == CastBatchSegments)) + { + yield return batch; + batch = []; + } + + batch.Add(segment); + } + + if (batch.Count > 0) + { + yield return batch; + } + } + + /// + /// A recorded segment's name without its extension, which is what the transcoding route takes. + /// The extension changes — .m4s in, .ts out — so it cannot be part of the name. + /// + public static string CastSegmentName(string fileName) => + fileName.EndsWith(".m4s", StringComparison.Ordinal) + ? fileName[..^4] + : fileName; + public static string BuildVod( IReadOnlyList segments, DateTimeOffset? from = null, @@ -96,6 +251,148 @@ public static string BuildVod( return sb.ToString(); } + /// + /// A live playlist over the newest few segments: the same files serves, + /// presented as a stream that has not finished. + /// + /// Why ffmpeg's own live.m3u8 is not simply served instead. Two reasons, + /// both deliberate elsewhere. It is written with hls_list_size 0 so that nothing is ever + /// deleted from it — the retention worker prunes by age instead — which means it names every + /// segment the session ever wrote, and a player handed it would open hours behind. And its + /// segment names carry no credential, so on any authenticated route the playlist would load and + /// every segment would then 401. See 's note on relative resolution. + /// + /// All segments must share one initialisation. The caller passes segments from a + /// single ffmpeg session, because a discontinuity at the live edge is exactly where players + /// give up rather than recover, and because EXT-X-MEDIA-SEQUENCE must never go backwards + /// across a refresh — which a session restart, whose numbering starts again at zero, would + /// otherwise make it do. + /// + /// Consecutive segments of one session, oldest first. + /// The sequence number of [0]. Taken + /// from ffmpeg's own filename counter, so it advances by one per segment and survives the + /// window sliding forward. + /// Appended to every segment and init URI, for the reason + /// gives. + public static string BuildLive( + IReadOnlyList segments, + int mediaSequence, + string? streamToken = null) + { + var sb = new StringBuilder(); + + string suffix = string.IsNullOrEmpty(streamToken) + ? string.Empty + : $"?stream_token={Uri.EscapeDataString(streamToken)}"; + + int target = segments.Count == 0 + ? 1 + : Math.Max(1, (int)Math.Ceiling(segments.Max(s => s.DurationSeconds))); + + sb.Append("#EXTM3U\n"); + sb.Append("#EXT-X-VERSION:7\n"); + sb.Append(CultureInfo.InvariantCulture, $"#EXT-X-TARGETDURATION:{target}\n"); + + // No EXT-X-PLAYLIST-TYPE. VOD would promise the list never changes and EVENT would promise + // it only ever grows; a sliding window is neither, and a player told either one stops + // refreshing. + sb.Append(CultureInfo.InvariantCulture, $"#EXT-X-MEDIA-SEQUENCE:{mediaSequence}\n"); + + if (segments.Count > 0) + { + sb.Append(CultureInfo.InvariantCulture, + $"#EXT-X-MAP:URI=\"{segments[0].InitFileName}{suffix}\"\n"); + } + + foreach (RecordingSegment segment in segments) + { + sb.Append(CultureInfo.InvariantCulture, + $"#EXTINF:{segment.DurationSeconds.ToString("0.######", CultureInfo.InvariantCulture)},\n"); + sb.Append(segment.FileName).Append(suffix).Append('\n'); + } + + // No EXT-X-ENDLIST: its absence is the whole difference. It is what tells the player to + // come back for this playlist again rather than stopping at the last segment in it. + return sb.ToString(); + } + + /// + /// The sequence number ffmpeg gave a segment, read out of its filename + /// (seg-{stamp}-{NNNNN}.m4s, written by -hls_segment_filename). + /// + /// It is taken from the name rather than counted here because it has to mean the same + /// thing on the next request, when the window has moved on and this playlist starts at a + /// different segment. ffmpeg's counter is the only numbering that both sides already agree on. + /// Returns 0 for anything that does not parse, which costs a player one reload rather than a + /// failure. + /// + public static int SequenceOf(string fileName) + { + int lastDash = fileName.LastIndexOf('-'); + int dot = fileName.LastIndexOf('.'); + + return lastDash >= 0 && dot > lastDash + && int.TryParse( + fileName.AsSpan(lastDash + 1, dot - lastDash - 1), + NumberStyles.None, + CultureInfo.InvariantCulture, + out int sequence) + ? sequence + : 0; + } + + /// + /// The same playlist with appended to every URI in it. + /// + /// For serving a playlist ffmpeg wrote — the preview ring's — rather than one built here. + /// A player resolves the relative names in it against the playlist's own URL, and RFC 3986 + /// drops the query when it does, so without this the playlist loads and every segment is then + /// refused. Exactly the trap describes; this is it applied to text + /// somebody else produced. + /// + /// A URI is any line that is not blank and does not begin with #, plus the one + /// inside EXT-X-MAP. Nothing else in a playlist names a file. + /// + public static string WithStreamToken(string playlist, string? streamToken) + { + if (string.IsNullOrEmpty(streamToken)) + { + return playlist; + } + + string suffix = $"?stream_token={Uri.EscapeDataString(streamToken)}"; + var sb = new StringBuilder(); + + foreach (string line in playlist.Split('\n')) + { + string trimmed = line.TrimEnd('\r'); + + if (trimmed.StartsWith("#EXT-X-MAP:", StringComparison.Ordinal)) + { + // URI="init-….mp4" — the only tag that names a file. + int open = trimmed.IndexOf('"'); + int close = open < 0 ? -1 : trimmed.IndexOf('"', open + 1); + + sb.Append(close < 0 + ? trimmed + : string.Concat( + trimmed.AsSpan(0, close), suffix, trimmed.AsSpan(close))); + } + else if (trimmed.Length > 0 && !trimmed.StartsWith('#')) + { + sb.Append(trimmed).Append(suffix); + } + else + { + sb.Append(trimmed); + } + + sb.Append('\n'); + } + + return sb.ToString(); + } + /// /// Reads the segments and their true durations out of a live playlist ffmpeg wrote, in order. /// diff --git a/Server/Serval.Server/Recordings/LiveWindow.cs b/Server/Serval.Server/Recordings/LiveWindow.cs new file mode 100644 index 0000000..0e455e2 --- /dev/null +++ b/Server/Serval.Server/Recordings/LiveWindow.cs @@ -0,0 +1,51 @@ +namespace Serval.Server.Recordings; + +/// +/// Which recorded segments make up "now" for a live HLS playlist. +/// +/// Used by the Google Home playback route, which is what a Cast receiver fetches when the +/// destination cannot do WebRTC. Separate from the recording index's own idea of a window because +/// this one answers "what can a player start from right now", not "what was recorded" — and a +/// player handed segments it has no init for shows nothing at all. +/// +public static class LiveWindow +{ + /// + /// How far back to look. Long enough to hold the three or four segments a player wants before + /// it starts, short enough that "live" means it: at the default four-second segment length this + /// is the most recent half-minute. + /// + public static readonly TimeSpan Span = TimeSpan.FromSeconds(32); + + /// + /// The most segments a playlist will name. A player buffers a few and then follows; naming more + /// only invites it to start further behind. + /// + public const int MaxSegments = 6; + + /// + /// The tail of that shares the newest initialisation segment, at + /// most of them. + /// + /// An fMP4 segment cannot be decoded without the init it was written with, and each + /// ffmpeg session writes a fresh one. A VOD playlist spans that boundary with a discontinuity; + /// a live one must not, because the player would have to reset its decoder at the live edge and + /// because the sequence numbering starts again on the far side of it. + /// + public static List NewestSession(List segments) + { + if (segments.Count == 0) + { + return segments; + } + + string init = segments[^1].InitFileName; + + return + [ + .. segments + .Where(s => string.Equals(s.InitFileName, init, StringComparison.Ordinal)) + .TakeLast(MaxSegments), + ]; + } +} diff --git a/Server/Serval.Server/Serval.Server.csproj b/Server/Serval.Server/Serval.Server.csproj index 04a4b16..1989ace 100644 --- a/Server/Serval.Server/Serval.Server.csproj +++ b/Server/Serval.Server/Serval.Server.csproj @@ -44,4 +44,15 @@ + + + + PreserveNewest + + + diff --git a/Server/Serval.Server/Storage/MongoContext.cs b/Server/Serval.Server/Storage/MongoContext.cs index 750fe2b..9fe6df4 100644 --- a/Server/Serval.Server/Storage/MongoContext.cs +++ b/Server/Serval.Server/Storage/MongoContext.cs @@ -6,6 +6,7 @@ using Serval.Server.Cameras; using Serval.Server.Clips; using Serval.Server.Configuration; +using Serval.Server.GoogleHome; using Serval.Server.Preferences; using Serval.Server.Push; using Serval.Server.Recordings; @@ -100,6 +101,33 @@ public MongoContext(IOptions options) public IMongoCollection PushKeys => _database.GetCollection("push_keys"); + /// + /// Authorization codes issued to Google during account linking. Short-lived and single-use; + /// see for why consuming one is an update + /// rather than a read. + /// + public IMongoCollection GoogleAuthorizationCodes => + _database.GetCollection("google_auth_codes"); + + /// Access and refresh tokens issued to Google, stored as hashes only. + public IMongoCollection GoogleTokens => + _database.GetCollection("google_tokens"); + + /// + /// The one Google account this deployment is linked to, keyed by the agent user id — so it + /// needs no index of its own, and there is at most one document. + /// + public IMongoCollection GoogleLinks => + _database.GetCollection("google_links"); + + /// + /// Per-camera on/off as set from the Google Home app. Google-facing only: it decides whether a + /// camera is offered a stream there, and never touches ingest — see + /// . Only cameras switched off have a row. + /// + public IMongoCollection GoogleCameraSwitches => + _database.GetCollection("google_camera_switches"); + /// /// Create indexes. Idempotent — Mongo ignores a CreateOne for an index that already /// exists, so this runs safely on every boot. @@ -228,5 +256,35 @@ await PushSubscriptions.Indexes.CreateOneAsync( new CreateIndexModel( Builders.IndexKeys.Ascending(s => s.UserId)), cancellationToken: cancellationToken); + + // Google's credentials are looked up by hash on every call — the code once at exchange, + // the access token on every fulfillment request. + await GoogleAuthorizationCodes.Indexes.CreateOneAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.CodeHash), + new CreateIndexOptions { Unique = true }), + cancellationToken: cancellationToken); + + await GoogleAuthorizationCodes.Indexes.CreateOneAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.ExpiresAt), + new CreateIndexOptions { ExpireAfter = TimeSpan.Zero }), + cancellationToken: cancellationToken); + + await GoogleTokens.Indexes.CreateOneAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(t => t.TokenHash), + new CreateIndexOptions { Unique = true }), + cancellationToken: cancellationToken); + + // Expiring access tokens clean themselves up. Refresh tokens carry no ExpiresAt at all and + // are therefore skipped by this index rather than being given a far-future date to dodge + // it — Mongo ignores a document whose indexed field is not a date. See GoogleToken for why + // they must not expire. + await GoogleTokens.Indexes.CreateOneAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(t => t.ExpiresAt), + new CreateIndexOptions { ExpireAfter = TimeSpan.Zero }), + cancellationToken: cancellationToken); } } diff --git a/deploy/.env.example b/deploy/.env.example index a60f298..f1d4a54 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -16,6 +16,10 @@ SERVAL_ADMIN_PASSWORD= # than one (e.g. LAN + a public address for outside access): #SERVAL_WEBRTC_CANDIDATES=192.168.1.20:8666 #SERVAL_WEBRTC_CANDIDATES=192.168.1.20:8666,203.0.113.4:8666 +# On a dual-stack network, offer both families. Google's guidance for the displays and televisions +# it streams to is to advertise IPv4 and IPv6 candidates, since which one connects is the far end's +# choice and a single family is a connection that silently never establishes: +#SERVAL_WEBRTC_CANDIDATES=192.168.1.20:8666,[2001:db8::20]:8666 # Days of recordings to keep. Age-based only, no size cap — size this against the disk behind # the video volume (six 4K cameras ≈ 300-400 GB/day). @@ -25,6 +29,46 @@ SERVAL_ADMIN_PASSWORD= # Output:ApiKey). Uncomment the matching line in docker-compose.yml too. #SERVAL_API_KEY= +# Cameras in Google Home — off by default, and it needs a public HTTPS URL with a real certificate +# reaching this server plus a Nest Hub on the same LAN as go2rtc. Google will not send a camera to +# a television by voice, whoever the vendor is; casting from the app is what does that, below. +# Read Docs/google-home.md first; most deployments cannot use this. +# +# The client id and secret are values you GENERATE, not values Google gives you. The client id is a +# secret here — it is the only thing deciding whose Google account may link to this server: +# openssl rand -base64 32 +#SERVAL_GOOGLE_ENABLED=true +#SERVAL_GOOGLE_PUBLIC_BASE_URL=https://serval.example.com +#SERVAL_GOOGLE_PROJECT_ID=your-home-developer-console-project-id +#SERVAL_GOOGLE_CLIENT_ID= +#SERVAL_GOOGLE_CLIENT_SECRET= +# +# Optional. The one file Google does give you: a HomeGraph service-account key, which lets Google +# hear about a camera you added or renamed without re-linking. Everything works without it. +# Put the downloaded JSON at ./secrets/homegraph.json (gitignored) and uncomment the secrets +# volume in docker-compose.yml. +#SERVAL_GOOGLE_HOMEGRAPH_KEY=/app/secrets/homegraph.json +# +# Optional. A PIN turns on the Home app's switch for each camera, and it is only asked for in the +# direction that matters: switching a camera OFF. Unset, the switch is not offered at all — which +# is the safe default, because a voice carries through an open window and out of a television. +#SERVAL_GOOGLE_VERIFICATION_PIN= +# +# Optional, and what puts cameras on a television at all — the Cast button in the app. Register a +# Cast application whose receiver URL is https:///api/google/camerastream/receiver and +# put its id here. The live view then plays over WebRTC, straight from go2rtc to the television; +# a recording you have scrubbed back to is transcoded and plays from where you are. Unset, there is +# no Cast button. See the "Cameras on a television" section of Docs/google-home.md. +#SERVAL_GOOGLE_CAST_RECEIVER_APP_ID= +# +# The tallest a cast recording is transcoded to. A television will happily report that it can +# display 2160p — that is true of the panel and says nothing about the network. At 2160p a +# four-second segment is 9.4 MB and takes 1.25s to encode, so keeping up needs 27 Mbit/s sustained +# through your public address; at 1080p it is 2.5 MB and about 6 Mbit/s. Raise this only where the +# path is known to carry it. Live casting is WebRTC and is not transcoded at all, so it is +# unaffected either way. +#SERVAL_GOOGLE_CAST_MAX_HEIGHT=1080 + # The server image. Override to pin a version (sha-) or to use your own build. #SERVAL_IMAGE=ghcr.io/flickersoft/serval-server:latest diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f6d9d1f..14f24ef 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -58,6 +58,39 @@ services: Serval__WebRtc__Enabled: "true" Serval__WebRtc__Go2RtcUrl: http://go2rtc:1984 + # --- Cameras in Google Home (optional, off by default) ------------------------------- + # The only feature here that needs this server reachable INBOUND from the internet, over + # HTTPS with a real certificate, and it also needs a Nest Hub on the same LAN as go2rtc. + # Google will not send a camera to a television by voice, whoever the vendor is; the app's + # own Cast button is what does that. Read Docs/google-home.md before turning it on. + # + # Wired rather than commented out, unlike the optional blocks below: every one of these + # defaults to empty, empty keeps the integration closed, and setting them in .env is then + # the whole of turning it on. The client id and secret are values you generate, not values + # Google gives you. + Serval__GoogleHome__Enabled: ${SERVAL_GOOGLE_ENABLED:-false} + Serval__GoogleHome__PublicBaseUrl: ${SERVAL_GOOGLE_PUBLIC_BASE_URL:-} + Serval__GoogleHome__ProjectId: ${SERVAL_GOOGLE_PROJECT_ID:-} + Serval__GoogleHome__ClientId: ${SERVAL_GOOGLE_CLIENT_ID:-} + Serval__GoogleHome__ClientSecret: ${SERVAL_GOOGLE_CLIENT_SECRET:-} + # Optional, and the one file Google does give you. Without it the integration works and + # Google simply does not hear about a camera you added or renamed until you re-link. + # Uncomment the ./secrets mount below to use it. + Serval__GoogleHome__HomeGraphKeyPath: ${SERVAL_GOOGLE_HOMEGRAPH_KEY:-} + # Optional. Empty leaves the Home app's per-camera switch out of the device list entirely, + # which is the safe default — with a PIN set, switching a camera off asks for it. + Serval__GoogleHome__VerificationPin: ${SERVAL_GOOGLE_VERIFICATION_PIN:-} + # Optional, and what puts cameras on a television at all — the Cast button in the app. + # Names the Cast application you registered against this server's own receiver page. Live + # then plays over WebRTC straight from go2rtc; a recording is transcoded as it plays. + # Empty is a working deployment with no Cast button. See Docs/google-home.md. + Serval__GoogleHome__CastReceiverAppId: ${SERVAL_GOOGLE_CAST_RECEIVER_APP_ID:-} + # The tallest a cast *recording* is transcoded to. 1080p because that is what the path can + # actually deliver: a 2160p segment is 9.4 MB and takes 1.25s to encode, which needs 27 Mbit/s + # sustained through your public address to keep up, and does not. Live casting is unaffected — + # it is WebRTC and never transcoded. + Serval__GoogleHome__CastMaxHeight: ${SERVAL_GOOGLE_CAST_MAX_HEIGHT:-1080} + # --- Hardware video encode (optional) ------------------------------------------------ # Nothing is re-encoded unless a camera's record stream asks for it, so this sits idle on # a stack that only records. To use the GPU (VAAPI, Intel or AMD) when a stream does ask, @@ -90,6 +123,9 @@ services: # Model weights, mounted read-only from ./models (empty until you enable the AI — an empty # directory costs nothing, the server just notes the models are absent and keeps recording). - ./models:/app/models:ro + # The Google Home HomeGraph service-account key, if you have one. Read-only: nothing here + # should ever write a credential back. See Docs/google-home.md. + # - ./secrets:/app/secrets:ro # Raw frames on their way to object detection: written by ffmpeg, read and deleted # immediately. Memory rather than disk on purpose — a busy host pushes tens of MB a second # through here and none of it is worth writing to the device holding the recordings.