From 6ee4aa30e48bef2ece1648ac39947ed6dbd1b795 Mon Sep 17 00:00:00 2001 From: Chandran Date: Wed, 10 Jun 2026 11:10:00 +0200 Subject: [PATCH] feat(teleop): show live camera feeds beside the 3D viewer Add an optional camera panel to the teleoperation page, off by default so landing on the page never calls getUserMedia (same consent pattern as the calibration camera toggle). Teleoperation opens no cv2 cameras, so the browser can stream them directly while the arm runs. The panel mirrors the selected robot's configured cameras: it shows a live feed for each camera on the robot record (e.g. wrist_cam, webcam), stacked vertically, and shows nothing when the robot has none -- teleop never surfaces a device that was not deliberately configured. --- .../src/components/control/CameraFeed.tsx | 46 +++++++++ .../components/control/TeleopCameraPanel.tsx | 94 +++++++++++++++++++ .../components/control/VisualizerPanel.tsx | 6 ++ frontend/src/pages/Teleoperation.tsx | 7 +- 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/control/CameraFeed.tsx create mode 100644 frontend/src/components/control/TeleopCameraPanel.tsx diff --git a/frontend/src/components/control/CameraFeed.tsx b/frontend/src/components/control/CameraFeed.tsx new file mode 100644 index 0000000..c1d3cbc --- /dev/null +++ b/frontend/src/components/control/CameraFeed.tsx @@ -0,0 +1,46 @@ +import React from "react"; +import { VideoOff } from "lucide-react"; +import { useCameraStream } from "@/hooks/useCameraStream"; + +interface CameraFeedProps { + /** Browser deviceId to stream. Empty string renders the "no camera" state. */ + deviceId: string; + /** Optional caption shown under the feed. */ + label?: string; +} + +/** Live browser-camera feed bound to a deviceId via getUserMedia. */ +const CameraFeed: React.FC = ({ deviceId, label }) => { + const { videoRef, hasError } = useCameraStream(deviceId, false); + const showVideo = deviceId && !hasError; + + return ( +
+
+ {showVideo ? ( +
+ {label && ( +
+ {label} +
+ )} +
+ ); +}; + +export default CameraFeed; diff --git a/frontend/src/components/control/TeleopCameraPanel.tsx b/frontend/src/components/control/TeleopCameraPanel.tsx new file mode 100644 index 0000000..cb48a71 --- /dev/null +++ b/frontend/src/components/control/TeleopCameraPanel.tsx @@ -0,0 +1,94 @@ +import React, { useState } from "react"; +import { RefreshCw } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { useRobots } from "@/hooks/useRobots"; +import CameraFeed from "./CameraFeed"; + +/** + * Optional live camera panel for the teleoperation page. Off by default so we + * never call getUserMedia just by landing on the page (same consent pattern as + * the calibration camera toggle). Teleoperation opens no cv2 cameras, so the + * browser can stream them directly while the arm runs. + * + * A strict mirror of the selected robot's configured cameras: one live feed per + * camera on the robot record (e.g. "wrist_cam", "webcam"), stacked vertically. + * If the robot has none configured it shows nothing — teleop never surfaces a + * device that wasn't deliberately added to the robot. + */ +const TeleopCameraPanel: React.FC = () => { + const [enabled, setEnabled] = useState(false); + // Bumped by the retry button to remount the feeds (a fresh getUserMedia + // attempt) — useful if a camera was unplugged and reconnected. + const [reloadKey, setReloadKey] = useState(0); + const { selectedRecord, isLoading: robotsLoading } = useRobots(); + + // Feeds come solely from the robot's configured cameras; each carries a stored + // browser device_id we stream directly. A configured camera whose device is + // currently absent still shows (name + failed-preview placeholder), so the + // user can tell it's expected but not detected. + const configured = selectedRecord?.cameras ?? []; + const feeds = configured.map((c) => ({ + key: c.id, + name: c.name, + deviceId: c.device_id, + })); + + return ( +
+
+

Cameras

+
+ {enabled && feeds.length > 0 && ( + + )} + + +
+
+ + {enabled ? ( + feeds.length > 0 ? ( +
+ {feeds.map((feed) => ( + + ))} +
+ ) : ( +

+ {robotsLoading + ? "Loading robot..." + : "No cameras configured for this robot. Add them during calibration to see live feeds here."} +

+ ) + ) : ( +

+ Turn on to watch your cameras while you teleoperate. +

+ )} +
+ ); +}; + +export default TeleopCameraPanel; diff --git a/frontend/src/components/control/VisualizerPanel.tsx b/frontend/src/components/control/VisualizerPanel.tsx index 93ef0d2..9dca786 100644 --- a/frontend/src/components/control/VisualizerPanel.tsx +++ b/frontend/src/components/control/VisualizerPanel.tsx @@ -8,11 +8,14 @@ import Logo from "@/components/Logo"; interface VisualizerPanelProps { onGoBack: () => void; className?: string; + /** Optional content rendered as a column beside the 3D viewer (e.g. a camera panel). */ + rightSlot?: React.ReactNode; } const VisualizerPanel: React.FC = ({ onGoBack, className, + rightSlot, }) => { return (
= ({
+ {rightSlot && ( +
{rightSlot}
+ )} ); }; diff --git a/frontend/src/pages/Teleoperation.tsx b/frontend/src/pages/Teleoperation.tsx index 745d5ea..b72db2a 100644 --- a/frontend/src/pages/Teleoperation.tsx +++ b/frontend/src/pages/Teleoperation.tsx @@ -1,6 +1,7 @@ import React, { useCallback, useEffect, useRef } from "react"; import { useNavigate } from "react-router-dom"; import VisualizerPanel from "@/components/control/VisualizerPanel"; +import TeleopCameraPanel from "@/components/control/TeleopCameraPanel"; import { useToast } from "@/hooks/use-toast"; import { useApi } from "@/contexts/ApiContext"; @@ -69,7 +70,11 @@ const TeleoperationPage = () => { return (
- + } + />
);