/** * @license * Copyright 2025-2026 Open Home Foundation * SPDX-License-Identifier: Apache-2.0 */ import { consume } from "@lit/context"; import type { MatterClient, WebRtcAnswerData, WebRtcCallbackData, WebRtcIceCandidatesData, WebRtcEndData, } from "@matter-server/ws-client"; import { mdiAlertCircleOutline, mdiVideoOutline } from "@mdi/js"; import { LitElement, css, html } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { clientContext } from "../client/client-context.js"; import { asObject, pickNumber } from "../util/attribute-shapes.js"; import "./ha-svg-icon.js"; // Spec values from @matter/types globals/StreamUsage.ts and // web-rtc-transport-definitions.ts WebRtcEndReason. Kept inline to avoid a new // @matter/types dep just for two numeric literals. const STREAM_USAGE_LIVE_VIEW = 3; const END_REASON_USER_HANGUP = 2; // Hardcoded to avoid pulling full cluster schema into the dashboard bundle. export const CAMERA_AV_STREAM_MANAGEMENT_CLUSTER_ID = 0x551; const WEBRTC_TRANSPORT_PROVIDER_CLUSTER_ID = 0x553; // AVSM FeatureMap bits (spec §11.2.4): WMARK=6 enables watermark overlay, // OSD=7 enables on-screen display. When advertised, VideoStreamAllocate and // SnapshotStreamAllocate REQUIRE watermarkEnabled / osdEnabled fields. export const AVSM_FEAT_WMARK = 1 << 6; export const AVSM_FEAT_OSD = 1 << 7; export const AVSM_FEATURE_MAP_ATTR_ID = 0xfffc; const DEFAULT_MAX_RESOLUTION = { width: 1920, height: 1080 }; const DEFAULT_MIN_RESOLUTION = { width: 640, height: 480 }; interface ProvideOfferResponse { webRtcSessionId: number; videoStreamId: number | null; audioStreamId: number | null; } function parseStreamAllocate(value: unknown, idKey: "videoStreamId" | "audioStreamId"): number | null { const obj = asObject(value); if (!obj) return null; return pickNumber(obj, idKey); } function parseSnapshotAllocateResponse(value: unknown): number | null { const obj = asObject(value); if (!obj) return null; return pickNumber(obj, "snapshotStreamId"); } interface SnapshotCapability { resolution: { width: number; height: number }; maxFrameRate: number; imageCodec: number; } const SNAPSHOT_DEFAULTS: SnapshotCapability = { resolution: { width: 1920, height: 1080 }, maxFrameRate: 30, imageCodec: 0, }; function parseSnapshotCapabilitiesFromList(list: unknown[]): SnapshotCapability { // SnapshotCapabilitiesStruct field IDs per Matter 1.5.1 §11.2.6.9: // 0=resolution (VideoResolutionStruct {0=width, 1=height}), 1=maxFrameRate, 2=imageCodec. // Cached attributes are tag-based (numeric keys); read_attribute responses are name-based. // Prefer the highest-resolution entry — Aqara G350 ships [VGA, 1080p] and 1080p is the useful one. const candidates = list.map(asObject).filter((c): c is Record => c !== undefined); if (candidates.length === 0) return SNAPSHOT_DEFAULTS; const parsed = candidates.map(cap => { const res = asObject(cap["resolution"] ?? cap["0"]); const width = res ? pickNumber(res, "width", "0") : null; const height = res ? pickNumber(res, "height", "1") : null; const maxFrameRate = pickNumber(cap, "maxFrameRate", "1"); const imageCodec = pickNumber(cap, "imageCodec", "2"); return { resolution: { width: width ?? SNAPSHOT_DEFAULTS.resolution.width, height: height ?? SNAPSHOT_DEFAULTS.resolution.height, }, maxFrameRate: maxFrameRate ?? SNAPSHOT_DEFAULTS.maxFrameRate, imageCodec: imageCodec ?? SNAPSHOT_DEFAULTS.imageCodec, }; }); return parsed.reduce((best, cur) => cur.resolution.width * cur.resolution.height > best.resolution.width * best.resolution.height ? cur : best, ); } interface CaptureSnapshotResult { dataBase64: string; imageCodec: number; resolution: { width: number; height: number }; } function bytesToBase64(bytes: Uint8Array): string { // Chunked to avoid stack overflow when spreading large arrays into String.fromCharCode. const CHUNK = 8192; let binary = ""; for (let i = 0; i < bytes.length; i += CHUNK) { binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)); } return btoa(binary); } function parseDataToBase64(data: unknown): string { if (typeof data === "string") return data; if (data instanceof Uint8Array) return bytesToBase64(data); if (data instanceof ArrayBuffer) return bytesToBase64(new Uint8Array(data)); if (Array.isArray(data)) return bytesToBase64(new Uint8Array(data as number[])); throw new Error(`Unexpected snapshot data shape: ${typeof data}`); } function parseCaptureSnapshotResponse(value: unknown): CaptureSnapshotResult { const obj = asObject(value); if (!obj) throw new Error("CaptureSnapshot returned no response"); const data = obj["data"]; if (data == null) throw new Error("CaptureSnapshot response missing data field"); const imageCodec = pickNumber(obj, "imageCodec") ?? 0; const res = asObject(obj["resolution"]); const width = res ? (pickNumber(res, "width") ?? 0) : 0; const height = res ? (pickNumber(res, "height") ?? 0) : 0; return { dataBase64: parseDataToBase64(data), imageCodec, resolution: { width, height } }; } function parseProvideOfferResponse(value: unknown): ProvideOfferResponse | null { const obj = asObject(value); if (!obj) return null; const sessionId = pickNumber(obj, "webRtcSessionId"); if (sessionId === null) return null; return { webRtcSessionId: sessionId, videoStreamId: parseStreamAllocate(value, "videoStreamId"), audioStreamId: parseStreamAllocate(value, "audioStreamId"), }; } type StreamState = "idle" | "connecting" | "streaming" | "error"; @customElement("webrtc-stream-view") export class WebRtcStreamView extends LitElement { @consume({ context: clientContext, subscribe: true }) @property({ attribute: false }) client?: MatterClient; @property({ attribute: false }) nodeId!: number | bigint; @property({ type: Number }) endpointId!: number; @property({ type: Object }) resolution: { width: number; height: number } | null = null; @property({ type: Boolean }) watermarkEnabled = false; @property({ type: Boolean }) osdEnabled = false; @state() private _state: StreamState = "idle"; @state() private _errorMessage: string | null = null; @property({ attribute: false }) snapshotResolution: { width: number; height: number } | null = null; private _snapshotStreamId: number | null = null; private _snapshotResolution: { width: number; height: number } | null = null; /** True when we allocated this stream ourselves and must Deallocate it on stop. False when reusing an existing allocation. */ private _videoStreamOwned = false; private _audioStreamOwned = false; private _snapshotStreamOwned = false; @query("video") private _video?: HTMLVideoElement; private _pc: RTCPeerConnection | null = null; private _localIceQueue: RTCIceCandidate[] = []; private _answerReceived = false; private _webRtcSessionId: number | null = null; private _videoStreamId: number | null = null; private _audioStreamId: number | null = null; private _unsubscribe: (() => void) | null = null; private _stopping = false; private _endReceivedFromPeer = false; private _preSessionQueue: WebRtcCallbackData[] = []; get state(): StreamState { return this._state; } get videoStreamId(): number | null { return this._videoStreamId; } get muted(): boolean { return this._video?.muted ?? true; } setMuted(muted: boolean): void { if (this._video) this._video.muted = muted; } override disconnectedCallback() { super.disconnectedCallback(); void this.deallocateSnapshot(); void this.stop(); } override render() { return html` ${this._state === "idle" ? html`
Click Start to begin streaming
` : null} ${this._state === "connecting" ? html`
Connecting…
` : null} ${this._state === "error" ? html`
${this._errorMessage ?? "Stream error"}
` : null} `; } async start(): Promise { if (!this.client) throw new Error("Matter client not available"); if (this._state === "connecting" || this._state === "streaming") return; this._fireStateChange("connecting", null); this._answerReceived = false; this._localIceQueue = []; this._stopping = false; this._endReceivedFromPeer = false; this._preSessionQueue = []; try { const pc = new RTCPeerConnection({ iceServers: [] }); this._pc = pc; pc.addTransceiver("video", { direction: "recvonly" }); pc.addTransceiver("audio", { direction: "recvonly" }); pc.onicecandidate = ev => { if (ev.candidate === null) { console.log("[webrtc-stream-view] local ICE gathering complete"); return; } console.log("[webrtc-stream-view] local ICE candidate", ev.candidate.candidate, { queued: !this._answerReceived, queueDepth: this._localIceQueue.length, }); if (!this._answerReceived) { this._localIceQueue.push(ev.candidate); return; } void this._sendLocalIceCandidates([ev.candidate]); }; pc.oniceconnectionstatechange = () => { console.log("[webrtc-stream-view] iceConnectionState ->", pc.iceConnectionState); }; pc.onconnectionstatechange = () => { console.log("[webrtc-stream-view] connectionState ->", pc.connectionState); }; pc.onsignalingstatechange = () => { console.log("[webrtc-stream-view] signalingState ->", pc.signalingState); }; pc.ontrack = ev => { console.log("[webrtc-stream-view] ontrack", { kind: ev.track.kind, readyState: ev.track.readyState, streams: ev.streams.length, }); const video = this._video; if (!video) { console.warn("[webrtc-stream-view] ontrack fired but