/* * Copyright 2020 - 2024 The Matrix.org Foundation C.I.C. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { EventEmitter } from "events"; import { ITransport } from "./transport/ITransport"; import { Widget } from "./models/Widget"; import { PostmessageTransport } from "./transport/PostmessageTransport"; import { WidgetApiDirection } from "./interfaces/WidgetApiDirection"; import { IWidgetApiRequest, IWidgetApiRequestEmptyData } from "./interfaces/IWidgetApiRequest"; import { IContentLoadedActionRequest } from "./interfaces/ContentLoadedAction"; import { WidgetApiFromWidgetAction, WidgetApiToWidgetAction } from "./interfaces/WidgetApiAction"; import { IWidgetApiErrorResponseData } from "./interfaces/IWidgetApiErrorResponse"; import { Capability, MatrixCapabilities, getTimelineRoomIDFromCapability, isTimelineCapability, } from "./interfaces/Capabilities"; import { IOpenIDUpdate, ISendEventDetails, ISendDelayedEventDetails, WidgetDriver } from "./driver/WidgetDriver"; import { ICapabilitiesActionResponseData, INotifyCapabilitiesActionRequestData, IRenegotiateCapabilitiesActionRequest, } from "./interfaces/CapabilitiesAction"; import { ISupportedVersionsActionRequest, ISupportedVersionsActionResponseData, } from "./interfaces/SupportedVersionsAction"; import { ApiVersion, CurrentApiVersions, UnstableApiVersion } from "./interfaces/ApiVersion"; import { IScreenshotActionResponseData } from "./interfaces/ScreenshotAction"; import { IVisibilityActionRequestData } from "./interfaces/VisibilityAction"; import { IWidgetApiAcknowledgeResponseData, IWidgetApiResponseData } from "./interfaces/IWidgetApiResponse"; import { IModalWidgetButtonClickedRequestData, IModalWidgetOpenRequestData, IModalWidgetOpenRequestDataButton, IModalWidgetReturnData, } from "./interfaces/ModalWidgetActions"; import { ISendEventFromWidgetActionRequest, ISendEventFromWidgetResponseData, ISendEventToWidgetRequestData, } from "./interfaces/SendEventAction"; import { ISendToDeviceFromWidgetActionRequest, ISendToDeviceFromWidgetResponseData, ISendToDeviceToWidgetRequestData, } from "./interfaces/SendToDeviceAction"; import { EventDirection, EventKind, WidgetEventCapability } from "./models/WidgetEventCapability"; import { IRoomEvent } from "./interfaces/IRoomEvent"; import { IGetOpenIDActionRequest, IGetOpenIDActionResponseData, IOpenIDCredentials, OpenIDRequestState, } from "./interfaces/GetOpenIDAction"; import { SimpleObservable } from "./util/SimpleObservable"; import { IOpenIDCredentialsActionRequestData } from "./interfaces/OpenIDCredentialsAction"; import { INavigateActionRequest } from "./interfaces/NavigateAction"; import { IReadEventFromWidgetActionRequest, IReadEventFromWidgetResponseData } from "./interfaces/ReadEventAction"; import { ITurnServer, IWatchTurnServersRequest, IUnwatchTurnServersRequest, IUpdateTurnServersRequestData, } from "./interfaces/TurnServerActions"; import { Symbols } from "./Symbols"; import { IReadRelationsFromWidgetActionRequest, IReadRelationsFromWidgetResponseData, } from "./interfaces/ReadRelationsAction"; import { IUserDirectorySearchFromWidgetActionRequest, IUserDirectorySearchFromWidgetResponseData, } from "./interfaces/UserDirectorySearchAction"; import { IReadRoomAccountDataFromWidgetActionRequest, IReadRoomAccountDataFromWidgetResponseData, } from "./interfaces/ReadRoomAccountDataAction"; import { IGetMediaConfigActionFromWidgetActionRequest, IGetMediaConfigActionFromWidgetResponseData, } from "./interfaces/GetMediaConfigAction"; import { IUpdateDelayedEventFromWidgetActionRequest, UpdateDelayedEventAction, } from "./interfaces/UpdateDelayedEventAction"; import { IUploadFileActionFromWidgetActionRequest, IUploadFileActionFromWidgetResponseData, } from "./interfaces/UploadFileAction"; import { IDownloadFileActionFromWidgetActionRequest, IDownloadFileActionFromWidgetResponseData, } from "./interfaces/DownloadFileAction"; import { IThemeChangeActionRequestData } from "./interfaces/ThemeChangeAction"; import { IUpdateStateToWidgetRequestData } from "./interfaces/UpdateStateAction"; import { IToDeviceMessage } from "./interfaces/IToDeviceMessage"; /** * API handler for the client side of widgets. This raises events * for each action received as `action:${action}` (eg: "action:screenshot"). * Default handling can be prevented by using preventDefault() on the * raised event. The default handling varies for each action: ones * which the SDK can handle safely are acknowledged appropriately and * ones which are unhandled (custom or require the client to do something) * are rejected with an error. * * Events which are preventDefault()ed must reply using the transport. * The events raised will have a default of an IWidgetApiRequest * interface. * * When the ClientWidgetApi is ready to start sending requests, it will * raise a "ready" CustomEvent. After the ready event fires, actions can * be sent and the transport will be ready. * * When the widget has indicated it has loaded, this class raises a * "preparing" CustomEvent. The preparing event does not indicate that * the widget is ready to receive communications - that is signified by * the ready event exclusively. * * This class only handles one widget at a time. */ export class ClientWidgetApi extends EventEmitter { public readonly transport: ITransport; private cachedWidgetVersions: ApiVersion[] | null = null; // contentLoadedActionSent is used to check that only one ContentLoaded request is send. private contentLoadedActionSent = false; private readonly allowedCapabilities = new Set(); private readonly allowedEvents: WidgetEventCapability[] = []; private isStopped = false; private turnServers: AsyncGenerator | null = null; private contentLoadedWaitTimer?: ReturnType; // Stores pending requests to push a room's state to the widget private readonly pushRoomStateTasks = new Set>(); // Room ID → event type → state key → events to be pushed private readonly pushRoomStateResult = new Map>>(); private flushRoomStateTask: Promise | null = null; /** * Creates a new client widget API. This will instantiate the transport * and start everything. When the iframe is loaded under the widget's * conditions, a "ready" event will be raised. * @param {Widget} widget The widget to communicate with. * @param {HTMLIFrameElement} iframe The iframe the widget is in. * @param {WidgetDriver} driver The driver for this widget/client. */ public constructor( public readonly widget: Widget, iframe: HTMLIFrameElement, private readonly driver: WidgetDriver, ) { super(); if (!iframe?.contentWindow) { throw new Error("No iframe supplied"); } if (!widget) { throw new Error("Invalid widget"); } if (!driver) { throw new Error("Invalid driver"); } this.transport = new PostmessageTransport( WidgetApiDirection.ToWidget, widget.id, iframe.contentWindow, globalThis, ); this.transport.targetOrigin = widget.origin; this.transport.on("message", this.handleMessage.bind(this)); iframe.addEventListener("load", this.onIframeLoad.bind(this)); this.transport.start(); } public hasCapability(capability: Capability): boolean { return this.allowedCapabilities.has(capability); } public canUseRoomTimeline(roomId: string | Symbols.AnyRoom): boolean { return ( this.hasCapability(`org.matrix.msc2762.timeline:${Symbols.AnyRoom}`) || this.hasCapability(`org.matrix.msc2762.timeline:${roomId}`) ); } public canSendRoomEvent(eventType: string, msgtype: string | null = null): boolean { return this.allowedEvents.some((e) => e.matchesAsRoomEvent(EventDirection.Send, eventType, msgtype)); } public canSendStateEvent(eventType: string, stateKey: string): boolean { return this.allowedEvents.some((e) => e.matchesAsStateEvent(EventDirection.Send, eventType, stateKey)); } public canSendToDeviceEvent(eventType: string): boolean { return this.allowedEvents.some((e) => e.matchesAsToDeviceEvent(EventDirection.Send, eventType)); } public canReceiveRoomEvent(eventType: string, msgtype: string | null = null): boolean { return this.allowedEvents.some((e) => e.matchesAsRoomEvent(EventDirection.Receive, eventType, msgtype)); } public canReceiveStateEvent(eventType: string, stateKey: string | null): boolean { return this.allowedEvents.some((e) => e.matchesAsStateEvent(EventDirection.Receive, eventType, stateKey)); } public canReceiveToDeviceEvent(eventType: string): boolean { return this.allowedEvents.some((e) => e.matchesAsToDeviceEvent(EventDirection.Receive, eventType)); } public canReceiveRoomAccountData(eventType: string): boolean { return this.allowedEvents.some((e) => e.matchesAsRoomAccountData(EventDirection.Receive, eventType)); } public stop(): void { this.isStopped = true; this.transport.stop(); } public async getWidgetVersions(): Promise { if (Array.isArray(this.cachedWidgetVersions)) { return this.cachedWidgetVersions; } try { const r = await this.transport.send( WidgetApiToWidgetAction.SupportedApiVersions, {}, ); this.cachedWidgetVersions = r.supported_versions; return r.supported_versions; } catch (e) { console.warn("non-fatal error getting supported widget versions: ", e); return []; } } private beginCapabilities(): void { // widget has loaded - tell all the listeners that this.emit("preparing"); let requestedCaps: Capability[]; this.transport .send(WidgetApiToWidgetAction.Capabilities, {}) .then((caps) => { requestedCaps = caps.capabilities; return this.driver.validateCapabilities(new Set(caps.capabilities)); }) .then((allowedCaps) => { this.allowCapabilities([...allowedCaps], requestedCaps); this.emit("ready"); }) .catch((e) => { this.emit("error:preparing", e); }); } private allowCapabilities(allowed: string[], requested: string[]): void { console.log(`Widget ${this.widget.id} is allowed capabilities:`, allowed); for (const c of allowed) this.allowedCapabilities.add(c); const allowedEvents = WidgetEventCapability.findEventCapabilities(allowed); this.allowedEvents.push(...allowedEvents); this.transport .send(WidgetApiToWidgetAction.NotifyCapabilities, { requested, approved: Array.from(this.allowedCapabilities), }) .catch((e) => { console.warn("non-fatal error notifying widget of approved capabilities:", e); }) .then(() => { this.emit("capabilitiesNotified"); }); // Push the initial room state for all rooms with a timeline capability for (const c of allowed) { if (isTimelineCapability(c)) { const roomId = getTimelineRoomIDFromCapability(c); if (roomId === Symbols.AnyRoom) { for (const roomId of this.driver.getKnownRooms()) { this.pushRoomState(roomId); } } else { this.pushRoomState(roomId); } } } if (allowed.includes(MatrixCapabilities.MSC4407ReceiveStickyEvent)) { console.debug(`Widget ${this.widget.id} is allowed to receive sticky events, check current sticky state.`); // If the widget can receive sticky events, push all sticky events in known rooms now. // Sticky events are like a state, and passed history is needed to get the full state. const roomIds = allowed .filter((capability) => isTimelineCapability(capability)) .map((timelineCapability) => getTimelineRoomIDFromCapability(timelineCapability)) .flatMap((roomIdOrWildcard) => { if (roomIdOrWildcard === Symbols.AnyRoom) { // Do we support getting sticky state for any room? return this.driver.getKnownRooms(); } else { return roomIdOrWildcard; } }); console.debug(`Widget ${this.widget.id} is allowed to receive sticky events in rooms:`, roomIds); for (const roomId of roomIds) { this.pushStickyState(roomId).catch((err) => { console.error(`Failed to push sticky events to widget ${this.widget.id} for room ${roomId}:`, err); }); } } // If new events are allowed and the currently viewed room isn't covered // by a timeline capability, then we know that there could be some state // in the viewed room that the widget hasn't learned about yet- push it. if (allowedEvents.length > 0 && this.viewedRoomId !== null && !this.canUseRoomTimeline(this.viewedRoomId)) { this.pushRoomState(this.viewedRoomId); } } private onIframeLoad(ev: Event): void { if (this.widget.waitForIframeLoad) { // If the widget is set to waitForIframeLoad the capabilities immediately get setup after load. // The client does not wait for the ContentLoaded action. this.beginCapabilities(); } else { // Reaching this means, that the Iframe got reloaded/loaded and // the clientApi is awaiting the FIRST ContentLoaded action. console.log("waitForIframeLoad is false: waiting for widget to send contentLoaded"); this.contentLoadedWaitTimer = setTimeout(() => { console.error( "Widget specified waitForIframeLoad=false but timed out waiting for contentLoaded event!", ); }, 10000); this.contentLoadedActionSent = false; } } private handleContentLoadedAction(action: IContentLoadedActionRequest): void { if (this.contentLoadedWaitTimer !== undefined) { clearTimeout(this.contentLoadedWaitTimer); this.contentLoadedWaitTimer = undefined; } if (this.contentLoadedActionSent) { throw new Error( "Improper sequence: ContentLoaded Action can only be sent once after the widget loaded " + "and should only be used if waitForIframeLoad is false (default=true)", ); } if (this.widget.waitForIframeLoad) { this.transport.reply(action, { error: { message: "Improper sequence: not expecting ContentLoaded event if " + "waitForIframeLoad is true (default=true)", }, }); } else { this.transport.reply(action, {}); this.beginCapabilities(); } this.contentLoadedActionSent = true; } private replyVersions(request: ISupportedVersionsActionRequest): void { this.transport.reply(request, { supported_versions: CurrentApiVersions, }); } private async supportsUpdateState(): Promise { return (await this.getWidgetVersions()).includes(UnstableApiVersion.MSC2762_UPDATE_STATE); } private handleCapabilitiesRenegotiate(request: IRenegotiateCapabilitiesActionRequest): void { // acknowledge first this.transport.reply(request, {}); const requested = request.data?.capabilities || []; const newlyRequested = new Set(requested.filter((r) => !this.hasCapability(r))); if (newlyRequested.size === 0) { // Nothing to do - skip validation this.allowCapabilities([], []); } this.driver .validateCapabilities(newlyRequested) .then((allowed) => this.allowCapabilities([...allowed], [...newlyRequested])); } private handleNavigate(request: INavigateActionRequest): void { if (!this.hasCapability(MatrixCapabilities.MSC2931Navigate)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } if (!request.data?.uri.startsWith("https://matrix.to/#")) { return this.transport.reply(request, { error: { message: "Invalid matrix.to URI" }, }); } const onErr = (e: unknown): void => { console.error("[ClientWidgetApi] Failed to handle navigation: ", e); this.handleDriverError(e, request, "Error handling navigation"); }; try { this.driver .navigate(request.data.uri.toString()) .catch((e: unknown) => onErr(e)) .then(() => { return this.transport.reply(request, {}); }); } catch (e) { return onErr(e); } } private handleOIDC(request: IGetOpenIDActionRequest): void { let phase = 1; // 1 = initial request, 2 = after user manual confirmation const replyState = ( state: OpenIDRequestState, credential?: IOpenIDCredentials, ): void | Promise => { credential = credential || {}; if (phase > 1) { return this.transport.send( WidgetApiToWidgetAction.OpenIDCredentials, { state: state, original_request_id: request.requestId, ...credential, }, ); } else { return this.transport.reply(request, { state: state, ...credential, }); } }; const replyError = (msg: string): void | Promise => { console.error("[ClientWidgetApi] Failed to handle OIDC: ", msg); if (phase > 1) { // We don't have a way to indicate that a random error happened in this flow, so // just block the attempt. return replyState(OpenIDRequestState.Blocked); } else { return this.transport.reply(request, { error: { message: msg }, }); } }; const observer = new SimpleObservable((update) => { if (update.state === OpenIDRequestState.PendingUserConfirmation && phase > 1) { observer.close(); return replyError("client provided out-of-phase response to OIDC flow"); } if (update.state === OpenIDRequestState.PendingUserConfirmation) { replyState(update.state); phase++; return; } if (update.state === OpenIDRequestState.Allowed && !update.token) { return replyError("client provided invalid OIDC token for an allowed request"); } if (update.state === OpenIDRequestState.Blocked) { update.token = undefined; // just in case the client did something weird } observer.close(); return replyState(update.state, update.token); }); this.driver.askOpenID(observer); } private handleReadRoomAccountData(request: IReadRoomAccountDataFromWidgetActionRequest): void | Promise { const events = this.driver.readRoomAccountData(request.data.type); if (!this.canReceiveRoomAccountData(request.data.type)) { return this.transport.reply(request, { error: { message: "Cannot read room account data of this type" }, }); } return events.then((evs) => { this.transport.reply(request, { events: evs }); }); } private async handleReadEvents(request: IReadEventFromWidgetActionRequest): Promise { if (!request.data.type) { return this.transport.reply(request, { error: { message: "Invalid request - missing event type" }, }); } if (request.data.limit !== undefined && (!request.data.limit || request.data.limit < 0)) { return this.transport.reply(request, { error: { message: "Invalid request - limit out of range" }, }); } let askRoomIds: string[]; if (request.data.room_ids === undefined) { askRoomIds = this.viewedRoomId === null ? [] : [this.viewedRoomId]; } else if (request.data.room_ids === Symbols.AnyRoom) { askRoomIds = this.driver.getKnownRooms().filter((roomId) => this.canUseRoomTimeline(roomId)); } else { askRoomIds = request.data.room_ids; for (const roomId of askRoomIds) { if (!this.canUseRoomTimeline(roomId)) { return this.transport.reply(request, { error: { message: `Unable to access room timeline: ${roomId}` }, }); } } } const limit = request.data.limit || 0; const since = request.data.since; let stateKey: string | undefined = undefined; let msgtype: string | undefined = undefined; if (request.data.state_key !== undefined) { stateKey = request.data.state_key === true ? undefined : request.data.state_key.toString(); if (!this.canReceiveStateEvent(request.data.type, stateKey ?? null)) { return this.transport.reply(request, { error: { message: "Cannot read state events of this type" }, }); } } else { msgtype = request.data.msgtype; if (!this.canReceiveRoomEvent(request.data.type, msgtype)) { return this.transport.reply(request, { error: { message: "Cannot read room events of this type" }, }); } } let events: IRoomEvent[]; if (request.data.room_ids === undefined && askRoomIds.length === 0) { // For backwards compatibility we still call the deprecated // readRoomEvents and readStateEvents methods in case the client isn't // letting us know the currently viewed room via setViewedRoomId // // This can be considered as a deprecated implementation. // A driver should call `setViewedRoomId` on the widget messaging and implement the new readRoomState and readRoomTimeline // Methods. // This block makes sure that it is also possible to not use setViewedRoomId. // readRoomTimeline and readRoomState are required however! Otherwise widget requests that include // `room_ids` will fail. console.warn( "The widgetDriver uses deprecated behaviour:\n It does not set the viewedRoomId using `setViewedRoomId`", ); events = await // This returns [] with the current driver of Element Web. // Add default implementations of the `readRoomEvents` and `readStateEvents` // methods to use `readRoomTimeline` and `readRoomState` if they are not overwritten. (request.data.state_key === undefined ? this.driver.readRoomEvents(request.data.type, msgtype, limit, null, since) : this.driver.readStateEvents(request.data.type, stateKey, limit, null)); } else if (await this.supportsUpdateState()) { // Calling read_events with a stateKey still reads from the rooms timeline (not the room state). events = ( await Promise.all( askRoomIds.map((roomId) => this.driver.readRoomTimeline(roomId, request.data.type, msgtype, stateKey, limit, since), ), ) ).flat(1); } else { // TODO: remove this once `UnstableApiVersion.MSC2762_UPDATE_STATE` becomes stable. // Before version `MSC2762_UPDATE_STATE` we used readRoomState for read_events actions. events = ( request.data.state_key === undefined ? await Promise.all( askRoomIds.map((roomId) => this.driver.readRoomTimeline(roomId, request.data.type, msgtype, stateKey, limit, since), ), ) : await Promise.all( askRoomIds.map((roomId) => this.driver.readRoomState(roomId, request.data.type, stateKey)), ) ).flat(1); } this.transport.reply(request, { events }); } private handleSendEvent(request: ISendEventFromWidgetActionRequest): void { if (!request.data.type) { return this.transport.reply(request, { error: { message: "Invalid request - missing event type" }, }); } if (!!request.data.room_id && !this.canUseRoomTimeline(request.data.room_id)) { return this.transport.reply(request, { error: { message: `Unable to access room timeline: ${request.data.room_id}` }, }); } const isDelayedEvent = request.data.delay !== undefined || request.data.parent_delay_id !== undefined; if (isDelayedEvent && !this.hasCapability(MatrixCapabilities.MSC4157SendDelayedEvent)) { return this.transport.reply(request, { error: { message: `Missing capability for ${MatrixCapabilities.MSC4157SendDelayedEvent}` }, }); } const isStickyEvent = request.data.sticky_duration_ms !== undefined; if (isStickyEvent && !this.hasCapability(MatrixCapabilities.MSC4407SendStickyEvent)) { return this.transport.reply(request, { error: { message: `Missing capability for ${MatrixCapabilities.MSC4407SendStickyEvent}` }, }); } let sendEventPromise: Promise; if (request.data.state_key !== undefined) { if (!this.canSendStateEvent(request.data.type, request.data.state_key)) { return this.transport.reply(request, { error: { message: "Cannot send state events of this type" }, }); } if (isStickyEvent) { return this.transport.reply(request, { error: { message: "Cannot send a state event with a sticky duration" }, }); } if (isDelayedEvent) { sendEventPromise = this.driver.sendDelayedEvent( request.data.delay ?? null, request.data.parent_delay_id ?? null, request.data.type, request.data.content || {}, request.data.state_key, request.data.room_id, ); } else { sendEventPromise = this.driver.sendEvent( request.data.type, request.data.content || {}, request.data.state_key, request.data.room_id, ); } } else { const content = (request.data.content as { msgtype?: string }) || {}; const msgtype = content["msgtype"]; if (!this.canSendRoomEvent(request.data.type, msgtype)) { return this.transport.reply(request, { error: { message: "Cannot send room events of this type" }, }); } // Events can be sticky, delayed, both, or neither. The following // section of code takes the common parameters and uses the correct // function depending on the request type. const params: Parameters = [ request.data.type, content, null, // not sending a state event request.data.room_id, ]; if (isDelayedEvent && request.data.sticky_duration_ms) { sendEventPromise = this.driver.sendDelayedStickyEvent( request.data.delay ?? null, request.data.parent_delay_id ?? null, request.data.sticky_duration_ms, request.data.type, content, request.data.room_id, ); } else if (isDelayedEvent) { sendEventPromise = this.driver.sendDelayedEvent( request.data.delay ?? null, request.data.parent_delay_id ?? null, ...params, ); } else if (request.data.sticky_duration_ms) { sendEventPromise = this.driver.sendStickyEvent( request.data.sticky_duration_ms, request.data.type, content, request.data.room_id, ); } else { sendEventPromise = this.driver.sendEvent(...params); } } sendEventPromise .then((sentEvent) => { return this.transport.reply(request, { room_id: sentEvent.roomId, ...("eventId" in sentEvent ? { event_id: sentEvent.eventId, } : { delay_id: sentEvent.delayId, }), }); }) .catch((e: unknown) => { console.error("error sending event: ", e); this.handleDriverError(e, request, "Error sending event"); }); } private handleUpdateDelayedEvent(request: IUpdateDelayedEventFromWidgetActionRequest): void { if (!request.data.delay_id) { return this.transport.reply(request, { error: { message: "Invalid request - missing delay_id" }, }); } if (!this.hasCapability(MatrixCapabilities.MSC4157UpdateDelayedEvent)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } let updateDelayedEvent: (delayId: string) => Promise; switch (request.data.action) { case UpdateDelayedEventAction.Cancel: updateDelayedEvent = this.driver.cancelScheduledDelayedEvent; break; case UpdateDelayedEventAction.Restart: updateDelayedEvent = this.driver.restartScheduledDelayedEvent; break; case UpdateDelayedEventAction.Send: updateDelayedEvent = this.driver.sendScheduledDelayedEvent; break; default: return this.transport.reply(request, { error: { message: "Invalid request - unsupported action" }, }); } updateDelayedEvent .call(this.driver, request.data.delay_id) .then(() => { return this.transport.reply(request, {}); }) .catch((e: unknown) => { console.error("error updating delayed event: ", e); this.handleDriverError(e, request, "Error updating delayed event"); }); } private async handleSendToDevice(request: ISendToDeviceFromWidgetActionRequest): Promise { if (!request.data.type) { this.transport.reply(request, { error: { message: "Invalid request - missing event type" }, }); } else if (!request.data.messages) { this.transport.reply(request, { error: { message: "Invalid request - missing event contents" }, }); } else if (typeof request.data.encrypted !== "boolean") { this.transport.reply(request, { error: { message: "Invalid request - missing encryption flag" }, }); } else if (!this.canSendToDeviceEvent(request.data.type)) { this.transport.reply(request, { error: { message: "Cannot send to-device events of this type" }, }); } else { try { await this.driver.sendToDevice(request.data.type, request.data.encrypted, request.data.messages); this.transport.reply(request, {}); } catch (e) { console.error("error sending to-device event", e); this.handleDriverError(e, request, "Error sending event"); } } } private async pollTurnServers(turnServers: AsyncGenerator, initialServer: ITurnServer): Promise { try { await this.transport.send( WidgetApiToWidgetAction.UpdateTurnServers, initialServer as IUpdateTurnServersRequestData, // it's compatible, but missing the index signature ); // Pick the generator up where we left off for await (const server of turnServers) { await this.transport.send( WidgetApiToWidgetAction.UpdateTurnServers, server as IUpdateTurnServersRequestData, // it's compatible, but missing the index signature ); } } catch (e) { console.error("error polling for TURN servers", e); } } private async handleWatchTurnServers(request: IWatchTurnServersRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC3846TurnServers)) { this.transport.reply(request, { error: { message: "Missing capability" }, }); } else if (this.turnServers) { // We're already polling, so this is a no-op this.transport.reply(request, {}); } else { try { const turnServers = this.driver.getTurnServers(); // Peek at the first result, so we can at least verify that the // client isn't banned from getting TURN servers entirely const { done, value } = await turnServers.next(); if (done) throw new Error("Client refuses to provide any TURN servers"); this.transport.reply(request, {}); // Start the poll loop, sending the widget the initial result this.pollTurnServers(turnServers, value); this.turnServers = turnServers; } catch (e) { console.error("error getting first TURN server results", e); this.transport.reply(request, { error: { message: "TURN servers not available" }, }); } } } private async handleUnwatchTurnServers(request: IUnwatchTurnServersRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC3846TurnServers)) { this.transport.reply(request, { error: { message: "Missing capability" }, }); } else if (!this.turnServers) { // We weren't polling anyways, so this is a no-op this.transport.reply(request, {}); } else { // Stop the generator, allowing it to clean up await this.turnServers.return(undefined); this.turnServers = null; this.transport.reply(request, {}); } } private async handleReadRelations(request: IReadRelationsFromWidgetActionRequest): Promise { if (!request.data.event_id) { return this.transport.reply(request, { error: { message: "Invalid request - missing event ID" }, }); } if (request.data.limit !== undefined && request.data.limit < 0) { return this.transport.reply(request, { error: { message: "Invalid request - limit out of range" }, }); } if (request.data.room_id !== undefined && !this.canUseRoomTimeline(request.data.room_id)) { return this.transport.reply(request, { error: { message: `Unable to access room timeline: ${request.data.room_id}` }, }); } try { const result = await this.driver.readEventRelations( request.data.event_id, request.data.room_id, request.data.rel_type, request.data.event_type, request.data.from, request.data.to, request.data.limit, request.data.direction, ); // only return events that the user has the permission to receive const chunk = result.chunk.filter((e) => { if (e.state_key !== undefined) { return this.canReceiveStateEvent(e.type, e.state_key); } else { return this.canReceiveRoomEvent(e.type, (e.content as { msgtype?: string })["msgtype"]); } }); return this.transport.reply(request, { chunk, prev_batch: result.prevBatch, next_batch: result.nextBatch, }); } catch (e) { console.error("error getting the relations", e); this.handleDriverError(e, request, "Unexpected error while reading relations"); } } private async handleUserDirectorySearch(request: IUserDirectorySearchFromWidgetActionRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC3973UserDirectorySearch)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } if (typeof request.data.search_term !== "string") { return this.transport.reply(request, { error: { message: "Invalid request - missing search term" }, }); } if (request.data.limit !== undefined && request.data.limit < 0) { return this.transport.reply(request, { error: { message: "Invalid request - limit out of range" }, }); } try { const result = await this.driver.searchUserDirectory(request.data.search_term, request.data.limit); return this.transport.reply(request, { limited: result.limited, results: result.results.map((r) => ({ user_id: r.userId, display_name: r.displayName, avatar_url: r.avatarUrl, })), }); } catch (e) { console.error("error searching in the user directory", e); this.handleDriverError(e, request, "Unexpected error while searching in the user directory"); } } private async handleGetMediaConfig(request: IGetMediaConfigActionFromWidgetActionRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC4039UploadFile)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } try { const result = await this.driver.getMediaConfig(); return this.transport.reply(request, result); } catch (e) { console.error("error while getting the media configuration", e); this.handleDriverError(e, request, "Unexpected error while getting the media configuration"); } } private async handleUploadFile(request: IUploadFileActionFromWidgetActionRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC4039UploadFile)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } try { const result = await this.driver.uploadFile(request.data.file); return this.transport.reply(request, { content_uri: result.contentUri, }); } catch (e) { console.error("error while uploading a file", e); this.handleDriverError(e, request, "Unexpected error while uploading a file"); } } private async handleDownloadFile(request: IDownloadFileActionFromWidgetActionRequest): Promise { if (!this.hasCapability(MatrixCapabilities.MSC4039DownloadFile)) { return this.transport.reply(request, { error: { message: "Missing capability" }, }); } try { const result = await this.driver.downloadFile(request.data.content_uri); return this.transport.reply(request, { file: result.file }); } catch (e) { console.error("error while downloading a file", e); this.handleDriverError(e, request, "Unexpected error while downloading a file"); } } private handleDriverError(e: unknown, request: IWidgetApiRequest, message: string): void { const data = this.driver.processError(e); this.transport.reply(request, { error: { message, ...data, }, }); } private handleMessage(ev: CustomEvent): void | Promise { if (this.isStopped) return; const actionEv = new CustomEvent(`action:${ev.detail.action}`, { detail: ev.detail, cancelable: true, }); this.emit(`action:${ev.detail.action}`, actionEv); if (!actionEv.defaultPrevented) { switch (ev.detail.action) { case WidgetApiFromWidgetAction.ContentLoaded: return this.handleContentLoadedAction(ev.detail); case WidgetApiFromWidgetAction.SupportedApiVersions: return this.replyVersions(ev.detail); case WidgetApiFromWidgetAction.SendEvent: return this.handleSendEvent(ev.detail); case WidgetApiFromWidgetAction.SendToDevice: return this.handleSendToDevice(ev.detail); case WidgetApiFromWidgetAction.GetOpenIDCredentials: return this.handleOIDC(ev.detail); case WidgetApiFromWidgetAction.MSC2931Navigate: return this.handleNavigate(ev.detail); case WidgetApiFromWidgetAction.MSC2974RenegotiateCapabilities: return this.handleCapabilitiesRenegotiate(ev.detail); case WidgetApiFromWidgetAction.MSC2876ReadEvents: return this.handleReadEvents(ev.detail); case WidgetApiFromWidgetAction.WatchTurnServers: return this.handleWatchTurnServers(ev.detail); case WidgetApiFromWidgetAction.UnwatchTurnServers: return this.handleUnwatchTurnServers(ev.detail); case WidgetApiFromWidgetAction.MSC3869ReadRelations: return this.handleReadRelations(ev.detail); case WidgetApiFromWidgetAction.MSC3973UserDirectorySearch: return this.handleUserDirectorySearch(ev.detail); case WidgetApiFromWidgetAction.BeeperReadRoomAccountData: return this.handleReadRoomAccountData(ev.detail); case WidgetApiFromWidgetAction.MSC4039GetMediaConfigAction: return this.handleGetMediaConfig(ev.detail); case WidgetApiFromWidgetAction.MSC4039UploadFileAction: return this.handleUploadFile(ev.detail); case WidgetApiFromWidgetAction.MSC4039DownloadFileAction: return this.handleDownloadFile(ev.detail); case WidgetApiFromWidgetAction.MSC4157UpdateDelayedEvent: return this.handleUpdateDelayedEvent(ev.detail); default: return this.transport.reply(ev.detail, { error: { message: "Unknown or unsupported from-widget action: " + ev.detail.action, }, }); } } } /** * Informs the widget that the client's theme has changed. * @param theme The theme data, as an object with arbitrary contents. */ public updateTheme(theme: IThemeChangeActionRequestData): Promise { return this.transport.send(WidgetApiToWidgetAction.ThemeChange, theme); } /** * Informs the widget that the client's language has changed. * @param lang The BCP 47 identifier representing the client's current language. */ public updateLanguage(lang: string): Promise { return this.transport.send(WidgetApiToWidgetAction.LanguageChange, { lang }); } /** * Takes a screenshot of the widget. * @returns Resolves to the widget's screenshot. * @throws Throws if there is a problem. */ public takeScreenshot(): Promise { return this.transport.send(WidgetApiToWidgetAction.TakeScreenshot, {}); } /** * Alerts the widget to whether or not it is currently visible. * @param {boolean} isVisible Whether the widget is visible or not. * @returns {Promise} Resolves when the widget acknowledges the update. */ public updateVisibility(isVisible: boolean): Promise { return this.transport.send(WidgetApiToWidgetAction.UpdateVisibility, { visible: isVisible, }); } public sendWidgetConfig(data: IModalWidgetOpenRequestData): Promise { return this.transport.send(WidgetApiToWidgetAction.WidgetConfig, data).then(); } public notifyModalWidgetButtonClicked(id: IModalWidgetOpenRequestDataButton["id"]): Promise { return this.transport .send(WidgetApiToWidgetAction.ButtonClicked, { id }) .then(); } public notifyModalWidgetClose(data: IModalWidgetReturnData): Promise { return this.transport.send(WidgetApiToWidgetAction.CloseModalWidget, data).then(); } /** * Feeds an event to the widget. As a client you are expected to call this * for every new event in every room to which you are joined or invited. * @param {IRoomEvent} rawEvent The event to (try to) send to the widget. * @param {string} currentViewedRoomId The room ID the user is currently * interacting with. Not the room ID of the event. * @returns {Promise} Resolves when delivered or if the widget is not * able to read the event due to permissions, rejects if the widget failed * to handle the event. * @deprecated It is recommended to communicate the viewed room ID by calling * {@link ClientWidgetApi.setViewedRoomId} rather than passing it to this * method. */ public async feedEvent(rawEvent: IRoomEvent, currentViewedRoomId: string): Promise; /** * Feeds an event to the widget. As a client you are expected to call this * for every new event (including state events) in every room to which you are joined or invited. * @param {IRoomEvent} rawEvent The event to (try to) send to the widget. * @returns {Promise} Resolves when delivered or if the widget is not * able to read the event due to permissions, rejects if the widget failed * to handle the event. */ public async feedEvent(rawEvent: IRoomEvent): Promise; public async feedEvent(rawEvent: IRoomEvent, currentViewedRoomId?: string): Promise { if (currentViewedRoomId !== undefined) this.setViewedRoomId(currentViewedRoomId); if (rawEvent.room_id !== this.viewedRoomId && !this.canUseRoomTimeline(rawEvent.room_id)) { return; // no-op } if (rawEvent.state_key !== undefined && rawEvent.state_key !== null) { // state event if (!this.canReceiveStateEvent(rawEvent.type, rawEvent.state_key)) { return; // no-op } } else { // message event if (!this.canReceiveRoomEvent(rawEvent.type, (rawEvent.content as { msgtype?: string })?.["msgtype"])) { return; // no-op } } // Feed the event into the widget await this.transport.send( WidgetApiToWidgetAction.SendEvent, // it's compatible, but missing the index signature rawEvent as ISendEventToWidgetRequestData, ); } /** * Feeds a to-device event to the widget. As a client you are expected to * call this for every to-device event you receive. * @param {IRoomEvent} rawEvent The event to (try to) send to the widget. * @param {boolean} encrypted Whether the event contents were encrypted. * @returns {Promise} Resolves when delivered or if the widget is not * able to receive the event due to permissions, rejects if the widget * failed to handle the event. */ public async feedToDevice(message: IToDeviceMessage, encrypted: boolean): Promise { if (this.canReceiveToDeviceEvent(message.type)) { await this.transport.send(WidgetApiToWidgetAction.SendToDevice, { ...message, encrypted, }); } } private viewedRoomId: string | null = null; /** * Indicate that a room is being viewed (making it possible for the widget * to interact with it). */ public setViewedRoomId(roomId: string | null): void { this.viewedRoomId = roomId; // If the widget doesn't have timeline permissions for the room then // this is its opportunity to learn the room state. We push the entire // room state, which could be redundant if this room had been viewed // once before, but it's easier than selectively pushing just the bits // of state that changed while the room was in the background. if (roomId !== null && !this.canUseRoomTimeline(roomId)) this.pushRoomState(roomId); } private async flushRoomState(): Promise { try { // Only send a single action once all concurrent tasks have completed do await Promise.all(this.pushRoomStateTasks); while (this.pushRoomStateTasks.size > 0); const events: IRoomEvent[] = []; for (const eventTypeMap of this.pushRoomStateResult.values()) { for (const stateKeyMap of eventTypeMap.values()) { events.push(...stateKeyMap.values()); } } if ((await this.getWidgetVersions()).includes(UnstableApiVersion.MSC2762_UPDATE_STATE)) { // Only send state updates when using UpdateState. Otherwise the SendEvent action will be responsible for state updates. await this.transport.send(WidgetApiToWidgetAction.UpdateState, { state: events, }); } } finally { this.flushRoomStateTask = null; } } /** * Reads the current sticky state of the room and pushes it to the widget. * * It will only push events that the widget is allowed to receive. * @param roomId * @private */ private async pushStickyState(roomId: string): Promise { console.debug("Pushing sticky state to widget for room", roomId); return this.driver .readStickyEvents(roomId) .then((events) => { // filter to the allowed sticky events const filtered = events.filter((e) => { return this.canReceiveRoomEvent( e.type, typeof e.content?.msgtype === "string" ? e.content.msgtype : null, ); }); return { roomId, stickyEvents: filtered }; }) .then(async ({ roomId, stickyEvents }) => { console.debug("Pushing", stickyEvents.length, "sticky events to widget for room", roomId); const promises = stickyEvents.map((rawEvent) => { return this.transport.send( WidgetApiToWidgetAction.SendEvent, // copied from feedEvent; it's compatible, but missing the index signature rawEvent as ISendEventToWidgetRequestData, ); }); await Promise.all(promises); }); } /** * Read the room's state and push all entries that the widget is allowed to * read through to the widget. */ private pushRoomState(roomId: string): void { for (const cap of this.allowedEvents) { if (cap.kind === EventKind.State && cap.direction === EventDirection.Receive) { // Initiate the task const events = this.driver.readRoomState(roomId, cap.eventType, cap.keyStr ?? undefined); const task = events .then( (events) => { // When complete, queue the resulting events to be // pushed to the widget for (const event of events) { let eventTypeMap = this.pushRoomStateResult.get(roomId); if (eventTypeMap === undefined) { eventTypeMap = new Map(); this.pushRoomStateResult.set(roomId, eventTypeMap); } let stateKeyMap = eventTypeMap.get(cap.eventType); if (stateKeyMap === undefined) { stateKeyMap = new Map(); eventTypeMap.set(cap.eventType, stateKeyMap); } if (!stateKeyMap.has(event.state_key!)) stateKeyMap.set(event.state_key!, event); } }, (e) => console.error( `Failed to read room state for ${roomId} (${cap.eventType}, ${cap.keyStr})`, e, ), ) .then(() => { // Mark request as no longer pending this.pushRoomStateTasks.delete(task); }); // Mark task as pending this.pushRoomStateTasks.add(task); // Assuming no other tasks are already happening concurrently, // schedule the widget action that actually pushes the events this.flushRoomStateTask ??= this.flushRoomState(); this.flushRoomStateTask.catch((e) => console.error("Failed to push room state", e)); } } } /** * Feeds a room state update to the widget. As a client you are expected to * call this for every state update in every room to which you are joined or * invited. * @param {IRoomEvent} rawEvent The state event corresponding to the updated * room state entry. * @returns {Promise} Resolves when delivered or if the widget is not * able to receive the room state due to permissions, rejects if the * widget failed to handle the update. */ public async feedStateUpdate(rawEvent: IRoomEvent): Promise { if (rawEvent.state_key === undefined) throw new Error("Not a state event"); if ( (rawEvent.room_id === this.viewedRoomId || this.canUseRoomTimeline(rawEvent.room_id)) && this.canReceiveStateEvent(rawEvent.type, rawEvent.state_key) ) { // Updates could race with the initial push of the room's state if (this.pushRoomStateTasks.size === 0) { // No initial push tasks are pending; safe to send immediately if ((await this.getWidgetVersions()).includes(UnstableApiVersion.MSC2762_UPDATE_STATE)) { // Only send state updates when using UpdateState. Otherwise the SendEvent action will be responsible for state updates. await this.transport.send(WidgetApiToWidgetAction.UpdateState, { state: [rawEvent], }); } } else { // Lump the update in with whatever data will be sent in the // initial push later. Even if we set it to an "outdated" entry // here, we can count on any newer entries being passed to this // same method eventually; this won't cause stuck state. let eventTypeMap = this.pushRoomStateResult.get(rawEvent.room_id); if (eventTypeMap === undefined) { eventTypeMap = new Map(); this.pushRoomStateResult.set(rawEvent.room_id, eventTypeMap); } let stateKeyMap = eventTypeMap.get(rawEvent.type); if (stateKeyMap === undefined) { stateKeyMap = new Map(); eventTypeMap.set(rawEvent.type, stateKeyMap); } if (!stateKeyMap.has(rawEvent.type)) stateKeyMap.set(rawEvent.state_key, rawEvent); do await Promise.all(this.pushRoomStateTasks); while (this.pushRoomStateTasks.size > 0); await this.flushRoomStateTask; } } } }