/** * @module smwebsdk */ import { SmEvent } from './SmEvent'; import { LocalSession } from './LocalSession'; import { Session } from './Session'; import { WebSocketSession } from './WebSocketSession'; import { WebsocketResponse } from './websocket-message/index'; import { SceneRequestBody, SceneResponse } from './websocket-message/scene/index'; import { AudioSourceTypes } from './enums/index'; import { ConfigurationModel, PersonaEventMap } from './models/index'; import { SceneResponseBody } from './websocket-message/scene/SceneResponse'; import { ContentAwareness } from './ContentAwareness'; import { LogLevel } from './utils/Logger'; import { Context } from '@opentelemetry/api'; import { LoggingConfig, MediaDeviceOptions, RetryOptions, UserMedia } from './types/scene'; import { Conversation } from './Conversation'; import { PersonaId } from './models/PersonaId'; import { ConnectionState } from './ConnectionState'; import { ChromaKeyOptions } from './dedicated-workers/chroma-key.worker'; /** * Configuration to use when constructing a Scene * * @public */ export interface SceneOptions { /** The HTMLVideoElement where the digital person should be displayed */ videoElement?: HTMLVideoElement; /** Whether the connection should be established with audio but no video */ audioOnly?: boolean; /** Default true. True means that the DP's speech should be interrupted when the user changes tabs */ stopSpeakingWhenNotVisible?: boolean; /** Media devices (camera and microphone) to request from the user before connecting */ requestedMediaDevices?: MediaDeviceOptions; /** Media devices (camera and microphone) that the user must approve access to for the connection to succeed */ requiredMediaDevices?: MediaDeviceOptions; /** Frequency of sending on-screen content measurements to the server in milliseconds */ contentAwarenessDebounceTime?: number; /** Preferred logging levels for websdk */ loggingConfig?: LoggingConfig; /** The API key used to authenticate with the server */ apiKey?: string; /** The config to allow sending current page url when connection succeeds */ sendMetadata?: { pageUrl: boolean; }; /** Configuration of the OpenTelemetry tracer */ tracerOptions?: TracerOptions; /** Chroma key options * @internal */ chromaKeyOptions?: ChromaKeyOptions; } /** * Configuration to use when connecting to a Scene * * @public */ export interface ConnectOptions { /** Options for customizing connection error retries */ retryOptions?: RetryOptions; /** A custom text string that is sent to the orchestration server */ userText?: string; /** The custom token server config */ tokenServer?: { /** The server websocket uri */ uri: string; /** A jwt access token issued to permit access to the server */ token: string; }; } /** * Available configuration options for the `Scene.connect` tracer. */ export type TracerOptions = { /** Suppress tracing by OpenTelemetry. */ disableTracing: boolean; /** The context to use as the parent for tracing. */ parentCtx: Context; /** The endpoint to use for sending traces */ url: string; }; /** * Scene class to hold a webrtc connection to a scene containing a persona. * @public */ export declare class Scene { private _apiKey; private _videoElement; private _audioOnly; private _requestedUserMedia; private _requiredUserMedia; /** * set to function(scene, state) called when a state message is received as per the scene protocol * (DEPRECATED: onStateEvent.addListener allows for multiple listeners). */ private _onState; private _onStateEvent; /** * set to function(scene, results) called when speech to text results are recognized, * results are documented in scene protocol * (DEPRECATED: onRecognizeResultsEvent.addListener allows for multiple listeners). */ private _onRecognizeResults; private _onRecognizeResultsEvent; private _metadataSender; /** * set to function(scene, sessionId, reason) called when the session is disconnected. * 'reason' can be one of 'normal' or 'sessionTimeout' * (DEPRECATED: onDisconnectedEvent.addListener allows for multiple listeners). */ private _onDisconnected; private _onDisconnectedEvent; /** set to function(scene, text) called when a custom text message is sent from the orchestration server */ private _onUserText; private _onUserTextEvent; /** Demo mode events */ private _onDemoMode; private _onDemoModeEvent; private _onConversationResultEvents; private _onSpeechMarkerEvents; private _session; private _underRuntimeHost; private _isWebSocketOnly; private _transactionId; private _pendingResponses; private _sceneId; private _microphoneUnmuteTimer; private _echoCancellationEnabled; private _serverControlledCameras; private _stopSpeakingWhenNotVisible; private _loggingConfig; private _logger; private _tracerOptions; private _chromaKeyOptions; connectionResult: any; contentAwareness: ContentAwareness | undefined; contentAwarenessDebounceTime: number | undefined; private _sessionResumeEnabled; private _isResumedSession; private _sendMetadata; private _onMicrophoneActive; private _onCameraActive; conversation: Conversation; connectionState: ConnectionState; currentPersonaId: PersonaId; /** Returns the version of the webSdk and platformSdk */ version: { webSdk: string; platformSdk: string; }; /** * Tests the first value of the Scene construtor to decide if * it matches the new-style config options format. */ private isSceneOptions; /** * Construct Scene from options object * @param options - {@link SceneOptions} Optional configuration for the Scene. May be an empty object. */ constructor(options: SceneOptions); /** * Construct Scene with parameters * @deprecated Use `new Scene(options: SceneOptions)` instead * @param videoElement - A video element that will display the connected scene * @param audioOnly - This streaming should be audio streaming only (no video streaming) * @param requestedUserMedia - The user media devices (microphone/camera) that should be requested, one of: * UserMedia.None, UserMedia.Microphone, UserMedia.MicrophoneAndCamera (default) * @param requiredUserMedia - Required user media devices, one of: * UserMedia.None, UserMedia.Microphone, UserMedia.MicrophoneAndCamera * If less user media devices are requested then are required then the requirements takes precedence. * If this user media requirements is not met then Connect() will fail. * @param contentAwarenessDebounceTime - The timeout period used for debouncing messaging within the content awareness class * @param loggingConfig - Options to configure different log levels for different classes */ constructor(videoElement?: HTMLVideoElement, audioOnly?: boolean, requestedUserMedia?: UserMedia, requiredUserMedia?: UserMedia, contentAwarenessDebounceTime?: number, loggingConfig?: Partial, tracerOptions?: TracerOptions, chromaKeyOptions?: ChromaKeyOptions); connectionValid(): boolean; /** * Check if the scene connection is open and valid. * * @returns Returns true if the connection is open and valid otherwise false. */ isConnected(): boolean; /** * Extends the server side timeout. This also happens automatically whenever the persona speaks. */ keepAlive(): void; /** * Disconnects the session */ disconnect(): void; private iosVisibilityChange; /** * Connect to a scene using options object */ connect(options?: ConnectOptions): Promise; /** * Connect to a scene at the given server uri. * * @param serverUri - The server websocket uri to connect to. * @param userText - A custom text string that is sent to the orchestration server. * @param accessToken - A jwt access token issued to permit access to this server. * @param retryOptions - Options for customizing connection error retry. * @returns Returns a promise that holds success/failure callbacks. If the promise * is rejected then an Eror is given as the argument * converts to a string message. The error result has two fields 'message' * which is the string message and 'name' which is one of the following * error name/reason codes: * - **notSupported** - the browser does not support getUserMedia * - **noUserMedia** - the microphone/camera is either not available, not usable or the user declined permission to use them * - **serverConnectionFailed** - the connection to the server failed * - **noScene** - no persona was available * - **mediaStreamFailed** - the audio/video stream failed * - **sessionTimeout** - the session timed out before it was fully available */ connect(serverUri?: string, userText?: string, accessToken?: string, retryOptions?: RetryOptions): Promise; private initializeTracer; private initTelemetryToken; onMessage(message: WebsocketResponse): void; sendOnewaySceneRequest(name: string, body: SceneRequestBody): void; /** * The internal method used for sending request messages. * * All offically supported message have their own public methods (e.g. `conversationSend()` or `scene.startRecognize()`). \ * Please use those instead. * * @internal */ sendRequest(name: string, body?: SceneRequestBody): Promise; onSceneMessage(message: SceneResponse): void; protected processResponse(body: SceneResponseBody, name: string, status: number, transaction: string): void; private controlMicrophoneMute; /** Close the current scene connection */ private close; private stopSpeakingWhenNotVisible; private stopSpeakingWhenUnloaded; private sessionConnected; private cleanupEventListeners; private sessionClosed; private rtcUserText; private enableFlaggedFeatures; sendContent(): void; /** * Sends updated video element size to server * this gives the app the chance to choose what size should be rendered on server * and the application is responsible to register for a video element size change * event and call this method to maintain best possible video quality for the size * and/or to set an updated video element size and then call this method. * @param width - The width in pixels to render the video * @param height - The height in pixels to render the video */ sendVideoBounds(width: number, height: number): void; /** * Send configuration to the scene * @param configuration - Scene configuration as per the scene protocol */ configure(configuration: ConfigurationModel): Promise; /** * Send a custom user text message to the orchestration server * @param text - Custom text sent to the orchestration server */ private sendUserText; /** * Start the speech to text recognizer * @param audioSource - The audio source either smwebsdk.audioSource.processed or * smwebsdk.audioSource.squelched, defaults to processed. */ startRecognize(audioSource?: AudioSourceTypes): Promise; /** Stop the speech to text reconizer */ stopRecognize(): Promise; /** Is the microphone connected in the session */ isMicrophoneConnected(): boolean | null; /** Is the camera connected in the session */ isCameraConnected(): boolean | null; session(): Session | LocalSession | WebSocketSession | undefined; hasContentAwareness(): boolean; hasServerControlledCameras(): boolean; /** * Check if session persistence feature is supported in current session * * @returns `boolean` * * Usage: * ```javascript * const isSessionPersistenceSupported = scene.supportsSessionPersistence(); * ``` */ supportsSessionPersistence(): boolean; /** * Check if current session is a new session or a resumed session * * @returns `boolean` * * Usage: * ```javascript * const isResumedSession = scene.isResumedSession(); * ``` */ isResumedSession(): boolean; get onConversationResultEvents(): PersonaEventMap; get onSpeechMarkerEvents(): PersonaEventMap; /** Get the current scene state */ getState(): Promise; get onStateEvent(): SmEvent; set onState(onState: Function); get onDisconnectedEvent(): SmEvent; set onDisconnected(onDisconnected: Function); get onRecognizeResultsEvent(): SmEvent; set onRecognizeResults(onRecognizeResults: Function); get onUserTextEvent(): SmEvent; set onUserText(onUserText: Function); get echoCancellationEnabled(): boolean; set echoCancellationEnabled(enabled: boolean); /** * @internal */ get onDemoModeEvent(): SmEvent; get videoElement(): HTMLVideoElement | undefined; set videoElement(videoElement: HTMLVideoElement | undefined); get viewerOffsetX(): number; get viewerOffsetY(): number; get isWebSocketOnly(): boolean; set isWebSocketOnly(isWebSocketOnly: boolean); /** * @returns an {@link SmEvent} associated with the microphone. * * Listeners can then be added to this event allowing you to call functions when the microphone active status changes. * * Usage: * ```javascript * scene.onMicrophoneActive.addListener( * (active) => console.log('Microphone Active: ', active)); * ``` */ get onMicrophoneActive(): SmEvent; /** * Specifies if the microphone is currently active and streaming audio * to the server. * * @returns `boolean` * * Usage: * ```javascript * const isMicrophoneActive = scene.isMicrophoneActive(); * ``` */ isMicrophoneActive(): boolean; /** * @returns an {@link SmEvent} associated with the camera. * * Listeners can then be added to this event allowing you to call functions when the camera active status changes. * * Usage: * ```javascript * scene.onCameraActive.addListener( * (active) => console.log('Camera Active: ', active)); * ``` */ get onCameraActive(): SmEvent; /** * Specifies if the camera is currently active and streaming video * to the server. * * @returns `boolean` * * Usage: * ```javascript * const isCameraActive = scene.isCameraActive(); * ``` */ isCameraActive(): boolean; /** * On success, starts or stops streaming video/audio to the server based on the values of `microphone` and `camera`. * * @param options.microphone - If `true`, activates the microphone and starts streaming audio. \ * If `false` deactivates the microphone and stops streaming audio. \ * If not set, microphone will retain its existing state. * @param options.camera - If `true`, activates the camera and starts streaming video. \ * If `false` deactivates the camera and stops streaming video. \ * If not set, microphone will retain its existing state. * * @returns Returns a promise which is fulfilled when the media active state has been successfully changed. \ * If the session is not defined it will return `undefined`. \ * If the active state could not be changed, the promise is rejected with an Error object having the format: * ```javascript * { * message: string; * name: errorCode; * } * ``` * Where `errorCode` is one of: * - `noUserMedia` - the microphone/camera is either not available, not usable or the user declined permission to use them * - `failedUpgrade` - the media upgrade failed * - `notSupported` - user’s browser does not support the getUserMedia API * - `noConnection` - connection has not been established - ensure scene.connect() has been called previously * * Usage: * ```javascript * scene.setMediaDeviceActive({ microphone: true, camera: false }) * .then(console.log('microphone activated, camera deactivated')); * .catch((error) => console.log('error occurred: ', error); * ``` */ setMediaDeviceActive(options: { microphone?: boolean; camera?: boolean; }): Promise; /** * Play the video element and return results. Different browsers have different restrictions on autoplay. * Using this method can handle all the cases browsers can have on inital video playback. * @param videoElement - Optional parameter specifying the video element hosting the Digital Person. If not specified the video element passed to the Scene constructor will be used. * @returns Returns a promise which is fulfilled when the video playback is successful, with indication of video and audio status. * If the video element is not defined or video play fails the promise is rejected with an Error object having the format: * ```javascript * { * message: string; * name: errorCode; * } * ``` * Where `errorCode` is one of: * - `noVideoElement` - no HTMLVideoElement found from `videoElement` or `Scene` constructor * - `userInteractionRequired` - cannot start media playback due to browser restriction; user interaction is required before playing again * * Usage: * ```javascript * scene.startVideo() * .then(({ video, audio }) => { * if (!audio) { * //video is muted, ask user to unmute video * } * }) * .catch((error) => { * if (error.name === 'userInteractionRequired') { * //ask user to interact with the UI * //unmute video and play again * video.muted = false; * video.play(); * } * }); * ``` */ startVideo(videoElement?: HTMLVideoElement): Promise<{ video: boolean; audio: boolean; } | Error>; private playVideo; private fetchAuthConfig; private connectArgsToConfig; /** * Check if the session logging is enabled. * * @returns Returns true if the session logging is enabled otherwise false. */ isLoggingEnabled(): boolean; /** * Check minimal log level of session logging. * * @returns Returns minimal log setting of session logging, type is LogLevel. */ getMinLogLevel(): "debug" | "log" | "warn" | "error"; /** * Enable/disable session logging * @param enable - set true to enable session log, false to disable */ setLogging(enable: boolean): void; /** * Set minimal log level of session logging. * @param level - use LogLevel type to set minimal log level of session logging */ setMinLogLevel(level: LogLevel): void; } //# sourceMappingURL=Scene.d.ts.map