import { Animated, EasingFunction, StyleProp, ViewStyle } from 'react-native'; import React, { PureComponent } from 'react'; import { Bubble } from './components/SpeechBubbles'; import { PipelineProfile, SpokestackConfig, SpokestackErrorEvent, SpokestackNLUResult, TTSFormat } from 'react-native-spokestack'; export interface IntentResult { /** * A user-defined key to indicate where the user is in the conversation * Include this in the `exitNodes` prop if Spokestack should not listen * again after saying the prompt. */ node: string; /** Will be processed into Speech unless the tray is in silent mode */ prompt: string; /** * Set to `true` to stop the wakeword recognizer * during playback of the prompt. */ noInterrupt?: boolean; /** Any other data you might want to add */ data?: any; } export interface SpokestackTrayProps { /** * Your Spokestack tokens generated in your Spokestack account * at https://spokestack.io/account. * Create an account for free then generate a token. * This is from the "ID" field. */ clientId: string; /** * Your Spokestack tokens generated in your Spokestack account * at https://spokestack.io/account. * Create an account for free then generate a token. * This is from the "secret" field. */ clientSecret: string; /** * This function takes an intent from the NLU * and returns an object with a unique conversation * node name (that you define) and a prompt * to be processed by TTS and spoken. * * Note: the prompt is only shown in a chat bubble * if sound has been turned off. */ handleIntent: (intent: string, slots?: SpokestackNLUResult['slots'], utterance?: string) => IntentResult; /** * The NLU Tensorflow Lite model (.tflite), JSON metadata, and NLU vocabulary (.txt) * * All 3 fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://spokestack.io/docs/concepts/nlu to learn more about NLU. * * ```js * // ... * nlu={{ * model: 'https://somecdn.com/nlu.tflite', * vocab: 'https://somecdn.com/vocab.txt', * metadata: 'https://somecdn.com/metadata.json' * }} * ``` * * You can also pass local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * nlu={{ * model: require('./nlu.tflite'), * vocab: require('./vocab.txt'), * // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import` * // so the file is not parsed when included but instead imported as a source object. This makes it so the * // file is read and parsed by the underlying native libraries instead. * metadata: require('./metadata.sjson') * }} * ``` */ nlu: SpokestackConfig['nlu']; /** Width (and height) of the mic button */ buttonWidth?: number; /** How long to wait to close the tray after speaking (ms) */ closeDelay?: number; /** Show debug messages from react-native-spokestack */ debug?: boolean; /** Duration for the tray animation (ms) */ duration?: number; /** Easing function for the tray animation */ easing?: EasingFunction; /** * Edit the transcript before classification * and before the user response bubble is shown. */ editTranscript?: (transcript: string) => string; /** * All nodes in this array should end * the conversation and close the tray */ exitNodes?: string[]; /** * Font to use for "LISTENING...", "LOADING...", * and chat bubble text. */ fontFamily?: string; /** * Colors for the linear gradient shown when listening * Can be any number of colors (recommended: 2-3) */ gradientColors?: string[]; /** * Whether to greet the user with a welcome message * when the tray opens. * Note: `handleIntent` must respond to the "greet" intent. */ greet?: boolean; /** * Set this to false to disable the haptic * that gets played whenever the tray starts listening. */ haptic?: boolean; /** * Configuration for keyword recognition * * The filter, detect, encode, and metadata fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://www.spokestack.io/docs/concepts/keywords to learn more about keyword recognition. * * @example * ```js * // ... * keyword={{ * detect: 'https://s.spokestack.io/u/UbMeX/detect.tflite', * encode: 'https://s.spokestack.io/u/UbMeX/encode.tflite', * filter: 'https://s.spokestack.io/u/UbMeX/filter.tflite', * metadata: 'https://s.spokestack.io/u/UbMeX/metadata.json' * }} * ``` * * You can also download models ahead of time and include them from local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * keyword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite'), * // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import` * // so the file is not parsed when included but instead imported as a source object. This makes it so the * // file is read and parsed by the underlying native libraries instead. * metadata: require('./metadata.sjson') * }} * ``` * * Keyword configuration also accepts a classes field for when metadata is not specified. * * ```js * // ... * keyword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite'), * classes: ['one', 'two', 'three] * }} * ``` */ keyword?: SpokestackConfig['keyword']; /** Minimum height for the tray */ minHeight?: number; /** * Called whenever the tray has closed */ onClose?: () => void; /** Called whenever there's an error from Spokestack */ onError?: (e: SpokestackErrorEvent) => void; /** Called whenever the tray has opened */ onOpen?: () => void; /** * The tray button can be oriented on either side of the screen */ orientation?: 'left' | 'right'; /** * This color is used to theme the tray * and is used in the mic button and speech bubbles. */ primaryColor?: string; /** * The Spokestack config profile to pass to * react-native-spokestack. * These are available from react-native-spokestack * starting in version 4.0.0. * * If Wakeword config files are specified, the default will be * `TFLITE_WAKEWORD_NATIVE_ASR`. * Otherwise, the default is `PTT_NATIVE_ASR`. * * ```js * import SpokestackTray from 'react-native-spokestack-tray' * import { PipelineProfile } from 'react-native-spokestack' * * // ... * ` */ refreshModels?: boolean; /** * Whether to speak the greeting or only display * a chat bubble with the greet message, * even if sound is on. */ sayGreeting?: boolean; /** Replace the sound on image by passing a React Image component */ soundOnImage?: React.ReactNode; /** Replace the sound off image by passing a React Image component */ soundOffImage?: React.ReactNode; /** * Pass options directly to the Spokestack.initialize() * function from react-native-spokestack. * See https://github.com/spokestack/react-native-spokestack * for available options. */ spokestackConfig?: Partial; /** Starting height for tray */ startHeight?: number; /** This style prop is passed to the tray's container */ style?: Animated.WithAnimatedValue>; /** The format for the text passed to Spokestack.synthesize */ ttsFormat?: TTSFormat; /** * A key for a voice in Spokestack TTS, passed to Spokestack.synthesize. * This may only be changed if you have created a custom voice using a * Spokestack Maker account. See https://spokestack.io/pricing#maker. * If not specified, Spokestack's Free "demo-male" voice is used. */ voice?: string; /** * The NLU Tensorflow Lite models (.tflite) for wakeword. * * All 3 fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://spokestack.io/docs/concepts/wakeword-models to learn more about Wakeword * * Spokestack offers sample wakeword model files ("Spokestack"): * * ```js * // ... * wakeword={{ * detect: 'https://s.spokestack.io/u/hgmYb/detect.tflite', * encode: 'https://s.spokestack.io/u/hgmYb/encode.tflite', * filter: 'https://s.spokestack.io/u/hgmYb/filter.tflite' * }} * ``` * * You can also download these models ahead of time and include them from local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * wakeword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite') * }} * ``` */ wakeword?: SpokestackConfig['wakeword']; } interface State { bubbles: Bubble[]; height: number; listening: boolean; listeningWidth: number; loading: boolean; open: boolean; playerSource: string; pressed: boolean; silent: boolean; startHeight: number; } export default class SpokestackTray extends PureComponent { private wentToBackground; private hasStarted; private windowWidth; private windowHeight; private panX; private shadowOpacity; private listenWhenDone; private utterance; private openPanResponder; private expandPanResponder; static defaultProps: Partial; state: State; componentDidMount(): Promise; componentWillUnmount(): void; private initState; private addListeners; private appStateChange; private handleIntent; private onRecognize; private onActivate; private onDeactivate; private onStart; private onError; private handleError; private showHandle; private onEnd; private constrainX; private constrainHeight; private listen; private opened; private openOrClose; /** * Open the tray, greet (if applicable), and listen */ open: () => void; /** * Close the tray, stop listening, and restart wakeword */ close: () => void; /** * Passes the input to Spokestack.synthesize(), * plays the audio, and adds a speech bubble. */ say: (input: string) => Promise; /** * Add a bubble (system or user) * to the chat interface */ addBubble: (bubble: Bubble) => void; /** * Toggle silent mode */ toggleSilent: () => Promise; /** * Returns whether the tray is in silent mode */ isSilent: () => boolean; render(): JSX.Element; } export {};