import { Animated, AppState, AppStateStatus, Dimensions, Easing, EasingFunction, GestureResponderEvent, Image, Keyboard, PanResponder, PanResponderGestureState, SafeAreaView, StyleProp, StyleSheet, Text, TouchableOpacity, View, ViewStyle } from 'react-native' import React, { PureComponent } from 'react' import SpeechBubbles, { Bubble } from './components/SpeechBubbles' import Spokestack, { PipelineProfile, SpokestackConfig, SpokestackErrorEvent, SpokestackNLUResult, SpokestackRecognizeEvent, TTSFormat, TraceLevel } from 'react-native-spokestack' import { getSilent, setSilent } from './utils/settings' import { listen, stopListening } from './Spokestack' import Color from 'color' import HapticFeedback from 'react-native-haptic-feedback' import Video from 'react-native-video' import arrowImage from './images/icon-arrow-left.png' import { checkSpeech } from './utils/permissions' import merge from 'lodash/merge' import micImage from './images/icon-mic.png' import poweredImage from './images/powered-by-spokestack.png' import soundOffImage from './images/icon-sound-off.png' import soundOnImage from './images/icon-sound-on.png' const errorMessage = 'Sorry! We hit an error. Please check your network or restart the app and try again.' export interface IntentResult { /** * A user-defined key to indicate where the user is in the conversation * Include this in the `exitNodes` prop if Spokestack should not listen * again after saying the prompt. */ node: string /** Will be processed into Speech unless the tray is in silent mode */ prompt: string /** * Set to `true` to stop the wakeword recognizer * during playback of the prompt. */ noInterrupt?: boolean /** Any other data you might want to add */ data?: any } export interface SpokestackTrayProps { /** * Your Spokestack tokens generated in your Spokestack account * at https://spokestack.io/account. * Create an account for free then generate a token. * This is from the "ID" field. */ clientId: string /** * Your Spokestack tokens generated in your Spokestack account * at https://spokestack.io/account. * Create an account for free then generate a token. * This is from the "secret" field. */ clientSecret: string /** * This function takes an intent from the NLU * and returns an object with a unique conversation * node name (that you define) and a prompt * to be processed by TTS and spoken. * * Note: the prompt is only shown in a chat bubble * if sound has been turned off. */ handleIntent: ( intent: string, slots?: SpokestackNLUResult['slots'], utterance?: string ) => IntentResult /** * The NLU Tensorflow Lite model (.tflite), JSON metadata, and NLU vocabulary (.txt) * * All 3 fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://spokestack.io/docs/concepts/nlu to learn more about NLU. * * ```js * // ... * nlu={{ * model: 'https://somecdn.com/nlu.tflite', * vocab: 'https://somecdn.com/vocab.txt', * metadata: 'https://somecdn.com/metadata.json' * }} * ``` * * You can also pass local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * nlu={{ * model: require('./nlu.tflite'), * vocab: require('./vocab.txt'), * // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import` * // so the file is not parsed when included but instead imported as a source object. This makes it so the * // file is read and parsed by the underlying native libraries instead. * metadata: require('./metadata.sjson') * }} * ``` */ nlu: SpokestackConfig['nlu'] /** Width (and height) of the mic button */ buttonWidth?: number /** How long to wait to close the tray after speaking (ms) */ closeDelay?: number /** Show debug messages from react-native-spokestack */ debug?: boolean /** Duration for the tray animation (ms) */ duration?: number /** Easing function for the tray animation */ easing?: EasingFunction /** * Edit the transcript before classification * and before the user response bubble is shown. */ editTranscript?: (transcript: string) => string /** * All nodes in this array should end * the conversation and close the tray */ exitNodes?: string[] /** * Font to use for "LISTENING...", "LOADING...", * and chat bubble text. */ fontFamily?: string /** * Colors for the linear gradient shown when listening * Can be any number of colors (recommended: 2-3) */ gradientColors?: string[] /** * Whether to greet the user with a welcome message * when the tray opens. * Note: `handleIntent` must respond to the "greet" intent. */ greet?: boolean /** * Set this to false to disable the haptic * that gets played whenever the tray starts listening. */ haptic?: boolean /** * Configuration for keyword recognition * * The filter, detect, encode, and metadata fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://www.spokestack.io/docs/concepts/keywords to learn more about keyword recognition. * * @example * ```js * // ... * keyword={{ * detect: 'https://s.spokestack.io/u/UbMeX/detect.tflite', * encode: 'https://s.spokestack.io/u/UbMeX/encode.tflite', * filter: 'https://s.spokestack.io/u/UbMeX/filter.tflite', * metadata: 'https://s.spokestack.io/u/UbMeX/metadata.json' * }} * ``` * * You can also download models ahead of time and include them from local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * keyword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite'), * // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import` * // so the file is not parsed when included but instead imported as a source object. This makes it so the * // file is read and parsed by the underlying native libraries instead. * metadata: require('./metadata.sjson') * }} * ``` * * Keyword configuration also accepts a classes field for when metadata is not specified. * * ```js * // ... * keyword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite'), * classes: ['one', 'two', 'three] * }} * ``` */ keyword?: SpokestackConfig['keyword'] /** Minimum height for the tray */ minHeight?: number /** * Called whenever the tray has closed */ onClose?: () => void /** Called whenever there's an error from Spokestack */ onError?: (e: SpokestackErrorEvent) => void /** Called whenever the tray has opened */ onOpen?: () => void /** * The tray button can be oriented on either side of the screen */ orientation?: 'left' | 'right' /** * This color is used to theme the tray * and is used in the mic button and speech bubbles. */ primaryColor?: string /** * The Spokestack config profile to pass to * react-native-spokestack. * These are available from react-native-spokestack * starting in version 4.0.0. * * If Wakeword config files are specified, the default will be * `TFLITE_WAKEWORD_NATIVE_ASR`. * Otherwise, the default is `PTT_NATIVE_ASR`. * * ```js * import SpokestackTray from 'react-native-spokestack-tray' * import { PipelineProfile } from 'react-native-spokestack' * * // ... * ` */ refreshModels?: boolean /** * Whether to speak the greeting or only display * a chat bubble with the greet message, * even if sound is on. */ sayGreeting?: boolean /** Replace the sound on image by passing a React Image component */ soundOnImage?: React.ReactNode /** Replace the sound off image by passing a React Image component */ soundOffImage?: React.ReactNode /** * Pass options directly to the Spokestack.initialize() * function from react-native-spokestack. * See https://github.com/spokestack/react-native-spokestack * for available options. */ spokestackConfig?: Partial /** Starting height for tray */ startHeight?: number /** This style prop is passed to the tray's container */ style?: Animated.WithAnimatedValue> /** The format for the text passed to Spokestack.synthesize */ ttsFormat?: TTSFormat /** * A key for a voice in Spokestack TTS, passed to Spokestack.synthesize. * This may only be changed if you have created a custom voice using a * Spokestack Maker account. See https://spokestack.io/pricing#maker. * If not specified, Spokestack's Free "demo-male" voice is used. */ voice?: string /** * The NLU Tensorflow Lite models (.tflite) for wakeword. * * All 3 fields accept 2 types of values. * 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN). * 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`) * * See https://spokestack.io/docs/concepts/wakeword-models to learn more about Wakeword * * Spokestack offers sample wakeword model files ("Spokestack"): * * ```js * // ... * wakeword={{ * detect: 'https://s.spokestack.io/u/hgmYb/detect.tflite', * encode: 'https://s.spokestack.io/u/hgmYb/encode.tflite', * filter: 'https://s.spokestack.io/u/hgmYb/filter.tflite' * }} * ``` * * You can also download these models ahead of time and include them from local files. * Note: this requires a change to your metro.config.js. For more info, see * "Including model files in your app bundle" in the README.md. * * ```js * // ... * wakeword={{ * detect: require('./detect.tflite'), * encode: require('./encode.tflite'), * filter: require('./filter.tflite') * }} * ``` */ wakeword?: SpokestackConfig['wakeword'] } interface State { bubbles: Bubble[] height: number listening: boolean listeningWidth: number loading: boolean open: boolean playerSource: string pressed: boolean silent: boolean startHeight: number } export default class SpokestackTray extends PureComponent< SpokestackTrayProps, State > { private wentToBackground: boolean private hasStarted = false private windowWidth = Dimensions.get('window').width private windowHeight: number private panX = new Animated.Value(0) private shadowOpacity = new Animated.Value(0) private listenWhenDone = false private utterance = '' private openPanResponder = PanResponder.create({ onStartShouldSetPanResponder: () => true, onMoveShouldSetPanResponder: () => true, onPanResponderGrant: () => { const { height } = this.state this.setState({ pressed: true, startHeight: height }) const { width: windowWidth, height: windowHeight } = Dimensions.get('window') this.windowWidth = windowWidth this.windowHeight = windowHeight }, onPanResponderMove: (_event, { dx, dy }) => { this.panX.setValue(this.constrainX(dx)) this.setState({ height: this.constrainHeight(dy) }) }, onPanResponderRelease: (_event, { dx, dy }) => { const { orientation } = this.props const shouldOpen = (Math.abs(dx) < 2 && Math.abs(dy) < 2) || (orientation === 'left' ? dx > this.windowWidth / 3 : dx < -this.windowWidth / 3) this.openOrClose(shouldOpen) }, onPanResponderTerminate: () => { this.openOrClose(false) } }) private expandPanResponder = PanResponder.create({ onMoveShouldSetPanResponder: () => true, onPanResponderGrant: () => { const { height } = this.state this.setState({ startHeight: height }) this.windowHeight = Dimensions.get('window').height }, onPanResponderMove: ( _event: GestureResponderEvent, { dy }: PanResponderGestureState ) => { this.setState({ height: this.constrainHeight(dy) }) } }) static defaultProps: Partial = { buttonWidth: 60, closeDelay: 0, duration: 500, editTranscript: (transcript) => transcript, easing: Easing.bezier(0.77, 0.41, 0.2, 0.84), gradientColors: ['#61fae9', '#2F5BEA'], greet: false, haptic: true, minHeight: 170, orientation: 'left', primaryColor: '#2f5bea', sayGreeting: true, soundOnImage: ( ), soundOffImage: ( ), startHeight: 220, ttsFormat: TTSFormat.TEXT, voice: 'demo-male' } state: State = { bubbles: [], height: this.props.startHeight, listening: false, listeningWidth: 0, loading: false, open: false, playerSource: '', pressed: false, silent: false, startHeight: this.props.startHeight } async componentDidMount() { const { clientId, clientSecret, debug, keyword, nlu, profile, refreshModels, spokestackConfig = {}, wakeword } = this.props await this.initState() this.addListeners() try { await Spokestack.initialize( clientId, clientSecret, merge(spokestackConfig, { traceLevel: debug ? TraceLevel.DEBUG : TraceLevel.NONE, refreshModels, pipeline: { profile }, keyword, nlu, wakeword }) ) } catch (error) { console.error('Error during Spokestack.initialize()') this.handleError(error) } if (await checkSpeech()) { try { await Spokestack.start() } catch (error) { console.error('Error attempting to start on componentDidMount') this.handleError(error) } } this.showHandle() } componentWillUnmount() { AppState.removeEventListener('change', this.appStateChange) Spokestack.destroy() } private async initState() { this.setState({ silent: await getSilent() }) } private addListeners() { AppState.addEventListener('change', this.appStateChange) Spokestack.addEventListener('recognize', this.onRecognize) Spokestack.addEventListener('activate', this.onActivate) Spokestack.addEventListener('deactivate', this.onDeactivate) Spokestack.addEventListener('start', this.onStart) Spokestack.addEventListener('timeout', this.close) Spokestack.addEventListener('error', this.onError) if (this.props.debug) { Spokestack.addEventListener('trace', console.log.bind(console)) } } private appStateChange = async (nextAppState: AppStateStatus) => { console.log( `App state changed to ${nextAppState}, and wentToBackground is ${this.wentToBackground}` ) // Enable/disable wakeword based on app state if (nextAppState === 'active') { // Needs to be coming from the background // and the pipeline must have been started at least once already if (this.wentToBackground && this.hasStarted && (await checkSpeech())) { console.log('App became active. Starting speech pipeline.') Spokestack.start() } this.wentToBackground = false } else if (nextAppState === 'background') { this.wentToBackground = true console.log('App went to background. Stopping speech pipeline.') Spokestack.stop() } } private async handleIntent(result: SpokestackNLUResult) { const { closeDelay, exitNodes, handleIntent, onError, sayGreeting } = this.props const { silent } = this.state const response = handleIntent(result.intent, result.slots, this.utterance) console.log(`Processed intent ${result.intent}`, response) const shouldListen = exitNodes.indexOf(response.node) === -1 if (response.prompt) { const isGreeting = result.intent === 'greet' if (silent || (isGreeting && !sayGreeting)) { this.addBubble({ text: response.prompt, isLeft: true }) if (shouldListen) { setTimeout(this.listen, 200) } else { setTimeout(this.close, closeDelay) } } else { if (response.noInterrupt) { await Spokestack.stop() } this.listenWhenDone = shouldListen this.say(response.prompt) } } else if (onError) { onError({ error: 'No prompt returned in the response' }) } } private onRecognize = async ({ transcript }: SpokestackRecognizeEvent) => { const { editTranscript } = this.props console.log('[Spokestack onRecognize]:', transcript) const edited = editTranscript(transcript) console.log('Transcript after editing: ', edited) this.utterance = edited this.addBubble({ text: edited, isLeft: false }) // Only call listeners if there's a transcript if (edited.length > 0) { try { const result = await Spokestack.classify(edited) if (result) { this.handleIntent(result) } } catch (error) { console.error('Error during classification') this.handleError(error) } } } private onActivate = () => { const { haptic } = this.props this.setState({ listening: true, loading: false }, this.open) if (haptic) { HapticFeedback.trigger('impactHeavy', { enableVibrateFallback: true }) } } private onDeactivate = () => { this.setState({ listening: false }) } private onStart = () => { this.hasStarted = true } private onError = (error: SpokestackErrorEvent) => { this.setState({ listening: false }) this.handleError(error) } private handleError = (error: SpokestackErrorEvent | Error) => { console.error(error) const { onError } = this.props if (onError) { const errorEvent = error as SpokestackErrorEvent if (errorEvent.error) { onError(errorEvent) } else { onError({ error: (error as Error).message }) } } } private showHandle = () => { const { buttonWidth, easing, orientation } = this.props Animated.timing(this.panX, { duration: 200, easing, useNativeDriver: true, toValue: (buttonWidth / 2) * (orientation === 'right' ? -1 : 1) }).start() } private onEnd = () => { console.log('onEnd listenWhenDone', this.listenWhenDone) this.setState({ playerSource: null }, async () => { if (this.listenWhenDone) { this.listenWhenDone = false setTimeout(this.listen, 200) } else { setTimeout(this.close, this.props.closeDelay) } }) } private constrainX = (dx: number) => { const { buttonWidth, orientation } = this.props const closedXValue = buttonWidth / 2 if (orientation === 'left') { return Math.min( Math.max(closedXValue, dx), this.windowWidth + closedXValue ) } return Math.min( Math.max(-this.windowWidth - closedXValue, dx), -closedXValue ) } private constrainHeight = (dy: number) => { const { minHeight } = this.props const { startHeight } = this.state return Math.min(Math.max(minHeight, startHeight - dy), this.windowHeight) } private listen = async () => { try { await listen() } catch (e) { console.error('Error attempting to listen.') this.handleError(e) } } private async opened() { const { greet } = this.props const { listening } = this.state // Could already be listening in response to wakeword if (listening) { return } if (greet) { this.handleIntent({ intent: 'greet', confidence: 100, slots: {} }) } else { this.listen() } } private async openOrClose(shouldOpen: boolean) { if (!shouldOpen) { // Set open to false to immediately show the mic button // Clear the player source as well to stop playing this.setState({ open: shouldOpen, playerSource: null }) } this.setState({ pressed: false }) const { buttonWidth, duration, easing, orientation } = this.props const { listening } = this.state this.windowWidth = Dimensions.get('window').width const closedXValue = buttonWidth / 2 Animated.parallel([ Animated.timing(this.panX, { duration, easing, useNativeDriver: true, toValue: orientation === 'left' ? shouldOpen ? this.windowWidth + closedXValue : closedXValue : shouldOpen ? -this.windowWidth - closedXValue : -closedXValue }), Animated.timing(this.shadowOpacity, { duration, easing, useNativeDriver: true, toValue: shouldOpen ? 0.25 : 0 }) ]).start(() => { const { onClose, onOpen } = this.props this.setState({ open: shouldOpen }) if (shouldOpen) { Keyboard.dismiss() // Check here as well as it could // have tried to start listening // and then stopped due to error // during the animation. // opened() would then try to listen again, // causing an infinite loop if (!listening) { this.opened() } if (onOpen) { onOpen() } } else { stopListening() .then(Spokestack.start) .then(() => { if (onClose) { onClose() } }) .catch((error) => { if (onClose) { onClose() } this.handleError(error) }) } }) } /** * Open the tray, greet (if applicable), and listen */ open = () => { const { open } = this.state if (open) { return } this.openOrClose(true) } /** * Close the tray, stop listening, and restart wakeword */ close = () => { const { open } = this.state if (!open) { return } this.openOrClose(false) } /** * Passes the input to Spokestack.synthesize(), * plays the audio, and adds a speech bubble. */ say = async (input: string) => { const { ttsFormat: format, voice } = this.props // Don't listen for wakeword if we're about to close the tray // This also helps with false positives on // goodbye messages that may include the wakeword. if (!this.listenWhenDone) { await Spokestack.stop() } this.setState({ loading: true }, () => { Spokestack.synthesize(input, format, voice) .then((url) => { if (url) { this.setState({ playerSource: url }) } else { console.log( `Synthesize unsuccessful for input ${input}, going to onEnd` ) this.setState({ loading: false }, this.onEnd) } this.addBubble({ text: input, isLeft: true }) }) .catch((error: Error) => { console.error(`Error synthesizing ${input}`) this.setState({ loading: false }) this.handleError(error) }) }) } /** * Add a bubble (system or user) * to the chat interface */ addBubble = (bubble: Bubble) => { const { bubbles } = this.state // Avoid repeating a bubble const last = bubbles[bubbles.length - 1] if (!last || last.text !== bubble.text || last.isLeft !== bubble.isLeft) { // Only add one bubble with an error message this.setState({ bubbles: bubbles .filter((bubble) => bubble.text !== errorMessage) .concat(bubble) }) } } /** * Toggle silent mode */ toggleSilent = () => { const { playerSource, silent } = this.state if (!silent && playerSource) { this.onEnd() } this.setState({ silent: !silent }) return setSilent(!silent) } /** * Returns whether the tray is in silent mode */ isSilent = () => { return this.state.silent } render() { const { buttonWidth, fontFamily, gradientColors, orientation, primaryColor, soundOnImage: soundOn, soundOffImage: soundOff, style } = this.props const { bubbles, height, listening, loading, open, playerSource, pressed, silent } = this.state const closedXValue = buttonWidth / 2 return ( {!!playerSource && !silent && ( ) } } const styles = StyleSheet.create({ container: { position: 'absolute', bottom: 0 }, buttonView: { position: 'absolute', top: 7, padding: 8, flexDirection: 'column', justifyContent: 'center' }, tray: { flex: 1, backgroundColor: 'white', borderTopLeftRadius: 7, borderTopRightRadius: 7, shadowColor: '#262226', shadowOffset: { width: 0, height: -1 }, shadowRadius: 20, elevation: 20 }, content: { flex: 1, paddingBottom: 30 }, header: { position: 'relative', width: '100%', justifyContent: 'space-between', alignItems: 'center', paddingTop: 10, height: 55, backgroundColor: 'white', borderTopLeftRadius: 7, borderTopRightRadius: 7, overflow: 'hidden', borderBottomWidth: 1, borderBottomColor: '#e7ebee' }, resizer: { position: 'absolute', top: 0, left: 0, right: 0, bottom: 0, paddingTop: 10, justifyContent: 'center', alignItems: 'center', // Support: Android // For some reason, a border is needed // for the this view to actually take up space borderWidth: 1, borderColor: 'transparent' }, touchbar: { position: 'absolute', width: 40, height: 3, top: 10, left: '50%', marginLeft: -20, backgroundColor: '#e7ebee' }, loadingText: { fontSize: 14, color: 'rgba(0, 0, 0, 0.5)' }, headerButton: { justifyContent: 'center', alignItems: 'center', width: 44, height: 44 }, silentButton: { marginHorizontal: 5 }, arrow: { width: 14, height: 14 }, mic: { width: 20, height: 20 }, powered: { position: 'absolute', bottom: 10, left: '50%', marginLeft: -59 }, poweredImage: { width: 118, height: 16 } })