import {
Animated,
AppState,
AppStateStatus,
Dimensions,
Easing,
EasingFunction,
GestureResponderEvent,
Image,
Keyboard,
PanResponder,
PanResponderGestureState,
SafeAreaView,
StyleProp,
StyleSheet,
Text,
TouchableOpacity,
View,
ViewStyle
} from 'react-native'
import React, { PureComponent } from 'react'
import SpeechBubbles, { Bubble } from './components/SpeechBubbles'
import Spokestack, {
PipelineProfile,
SpokestackConfig,
SpokestackErrorEvent,
SpokestackNLUResult,
SpokestackRecognizeEvent,
TTSFormat,
TraceLevel
} from 'react-native-spokestack'
import { getSilent, setSilent } from './utils/settings'
import { listen, stopListening } from './Spokestack'
import Color from 'color'
import HapticFeedback from 'react-native-haptic-feedback'
import Video from 'react-native-video'
import arrowImage from './images/icon-arrow-left.png'
import { checkSpeech } from './utils/permissions'
import merge from 'lodash/merge'
import micImage from './images/icon-mic.png'
import poweredImage from './images/powered-by-spokestack.png'
import soundOffImage from './images/icon-sound-off.png'
import soundOnImage from './images/icon-sound-on.png'
const errorMessage =
'Sorry! We hit an error. Please check your network or restart the app and try again.'
export interface IntentResult {
/**
* A user-defined key to indicate where the user is in the conversation
* Include this in the `exitNodes` prop if Spokestack should not listen
* again after saying the prompt.
*/
node: string
/** Will be processed into Speech unless the tray is in silent mode */
prompt: string
/**
* Set to `true` to stop the wakeword recognizer
* during playback of the prompt.
*/
noInterrupt?: boolean
/** Any other data you might want to add */
data?: any
}
export interface SpokestackTrayProps {
/**
* Your Spokestack tokens generated in your Spokestack account
* at https://spokestack.io/account.
* Create an account for free then generate a token.
* This is from the "ID" field.
*/
clientId: string
/**
* Your Spokestack tokens generated in your Spokestack account
* at https://spokestack.io/account.
* Create an account for free then generate a token.
* This is from the "secret" field.
*/
clientSecret: string
/**
* This function takes an intent from the NLU
* and returns an object with a unique conversation
* node name (that you define) and a prompt
* to be processed by TTS and spoken.
*
* Note: the prompt is only shown in a chat bubble
* if sound has been turned off.
*/
handleIntent: (
intent: string,
slots?: SpokestackNLUResult['slots'],
utterance?: string
) => IntentResult
/**
* The NLU Tensorflow Lite model (.tflite), JSON metadata, and NLU vocabulary (.txt)
*
* All 3 fields accept 2 types of values.
* 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN).
* 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`)
*
* See https://spokestack.io/docs/concepts/nlu to learn more about NLU.
*
* ```js
* // ...
* nlu={{
* model: 'https://somecdn.com/nlu.tflite',
* vocab: 'https://somecdn.com/vocab.txt',
* metadata: 'https://somecdn.com/metadata.json'
* }}
* ```
*
* You can also pass local files.
* Note: this requires a change to your metro.config.js. For more info, see
* "Including model files in your app bundle" in the README.md.
*
* ```js
* // ...
* nlu={{
* model: require('./nlu.tflite'),
* vocab: require('./vocab.txt'),
* // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import`
* // so the file is not parsed when included but instead imported as a source object. This makes it so the
* // file is read and parsed by the underlying native libraries instead.
* metadata: require('./metadata.sjson')
* }}
* ```
*/
nlu: SpokestackConfig['nlu']
/** Width (and height) of the mic button */
buttonWidth?: number
/** How long to wait to close the tray after speaking (ms) */
closeDelay?: number
/** Show debug messages from react-native-spokestack */
debug?: boolean
/** Duration for the tray animation (ms) */
duration?: number
/** Easing function for the tray animation */
easing?: EasingFunction
/**
* Edit the transcript before classification
* and before the user response bubble is shown.
*/
editTranscript?: (transcript: string) => string
/**
* All nodes in this array should end
* the conversation and close the tray
*/
exitNodes?: string[]
/**
* Font to use for "LISTENING...", "LOADING...",
* and chat bubble text.
*/
fontFamily?: string
/**
* Colors for the linear gradient shown when listening
* Can be any number of colors (recommended: 2-3)
*/
gradientColors?: string[]
/**
* Whether to greet the user with a welcome message
* when the tray opens.
* Note: `handleIntent` must respond to the "greet" intent.
*/
greet?: boolean
/**
* Set this to false to disable the haptic
* that gets played whenever the tray starts listening.
*/
haptic?: boolean
/**
* Configuration for keyword recognition
*
* The filter, detect, encode, and metadata fields accept 2 types of values.
* 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN).
* 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`)
*
* See https://www.spokestack.io/docs/concepts/keywords to learn more about keyword recognition.
*
* @example
* ```js
* // ...
* keyword={{
* detect: 'https://s.spokestack.io/u/UbMeX/detect.tflite',
* encode: 'https://s.spokestack.io/u/UbMeX/encode.tflite',
* filter: 'https://s.spokestack.io/u/UbMeX/filter.tflite',
* metadata: 'https://s.spokestack.io/u/UbMeX/metadata.json'
* }}
* ```
*
* You can also download models ahead of time and include them from local files.
* Note: this requires a change to your metro.config.js. For more info, see
* "Including model files in your app bundle" in the README.md.
*
* ```js
* // ...
* keyword={{
* detect: require('./detect.tflite'),
* encode: require('./encode.tflite'),
* filter: require('./filter.tflite'),
* // IMPORTANT: a special extension is used for local metadata JSON files (`.sjson`) when using `require` or `import`
* // so the file is not parsed when included but instead imported as a source object. This makes it so the
* // file is read and parsed by the underlying native libraries instead.
* metadata: require('./metadata.sjson')
* }}
* ```
*
* Keyword configuration also accepts a classes field for when metadata is not specified.
*
* ```js
* // ...
* keyword={{
* detect: require('./detect.tflite'),
* encode: require('./encode.tflite'),
* filter: require('./filter.tflite'),
* classes: ['one', 'two', 'three]
* }}
* ```
*/
keyword?: SpokestackConfig['keyword']
/** Minimum height for the tray */
minHeight?: number
/**
* Called whenever the tray has closed
*/
onClose?: () => void
/** Called whenever there's an error from Spokestack */
onError?: (e: SpokestackErrorEvent) => void
/** Called whenever the tray has opened */
onOpen?: () => void
/**
* The tray button can be oriented on either side of the screen
*/
orientation?: 'left' | 'right'
/**
* This color is used to theme the tray
* and is used in the mic button and speech bubbles.
*/
primaryColor?: string
/**
* The Spokestack config profile to pass to
* react-native-spokestack.
* These are available from react-native-spokestack
* starting in version 4.0.0.
*
* If Wakeword config files are specified, the default will be
* `TFLITE_WAKEWORD_NATIVE_ASR`.
* Otherwise, the default is `PTT_NATIVE_ASR`.
*
* ```js
* import SpokestackTray from 'react-native-spokestack-tray'
* import { PipelineProfile } from 'react-native-spokestack'
*
* // ...
* `
*/
refreshModels?: boolean
/**
* Whether to speak the greeting or only display
* a chat bubble with the greet message,
* even if sound is on.
*/
sayGreeting?: boolean
/** Replace the sound on image by passing a React Image component */
soundOnImage?: React.ReactNode
/** Replace the sound off image by passing a React Image component */
soundOffImage?: React.ReactNode
/**
* Pass options directly to the Spokestack.initialize()
* function from react-native-spokestack.
* See https://github.com/spokestack/react-native-spokestack
* for available options.
*/
spokestackConfig?: Partial
/** Starting height for tray */
startHeight?: number
/** This style prop is passed to the tray's container */
style?: Animated.WithAnimatedValue>
/** The format for the text passed to Spokestack.synthesize */
ttsFormat?: TTSFormat
/**
* A key for a voice in Spokestack TTS, passed to Spokestack.synthesize.
* This may only be changed if you have created a custom voice using a
* Spokestack Maker account. See https://spokestack.io/pricing#maker.
* If not specified, Spokestack's Free "demo-male" voice is used.
*/
voice?: string
/**
* The NLU Tensorflow Lite models (.tflite) for wakeword.
*
* All 3 fields accept 2 types of values.
* 1. A string representing a remote URL from which to download and cache the file (presumably from a CDN).
* 2. A source object retrieved by a `require` or `import` (e.g. `model: require('./nlu.tflite')`)
*
* See https://spokestack.io/docs/concepts/wakeword-models to learn more about Wakeword
*
* Spokestack offers sample wakeword model files ("Spokestack"):
*
* ```js
* // ...
* wakeword={{
* detect: 'https://s.spokestack.io/u/hgmYb/detect.tflite',
* encode: 'https://s.spokestack.io/u/hgmYb/encode.tflite',
* filter: 'https://s.spokestack.io/u/hgmYb/filter.tflite'
* }}
* ```
*
* You can also download these models ahead of time and include them from local files.
* Note: this requires a change to your metro.config.js. For more info, see
* "Including model files in your app bundle" in the README.md.
*
* ```js
* // ...
* wakeword={{
* detect: require('./detect.tflite'),
* encode: require('./encode.tflite'),
* filter: require('./filter.tflite')
* }}
* ```
*/
wakeword?: SpokestackConfig['wakeword']
}
interface State {
bubbles: Bubble[]
height: number
listening: boolean
listeningWidth: number
loading: boolean
open: boolean
playerSource: string
pressed: boolean
silent: boolean
startHeight: number
}
export default class SpokestackTray extends PureComponent<
SpokestackTrayProps,
State
> {
private wentToBackground: boolean
private hasStarted = false
private windowWidth = Dimensions.get('window').width
private windowHeight: number
private panX = new Animated.Value(0)
private shadowOpacity = new Animated.Value(0)
private listenWhenDone = false
private utterance = ''
private openPanResponder = PanResponder.create({
onStartShouldSetPanResponder: () => true,
onMoveShouldSetPanResponder: () => true,
onPanResponderGrant: () => {
const { height } = this.state
this.setState({ pressed: true, startHeight: height })
const { width: windowWidth, height: windowHeight } =
Dimensions.get('window')
this.windowWidth = windowWidth
this.windowHeight = windowHeight
},
onPanResponderMove: (_event, { dx, dy }) => {
this.panX.setValue(this.constrainX(dx))
this.setState({ height: this.constrainHeight(dy) })
},
onPanResponderRelease: (_event, { dx, dy }) => {
const { orientation } = this.props
const shouldOpen =
(Math.abs(dx) < 2 && Math.abs(dy) < 2) ||
(orientation === 'left'
? dx > this.windowWidth / 3
: dx < -this.windowWidth / 3)
this.openOrClose(shouldOpen)
},
onPanResponderTerminate: () => {
this.openOrClose(false)
}
})
private expandPanResponder = PanResponder.create({
onMoveShouldSetPanResponder: () => true,
onPanResponderGrant: () => {
const { height } = this.state
this.setState({ startHeight: height })
this.windowHeight = Dimensions.get('window').height
},
onPanResponderMove: (
_event: GestureResponderEvent,
{ dy }: PanResponderGestureState
) => {
this.setState({ height: this.constrainHeight(dy) })
}
})
static defaultProps: Partial = {
buttonWidth: 60,
closeDelay: 0,
duration: 500,
editTranscript: (transcript) => transcript,
easing: Easing.bezier(0.77, 0.41, 0.2, 0.84),
gradientColors: ['#61fae9', '#2F5BEA'],
greet: false,
haptic: true,
minHeight: 170,
orientation: 'left',
primaryColor: '#2f5bea',
sayGreeting: true,
soundOnImage: (
),
soundOffImage: (
),
startHeight: 220,
ttsFormat: TTSFormat.TEXT,
voice: 'demo-male'
}
state: State = {
bubbles: [],
height: this.props.startHeight,
listening: false,
listeningWidth: 0,
loading: false,
open: false,
playerSource: '',
pressed: false,
silent: false,
startHeight: this.props.startHeight
}
async componentDidMount() {
const {
clientId,
clientSecret,
debug,
keyword,
nlu,
profile,
refreshModels,
spokestackConfig = {},
wakeword
} = this.props
await this.initState()
this.addListeners()
try {
await Spokestack.initialize(
clientId,
clientSecret,
merge(spokestackConfig, {
traceLevel: debug ? TraceLevel.DEBUG : TraceLevel.NONE,
refreshModels,
pipeline: { profile },
keyword,
nlu,
wakeword
})
)
} catch (error) {
console.error('Error during Spokestack.initialize()')
this.handleError(error)
}
if (await checkSpeech()) {
try {
await Spokestack.start()
} catch (error) {
console.error('Error attempting to start on componentDidMount')
this.handleError(error)
}
}
this.showHandle()
}
componentWillUnmount() {
AppState.removeEventListener('change', this.appStateChange)
Spokestack.destroy()
}
private async initState() {
this.setState({ silent: await getSilent() })
}
private addListeners() {
AppState.addEventListener('change', this.appStateChange)
Spokestack.addEventListener('recognize', this.onRecognize)
Spokestack.addEventListener('activate', this.onActivate)
Spokestack.addEventListener('deactivate', this.onDeactivate)
Spokestack.addEventListener('start', this.onStart)
Spokestack.addEventListener('timeout', this.close)
Spokestack.addEventListener('error', this.onError)
if (this.props.debug) {
Spokestack.addEventListener('trace', console.log.bind(console))
}
}
private appStateChange = async (nextAppState: AppStateStatus) => {
console.log(
`App state changed to ${nextAppState}, and wentToBackground is ${this.wentToBackground}`
)
// Enable/disable wakeword based on app state
if (nextAppState === 'active') {
// Needs to be coming from the background
// and the pipeline must have been started at least once already
if (this.wentToBackground && this.hasStarted && (await checkSpeech())) {
console.log('App became active. Starting speech pipeline.')
Spokestack.start()
}
this.wentToBackground = false
} else if (nextAppState === 'background') {
this.wentToBackground = true
console.log('App went to background. Stopping speech pipeline.')
Spokestack.stop()
}
}
private async handleIntent(result: SpokestackNLUResult) {
const { closeDelay, exitNodes, handleIntent, onError, sayGreeting } =
this.props
const { silent } = this.state
const response = handleIntent(result.intent, result.slots, this.utterance)
console.log(`Processed intent ${result.intent}`, response)
const shouldListen = exitNodes.indexOf(response.node) === -1
if (response.prompt) {
const isGreeting = result.intent === 'greet'
if (silent || (isGreeting && !sayGreeting)) {
this.addBubble({ text: response.prompt, isLeft: true })
if (shouldListen) {
setTimeout(this.listen, 200)
} else {
setTimeout(this.close, closeDelay)
}
} else {
if (response.noInterrupt) {
await Spokestack.stop()
}
this.listenWhenDone = shouldListen
this.say(response.prompt)
}
} else if (onError) {
onError({ error: 'No prompt returned in the response' })
}
}
private onRecognize = async ({ transcript }: SpokestackRecognizeEvent) => {
const { editTranscript } = this.props
console.log('[Spokestack onRecognize]:', transcript)
const edited = editTranscript(transcript)
console.log('Transcript after editing: ', edited)
this.utterance = edited
this.addBubble({ text: edited, isLeft: false })
// Only call listeners if there's a transcript
if (edited.length > 0) {
try {
const result = await Spokestack.classify(edited)
if (result) {
this.handleIntent(result)
}
} catch (error) {
console.error('Error during classification')
this.handleError(error)
}
}
}
private onActivate = () => {
const { haptic } = this.props
this.setState({ listening: true, loading: false }, this.open)
if (haptic) {
HapticFeedback.trigger('impactHeavy', { enableVibrateFallback: true })
}
}
private onDeactivate = () => {
this.setState({ listening: false })
}
private onStart = () => {
this.hasStarted = true
}
private onError = (error: SpokestackErrorEvent) => {
this.setState({ listening: false })
this.handleError(error)
}
private handleError = (error: SpokestackErrorEvent | Error) => {
console.error(error)
const { onError } = this.props
if (onError) {
const errorEvent = error as SpokestackErrorEvent
if (errorEvent.error) {
onError(errorEvent)
} else {
onError({ error: (error as Error).message })
}
}
}
private showHandle = () => {
const { buttonWidth, easing, orientation } = this.props
Animated.timing(this.panX, {
duration: 200,
easing,
useNativeDriver: true,
toValue: (buttonWidth / 2) * (orientation === 'right' ? -1 : 1)
}).start()
}
private onEnd = () => {
console.log('onEnd listenWhenDone', this.listenWhenDone)
this.setState({ playerSource: null }, async () => {
if (this.listenWhenDone) {
this.listenWhenDone = false
setTimeout(this.listen, 200)
} else {
setTimeout(this.close, this.props.closeDelay)
}
})
}
private constrainX = (dx: number) => {
const { buttonWidth, orientation } = this.props
const closedXValue = buttonWidth / 2
if (orientation === 'left') {
return Math.min(
Math.max(closedXValue, dx),
this.windowWidth + closedXValue
)
}
return Math.min(
Math.max(-this.windowWidth - closedXValue, dx),
-closedXValue
)
}
private constrainHeight = (dy: number) => {
const { minHeight } = this.props
const { startHeight } = this.state
return Math.min(Math.max(minHeight, startHeight - dy), this.windowHeight)
}
private listen = async () => {
try {
await listen()
} catch (e) {
console.error('Error attempting to listen.')
this.handleError(e)
}
}
private async opened() {
const { greet } = this.props
const { listening } = this.state
// Could already be listening in response to wakeword
if (listening) {
return
}
if (greet) {
this.handleIntent({ intent: 'greet', confidence: 100, slots: {} })
} else {
this.listen()
}
}
private async openOrClose(shouldOpen: boolean) {
if (!shouldOpen) {
// Set open to false to immediately show the mic button
// Clear the player source as well to stop playing
this.setState({ open: shouldOpen, playerSource: null })
}
this.setState({ pressed: false })
const { buttonWidth, duration, easing, orientation } = this.props
const { listening } = this.state
this.windowWidth = Dimensions.get('window').width
const closedXValue = buttonWidth / 2
Animated.parallel([
Animated.timing(this.panX, {
duration,
easing,
useNativeDriver: true,
toValue:
orientation === 'left'
? shouldOpen
? this.windowWidth + closedXValue
: closedXValue
: shouldOpen
? -this.windowWidth - closedXValue
: -closedXValue
}),
Animated.timing(this.shadowOpacity, {
duration,
easing,
useNativeDriver: true,
toValue: shouldOpen ? 0.25 : 0
})
]).start(() => {
const { onClose, onOpen } = this.props
this.setState({ open: shouldOpen })
if (shouldOpen) {
Keyboard.dismiss()
// Check here as well as it could
// have tried to start listening
// and then stopped due to error
// during the animation.
// opened() would then try to listen again,
// causing an infinite loop
if (!listening) {
this.opened()
}
if (onOpen) {
onOpen()
}
} else {
stopListening()
.then(Spokestack.start)
.then(() => {
if (onClose) {
onClose()
}
})
.catch((error) => {
if (onClose) {
onClose()
}
this.handleError(error)
})
}
})
}
/**
* Open the tray, greet (if applicable), and listen
*/
open = () => {
const { open } = this.state
if (open) {
return
}
this.openOrClose(true)
}
/**
* Close the tray, stop listening, and restart wakeword
*/
close = () => {
const { open } = this.state
if (!open) {
return
}
this.openOrClose(false)
}
/**
* Passes the input to Spokestack.synthesize(),
* plays the audio, and adds a speech bubble.
*/
say = async (input: string) => {
const { ttsFormat: format, voice } = this.props
// Don't listen for wakeword if we're about to close the tray
// This also helps with false positives on
// goodbye messages that may include the wakeword.
if (!this.listenWhenDone) {
await Spokestack.stop()
}
this.setState({ loading: true }, () => {
Spokestack.synthesize(input, format, voice)
.then((url) => {
if (url) {
this.setState({ playerSource: url })
} else {
console.log(
`Synthesize unsuccessful for input ${input}, going to onEnd`
)
this.setState({ loading: false }, this.onEnd)
}
this.addBubble({ text: input, isLeft: true })
})
.catch((error: Error) => {
console.error(`Error synthesizing ${input}`)
this.setState({ loading: false })
this.handleError(error)
})
})
}
/**
* Add a bubble (system or user)
* to the chat interface
*/
addBubble = (bubble: Bubble) => {
const { bubbles } = this.state
// Avoid repeating a bubble
const last = bubbles[bubbles.length - 1]
if (!last || last.text !== bubble.text || last.isLeft !== bubble.isLeft) {
// Only add one bubble with an error message
this.setState({
bubbles: bubbles
.filter((bubble) => bubble.text !== errorMessage)
.concat(bubble)
})
}
}
/**
* Toggle silent mode
*/
toggleSilent = () => {
const { playerSource, silent } = this.state
if (!silent && playerSource) {
this.onEnd()
}
this.setState({ silent: !silent })
return setSilent(!silent)
}
/**
* Returns whether the tray is in silent mode
*/
isSilent = () => {
return this.state.silent
}
render() {
const {
buttonWidth,
fontFamily,
gradientColors,
orientation,
primaryColor,
soundOnImage: soundOn,
soundOffImage: soundOff,
style
} = this.props
const {
bubbles,
height,
listening,
loading,
open,
playerSource,
pressed,
silent
} = this.state
const closedXValue = buttonWidth / 2
return (
{!!playerSource && !silent && (
)
}
}
const styles = StyleSheet.create({
container: {
position: 'absolute',
bottom: 0
},
buttonView: {
position: 'absolute',
top: 7,
padding: 8,
flexDirection: 'column',
justifyContent: 'center'
},
tray: {
flex: 1,
backgroundColor: 'white',
borderTopLeftRadius: 7,
borderTopRightRadius: 7,
shadowColor: '#262226',
shadowOffset: {
width: 0,
height: -1
},
shadowRadius: 20,
elevation: 20
},
content: {
flex: 1,
paddingBottom: 30
},
header: {
position: 'relative',
width: '100%',
justifyContent: 'space-between',
alignItems: 'center',
paddingTop: 10,
height: 55,
backgroundColor: 'white',
borderTopLeftRadius: 7,
borderTopRightRadius: 7,
overflow: 'hidden',
borderBottomWidth: 1,
borderBottomColor: '#e7ebee'
},
resizer: {
position: 'absolute',
top: 0,
left: 0,
right: 0,
bottom: 0,
paddingTop: 10,
justifyContent: 'center',
alignItems: 'center',
// Support: Android
// For some reason, a border is needed
// for the this view to actually take up space
borderWidth: 1,
borderColor: 'transparent'
},
touchbar: {
position: 'absolute',
width: 40,
height: 3,
top: 10,
left: '50%',
marginLeft: -20,
backgroundColor: '#e7ebee'
},
loadingText: {
fontSize: 14,
color: 'rgba(0, 0, 0, 0.5)'
},
headerButton: {
justifyContent: 'center',
alignItems: 'center',
width: 44,
height: 44
},
silentButton: {
marginHorizontal: 5
},
arrow: {
width: 14,
height: 14
},
mic: {
width: 20,
height: 20
},
powered: {
position: 'absolute',
bottom: 10,
left: '50%',
marginLeft: -59
},
poweredImage: {
width: 118,
height: 16
}
})