///
///
///
///
///
///
// namespace
namespace cf {
// interface
export interface IMicrophoneBridgeOptions{
el: HTMLElement;
button: UserInputSubmitButton;
microphoneObj: IUserInput;
eventTarget: EventDispatcher;
}
export const MicrophoneBridgeEvent = {
ERROR: "cf-microphone-bridge-error",
TERMNIAL_ERROR: "cf-microphone-bridge-terminal-error"
}
// class
export class MicrophoneBridge{
private equalizer: SimpleEqualizer;
private el: HTMLElement;
private button: UserInputSubmitButton;
private currentTextResponse: string = "";
private recordChunks: Array;
// private equalizer: SimpleEqualizer;
private promise: Promise;
private currentStream: MediaStream;
private _hasUserMedia: boolean = false;
private inputErrorCount: number = 0;
private inputCurrentError: string = "";
private microphoneObj: IUserInput;
private eventTarget: EventDispatcher;
private flowUpdateCallback: () => void;
private set hasUserMedia(value: boolean){
this._hasUserMedia = value;
if(!value){
// this.submitButton.classList.add("permission-waiting");
}else{
// this.submitButton.classList.remove("permission-waiting");
}
}
public set active(value: boolean){
if(this.equalizer){
this.equalizer.disabled = !value;
}
}
constructor(options: IMicrophoneBridgeOptions){
this.el = options.el;
this.button = options.button;
this.eventTarget = options.eventTarget;
// data object
this.microphoneObj = options.microphoneObj;
this.flowUpdateCallback = this.onFlowUpdate.bind(this);
this.eventTarget.addEventListener(FlowEvents.FLOW_UPDATE, this.flowUpdateCallback, false);
}
public cancel(){
this.button.loading = false;
if(this.microphoneObj.cancelInput){
this.microphoneObj.cancelInput();
}
}
public onFlowUpdate(){
this.currentTextResponse = null;
if(!this._hasUserMedia){
// check if user has granted
let hasGranted: boolean = false;
if(( window).navigator.mediaDevices){
( window).navigator.mediaDevices.enumerateDevices().then((devices: any) => {
devices.forEach((device: any) => {
if(!hasGranted && device.label !== ""){
hasGranted = true;
}
});
if(hasGranted){
// user has previously granted, so call getusermedia, as this wont prombt user
this.getUserMedia();
}else{
// await click on button, wait state
}
});
}
}else{
// user has granted ready to go go
if(!this.microphoneObj.awaitingCallback){
this.callInput();
}
}
}
public getUserMedia(){
try{
// from https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia#Using_the_new_API_in_older_browsers
// Older browsers might not implement mediaDevices at all, so we set an empty object first
if (navigator.mediaDevices === undefined) {
(navigator).mediaDevices = {};
}
// Some browsers partially implement mediaDevices. We can't just assign an object
// with getUserMedia as it would overwrite existing properties.
// Here, we will just add the getUserMedia property if it's missing.
if (navigator.mediaDevices.getUserMedia === undefined) {
navigator.mediaDevices.getUserMedia = function(constraints) {
// First get ahold of the legacy getUserMedia, if present
var getUserMedia = navigator.getUserMedia || (window).navigator.webkitGetUserMedia || (window).navigator.mozGetUserMedia;
// Some browsers just don't implement it - return a rejected promise with an error
// to keep a consistent interface
if (!getUserMedia) {
return Promise.reject(new Error('getUserMedia is not implemented in this browser'));
}
// Otherwise, wrap the call to the old navigator.getUserMedia with a Promise
return new Promise(function(resolve, reject) {
getUserMedia.call(navigator, constraints, resolve, reject);
});
}
}
( navigator.mediaDevices).getUserMedia( {audio: true})
.then((stream: MediaStream) => {
this.currentStream = stream;
if(stream.getAudioTracks().length > 0){
// interface is active and available, so call it immidiatly
this.hasUserMedia = true;
this.setupEqualizer();
if(!this.microphoneObj.awaitingCallback){
// microphone interface awaits speak out loud callback
this.callInput();
}
}else{
// code for when both devices are available
// interface is not active, button should be clicked
this.hasUserMedia = false;
}
})
.catch((error: any) => {
// Promise catch
this.hasUserMedia = false;
this.eventTarget.dispatchEvent(new Event(MicrophoneBridgeEvent.TERMNIAL_ERROR));
});
}catch(error){
// try catch
// whoops no getUserMedia, so roll back to standard UI
this.hasUserMedia = false;
this.eventTarget.dispatchEvent(new Event(MicrophoneBridgeEvent.TERMNIAL_ERROR));
}
}
public dealloc(){
this.cancel();
this.promise = null;
this.currentStream = null;
if(this.equalizer){
this.equalizer.dealloc();
}
this.equalizer = null;
this.eventTarget.removeEventListener(FlowEvents.FLOW_UPDATE, this.flowUpdateCallback, false);
this.flowUpdateCallback = null;
}
public callInput(messageTime: number = 0){
// remove current error message after x time
// clearTimeout(this.clearMessageTimer);
// this.clearMessageTimer = setTimeout(() =>{
// this.el.removeAttribute("message");
// }, messageTime);
this.button.loading = true;
if(this.equalizer){
this.equalizer.disabled = false;
}
// call API, SpeechRecognintion etc. you decide, passing along the stream from getUserMedia can be used.. as long as the resolve is called with string attribute
this.promise = new Promise((resolve: any, reject: any) => this.microphoneObj.input(resolve, reject, this.currentStream) )
.then((result) => {
// api contacted
this.promise = null;
// save response so it's available in getFlowDTO
this.currentTextResponse = result.toString();
if(!this.currentTextResponse || this.currentTextResponse == ""){
this.showError(Dictionary.get("user-audio-reponse-invalid"));
// invalid input, so call API again
this.callInput();
return;
}
this.inputErrorCount = 0;
this.inputCurrentError = "";
this.button.loading = false;
// continue flow
let dto: FlowDTO = {
text: this.currentTextResponse
};
ConversationalForm.illustrateFlow(this, "dispatch", UserInputEvents.SUBMIT, dto);
this.eventTarget.dispatchEvent(new CustomEvent(UserInputEvents.SUBMIT, {
detail: dto
}));
}).catch((error) => {
// API error
// ConversationalForm.illustrateFlow(this, "dispatch", MicrophoneBridgeEvent.ERROR, error);
// this.eventTarget.dispatchEvent(new CustomEvent(MicrophoneBridgeEvent.ERROR, {
// detail: error
// }));
if(this.isErrorTerminal(error)){
// terminal error, fallback to
this.eventTarget.dispatchEvent(new CustomEvent(MicrophoneBridgeEvent.TERMNIAL_ERROR,{
detail: Dictionary.get("microphone-terminal-error")
}));
if(!ConversationalForm.suppressLog) console.log("Conversational Form: Terminal error: ", error);
}else{
if(this.inputCurrentError != error){
// api failed ...
// show result in UI
this.inputErrorCount = 0;
this.inputCurrentError = error;
}else{
}
this.inputErrorCount++;
if(this.inputErrorCount > 2){
this.showError(error);
}else{
this.eventTarget.dispatchEvent(new CustomEvent(MicrophoneBridgeEvent.TERMNIAL_ERROR,{
detail: Dictionary.get("microphone-terminal-error")
}));
if(!ConversationalForm.suppressLog) console.log("Conversational Form: Terminal error: ", error);
}
}
});
}
protected isErrorTerminal(error: string): boolean{
const terminalErrors: Array = ["network"];
if(terminalErrors.indexOf(error) !== -1)
return true;
return false;
}
private showError(error: string){
const dto: FlowDTO = {
errorText: error
};
ConversationalForm.illustrateFlow(this, "dispatch", FlowEvents.USER_INPUT_INVALID, dto)
this.eventTarget.dispatchEvent(new CustomEvent(FlowEvents.USER_INPUT_INVALID, {
detail: dto
}));
this.callInput();
}
private setupEqualizer(){
const eqEl: HTMLElement = this.el.getElementsByTagName("cf-icon-audio-eq")[0];
if(SimpleEqualizer.supported && eqEl){
this.equalizer = new SimpleEqualizer({
stream: this.currentStream,
elementToScale: eqEl
});
}
}
}
class SimpleEqualizer{
private context: AudioContext;
private analyser: AnalyserNode;
private mic: MediaStreamAudioSourceNode;
private javascriptNode: ScriptProcessorNode;
private elementToScale: HTMLElement;
private maxBorderWidth: number = 0;
private _disabled: boolean = false;
public set disabled(value: boolean){
this._disabled = value;
this.elementToScale.style.borderWidth = 0 + "px";
}
constructor(options: any){
this.elementToScale = options.elementToScale;
this.context = new AudioContext();
this.analyser = this.context.createAnalyser();
this.mic = this.context.createMediaStreamSource(options.stream);
this.javascriptNode = this.context.createScriptProcessor(2048, 1, 1);
this.analyser.smoothingTimeConstant = 0.3;
this.analyser.fftSize = 1024;
this.mic.connect(this.analyser);
this.analyser.connect(this.javascriptNode);
this.javascriptNode.connect(this.context.destination);
this.javascriptNode.onaudioprocess = () => {
this.onAudioProcess();
};
}
private onAudioProcess(){
if(this._disabled)
return;
var array = new Uint8Array(this.analyser.frequencyBinCount);
this.analyser.getByteFrequencyData(array);
var values = 0;
var length = array.length;
for (var i = 0; i < length; i++) {
values += array[i];
}
var average = values / length;
const percent: number = Math.min(1, Math.max(0, 1 - ((50 - average) / 50)));
if(!this.maxBorderWidth){
this.maxBorderWidth = this.elementToScale.offsetWidth * 0.5;
}
this.elementToScale.style.borderWidth = (this.maxBorderWidth * percent) + "px";
}
public dealloc(){
this.javascriptNode.onaudioprocess = null;
this.javascriptNode = null;
this.analyser = null;
this.mic = null;
this.elementToScale = null;
this.context = null;
}
public static supported():boolean{
(window).AudioContext = (window).AudioContext || (window).webkitAudioContext;
if((window).AudioContext){
return true;
}
else {
return false;
}
}
}
}