Unable to Stop DetectIntentStream on DialogFlow Es - node.js

Question

I am trying to create a two-way communication voice bot using detectIntentStream Api of Dialogflow ES. But unable to stop streaming or end the session after communication.
I use example2 of this repository: https://github.com/dialogflow/selfservicekiosk-audio-streaming/tree/master/examples
here is the server code
const projectId = process.env.npm_config_PROJECT_ID;
const example = process.env.npm_config_EXAMPLE;
const port = process.env.npm_config_PORT || 3000;
const languageCode = "en-US";
let encoding = "AUDIO_ENCODING_LINEAR_16";
if (example > 3) {
// NOTE: ENCODING NAMING FOR SPEECH API IS DIFFERENT
encoding = "LINEAR16";
}
console.log(example);
if (example == 7) {
// NOTE: ENCODING NAMING FOR SPEECH API IS DIFFERENT
encoding = "linear16";
}
const singleUtterance = false;
const interimResults = false;
const sampleRateHertz = 16000;
const speechContexts = [
{
phrases: ["mail", "email"],
boost: 20.0,
},
];
console.log(example);
console.log(projectId);
// ----------------------
// load all the libraries for the server
const socketIo = require("socket.io");
const path = require("path");
const fs = require("fs");
const http = require("http");
const cors = require("cors");
const express = require("express");
const ss = require("socket.io-stream");
// load all the libraries for the Dialogflow part
const uuid = require("uuid");
const util = require("util");
const { Transform, pipeline } = require("stream");
const pump = util.promisify(pipeline);
const df = require("dialogflow").v2beta1;
// set some server variables
const app = express();
var server;
var sessionId, sessionClient, sessionPath, request;
var speechClient,
requestSTT,
ttsClient,
requestTTS,
mediaTranslationClient,
requestMedia;
// STT demo
const speech = require("#google-cloud/speech");
// TTS demo
const textToSpeech = require("#google-cloud/text-to-speech");
// Media Translation Demo
const mediatranslation = require("#google-cloud/media-translation");
/**
* Setup Express Server with CORS and SocketIO
*/
function setupServer() {
// setup Express
app.use(cors());
app.get("/", function (req, res) {
res.sendFile(path.join(__dirname + "/example" + example + ".html"));
});
server = http.createServer(app);
io = socketIo(server);
server.listen(port, () => {
console.log("Running server on port %s", port);
});
// Listener, once the client connect to the server socket
io.on("connect", (client) => {
console.log(`Client connected [id=${client.id}]`);
client.emit("server_setup", `Server connected [id=${client.id}]`);
ss(client).on("stream", function (stream, data) {
// get the name of the stream
const filename = path.basename(data.name);
// pipe the filename to the stream
stream.pipe(fs.createWriteStream(filename));
// make a detectIntStream call
detectIntentStream(stream, function (results) {
console.log(results);
console.log(results.outputAudio);
client.emit("results", results);
});
});
});
}
function setupDialogflow() {
// Dialogflow will need a session Id
sessionId = uuid.v4();
// Dialogflow will need a DF Session Client
// So each DF session is unique
sessionClient = new df.SessionsClient();
// Create a session path from the Session client,
// which is a combination of the projectId and sessionId.
sessionPath = sessionClient.sessionPath(projectId, sessionId);
// Create the initial request object
// When streaming, this is the first call you will
// make, a request without the audio stream
// which prepares Dialogflow in receiving audio
// with a certain sampleRateHerz, encoding and languageCode
// this needs to be in line with the audio settings
// that are set in the client
request = {
session: sessionPath,
queryInput: {
audioConfig: {
sampleRateHertz: sampleRateHertz,
encoding: encoding,
languageCode: languageCode,
speechContexts: speechContexts,
},
singleUtterance: singleUtterance,
},
outputAudioConfig: {
audioEncoding: "OUTPUT_AUDIO_ENCODING_LINEAR_16",
},
};
}
async function detectIntentStream(audio, cb) {
// execute the Dialogflow Call: streamingDetectIntent()
const stream = sessionClient
.streamingDetectIntent()
.on("data", function (data) {
// when data comes in
// log the intermediate transcripts
if (data.recognitionResult) {
console.log(
`Intermediate transcript:
${data.recognitionResult.transcript}`
);
} else {
// log the detected intent
console.log(`Detected intent:`);
cb(data);
}
})
.on("error", (e) => {
console.log(e);
})
.on("end", () => {
console.log("on end");
return null;
});
// Write request objects.
// Thee first message must contain StreamingDetectIntentRequest.session,
// [StreamingDetectIntentRequest.query_input] plus optionally
// [StreamingDetectIntentRequest.query_params]. If the client wants
// to receive an audio response, it should also contain
// StreamingDetectIntentRequest.output_audio_config.
// The message must not contain StreamingDetectIntentRequest.input_audio.
stream.write(request);
// pump is a small node module that pipes streams together and
// destroys all of them if one of them closes.
await pump(
audio,
// Format the audio stream into the request format.
new Transform({
objectMode: true,
transform: (obj, _, next) => {
next(null, {
inputAudio: obj,
outputAudioConfig: {
audioEncoding: `OUTPUT_AUDIO_ENCODING_LINEAR_16`,
},
});
},
}),
stream
);
}
setupDialogflow();
setupServer();
and here is client code
<!DOCTYPE html>
<!--
Copyright 2020 Google LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta charset="utf-8" />
<title>RecordRTC over Socket.io</title>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.indigo-pink.min.css">
<script src="https://www.WebRTC-Experiment.com/RecordRTC.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.3.0/socket.io.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io-stream/0.9.1/socket.io-stream.js"></script>
</head>
<body>
<div style="margin: 20px">
<h1 style="font-size: 18px;">Example 2: Dialogflow Speech Detection through streaming</h1>
<div>
<button id="start-recording" disabled>Start Streaming</button>
<button id="stop-recording" disabled>Stop Streaming</button>
</div>
<h2 style="font-size: 16px; margin-bottom: 10px;">Query Text</h2>
<code>data.queryResult.queryText</code><br/>
<input id="queryText" type="text" style="width: 400px;"/>
<h2 style="font-size: 16px; margin-bottom: 10px;">Intent</h2>
<code>data.queryResult.intent.displayName</code><br/>
<input id="intent" type="text" style="width: 400px;"/>
<h2 style="font-size: 16px;">Responses</h2>
<code>data.queryResult.fulfillmentText</code><br/>
<textarea id="results" style="width: 800px; height: 300px;"></textarea>
<code>data.queryResult.audio</code><br/>
<textarea id="result_audio" style="width: 800px; height: 300px;"></textarea>
</div>
<script type="text/javascript">
const startRecording = document.getElementById('start-recording');
const stopRecording = document.getElementById('stop-recording');
let recordAudio;
const socketio = io();
const socket = socketio.on('connect', function() {
startRecording.disabled = false;
});
startRecording.onclick = function() {
startRecording.disabled = true;
navigator.getUserMedia({
audio: true
}, function(stream) {
recordAudio = RecordRTC(stream, {
type: 'audio',
mimeType: 'audio/webm',
sampleRate: 44100,
desiredSampRate: 16000,
recorderType: StereoAudioRecorder,
numberOfAudioChannels: 1,
//1)
// get intervals based blobs
// value in milliseconds
// as you might not want to make detect calls every seconds
timeSlice: 4000,
//2)
// as soon as the stream is available
ondataavailable: function(blob) {
// 3
// making use of socket.io-stream for bi-directional
// streaming, create a stream
var stream = ss.createStream();
// stream directly to server
// it will be temp. stored locally
ss(socket).emit('stream', stream, {
name: 'stream.wav',
size: blob.size
});
// pipe the audio blob to the read stream
ss.createBlobReadStream(blob).pipe(stream);
}
});
recordAudio.stopRecording();
stopRecording.disabled = false;
}, function(error) {
console.error(JSON.stringify(error));
});
};
// 4)
// on stop button handler
stopRecording.onclick = function() {
// recording stopped
recordAudio.startRecording();
startRecording.disabled = false;
stopRecording.disabled = true;
// socketio.emit("close",function(){
// })
};
// const aduiopreview = document.getElementById('result_audio');
const resultpreview = document.getElementById('results');
const intentInput = document.getElementById('intent');
const textInput = document.getElementById('queryText');
socketio.on('results', function (data) {
console.log(data);
if(data.queryResult){
resultpreview.innerHTML += "" + data.queryResult.fulfillmentText;
intentInput.value = data.queryResult.intent.displayName;
textInput.value = "" + data.queryResult.queryText;
}
if(data.outputAudio){
console.log(data.outputAudio);
playOutput(data.outputAudio)
}
});
/*
* When working with Dialogflow and Dialogflow matched an intent,
* and returned an audio buffer. Play this output.
*/
function playOutput(arrayBuffer){
let audioContext = new AudioContext();
let outputSource;
try {
if(arrayBuffer.byteLength > 0){
console.log(arrayBuffer.byteLength);
audioContext.decodeAudioData(arrayBuffer,
function(buffer){
audioContext.resume();
outputSource = audioContext.createBufferSource();
outputSource.connect(audioContext.destination);
outputSource.buffer = buffer;
outputSource.start(0);
},
function(){
console.log(arguments);
});
}
} catch(e) {
console.log(e);
}
}
</script>
</body>
</html>