I'm trying to send the audio stream of a websocket connection to the Google Speech API. The websocket sends binary messages in 20ms increments. That it sends it in increments leads me to believe that I will somehow have to read and write the data to a local file temporarily to avoid killing the connection to Google. However, this is not ideal.
Is there a way to directly pipe the websocket stream into recognizeStream?
Google streamingRecognize example from the docs:
const request = {
config: {
encoding: encoding,
sampleRate: sampleRate
}
};
const recognizeStream = speech.createRecognizeStream(request)
.on('error', console.error)
.on('data', (data) => process.stdout.write(data.results));
record.start({
sampleRate: sampleRate,
threshold: 0
}).pipe(recognizeStream);
Websocket connection:
var HttpDispatcher = require('httpdispatcher');
var dispatcher = new HttpDispatcher();
var WebSocketServer = require('websocket').server;
var server = http.createServer(handleRequest);
var wsServer = new WebSocketServer({
httpServer: server,
autoAcceptConnections: true,
});
function handleRequest(request, response){
try {
//log the request on console
console.log(request.url);
//Dispatch
dispatcher.dispatch(request, response);
} catch(err) {
console.log(err);
}
}
wsServer.on('connect', function(connection) {
console.log((new Date()) + ' Connection accepted' + ' - Protocol Version ' + connection.webSocketVersion);
connection.on('message', function(message) {
if (message.type === 'utf8') {
console.log(message.utf8Data);
}
else if (message.type === 'binary') {
//Send to Google Speech API by passing into recognizeStream
}
});
connection.on('close', function(reasonCode, description) {
console.log((new Date()) + ' Peer ' + connection.remoteAddress + ' disconnected.');
});
});
This is actually quite simple. So simple that I feel a little sheepish for not seeing it. In accordance to exactly how the code was written in the OP this works perfectly:
else if (message.type === 'binary') {
//Send to Google Speech API by passing into recognizeStream
recognizeStream.write(message.binaryData)
}
The best solution would be to use a specialized streaming solution rather than doing it yourself, this will handle all the buffers and give you a steady stream that's suitable for Google Speech API.Try using something like,
https://www.npmjs.com/package/websocket-stream
Related
I have managed to retrieve the Video Stream from my React Native client using mediaDevices, also the React Native Client is connecting to my NodeJS Server. Once connected the server responds to the Client saying that the Client has connected to the NodeJS server. But once connected i want to send a Video Stream that i get from the mediaDevices and send the Video Stream to the NodeJS server and then in return send the Video Stream back from the NodeJS Sever to the React Native client again.
Your probably asking why don't you just stream the Video straight onto the client rather than sending it to the server and back to the client again.
The reason is that i just want to learn how to use socket.io and socket.io-client to send and receive video streams.
Here is some of the code and how im trying to achieve this atm (Sevrer.js):
io.on("connection", socket => {
if(!patients[socket]){
patients[socket.id] = socket;
patients[socket.id].emit('PatientConnected', 'Success');
console.log(`a user connected :D ${socket.id}`);
}else{
}
patients[socket.id].on('stream', function(data) {
console.log("stream from client " + data.stream);
var bufArr = new ArrayBuffer(data.stream);
var bufView = new Uint8Array(bufArr);
patients[socket.id].emit("ReturnStream", bufView);
});
socket.on('error', function() {
delete patients[socket.id];
});
socket.on('disconnect', function() {
delete patients[socket.id];
});
});
And my Client (App.js):
const socket = io.connect("http://192.168.1.107:3000", {reconnect: true});
console.log("socket: " + socket);
socket.on("PatientConnected", (msg) => {
if(msg == 'Success'){
console.log("socket connected: " + socket.id);
}
});
socket.on("ReturnStream", (stream) => {
console.log("ReturnStream: " + stream);
var bufView = new Uint8Array(stream);
this.setState({ stream : bufView})
});
const isFront = true;
const facing = isFront ? 'front' : 'environment';
mediaDevices.enumerateDevices().then(sourceInfos => {
let videoSourceId;
for (let i = 0; i < sourceInfos.length; i++) {
const sourceInfo = sourceInfos[i];
if(sourceInfo.kind == "videoinput" && sourceInfo.facing == (isFront ? "front" : "back")) {
videoSourceId = sourceInfo.deviceId;
}
}
mediaDevices.getUserMedia({
audio: true,
video: {
mandatory: {
minWidth: 500,
minHeight: 300,
minFrameRate: 30
},
facingMode: (isFront ? "user" : "environment"),
optional: (videoSourceId ? [{sourceId: videoSourceId}] : [])
}
})
.then(stream => {
if(socket){
console.log('stream' + stream);
socket.emit('stream', {stream: stream});
}
})
.catch(error => {
// Log error
});
});
I am getting undefined in bufView from "var bufView = new Uint8Array(stream);", where am i going wrong or what am i missing something?. I am new to socket.io and socket.io-client, are these the libraries that allow the systems to send and receive Video Streams? and how can this be achieved?
So I am trying to setup a socket server in node.js using node-ipc, then send data from a client. I can connect perfectly fine, however when I send data I recieve the error Messages are large, You may want to consider smaller messages. I have followed all advice here, however, have still been unsuccessful in resolving the problem. I have tried sending from a socket connection in C and also from terminal. Any advice would be great, thanks in advance.
main.js
const ipc = require("node-ipc");
const socket = '/tmp/edufuse.sock';
ipc.serve(
socket,
function(){
ipc.server.on(
'myEvent',
function(data){
ipc.log('got a message : '.debug, data);
}
);
}
);
ipc.server.start();
test.json
```json
{ type: message, data: { foo: bar } }
command from terminal
pr -tF test.json | nc -U /tmp/edufuse.sock
Unfortunately, it appears this is an underlying problem with node-ipc. In order to get around this, I used net sockets and TCP.
const net = require('net');
const port = 8080;
const host = '127.0.0.1';
var server = net.createServer(function(socket) {
socket.on('data', function(data){
let str = data.toString('utf8');
console.log(str);
try {
let json = JSON.parse(str);
console.log(json);
} catch (e) {
console.log('error str: ' + str);
}
});
socket.on('error', function(err) {
console.log(err)
})
});
server.listen(port, host);
Trying to get a WebSocket working using rxjs webSocket
I set up a node server, that throws no errors at the moment, but it doesn't send messages to the server or connected users
It seems to work better when I use 'ws://echo.websocket.org/', I see messages echoed back to me, however, I don't see messages sent, in other open browsers, basically no communication between users, that are supposed to be subscribed to the websocket
All I want is a simple way of carrying simple info between users
index.js
var express = require('express');
var WebSocketServer = require('websocket').server;
var app2 = express()//There is already another app var for main app server
var server2 = http.createServer(app2);
var server2 = app2.listen(8081, function () {
console.log((new Date()) + ' Server is listening on port 8081');
})
wsServer = new WebSocketServer({
httpServer: server2,
autoAcceptConnections: false
});
function originIsAllowed(origin) {
if (origin === 'http://localhost:1234'){
return true;
}
// put logic here to detect whether the specified origin is allowed.
}
wsServer.on('request', function(request) {
if (!originIsAllowed(request.origin)) {
// Make sure we only accept requests from an allowed origin
request.reject();
console.log((new Date()) + ' Connection from origin ' + request.origin + ' rejected.');
return;
}
var connection = request.accept(null, request.origin);
console.log((new Date()) + ' Connection accepted.');
connection.on('message', function(message) {
// console.log(message)
if (message.type === 'utf8') {
console.log('Received Message from: ' + connection.remoteAddress);
connection.sendUTF(message.utf8Data);
}
else if (message.type === 'binary') {
console.log('Received Binary Message of ' + message.binaryData.length + ' bytes');
connection.sendBytes(message.binaryData);
}
});
connection.on('close', function(reasonCode, description) {
console.log((new Date()) + ' Peer ' + connection.remoteAddress + ' disconnected.');
});
});
in my clientside socket service:
import { Injectable } from '#angular/core';
import { Observable } from '../../../node_modules/rxjs';
#Injectable()
export class SocketService {
socket$ = Observable.webSocket( 'ws://localhost:8081');
// 'ws://echo.websocket.org/'
constructor() {
this.connectToSocket()
}
connectToSocket(){
this.socket$.subscribe(
(message) => console.log('message received: ' + message),
(err) => console.log(err),
() => console.log('complete')
);
}
}
And to send message, from another component, I use:
this.socketService.socket$.next(JSON.stringify('test'))
According to the documentation of WebSocket-Node (https://github.com/theturtle32/WebSocket-Node/blob/master/docs/WebSocketServer.md), you have to programatically accept each connection before continuing the handshake.
You can instead set the autoAcceptConnections option of your server config to true, like this:
const wss = new WebSocketServer({ httpServer: server2, autoAcceptConnections: true });
If you do indeed have custom request checking requirements, you can use either accept or reject inside your request handler, like this:
request.accept('chosen-protocol', 'accepted-origin')
or
request.reject()
My requirement is a little bit different, don't know even if it's achievable.
I am using Node.js for developing backend application server. This server basically does two jobs:
(1) Serving clients: My clients are all mobile phones who will be sending HTTP(S) request and after receiving the response will close the session.
(2) Calling some other asynchronously working service: The server, on the other hand, will be connected to some other server which works over just TCP/IP connection and not HTTP. Asynchronous here means, the server will send a request and should not wait for a response. The response will be received through same TCP/IP connection.
So the flow I want to achieve is:
Mobile phone sends the HTTP request to server
Server after receiving the HTTP request, does a call to service which is on TCP/IP
Server receives the response from TCP/IP service over the TCP/IP connection
Server responds to the phone with the response.
To represent the above flow I have attached the below image.
In the above image the TCP/IP Server is managed by some other provider.
I wrote the following code in node.js which works perfectly as per our requirement some times, but some times it sends incorrect response to the HTTP request. I did not write any code to handle this issue.
var net = require('net');
var client = new net.Socket();
client.connect(2202, 'example_ip', function () {
console.log('Connected');
// client.write('Hello, server! Love, Client.');
});
//Lets require/import the HTTP module
var http = require('http');
//Lets define a port we want to listen to
const PORT = 8080;
//We need a function which handles requests and send response
function handleRequest(request, response) {
var body = '';
request.on('data', function (chunk) {
body += chunk;
});
request.on('end', function () {
console.log('Received request from JMeter------------>>>');
// console.log(body);
client.write(body);
var count = 0;
client.on('data', function (data) {
console.log('<<<------------Received from SSM: ' + data);
response.end(data);
// client.destroy(); // kill client after server's response
});
});
client.on('close', function () {
console.log('Connection closed');
});
}
//Create a server
var server = http.createServer(handleRequest);
//Lets start our server
server.listen(PORT, function () {
//Callback triggered when server is successfully listening. Hurray!
console.log("Server listening on: http://localhost:%s", PORT);
});
Please some one guide me to solve this issue.
TCP streams don't work like WebSocket streams ( as you expect ). You need to use your own protocol to communicate with a TCP server. Keep in mind that HTTP clients are many and you have only one TCP connection to handle them, so use requestIds like below, code explains itself.
Not tested, but you can get the idea.
shared.js
exports.tcp = {
host: 'example_ip',
port: 2202
};
exports.http = {
host: 'localhost',
port: 8080
};
/**
* TCP "guarantees" that a receiver will receive the reconstituted
* stream of --> BYTES <-- as it was originally sent by the sender.
*
* eg. if written message = 'How are you today?'
* the messages can come to us as follows:
*
* 'How ar'
* 'e you '
* 'today?'
*
* so we need to use a simple protocol to handle messages
*/
exports.protocol = protocol;
function protocol(options) {
if (!options) options = {};
this.END_OF_MESSAGE = options.endOfMessage || '\0';
this.END_OF_PART = options.endOfPart || '\1';
this.dataBuffer = '';
}
protocol.prototype.packMessage = function(id, body) {
return [id, body].join( this.END_OF_PART ) + this.END_OF_MESSAGE;
};
protocol.prototype.unpackMessage = function(message) {
var parts = message.toString('utf8').split( this.END_OF_PART );
return {id: parts.shift(), body: parts.shift()};
};
protocol.prototype.extractMessages = function(data, callback) {
this.dataBuffer += data.toString('utf8');
if (this.dataBuffer.indexOf(this.END_OF_MESSAGE) !== -1)
{
var messages = this.dataBuffer.split(this.END_OF_MESSAGE);
var incomplete = this.dataBuffer.slice(-1) === this.END_OF_MESSAGE
? '' : messages.pop();
messages.forEach(function(message)
{
if (message !== '') {
callback( this.unpackMessage(message) );
}
});
this.dataBuffer = incomplete;
// rest of 'data'
}
/**
if (Buffer.byteLength(this.dataBuffer, 'utf8') > 10240) { // 10KB
console.log('[!] socket flooded');
this.dataBuffer = '';
}
*/
};
protocol.prototype.reset = function() {
this.dataBuffer = '';
};
httpServer.js
var http = require('http');
var net = require('net');
var shared = require('./shared.js');
var protocol = new shared.protocol();
var server = http.createServer(handleRequest);
server.listen(shared.http.port, shared.http.host, function() {
console.log('HTTP server listening: %s:%s', shared.http.host, shared.http.port);
});
function handleRequest(request, response) {
var body = '';
var requestId = nextId++;
var eventName = 'message' + requestId;
request.on('data', function(chunk) {
body += chunk.toString('utf8');
});
request.on('end', function()
{
// ref#2
client.write( protocol.packMessage(requestId, body) );
// ref#3
client.once(eventName, function(data) {
clearTimeout(timeoutId);
response.end(data);
});
});
var timeoutId = setTimeout(function() {
client.removeListener(eventName);
response.end('timeout');
}, 10000); // 10 sec.
/**
* [!] Don't do this; you are adding just another 'data' event to
* the TCP client for EVERY http request !?
*
* request: UNIQUE obj. for every http request
* client: a PERSISTENT (TCP) stream obj.
*
client.on('data', function() { });
**/
}
var client = new net.Socket();
// ref#1
client.connect(shared.tcp.port, shared.tcp.host, function() {
console.log('TCP conn. established to: ', shared.tcp.host, shared.tcp.port);
});
var nextId = 0;
// unique per http req.
/**
* [!] Do this ( once ) ( not for every request )
*/
client.on('data', function(data)
{
protocol.extractMessages(data, function(message) {
client.emit('message' + message.id, message.body);
// ref#3
});
});
client.on('close', function()
{
console.log('TCP conn. closed');
client.removeAllListeners();
})
client.on('error', function()
{
console.log('TCP conn. error', arguments);
// client.destroy(); // and reconnect here
});
tcpServer.js
var net = require('net');
var shared = require('./shared.js');
var protocol = new shared.protocol();
var server = net.createServer(handleConnection);
server.listen(shared.tcp, function() {
console.log('TCP server listening %s:%s', shared.tcp.host, shared.tcp.port);
});
// [!] CONNECTION handler ( ref#1 )
function handleConnection(client)
{
var this.dataBuffer = '';
// [!] DATA handler ( ref#2 )
client.on('data', function(data) {
protocol.extractMessages(data, function(message)
{
var requestId = message.id;
var body = message.body;
// Do whatever you want with 'body' here
/**
* And return back to 'client' with 'requestId' using same protocol again
* so the 'client' ( from httpServer.js ) can handle your response
*/
client.write( protocol.packMessage(requestId, body) );
});
});
}
Can you instantiate a new client for each incoming request? This way the TCP connection for each request will be unique.
What i tried to achieve with node.js/io.js, is to send a file from one server to another one via a proxy. To avoid memory buffering i want to use streams.
The proxy should be able to connect to multiple targets dynamically. The target connection information for the proxy should be send prior to the filedata.
With normal socket communication and buffering it is not a problem. But how or in general can this be done with streams??
var net = require('net');
var fs = require('fs');
//create readstream from file
var myFile = fs.createReadStream('E:/sample.tar.gz');
// Proxy server
//####################################################################################################
var proxy = net.createServer(function (socket) {
// Create a new connection to the TCP server
var client = net.connect('9010');
// 2-way pipe between client and TCP server
socket.pipe(client).pipe(socket);
}).listen(9000);
// Targetserver
//####################################################################################################
var server = net.createServer(function (socket) {
// create filestream to write data into file
var destfile = fs.createWriteStream('E:/sample_copy.tar.gz')
socket.on('data', function (buffer) {
console.log('Get data on targetserver...');
// write buffer to file
destfile.write(buffer);
});
socket.on('end', function () {
// release file from writestream
destfile.end();
});
}).listen(9010);
// Client
//####################################################################################################
// Send file to proxy
var client = new net.Socket();
// connect to proxy
client.connect('9000', '127.0.0.1', function () {
console.log('Connection to proxy opened');
});
// send data to proxy
myFile.pipe(client);
// read response from taget
client.on('data', function(data) {
console.log('Response: ' + data);
// close the client socket completely
client.destroy();
});
// Add a 'close' event handler for the client socket
client.on('close', function() {
console.log('Connection to proxy closed');
});
Any hint to a good tutorial is also welcome.
TMOE
socket.write() already uses streams under the hood so you don't need to do anything special. Just send it the usual Buffer object or string and it will use a stream.
From the current source code of io.js, here's what happens when you use socket.write():
Socket.prototype.write = function(chunk, encoding, cb) {
if (typeof chunk !== 'string' && !(chunk instanceof Buffer))
throw new TypeError('invalid data');
return stream.Duplex.prototype.write.apply(this, arguments);
};
And stream is declared like this:
const stream = require('stream');
Apologies if I've misunderstood your question/requirements! By all means, clarify if I have misunderstood you and I'll try again (or delete this answer so it's not a distraction).