Trying to learn more about streams and cassandra, so figured I'd try out the binary protocol. I can't even get it to respond though.
var net = require('net');
var util = require('util');
var stream = require('stream');
var session;
session = net.connect({ port: 9160 }, function () {
console.log('connected');
var header = new Buffer([ 0x01, 0x00, 0x01, 0x01 ]);
var length = new Buffer(4);
var body = new Buffer("{'CQL_VERSION':'3.0.0'}", 'utf8');
length.writeUInt32BE(body.length, 0);
session.write(header);
session.write(length);
session.write(body);
setTimeout(function () { session.end(); }, 5000);
});
session.on('error', function (err) {
console.log(err);
});
var client = new (stream.Writable);
client._write = function (chunk, _, next) {
console.log('response received');
next();
};
session.pipe(client);
The program runs for 5 seconds, printing "connected" almost immediately, but the writable stream never receives content from the database. Any help would be much appreciated!
I'm working off of this documentation for the binary protocol and this guide to node streams.
Cassandra is running, version 2.0.5.
9160 is the Thrift's port. Have you tried your code with port 9042?
Related
I'm using node with the mqtt-connection and aedes mqtt libraries. I am using aedes to run a mqtt server, and I wish to connect to the mqtt server from within node using a stream, rather than open a tcp socket. Both libraries will accept a Duplex stream.
Why does something like this work:
const mqttCon = require('mqtt-connection');
const duplex = require('net').createConnection(1883);
const client = mqttCon(duplex, {
protocolVersion: 3
});
While something like this fails (closes the stream)after the first data exchange?
const mqttCon = require('mqtt-connection');
const duplex = new stream.Transform();
duplex._transform = (chunk, encoding, callback) => {
duplex.push(chunk);
console.log(chunk);
callback();
};
const client = mqttCon(duplex, {
protocolVersion: 3
});
aedes.handle(duplex);
I feel like I must have some fundamental misconception about how streams are supposed to work. Basically I want to create something that acts like a TCP socket, allowing these two "processes" to communicate internal to node.
Typical use of aedes to create a mqtt server would looking this this:
const aedes = require('aedes')(
{
concurrency: 500,
maxClientsIdLength: 100
}
);
const server = require('net').createServer(aedes.handle);
Edit: More details about the failure.
Then client feeds data to the stream, then as soon as aedes finishes responding, duplex.on('close') fires. There are no error messages, and no indication from either side that there has been an error, the stream just closes so each side then closes gracefully. I'm guessing that one side or the other sees an "end" to the stream so it closes.
The problem is that "duplex" is a single stream, where it really needs two streams bridged. Here is what works:
const duplexAedes = new stream.Duplex({
write: function (chunk, encoding, next) {
setImmediate(function () {
duplexClient.push(chunk);
});
next();
},
read: function (size) {
// Placeholder
}
});
const duplexClient = new stream.Duplex({
write: function (chunk, encoding, next) {
setImmediate(function () {
duplexAedes.push(chunk);
});
next();
},
read: function (size) {
// Placeholder
}
});
duplexAedes.authentication = clientId;
const client = mqttCon(duplexClient, {
protocolVersion: 3
});
aedes_handle(duplexAedes);
I am making use of "socket.io-client" and "socket.io stream" to make a request and then stream some data. I have the following code that handles this logic
Client Server Logic
router.get('/writeData', function(req, res) {
var io = req.app.get('socketio');
var nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
var nameNodeData = {};
async.waterfall([
checkForDataNodes,
readFileFromS3
], function(err, result) {
if (err !== null) {
res.json(err);
}else{
res.json("Finished Writing to DN's");
}
});
function checkForDataNodes(cb) {
nameNodeSocket.on('nameNodeData', function(data) {
nameNodeData = data;
console.log(nameNodeData);
cb(null, nameNodeData);
});
if (nameNodeData.numDataNodes === 0) {
cb("No datanodes found");
}
}
function readFileFromS3(nameNodeData, cb) {
for (var i in nameNodeData['blockToDataNodes']) {
var IP = nameNodeData['blockToDataNodes'][i]['ipValue'];
var dataNodeSocket = io.connect('http://'+ IP +":5000");
var ss = require("socket.io-stream");
var stream = ss.createStream();
var byteStartRange = nameNodeData['blockToDataNodes'][i]['byteStart'];
var byteStopRange = nameNodeData['blockToDataNodes'][i]['byteStop'];
paramsWithRange['Range'] = "bytes=" + byteStartRange.toString() + "-" + byteStopRange.toString();
//var file = require('fs').createWriteStream('testFile' + i + '.txt');
var getFileName = nameNodeData['blockToDataNodes'][i]['key'].split('/');
var fileData = {
'mainFile': paramsWithRange['Key'].split('/')[1],
'blockName': getFileName[1]
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
s3.getObject(paramsWithRange).createReadStream().pipe(stream);
//dataNodeSocket.disconnect();
}
cb(null);
}
});
Server Logic (that gets the data)
var dataNodeIO = require('socket.io')(server);
var ss = require("socket.io-stream");
dataNodeIO.on('connection', function(socket) {
console.log("Succesfully connected!");
ss(socket).on('sendData', function(stream, data) {
var IP = data['ipValue'];
var blockName = data['blockName'];
var mainFile = data['mainFile'];
dataNode.makeDir(mainFile);
dataNode.addToReport(mainFile, blockName);
stream.pipe(fs.createWriteStream(mainFile + '/' + blockName));
});
});
How can I properly disconnect the connections in function readFileFromS3. I have noticed using dataNodeSocket.disconnect() at the end does not work as I cannot verify the data was received on the 2nd server. But if I comment it out, I can see the data being streamed to the second server.
My objective is to close the connections in Client Server side
It appears that the main problem with closing the socket is that you weren't waiting for the stream to be done writing before trying to close the socket. So, because the writing is all asynchronous and finishes sometime later, you were trying to close the socket before the data had been written.
Also because you were putting asynchronous operations inside a for loop, you were also running all your operations in parallel which may not be exactly what you want as it makes error handling more difficult and server load more difficult.
Here's the code I would suggest that does the following:
Create a function streamFileFromS3() that streams a single file and returns a promise that will notify when it's done.
Use await in a for loop with that streamFileFromS3() to serialize the operations. You don't have to serialize them, but then you would have to change your error handling to figure out what to do if one errors while the others are already running and you'd have to be more careful about concurrency issues.
Use try/catch to catch any errors from streamFileFromS3().
Add error handling on the stream.
Change all occurrences of data['propertyName'] to data.propertyName. The only time you need to use brackets is if the property name contains a character that is not allowed in a Javascript identifier or if the property name is in a variable. Otherwise, the dot notation is preferred.
Add socket.io connection error handling logic for both socket.io connections.
Set returned status to 500 when there's an error processing the request
So, here's the code for that:
const ss = require("socket.io-stream");
router.get('/writeData', function(req, res) {
const io = req.app.get('socketio');
function streamFileFromS3(ip, data) {
return new Promise((resolve, reject) => {
const dataNodeSocket = io.connect(`http://${ip}:5000`);
dataNodeSocket.on('connect_error', reject);
dataNodeSocket.on('connect_timeout', () {
reject(new Error(`timeout connecting to http://${ip}:5000`));
});
dataNodeSocket.on('connection', () => {
// dataNodeSocket connected now
const stream = ss.createStream().on('error', reject);
paramsWithRange.Range = `bytes=${data.byteStart}-${data.byteStop}`;
const filename = data.key.split('/')[1];
const fileData = {
'mainFile': paramsWithRange.Key.split('/')[1],
'blockName': filename
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
// get S3 data and pipe it to the socket.io stream
s3.getObject(paramsWithRange).createReadStream().on('error', reject).pipe(stream);
stream.on('close', () => {
dataNodeSocket.disconnect();
resolve();
});
});
});
}
function connectError(msg) {
res.status(500).send(`Error connecting to ${NAMENODE_ADDRESS}`);
}
const nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
nameNodeSocket.on('connect_error', connectError).on('connect_timeout', connectError);
nameNodeSocket.on('nameNodeData', async (nameNodeData) => {
try {
for (let item of nameNodeData.blockToDataNodes) {
await streamFileFromS3(item.ipValue, item);
}
res.json("Finished Writing to DN's");
} catch(e) {
res.status(500).json(e);
}
});
});
Other notes:
I don't know what paramsWithRange is as it is not declared here and when you were doing everything in parallel, it was getting shared among all the connections which is asking for a concurrency issue. In my serialized implementation, it's probably safe to share it, but the way it is now bothers me as it's a concurrency issue waiting to happen.
I have socket.io sending a basic object from server to client. This bit works fine.
Now want to send a stream from server to client, using event-stream (specifically the results of a block-chain query). I am getting unexpected results in the browser console..
var io = require('socket.io')(server);
var dsteem = require('dsteem')
var es = require('event-stream')
var util = require('util')
var client = new dsteem.Client('https://api.steemit.com')
var stream = client.blockchain.getBlockStream()
/* This sends results to stdout, fine
io.on('connection', function(socket){
stream.pipe(es.map(function(block, callback) {
callback(null, util.inspect(block) + '\n')
})).pipe(process.stdout);
// And this sends a simple object to the client
socket.emit('blockchainOps', {"Foo!":"Doo!"} );
});
*/
// Putting both together sends strange connection data to client
io.on('connection', function(socket){
socket.emit('blockchainOps', function() {
stream.pipe(es.map(function(block, callback) {
callback(null, util.inspect(block) + '\n');
}))
})
});
What I get in the client console appears to be some kind of TCP socket function,
ƒ (){if(!n){n=!0;var r=a(arguments);u("sending ack %j",r),e.packet({type:i.ACK,id:t,data:r})}}
Can anyone help me understand what's going on and what I'm doing wrong?
== EDIT UPDATE ==
As suggested in comments, I've tried socket.io-stream to augment event-stream.
var es = require('event-stream')
var util = require('util')
var ss = require('socket.io-stream');
var stream = ss.createStream();
io.on('connection', function(socket){
ss(socket).emit('blockchainOps', stream, function(){
client.blockchain.getBlockStream()
.pipe(es.map(function(block, callback) {
callback(null, util.inspect(block) + '\n')
}))
.pipe(process.stdout)
}());
});
This time I get a socket object returned in the browser console which does not seem to be the stream data I was hoping for.
If anyone is looking for a working socket.io stream example
// server side
const { pipeline } = require('stream')
const server = require('http').Server().listen(8080)
const io = require('socket.io')(server)
const ss = require('socket.io-stream')
io.on('connection', (socket) => ss(socket).on('stream', (stream) => {
pipeline(stream, process.stdout, (err) => err && console.log(err))
}));
// client side
const client = require('socket.io-client')
const socket = client.connect('http://localhost:8080')
socket.on('connect', () => {
const stream = ss.createStream()
ss(socket).emit('stream', stream)
pipeline(process.stdin, stream, (err) => err && console.log(err))
});
You're using socket.emit wrong, you're passing the ACK callback to the client instead of your stream. Have a look at socket.emit signature :socket.emit(eventName[, ...args][, ack]).
You probably want something like
socket.emit('blockchainOps', client.blockchain.getBlockStream());
However, I don't think plain socket io supports passing a Stream like that. To pipe a stream down to the client you could use socketio-stream. It would look like this:
var ss = require('socket.io-stream');
var stream = ss.createStream();
ss(socket).emit('blockchainOps', stream);
client.blockchain.getBlockStream().pipe(stream);
EDIT:
On the client, you should be able to read your stream like this:
<script src="socket.io/socket.io.js"></script>
<script src="socket.io-stream.js"></script>
...
ss(socket).on('blockchainOps', function(stream) {
var binaryString = "";
stream.on('data', function(data) {
for(var i=0;i<data.length;i++) {
binaryString+=String.fromCharCode(data[i]);
}
});
stream.on('end', function(data) {
console.log(binaryString);
binaryString = "";
});
});
Trying to write a TCP client in Node v0.10.15 and I am having a little trouble getting data back from the server. I know that the server is working properly because I have 3-4 different clients written in different languages communicating with it.
Below is a snippet of a larger piece of code but this should get the point across.
The problem is: I'm expecting 2 packets coming back after writing to the socket (this part is not included in this example). I'm only seeing the "data" event being fired once. Is there something that I need to do to get node to resume reading from the Tcp stream? I can confirm that the server is sending 2 packets(The length and then the actual data) Any help would be appreciated.
var dc = require('./DataContracts.js');
var net = require('net');
require('buffertools').extend();
var client = net.Socket();
var isConnected = false;
var serverHost = '10.2.2.21';
var dataCallback;
var receivedBuffer = new Array();
function InitComm(buffer) {
if (!isConnected) {
client.connect(4987, serverHost, function() {
client.on('data', function(data) {
console.log('Received server packet...');
var buf = new Buffer(data);
receivedBuffer.push(buf);
client.resume();
});
client.on('end', function() {
if (receivedBuffer.length > 1) {
if (dataCallback !== undefined)
dataCallback(receivedBuffer);
}
});
client.on('close', function() {
//clean up
});
client.on('error', function(err) {
console.log('Error!: ' + err);
});
Communicate(buffer);
});
} else {
Communicate(buffer);
}
}
Turns out that node was combining both of the packets together. I must be missing a Carriage return on the first packet
var server = net.createServer(function(c) {
...
})
server.getConnections(function(err, count){
console.log("count", count);
})
I get the following error.
Object #<Server> has no method 'getConnections'
How to use getConnections of a tcp server?
I'm using node version v0.10.16
I'm not sure exactly why your code doesn't work. While having the request to getConnections outside of the connection callback is not necessary typical, it worked in my tests. server.connections is deprecated per the documentation, so it's use is discouraged.
Using telnet localhost 1337, a really poor echo socket is emulated below, and the current count of connections is displayed. The code below worked in my tests:
var server = require('net').createServer();
server.on('connection', function (socket) {
socket.on('data', function(data) {
socket.write(data);
});
server.getConnections(function(err, count) {
console.log("Connections: " + count);
});
});
server.listen(1337);
You can try like this:
var net = require('net');
var events = require('events');
var eventEmitter = new events.EventEmitter();
var server = net.createServer(function(c) {
...`enter code here`
eventEmitter.emit('event1');
})
eventEmitter.on('event1', function(){
server.getConnections(function(err, count){
console.log("count", count);
})
});