I am developing an API using Node.js. In my application when i hit an URL though browser i gto JSON response in my browser perfectly.. But when i get the response through my node.js code, its coming as junk.
Consider i am hitting the following url in browser:
localhost:2000/xxxxx/rrrrr/ggggg
I am receiving perfect output.
The following is the node.js code:
proxyReq.on("response", function(proxyRes) {
var body = ''
try{
proxyRes.on("data", function(chunk) { //Capture API response here---revisit
body += new Buffer(chunk, 'binary').toString();
//console.log("cccc=" +chunk)
/*zlib.unzip(chunk.toString(), function(err, chunk){
console.log("Inside zliib");
if (!err){
console.log('Response'+chunk.toString())
} else {
console.log("Inside zlib error");
}
});*/
//body = chunk.toString();
//console.log(chunk.toString('utf-8'));
console.log('cccccccc=' +body);
});
}catch(err){
console.log("errrr=" +err.stack);
}
}
Here the 'body' is printing as some junked data. I tried for "utf-8" and "binar" nothing works. Help me to solve this. Thanks in advance.
Mu Junked Data:
Ys��ǿJ����}/c�؎}�%�ld� a�T��mIL�Z� ]�T��I�r����ߕ�h4��ϕ�7t{�Q4��8���\�L�N؛T��VWM�r
?W&Q��N&/Q����|�W9??W��t�b�n:t>:��(t��G��K��w��=��r\���_��W�N�c��{���u�ۺU���m�^���z�'�ǫ��LFQ�uc���s�>��f
I didn't install using npm:
Third party code regarding ProxyReq:
var ended, mod, proxyReq, req_options;
if (err) {
return _this.error(err, req, res);
}
mod = req.api.data.protocol === "https" ? https : http;
req_options = _this.getHttpProxyOptions(req);
req_options.agent || (req_options.agent = new mod.Agent({
maxSockets: 100,
rejectUnauthorized: req.api.data.strictSSL
}));
_this.logger.debug(("Backend: " + req_options.method + " to ") + ("'" + req.api.data.protocol + "://") + ("" + req_options.host + ":" + req_options.port + req_options.path));
proxyReq = mod.request(req_options);
proxyReq.setTimeout(req.api.data.endPointTimeout * 1000, function() {
var e;
e = new Error("ETIMEDOUT");
e.code = "ETIMEDOUT";
proxyReq.emit("error", e);
return proxyReq.abort();
});
ended = false;
I found your problem. This is the documentation of event response for http/https
Emitted when a response is received to this request. This event is emitted only once. The response argument will be an instance of http.IncomingMessage.
And http.IncomingMessage is a ReadableStream.
After that, you should use this two events of stream to convert this into String :
Event: 'data'
Event: 'end'
For stream.on('data', ...) you should collect your data data into either a Buffer (if it is binary) or into a string.
For on('end', ...) you should call a callback with you completed buffer, or if you can inline it and use return using a Promises library.
Example, for you, you can change your proxyReq.on("response") callback content by this :
var Buffer = require('buffer').Buffer;
proxyReq.on("response", function(chunk) {
//proxyRes.setEncoding ('utf8');
var body = '';
chunk.on('data', function(data) {
body += new Buffer(data, 'binary').toString();
});
chunk.on('end', function () {
// callback
});
chunk.on('error', function (err) {
// catchable error
});
});
To simplify you request part, I recommand you to use the node-request package : https://github.com/mikeal/request
Related
I'd like parse a log file and POST what is read to a request endpoint. I've managed to build a solution that generates a request for every log line read. However, it doesn't create any back pressure so it just flogs the server and I'd like to slow it down.
This lead me to investigate using stream pipes to see if I could route data from a file directly into request.post(). I can't get the post call to post a body object though.
var stream = require('stream');
var request = require('request');
var liner = new stream.Transform( { objectMode: true } );
liner._transform = function (chunk, encoding, done) {
var data = chunk.toString()
if (this._lastLineData) data = this._lastLineData + data
var lines = data.split('\n')
this._lastLineData = lines.splice(lines.length-1,1)[0]
var that = this;
lines.forEach(function(line) {
var line_obj = JSON.parse(line);
if( line_obj.url === "/api/usages" && line_obj.method === 'POST' ) {
var req_body = line_obj.body.body;
that.push.bind(req_body);
}
});
done();
}
var file_name = process.argv[2];
console.log('Reading from ' + file_name);
var fs = require('fs')
var liner = require('./liner')
var source = fs.createReadStream(file_name)
source.pipe(liner).pipe(request
.post("http://localhost:8081/api/usages")
.on('response', function(response) {
console.log(response.statusCode) // 200
})
.on('error', function(err) {
console.log(err);
}));
The push call in the transform function is working correctly, but it's not posting that object via the body in request.post().
What am I missing?
Will this provide the back pressure I'm looking for to throttle the POST calls before all of the file reads are completed?
I've discovered that you cannot pipe a stream to an HTTP request because you would need the Content-Length known before hand (as per spec). The less pleasant alternative is to multipart the upload - as chunks are read from your transform they would marshal parts to the receiving API. This also means the receiving API needs to be able to receive multipart uploads and reassemble the whole file after all parts have been received and confirmed. AWS S3 has multipart uploads and it might be a nice example: http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html
I wanted to pipe my transform data to another API that I manage but it seems the effort is not likely worth it considering my files really aren't that big. I'll update this answer if I change my mind :)
Although I wasn't able to find a solution to the streaming question, I found a simple solution to the back pressure question.
I used async.queue to push work into a simple task queue.
// build the send queue
var pool = new http.Agent({keepAlive: true, keepAliveMsecs: 10000, maxSockets: Math.floor(send_queue_concurrency*1.5)});
var q = async.queue(function(task, callback){
request({
url : 'http://localhost:8081/xxxxxx',
method : 'POST',
json : task.req_body,
gzip : true,
pool : pool,
timeout: 30000
}, function(error, response, body){
if(error) {
console.log('request error : ' + error);
post_status.fail++;
} else {
if( response.statusCode === 400 ) {
console.dir(body);
}
}
callback();
});
}, send_queue_concurrency);
q.drain = done;
send_queue_concurrency is the primary lever for controlling request pressure.
i'm pushing work into the queue with a file parsing routine :
rl.on('line', function(line) {
line_count++;
try {
var line_object = JSON.parse(line);
var req_body = line_object.body.body;
q.push({req_body:req_body, line_object:line_object}, function(err){
if (err){
console.log('queue error! '+JSON.stringify(err));
}
});
} catch( e ) {
console.dir(e);
}
});
var done = function() {
// print out some reporting stats...
// console.log('xxxxxx');
console.log('\ndone.');
process.exit(0);
};
I'm writing an upload script. If there's an error while writing to disk I want to abort the request and return a 500 status code.
My code (below) sends a 500 as expected, but the upload doesn't stop. I keep getting "data" events until the upload is complete (even though my write pipe is already broken) and then the req.on('end' event fires which tries to send a 204 even though I've already sent a 500.
How can I abort the request so that I stop getting event notifications?
var filename = req.headers['wx-filename'];
var docType = req.headers['wx-doc-type'];
var clientId = req.headers['wx-client-id'];
var dir = Path.join(config.storage_path, docType, clientId);
var filePath = Path.join(dir, filename);
mkdirp.sync(dir);
var destFile = FileSystem.createWriteStream(filePath, {
flags: 'wx' // don't overwrite files
});
if (config.env === 'dev') {
(function () {
var fileSize = req.headers['content-length'];
var uploadedBytes = 0;
req.on('data', function (data) {
uploadedBytes += data.length;
var p = uploadedBytes / fileSize * 100;
var fn = Path.relative(config.storage_path, filePath);
util.log(fn + ': ' + p.toFixed(1) + '%');
});
})();
}
req.on('end', function () {
util.log("END");
resp.writeHead(204);
resp.end();
});
destFile.on('error', function (err) {
util.log("ERROR", err);
resp.writeHead(500, err.code);
resp.end();
// HOW TO STOP REQUEST?
});
req.pipe(destFile);
You need to remove the listeners for data and end, after that send a Connection: close header and at the end send the 500 error.
I am using express to create a webservice that will read string data from a stream, and respond to the HTTP POST request with that value. Here is the code for the S3Store.js file that defines the readFileFromS3(.) function:
S3Store.js
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = '';
readStream.on('data', function(data) {
//data = Buffer.concat([allData, data]);
data = allData + data;
console.log("data: " + data);
});
readStream.on('error', function(err) {
callback(err, null);
});
Now, if I call this method from a terminal like this:
s3Instance.readFileFromS3('123.json', function(err, data) {
console.log(data);
});
I see the appropriate string for data logged to the console. However, when I call the same method from inside one of the routes in express for HTTP POST requests, the service responds with a value of data set to empty string. Code for the POST request:
router.post('/resolve', function(req, res) {
var commandJson = req.body;
var appId = commandJson['appId'];
var command = commandJson['text'];
if (appId == undefined || command == undefined) {
res.status(400).send("Malformed Request: appId: " + appId + ", command: " + command);
};
s3Store.readFileFromS3('123.json', function(err, data) {
res.send(data);
});
});
Why does it return an empty string when calling the readFileFromS3(.) from the HTTP POST method and not when I ran the same method directly from the node console?
You're logging the data but you're not passing anything to the completion callback (see below for some more explanation):
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = [];
// Keep collecting data.
readStream.on('data', function(data) {
allData.push(data);
});
// Done reading, concatenate and pass to completion callback.
readStream.on('end', function() {
callback(null, Buffer.concat(allData));
});
// Handle any stream errors.
readStream.on('error', function(err) {
callback(err, null);
});
};
I took the liberty to rewrite the data collection to use Buffer's instead of strings, but this obviously isn't a requirement.
The callback argument is a completion function, meant to be called when either reading the S3 stream is done, or when it has thrown an error. The error handling was already in place, but not the part where you would call back when all the data from the stream was read, which is why I added the end handler.
At that point, the readStream is exhausted (everything from it has been read into allData), and you call the completion callback when the collected data as second argument.
The common idiom throughout Node is that completion callbacks take (at least) two arguments: the first is either an error, or null when there aren't errors, and the second is the data you want to pass back to the caller (in your case, the anonymous function in your route handler that calls res.send()).
I'm trying to access an Rserve server using Meteor through a server side route to call R code. The Node.js module that allows access to Rserve is node-rio I have used the Async.wrap function in the meteorhacks:npm Meteor package to wrap the "evaluate" method. When I try to access the route path "/rio" in a browser, I get "1" written to the console log, which is correct for the evaluation of the command "1" in R, but Chrome hangs with the message "Waiting for localhost". It doesn't proceed to the next line and "Result: 1" is never shown. The message "net::ERR_EMPTY_RESPONSE eventually shows up in Chrome.
Router.route('/rio', function() {
var rio = Meteor.npmRequire('rio');
var evalSync = Async.wrap(rio, 'evaluate');
var result = evalSync('1');
console.log("Result: " + result);
// JSON
this.response.writeHead(200, {'Content-Type': 'application/json'});
this.response.end(result);
}, {
where: 'server'
});
"evaluate" has a callback function parameter, but it is wrapped in an options parameter and accessed:
rio.evaluate(R_COMMAND, {callback: CALLBACK_FUNCTION})
Two errors:
as you already noted rio expects the callback as a value to the callback field in the parameters, not straight in the arguments themselves.
your result ("1") wasn't actually valid JSON.
This works:
Router.route('/rio', function() {
var rio = Meteor.npmRequire('rio');
var evalSync = Async.wrap(function(exp, callback) {
rio.evaluate(exp, {callback: callback});
});
var result = evalSync('1');
console.log("Result: " + result);
// JSON
this.response.writeHead(200, {'Content-Type': 'application/json'});
this.response.end(JSON.stringify({result: result}));
}, {
where: 'server'
});
This answer will produce the expected result for 'packageVersion("base")', it uses the Node module rserve-client instead of rio to connect to Rserve. The route also handles the parameter 'pkg'. This is the StackOverflow question that pointed me in the right direction:
How to call async method from Meteor own callbacks?
Router.route('rserve', {
path: '/rserve/:pkg',
where: 'server',
action: function() {
var r = Meteor.npmRequire("rserve-client");
var Future = Meteor.npmRequire("fibers/future");
var fut = new Future();
var cmd = 'packageVersion("' + this.params.pkg + '")';
var callR = function (input) {
r.connect('127.0.0.1', 6311, function (err, client) {
client.evaluate(input, function (err, ans) {
console.log("Result: " + ans);
client.end();
fut.return(ans);
});
});
return fut.wait();
};
var result = callR(cmd);
this.response.writeHead(200, {'Content-Type': 'application/json'});
this.response.end(JSON.stringify(result));
}
});
I'm relatively new to callbacks and have been unsuccessful in getting the following code to work. I have used the async.map function to return the data from each web call to its respective page. However, my console.log(return) is returning [ , undefined] even though the console.log(data) prior to the callback(data) is returning the web page's html. Here's my code:
var http = require("http"),
fs = require("fs"),
page, chap, ot,
async = require("async");
ot = fs.open('ot.txt', 'w');
page = "test";
chap = 2;
function getData(url, callback) {
var data = "";
var options = {
host: "rs.au.com",
port: 80
}
options.path = url;
console.log("request sent to: http://" + options.host + options.path);
var req = http.request(options, function(res) {
console.log("Response received " + res.statusCode);
res.on('data', function(chunk) {
data += chunk;
});
res.on('end', function(e) {
console.log(data);
callback(e, data);
});
}).end();
}
function main() {
var pathArr = [];
for ( var i = 1; i <= chap; i++ ) {
pathArr[i] = "/".concat(page, "/", i, ".html");
}
async.map(pathArr, getData, function(err, result) {
console.log("The result is :" + result);
});
}
main();
Could anyone please point out why my code is not working and how I can correct it?
Much appreciated!
EDIT: After Brandon Tilley's response I amended the callback function from callback(data) to callback(e, data), however I'm getting no response now from the last console.log output.
The Async library assumes your callbacks adhere to the standard Node.js callback signature, which is callback(err, others...). Since you are passing data as the first argument, Async assumes it's an error. You should use callback(e, data) instead (since e will be null in the case of no errors).
[Update]
The other problem is that your array is not correct. Since i starts at 1 and goes up to chap, pathArr[0] is undefined. Change:
pathArr[i] = "/".concat(page, "/", i, ".html");
to
pathArr[i-1] = "/".concat(page, "/", i, ".html");