I'm trying to fetch json from an api but only half of the response is received. So how to get the full response?
var request = require('request');
var url_to_check = 'http://example.com/api/test';
request.get(url_to_check).on('data', function(data) {
// Only half of the data is printed (8192). Remaining bytes are lost.
console.log(data.toString());
})
Your code is correct the only mistake you made is that you are streaming request data so you won't get whole data on event 'data' if a response is large. You will have to collect chunk and consolidate on event 'end'. check this code snippet
var request = require('request');
var url = 'https://reqres.in/api/users';
var req = request.get(url)
var data = []
req.on('data',function(chunk){
data.push(chunk))
})
req.on('end',function(){
console.log(Buffer.concat(data).toString())
})
And If you don't want to stream and pipe data and also response size is small then you can try this:
request.get(url, function(err, response, responseBody) {
if (!err) {
// var jsonRes = JSON.parse(JSON.stringify(response))
// responseBody = jsonRes.body
console.log(responseBody)
} else {
// handle error here
}
})
Related
I am trying to fetch a JSON file from a api website using the below code but i am getting an error saying "Unexpected end of JSON input" when I fetch using the below code
var express = require("express");
var app = express();
var body = require("body-parser");
var https = require("https");
app.get("/results", function (req, res) {
https.get("https://www.omdbapi.com/?apikey=d49698c3&s=harry", function (response) {
response.on("data", function (data) {
var got = JSON.parse(data);
res.send(got.Title);
})
})
});
There are a couple of wrong assumptions in your code.
1 - Looking at the data, you can see that the property Title is in every single object inside the Search property, which is an Array, thus when the JSON gets correctly parsed, you'll have an issue with got.Title
2 - The event data of the object response can be called multiple times until you can get all the data, every time it gets called, you receive a chunk of the data.
Once all the data has been sent the end event is called and there is the place to parse the JSON.
Your code is trying to parse an incomplete JSON string, just the first chunk.
The most common approach is to declare an array outside the functions that will handle the events data and end. For every data event you push the chunk into the outside array and on the end event you concatenate it.
Check this out:
var express = require('express')
var app = express()
var body = require('body-parser')
var https = require('https')
app.get('/results', function (req, res) {
https.get('https://www.omdbapi.com/?apikey=d49698c3&s=harry', function (response) {
const chunks = []
response.on('data', function (chunk) {
chunks.push(chunk)
})
response.on('end', function () {
const data = Buffer.concat(chunks)
var got = JSON.parse(data)
// Try this one out as well
// res.json(got)
res.send(got.Search[0].Title)
})
})
})
app.listen(3000)
I just tested the code above and it works like a charm.
In case of large json object you would want to parse the data in chunks.
Receive all the chunks on data event and club the data from as soon as it arrives and parse the concatenated json on the end event
const express = require("express");
const app = express();
const body = require("body-parser");
const https = require("https");
app.get("/results", function (req, res) {
https.get("https://www.omdbapi.com/?apikey=d49698c3&s=harry", function (response) {
let finalData = '';
response.on("data", function (data) {
finalData += data.toString();
});
response.on("end", function() {
const parsedData = JSON.parse(finalData);
res.send(parsedData.Title);
}
})
});
app.listen(3000)
For everyone who's having the same issue. I found a solution:
As what #Maestre San - has explained, you're parsing an incomplete JSON data. That's why you're getting the error. I've tried both solutions suggested by both but it still wasn't working for me. Hence, I researched and found out that first of all, you will need to store the data in an empty variable, once the data stream is done, you can then parse it by doing the following:
response.on("end", function () {
var jsonParse = JSON.parse(newsItems);
});
The full code is:
app.get("/", function (req, res) {
const queryString = "mamamoo";
const url = "https://newsapi.org/v2/everything?apiKey=<API_KEY>&qInTitle=" + queryString;
https.get(url, function (response) {
var newsItems = '';
response.on("data", function (data) {
newsItems += data;
});
response.on("end", function () {
var jsonParse = JSON.parse(newsItems);
console.log(jsonParse);
});
});
});
So as to explain what I did:
First, I make sure to catch any request made to my server. I then used the native node https request to make a request to the api, catching the response (the response contains statusCode and other response body).
However, I want to access the data body, so I performed "response.on". First, I captured the data in chunks and stored it in an empty variable.
Why store it in an empty variable? Because if I didn't and continued to parse it, it will throw an error saying "Unexpected end of JSON input", meaning, I'm parsing an incomplete JSON data.
Next step I did is, parse the data WHEN the data stream is done by specifying the "end".
Hope this helped.
I'd like parse a log file and POST what is read to a request endpoint. I've managed to build a solution that generates a request for every log line read. However, it doesn't create any back pressure so it just flogs the server and I'd like to slow it down.
This lead me to investigate using stream pipes to see if I could route data from a file directly into request.post(). I can't get the post call to post a body object though.
var stream = require('stream');
var request = require('request');
var liner = new stream.Transform( { objectMode: true } );
liner._transform = function (chunk, encoding, done) {
var data = chunk.toString()
if (this._lastLineData) data = this._lastLineData + data
var lines = data.split('\n')
this._lastLineData = lines.splice(lines.length-1,1)[0]
var that = this;
lines.forEach(function(line) {
var line_obj = JSON.parse(line);
if( line_obj.url === "/api/usages" && line_obj.method === 'POST' ) {
var req_body = line_obj.body.body;
that.push.bind(req_body);
}
});
done();
}
var file_name = process.argv[2];
console.log('Reading from ' + file_name);
var fs = require('fs')
var liner = require('./liner')
var source = fs.createReadStream(file_name)
source.pipe(liner).pipe(request
.post("http://localhost:8081/api/usages")
.on('response', function(response) {
console.log(response.statusCode) // 200
})
.on('error', function(err) {
console.log(err);
}));
The push call in the transform function is working correctly, but it's not posting that object via the body in request.post().
What am I missing?
Will this provide the back pressure I'm looking for to throttle the POST calls before all of the file reads are completed?
I've discovered that you cannot pipe a stream to an HTTP request because you would need the Content-Length known before hand (as per spec). The less pleasant alternative is to multipart the upload - as chunks are read from your transform they would marshal parts to the receiving API. This also means the receiving API needs to be able to receive multipart uploads and reassemble the whole file after all parts have been received and confirmed. AWS S3 has multipart uploads and it might be a nice example: http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html
I wanted to pipe my transform data to another API that I manage but it seems the effort is not likely worth it considering my files really aren't that big. I'll update this answer if I change my mind :)
Although I wasn't able to find a solution to the streaming question, I found a simple solution to the back pressure question.
I used async.queue to push work into a simple task queue.
// build the send queue
var pool = new http.Agent({keepAlive: true, keepAliveMsecs: 10000, maxSockets: Math.floor(send_queue_concurrency*1.5)});
var q = async.queue(function(task, callback){
request({
url : 'http://localhost:8081/xxxxxx',
method : 'POST',
json : task.req_body,
gzip : true,
pool : pool,
timeout: 30000
}, function(error, response, body){
if(error) {
console.log('request error : ' + error);
post_status.fail++;
} else {
if( response.statusCode === 400 ) {
console.dir(body);
}
}
callback();
});
}, send_queue_concurrency);
q.drain = done;
send_queue_concurrency is the primary lever for controlling request pressure.
i'm pushing work into the queue with a file parsing routine :
rl.on('line', function(line) {
line_count++;
try {
var line_object = JSON.parse(line);
var req_body = line_object.body.body;
q.push({req_body:req_body, line_object:line_object}, function(err){
if (err){
console.log('queue error! '+JSON.stringify(err));
}
});
} catch( e ) {
console.dir(e);
}
});
var done = function() {
// print out some reporting stats...
// console.log('xxxxxx');
console.log('\ndone.');
process.exit(0);
};
I am using express to create a webservice that will read string data from a stream, and respond to the HTTP POST request with that value. Here is the code for the S3Store.js file that defines the readFileFromS3(.) function:
S3Store.js
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = '';
readStream.on('data', function(data) {
//data = Buffer.concat([allData, data]);
data = allData + data;
console.log("data: " + data);
});
readStream.on('error', function(err) {
callback(err, null);
});
Now, if I call this method from a terminal like this:
s3Instance.readFileFromS3('123.json', function(err, data) {
console.log(data);
});
I see the appropriate string for data logged to the console. However, when I call the same method from inside one of the routes in express for HTTP POST requests, the service responds with a value of data set to empty string. Code for the POST request:
router.post('/resolve', function(req, res) {
var commandJson = req.body;
var appId = commandJson['appId'];
var command = commandJson['text'];
if (appId == undefined || command == undefined) {
res.status(400).send("Malformed Request: appId: " + appId + ", command: " + command);
};
s3Store.readFileFromS3('123.json', function(err, data) {
res.send(data);
});
});
Why does it return an empty string when calling the readFileFromS3(.) from the HTTP POST method and not when I ran the same method directly from the node console?
You're logging the data but you're not passing anything to the completion callback (see below for some more explanation):
S3Store.prototype.readFileFromS3 = function(filename, callback) {
var readConfig = {
'Bucket': 'shubham-test',
'Key': filename
};
var readStream = this.s3.getObject(readConfig).createReadStream();
var allData = [];
// Keep collecting data.
readStream.on('data', function(data) {
allData.push(data);
});
// Done reading, concatenate and pass to completion callback.
readStream.on('end', function() {
callback(null, Buffer.concat(allData));
});
// Handle any stream errors.
readStream.on('error', function(err) {
callback(err, null);
});
};
I took the liberty to rewrite the data collection to use Buffer's instead of strings, but this obviously isn't a requirement.
The callback argument is a completion function, meant to be called when either reading the S3 stream is done, or when it has thrown an error. The error handling was already in place, but not the part where you would call back when all the data from the stream was read, which is why I added the end handler.
At that point, the readStream is exhausted (everything from it has been read into allData), and you call the completion callback when the collected data as second argument.
The common idiom throughout Node is that completion callbacks take (at least) two arguments: the first is either an error, or null when there aren't errors, and the second is the data you want to pass back to the caller (in your case, the anonymous function in your route handler that calls res.send()).
I have an http server with a handleRequest callback that runs another script in vm.runInNewContext for each request. The script that runs inside vm.runInNewContext makes some asynchronous http post requests and writes the server response only after getting the responses from the posts.
As a result, the code of handleRequest callback ends before the server response is written.
Is it safe? or is there a way to avoid this situation?
Here is some code:
var server = http.createServer(handleRequest);
server.listen(8080);
var handleRequest = function (request, response) {
// get request data...
var context = {
ServerRequest : request,
ServerResponse : response
};
var stringScript = // a string with the script that posts data
var script = vm.createScript(stringScript);
script.runInNewContext({ context: context });
}
the script string does this:
var request = require('request');
var options = {....}
var req = request.get(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
});
res.on('end', function() {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function(err, decoded) {
// set response headers and write the response
context.ServerResponse.end(decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function(err, decoded) {
// set response headers and write the response
context.ServerResponse.end(decoded.toString());
})
} else {
// set response headers and write the response
context.ServerResponse.end(buffer.toString());
}
});
});
Simple solution: Return a promise (e.g. use the Q-library) from the VM-script.
script.runInNewContext will return whatever you return from the VM-script. That way you have a "callback" for when the VM code finishes.
// Script for VM
// I simplified it. Just resolve or reject the promise whenever you are done with your work
'use strict';
var defer = q.defer();
doABarrelRoll(function() {
defer.resolve('RESULT');
});
defer.promise; // This line will return the promise.
When returning a value from a VM-script, you do not need any return construction. Just write the thing you want and let the magic happen.
// Script for current context
'use strict';
var server = http.createServer(handleRequest);
server.listen(8080);
var handleRequest = function (request, response) {
// get request data...
var context = {
ServerRequest : request,
ServerResponse : response
};
var stringScript = // a string with the script that posts data
var script = vm.createScript(stringScript);
var prom = script.runInNewContext({
context: context,
q: require('q'),
});
prom.done(function ($result) {
console.log('VM finished with result: ' + $result);
});
}
I'm new to node.js, and I'm trying to call a service, parse its data and return it as part of a view. I can't seem to get the request to block until the response is complete. The console always logs 'wrong' before 'right' (returning the 1,2,3 array). What am I missing?
app.js
var reading = require('./reading');
app.get('/reading', function(req, res){
res.render('reading/index.stache',
{
locals : { ids : reading.list},
partials : {
list : '{{#ids}}{{.}}<br />{{/ids}}'
}
});
});
reading.js
var request,
http = require('http'),
host = 'google.com',
path ='/';
var list = function(){
var connection = http.createClient(80, host),
request = connection.request(path);
request.addListener('response', function(response){
var data = '';
response.addListener('data', function(chunk){
data += chunk;
});
response.addListener('end', function(){
console.log('right')
//var results = JSON.parse(data);
// i need results from json
return [88234,883425,234588];
});
});
request.end();
console.log('wrong');
return [1,2,3];
}
module.exports.list = list;
Of course you can't get the request to block until the response is back.
That's because there is communication latency between sending the request of and getting the response back. It would be stupid to wait and do nothing whilst that latency is happening.
Use callbacks and asynchronous control flow.
var list = function(callback){
var connection = http.createClient(80, host),
request = connection.request(path);
request.addListener('response', function(response){
var data = '';
response.addListener('data', function(chunk){
data += chunk;
});
response.addListener('end', function(){
console.log('right')
// USE A CALLBACK >:(
callback([88234,883425,234588]);
});
});
request.end();
}
If you wan't to run anything in sync have a look at the sync module. It's based on fibers.