Whats the best way to read a remote file? I want to get the whole file (not chunks).
I started with the following example
var get = http.get(options).on('response', function (response) {
response.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
I want to parse the file as csv, however for this I need the whole file rather than chunked data.
I'd use request for this:
request('http://google.com/doodle.png').pipe(fs.createWriteStream('doodle.png'))
Or if you don't need to save to a file first, and you just need to read the CSV into memory, you can do the following:
var request = require('request');
request.get('http://www.whatever.com/my.csv', function (error, response, body) {
if (!error && response.statusCode == 200) {
var csv = body;
// Continue with your processing here.
}
});
etc.
You can do something like this, without using any external libraries.
const fs = require("fs");
const https = require("https");
const file = fs.createWriteStream("data.txt");
https.get("https://www.w3.org/TR/PNG/iso_8859-1.txt", response => {
var stream = response.pipe(file);
stream.on("finish", function() {
console.log("done");
});
});
http.get(options).on('response', function (response) {
var body = '';
var i = 0;
response.on('data', function (chunk) {
i++;
body += chunk;
console.log('BODY Part: ' + i);
});
response.on('end', function () {
console.log(body);
console.log('Finished');
});
});
Changes to this, which works. Any comments?
function(url,callback){
request(url).on('data',(data) => {
try{
var json = JSON.parse(data);
}
catch(error){
callback("");
}
callback(json);
})
}
You can also use this. This is to async flow. The error comes when the response is not a JSON. Also in 404 status code .
Related
I'm fumbling my way through node.js with massive help from people on here and I'm struggling getting the body of a GET request into a variable.
Here's the code so far:
var speechOutput;
var myCallback = function(data) {
console.log('got data: '+data);
speechOutput = data;
};
var usingItNow = function(callback) {
var http = require('http');
var url = 'http://services.groupkt.com/country/get/iso2code/IN';
var req = http.get(url, (res) => {
var body = "";
res.on("data", (chunk) => {
body += chunk;
});
res.on("end", () => {
var result = JSON.parse(body);
callback(result);
});
}).on('error', function(e){
console.log("Got an error: ", e);
});
};
usingItNow(myCallback);
I'm using examples from other posts to try and get the body of the GET request into the speechOutput variable but it is coming out as undefined.
Ultimately I want the RestResponse.result.name in speechOutput, but I thought I would take this one step at a time. Can anyone offer any pointers?
Further to this, I have tried the following, which still came back undefined - maybe there is a bigger issue with the code? It doesn't even seem to be getting to the parse.
res.on("end", () => {
// var result = JSON.parse(body);
callback('result');
});
putting the line callback('result'); before the line var req = http.get(url, (res) => { returns 'result' but anything else is either undefined or causes an error.
Quoting Roy T. Fielding:
Server semantics for GET, however, are restricted such that a body,
if any, has no semantic meaning to the request. The requirements
on parsing are separate from the requirements on method semantics.
Don't use get request to send body parameters. Use post requests. If you want to send data within a get request, add them to the query string.
Read this for more info about bodies in get requests:
HTTP GET with request body
Update:
Try to log errors in the response, add this before you set up the listeners:
var body = "";
const { statusCode } = res;
const contentType = res.headers['content-type'];
let error;
if (statusCode !== 200) {
error = new Error('Request Failed.\n' +
`Status Code: ${statusCode}`);
} else if (!/^application\/json/.test(contentType)) {
error = new Error('Invalid content-type.\n' +
`Expected application/json but received ${contentType}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
res.on("data", (chunk) => {
body += chunk;
});
PROBLEM
I want to receive data from a device using IP Address via NodeJs. But I received the following data:
What I've Tried
This is the code that I've been able to get, which still produces the problem I described above.
var app = require('http').createServer(handler);
var url = require('url') ;
var statusCode = 200;
app.listen(6565);
function handler (req, res) {
var data = '';
req.on('data', function(chunk) {
data += chunk;
});
req.on('end', function() {
console.log(data.toString());
fs = require('fs');
fs.appendFile('helloworld.txt', data.toString(), function (err) {
if (err) return console.log(err);
});
});
res.writeHead(statusCode, {'Content-Type': 'text/plain'});
res.end();
}
And below is the result I received for console.log(req.headers)
So my question is, how do I decode the data? and anyone know what type of data are they?
Use Buffers to handle octet streams.
function handler (req, res) {
let body=[];
req.on('data', function(chunk) {
body.push(chunk);
});
req.on('end', function() {
body = Buffer.concat(body).toString('utf8');
...
var exports = module.exports = {};
var http = require('http');
exports.get = function(key, app, vari) {
http.get('<url here>/?key='+key+'&app='+app+'&var='+vari+'&req=0', function (response) {
response.setEncoding('utf8');
response.on('data', function(body) {
console.log(body);
return body;
});
});
};
My code (seen above) will output the response to the console just fine, but when trying to use the function in an export, it returns 'undefined' no matter what. The responses it receives are one line and are in the content type of "application/json". What's up with it? (And no, it's not the "url here", I just removed the URL for privacy reasons. If it helps, I can provide it.)
exports.get = function(key, app, vari) {
return
http.get('<url here>/?key='+key+'&app='+app+'&var='+vari+'&req=0', function (response) {
response.setEncoding('utf8');
response.on('data', function(body) {
console.log(body);
return body;
});
});
};
reference,and you need to listen end event and return a promise instead, just like:
var exports = module.exports = {};
var http = require('http');
exports.get = function(key, app, vari) {
return new Promise(function(resolve) {
http.get('<url here>/? key='+key+'&app='+app+'&var='+vari+'&req=0', function (response) {
response.setEncoding('utf8');
var data = '';
response.on('data', function(chunk) {
console.log(chunk);
data += chunk;
});
response.on('end', function() {
resolve(JSON.parse(data));
});
});
})
}
I figured it out, I just needed to have a call for an answer.
var exports = module.exports = {};
var http = require('http');
exports.get = function(key, app, vari, answ) {
http.get('http://<url here>/?key='+key+'&app='+app+'&var='+vari+'&req=0', function (response) {
response.setEncoding('utf8');
response.on('data', function(body) {
answ(body);
});
});
};
function media(req,res){
console.log(req.query.image);
var collectionName = 'imageTable';
var selector = MongoHelper.idSelector(req.query.image);
MongoHelper.findOne(selector, collectionName, function(err, image) {
console.log(image.picture);
var url_parts = url.parse(image.picture);
var options = {host: url_parts.hostname, path: url_parts.pathname};
http.get(options).on('response', function (response) {
var body = '';
var i = 0;
response.on('data', function (chunk) {
i++;
body += chunk;
console.log('BODY Part: ' + i);
});
response.on('end', function () {
console.log('Finished');
res.writeHead(200,{'Content-Type':'image/JPEG'});
res.write(body);
res.end();
});
});
});
}
I am fetching image from different server. I have url of that image. And I am writing the response. But here response image is get corrupted. Any idea about how to write jpeg image in response?
function media(req,res){
console.log(req.query.image);
var collectionName = 'facebook';
var selector = MongoHelper.idSelector(req.query.image);
MongoHelper.findOne(selector, collectionName, function(err, image) {
var url_parts = url.parse(image.picture);
var options = {host: url_parts.hostname, path: url_parts.pathname};
http.get(options).on('response', function (response) {
res.writeHead(200,{'Content-Type':'image/JPEG'});
response.on('data', function (chunk) {
res.write(chunk);
});
response.on('end', function () {
res.end();
});
});
});
}
Here I got the solution. Instead of writing whole data at the end. Write it each time you get and end the response when you reach to the end of file. But still if anyone have better idea can write here.
How can I use remote image in ImageMagick with Node.JS?
I want to achieve something like:
im.identify('http://www.website.com/image.jpg', function(error, features) {
console.log(features);
});
A quick code snippet of image resizing
http://nodejs.org/api/http.html
https://github.com/rsms/node-imagemagick
var thumb = '';
...
var request = http.get(options, function(response) {
var data = '';
response.setEncoding('binary');
response.on('data', function(chunk) {
data += chunk;
});
response.on('end', function () {
im.resize({
srcData: data,
width: 100,
height: 75,
format: 'jpg'
}, function(err, stdout, stderr) {
if (err) throw err;
thumb = stdout;
});
}
});
This is how I use remote images:
var srcUrl = 'http://domain.com/path/to/image.jpg';
var http = srcUrl.charAt(4) == 's' ? require("https") : require("http");
var url = require("url");
http.get(url.parse(srcUrl), function(res) {
if(res.statusCode !== 200) {
throw 'statusCode returned: ' + res.statusCode;
}
else {
var data = new Array;
var dataLen = 0;
res.on("data", function (chunk) {
data.push(chunk);
dataLen += chunk.length;
});
res.on("end", function () {
var buf = new Buffer(dataLen);
for(var i=0,len=data.length,pos=0; i<len; i++) {
data[i].copy(buf, pos);
pos += data[i].length;
}
im(buf).imFunctionYouWantToUse();
});
}
});
Credit go to https://stuk.github.io/jszip/documentation/howto/read_zip.html
It's hard to say if i understood you correctly (considering the amount of information you posted here).
The only way you can perform operations on a remote image using imagemagick is to download it to the local server first. This can be done using the http.ClientRequest class of node.js, afterwards you should be able to operate on the image as usual using Imagemagick.
This should work:
var request = require('request');
var fs = require('fs');
request({
'url': 'http://www.website.com/image.jpg',
'encoding':'binary'
},
function (error, response, body) {
if (!error && response.statusCode == 200) {
fs.writeFileSync('/mylocalpath/image.jpg', body, 'binary');
im.identify('/mylocalpath/image.jpg',
function(error, features) {
console.log(features);
}
);
}else{
console.error(error, response);
}
}
)