Node.js http.get with Node.js step module - node.js

I am new to Node.js world, kind of stuck in situation.
below code is for reference:
var http = require('http');
var step = require('step');
var request = require('request');
exports.readimage2 = function(req, res){
//res.send(200,'OK');
//var image_url = 'http://www.letsgodigital.org/images/artikelen/39/k20d-image.jpg'; //--- 10mb
//var image_url = 'http://upload.wikimedia.org/wikipedia/commons/2/2d/Snake_River_(5mb).jpg';
//var image_url = 'http://www.sandia.gov/images2005/f4_image1.jpg'; //--- 2mb
var image_url = 'http://www.fas.org/nuke/guide/pakistan/pakistan.gif'; // --- some KB
http.get(image_url,
function(responseData) {
var data = new Buffer(parseInt(responseData.headers['content-length'],10));
var pos = 0;
responseData.on('data', function(chunk) {
chunk.copy(data, pos);
pos += chunk.length;
});
responseData.on('end', function () {
res.send(200, data);
});
});
};
Above code fails working for large files if i use it with step module.
Anyone suggest how to do it properly with step.

Here how i did it using step..... although the request module did same for image buffer download thanks to a post on stackoverflow just need to set encoding to null in request to work for buffer response.
var canvas = new Canvas(3000, 3000),
ctx = canvas.getContext('2d'),
Image = Canvas.Image;
var image_url = "http://www.a2hosting.com/images/uploads/landing_images/node.js-hosting.png";
//var image_url = 'http://upload.wikimedia.org/wikipedia/commons/1/16/AsterNovi-belgii-flower-1mb.jpg';
step(
function() {
request.get({
url: image_url,
encoding: null
}, this);
},
function(err, response, body) {
var img = new Image;
img.src = body;
ctx.drawImage(img, 0, 0, img.width, img.height);
//res.send(200, data);
res.send(200, '<img src="' + canvas.toDataURL() + '" />');
}
);
Below is the code working for simple http module of node.
var http = require('http');
var step = require('step');
var request = require('request');
exports.imagedownload = function(req, res){
step(
function(){
console.log('*********** image download start ***********');
fndownload(this);
},
function(err, result){
if(err) {
}
console.log('*********** image download end ***********');
res.send(200, result);
}
);
};
function fndownload(callback) {
var image_url = 'http://upload.wikimedia.org/wikipedia/commons/2/2d/Snake_River_(5mb).jpg'; // --- some KB
http.get(image_url,
function(responseData) {
var data = new Buffer(parseInt(responseData.headers['content-length'],10));
var pos = 0;
responseData.on('data', function(chunk) {
chunk.copy(data, pos);
pos += chunk.length;
});
responseData.on('end', function () {
//res.send(200, data);
callback(null, data);
});
});
};

Related

make pdf file from jsPDF and send pdf file to server node js

i have code to make pdf and succeeded in downloading and opening it, but i want to send pdf to my server on node js, and i have made app.post on server but i can't make pdf become base64 and save it on server
in frontend
<script type="text/javascript">
function genPDF() {
html2canvas(document.getElementById('testDiv')).then(function (canvas) {
var img = canvas.toDataURL('image/png');
var doc = new jsPDF('landscape');
doc.addImage(img, 'png', 10, 10);
var temp = doc.save('test.pdf');
var post = new XMLHttpRequest();
post.open("POST", "/receive");
post.send(temp);
}
</script>
Download PDF
in server
app.post('/receive', function (request, respond) {
var body = '';
var filePath = './static' + '/document/Document.pdf';
//
request.on('data', function (data) {
body += data;
});
request.on('end', function () {
var data = body.replace(/^data:image\/\w+;base64,/, "");
var buf = new Buffer(data, 'base64');
fs.writeFile(filePath, buf, function (err) {
if (err) throw err
respond.end();
});
});
});
how to send var temp = doc.save('test.pdf'); server and generate pdf to base64?
Use the below code this will help you.
IN FE
<script type = "text/javascript">
function genPDF() {
html2canvas(document.getElementById('testDiv')).then(function (canvas) {
var img = canvas.toDataURL('image/png');
var doc = new jsPDF('landscape');
doc.addImage(img, 'png', 10, 10);
var temp = doc.save('test.pdf');
var data = new FormData();
data.append("pdf_file", temp);
var post = new XMLHttpRequest();
post.open("POST", "/receive");
post.send(data);
}
</script>
<a href = "javascript:genPDF()" > Download PDF </a>
IN BE
const fs = require('fs');
const multipartMiddleware = require('connect-multiparty')();
const express = require('express');
const app = express();
const port = 8000;
const filePath = './static' + '/document/Document.pdf';
app.post('/', multipartMiddleware, (request, response) => {
fs.readFile(request.files.pdf_file.path, (err, data) => {
fs.writeFile(filePath, data, function (err) {
if (err) throw err;
response.send('Done')
});
})
})
app.listen(port, (err) => {
if (err) {
return console.log('something bad happened', err)
}
console.log(`server is listening on ${port}`)
});

multiple http get calls nodejs

Thanks for looking into the code.
Here I am fetching some data using feed parser and taking out id's in navcodes array variable and wants to use these Id to make http call.Please find code below.
function processNavCode(){
var mfId = [53];
var preTitle = '';
var navCodes = [];
mfId.forEach(function(id){
var query = "http://portal.xyz.com/Rss.aspx?mf="+id;
feed(query, function(err, feeds) {
if (err) {
throw err;
}
feeds.forEach(function(feed){
var link = feed.link;
var title = feed.title;
var navCode = link.substr(link.length - 6);
if(title.split('-')[0].trim() != preTitle){
preTitle = title;
counter ++;
}
if(parseInt(navCode) != '')
navCodes.push = parseInt(navCode);
});
});
async.eachSeries(navCodes,insertbulkMFValues,function(){
console.log('I am done');
});
// insertbulkMFValues(navCode);
//Directly call insertbulkMFValues function
});
}
I have also tried to call the insertbulkMFValues directly as commented now but due to async nature of nodejs, I am getting the error of either 'Socket hang up' or 'read ECONNRESET'. I checked and used async but not able to work with that also.
var insertbulkMFValues =function(navCode,callback){
var options = {
host: 'www.quandl.com',
path: '/api/v3/datasets/AMFI/'+navCode+'.json?api_key=123456789&start_date=2013-08-30',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}
var req1 = https.request(options, function(response) {
var body = '';
response.setEncoding('utf8');
response.on('data', function(chunk) {
body += chunk;
});
response.on('end', function() {
if(typeof body === "string") {
var json = JSON.parse(body);
}
var mfData = json.dataset.data;
var schemeId = json.dataset.dataset_code;
var schemeName = json.dataset.name;
var isinCode = json.dataset.description;
var valueData=[];
for (var k = 0; k < mfData.length; k++) {
var myDate = new Date(mfData[k][0]);
valueData.push({
date:myDate,
NAV:parseFloat(mfData[k][1]).toFixed(2)
});
}
var query = { "navCode": schemeId };
var newData = {
createdDate: Date.now(),
navCode: schemeId,
schemeCode:count,
schemeName:schemeName,
ISINCode:isinCode,
values:valueData
};
MHistory.findOneAndUpdate(query, newData , {upsert:true}, function(err, doc){
if(err)
console.log('Errorr');
else
console.log('Success');
});
});
});
req1.on('error', function(e) {
console.log('problem with request: ' + e.message);
callback(true);
});
req1.end();
}
Thanks in advance..
J
You can directly call insertbulkMFValues for each navCode like:
if(parseInt(navCode) != '') {
insertbulkMFValues(navCode, function () {
console.log('something'}
});
}
Anything that you intend to do must be within the callback of the asynchronous function.
One option for you is to use the waterfall or parallel method of the async library to retrieve all feeds for each id and then invoke
async.eachSeries(navCodesAccumulated,insertbulkMFValues,function(){
console.log('I am done');
});
within the final result callback using the codes obtained.

Downloading image from the web with imagemagick and saving to parse

I'm able to download the image using http and and I am able to resize it using imagemagic, but I can't figure out how to upload it to Parse. Parse only lets me upload: an array of byte value numbers OR base64 string. What I'm confused on is how I can convert the stdout to base64 string so that I can upload it to parse. I tried using fs, but to no avail. I just keep getting errors when it tries to read the file and convert it. What is wrong with my code?
var _ = require('underscore');
var url = require("url");
var srcUrl = 'https://i.ytimg.com/vi/JxwwGtquGqw/maxresdefault.jpg';
var http = srcUrl.charAt(4) == 's' ? require("https") : require("http");
var im = require('imagemagick');
var Image = require("parse-image");
var image_card_2x = [540, 350];
var thumb = '';
var fs = require('fs');
var request = http.get(url.parse(srcUrl), function(response) {
console.log("got data");
var data = '';
response.setEncoding('binary');
response.on('data', function(chunk) {
data += chunk;
});
console.log("data is " + data);
response.on('end', function () {
// var image = new Image();
// image.setData(response.buffer);
// image.scale({
// width: image_card_2x,
// height: Math.floor((image.height() * arrayElement[0]) / image.width())
// });
// image.setFormat("JPEG");
// console.log("about to make image");
im.resize({
srcData: data,
width: 404,
height: 269,
format: 'jpg'
}, function(err, buffer) {
if (err) throw err;
var bitmap = fs.readFileSync(file);
var base64 = new Buffer(bitmap).toString('base64');
var file = new Parse.File("maa.jpg", {
base64: base64
});
console.log("about to save");
var VideoLinks = Parse.Object.extend("VideoLinks");
var videoLink = new VideoLinks();
videoLink.set("image", file);
return videoLink.save();
console.log("saved");
//fs.writeFileSync('kittens-resized.jpg', stdout, 'binary');
console.log("worked!");
});
})
});

how to check the progress of file uploading using formdata in node.js?

I want to check the percentage of file uploading in the third server.
below is my controller code.
which upload the file to third server.
BASE.APP.post('/uploadFile/:request/:file', function (req, res, next) {
//var form = new BASE.multiparty.Form();
var path = 'uploads/'+req.params.file;
var fstream = BASE.FS.createReadStream(path);
//form.on('part', function(part) {
// console.log('inside');
var url = req.usersession.webipAddress;
/* var formData = {
file: {
value: BASE.FS.createReadStream(path),
options: {
fileNameUnique: req.params.file
}
}
};
// Post the file to the upload server
BASE.request.post({url: url+'test/service/uploadFile/', formData: formData});*/
/*var form = new BASE.FormData();
form.append('fileNameUnique', req.params.file);
form.append( 'file',fstream);
*/
var formData = {
fileNameUnique: req.params.file,
file: fstream
};
var r = BASE.request.post({url: url+'test/service/uploadFile/', formData: formData}, function(err1, res1, body){
console.log('new method err' + err1);
console.log('new method' + res1);
clearInterval(tasktimeoutId);
tasktimeoutId = false;
res.send(res1);
});
var tasktimeoutId = null;
if(!tasktimeoutId){
tasktimeoutId = setInterval(function(){
console.log('interval inside');
interrupttable.findOne({ "filename": req.params.file }, function(err, thor) {
if (thor!=null)
{
console.log('null inside');
if(thor.status=='interrupt')
{
console.log('interrupt');
r.abort();
r = null;
clearInterval(tasktimeoutId);
tasktimeoutId = false;
res.send("interrupted");
//return
//next(err);
}
}
});
}, 1000);
}
});
is there any way to check the file progress in percentage. So that i can
show the progress bar in front end.
Check this repo: https://github.com/zeMirco/express-upload-progress. It should have what you are looking for :)

Need to send response after an action has completed

I am trying to make a web crawler which crawls IMDB and lists the movie name and rating. This is my index.js file.
Suppose i am crawling for 10 movies. I am then saving the crawled results in a different file say 'message.txt'. Now i want to send this message.txt file as a response to any request. But whenever I make a request it always send me an empty file to my browser initially. Then i notice that it takes some time before the crawled results are saved in the message.txt file. I think this is because all actions are asynchronous in nodejs. So is there a way to send the message.txt file only after crawling is complete?
var express = require('express');
var app = express();
var cheerio = require('cheerio');
var request = require('request');
var fs = require('fs');
app.listen(8080);
console.log('Running');
app.get('/', function(req, res) {
console.log('Recieved the get Request');
var i = 1;
var count = 0;
while (count < 10) {
var url = 'http://www.imdb.com/title/tt' + i + '/';
console.log(url);
count = count + 1;
i = i + 1;
request(url, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html);
var title, ratings, released;
var json = {
title: '',
ratings: '',
released: ''
};
$('.title_wrapper').filter(function() {
var data = $(this);
json.title = data.children().first().text().trim();
json.released = data.children().last().children().last().text().trim();
});
$('.ratingValue').filter(function() {
var data = $(this);
json.ratings = parseFloat(data.text().trim());
});
console.log(json);
fs.appendFile('message.txt', JSON.stringify(json, null, 4) + '\n', function(err) {});
};
});
};
res.sendFile(__dirname + '/index.js');
});
You can use the async package which is great for controlling flow, something like:
console.log('Recieved the get Request');
var i = 1;
var count = 0;
while (count < 10) {
var url = 'http://www.imdb.com/title/tt' + i + '/';
console.log(url);
count = count + 1;
i = i + 1;
async.waterfall([
function sendRequest (callback) {
if (!error) {
var $ = cheero.load(html);
var json = {
title: '',
ratings: '',
released: ''
}
}
$('.title_wrapper').filter(function() {
var data = $(this);
json.title = data.children().first().text().trim();
json.released = data.children().last().children().last().text().trim();
});
$('.ratingValue').filter(function() {
var data = $(this);
json.ratings = parseFloat(data.text().trim());
});
callback(null, JSON.stringify(json, null, 4) + '\n');
},
function appendFile (json, callback) {
fs.appendFile('message.txt', json, function(err) {
if (err) { callback(err); }
callback();
});
}
], function(err) {
res.sendFile(__dirname + '/index.js');
});
fs.appendFile('message.txt', JSON.stringify(json, null, 4) + '\n', function(err) {
//This part is executed after the process has been completed
});
You have to make a callback there as that part will be only called when your operation has been performed.
We are utilizing the callback feature here although there isn't any concrete callback except the err in our case, we don't need any other badly though.
Please try.
fs.appendFile() is asynchronous so the stuff you append to the file won't be there right away when the function returns. So if you want to read send that file to the user, you'll need to do it inside the callback you supply to fs.appendFile().
app.get('/', function(req, res) {
...
fs.appendFile(
'message.txt',
JSON.stringify(json, null, 4) + '\n',
function(err) {
if (err) {
// Log the error and send a message to the user here
return;
}
res.sendFile(__dirname + '/index.js')
}
);
};
});
};
});
You may be tempted to use fs.appendFileSync() instead. That would be fine for a command line tool, but since this is a web server, do not do that. It will lock up the thread while the I/O happens.

Resources