I am trying to use the azure-sdk-for-node to save a streamed image to Windows Azure blob storage but without success. Below is the function that I call and pass the video object with the thumbnail property. Initially I fetch the image using the request object which fetches the image from another website and turn that into a base64 object which in turn gets converted into a stream object because Azure blob service uses createBlockBlobFromStream method as I couldn't use createBlockBlobFromFile or createBlockBlobFromText to upload the image to blob storage.
var azure = require('azure')
, uuid = require('node-uuid')
, http = require('http')
, url = require('url')
, mmm = require('mmmagic')
, Magic = mmm.Magic
, stream = require('stream');
function createVideoThumbnail(video, callback){
var bs = azure.createBlobService(config.storageAccount, config.storageAccessKey, config.blobHost);
var sURL = video.Thumbnail;
var oURL = url.parse(sURL);
var client = http.createClient(80, oURL.hostname);
var request = client.request('GET', oURL.pathname, {'host': oURL.hostname});
request.end();
request.on('response', function (response) {
var type = response.headers["content-type"];
var prefix = "data:" + type + ";base64,";
var body = "";
response.setEncoding('binary');
response.on('end', function () {
var base64 = new Buffer(body, 'binary').toString('base64');
var data = prefix + base64;
console.log('base64 image data ' + video.Thumbnail + ': ' + data + '\n');
var decodedImage = new Buffer(data, 'base64');
var magic = new Magic(mmm.MAGIC_MIME_TYPE);
magic.detect(decodedImage, function(err, result) {
if(err) {
throw err;
}
var bytes = 0;
var imageStream = new stream.Stream();
imageStream.writable = true;
imageStream.write = function(buf) {
bytes += buf.length;
imageStream.emit('data', buf);
};
imageStream.end = function(buf) {
//if(arguments.length) {
imageStream.write(buf);
//}
imageStream.writable = false;
imageStream.emit('end');
console.log(bytes + ' bytes written');
};
var options = {}
console.log('mmm = ' + result + '\n');
options.contentType = result;
options.contentTypeHeader = result;
console.log('\n');
bs.createBlockBlobFromStream(config.imageContainer, uuid().replace(/-/gi, "").toLowerCase() + '.jpg', imageStream, decodedImage.length, options, function(error, blobResult, response) {
if (error)
console.log('got error = ' + JSON.stringify(error) + '\n');
if (blobResult)
console.log('blobResult = ' + JSON.stringify(blobResult) + '\n');
if (response)
console.log('response = ' + JSON.stringify(response) + '\n');
// now store in Azure blob storage
callback();
});
imageStream.end(decodedImage);
});
});
response.on('data', function (chunk) {
if (response.statusCode == 200) body += chunk;
});
});
}
and this is how I call it:
createVideoThumbnail(video, function(){
console.log("returning from create thumbnails\n\n");
});
The function is not working it hangs and won't print out the the final log statement:
console.log("returning from create thumbnails\n\n");
However the base64 does seem to work as I am getting this for the encoding:
data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAUDBAQEAwUEBAQFBQUGBwwIBwcHBw8LCwkMEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUFBQcGBw4ICA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh7/wAARCABaAKADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDx3QNShMv2IyqFcFrVieGHdCfUVFqDrY3BKsVU9favMba7uJn+XcoB3Lg42n1FbWdZu0G6eSQnjLLnNVLFJaMiOElLVGuNTZL8Wkaw7N+4llzmrtjek35RFtwCrdI8g8fWub13w34r03S11m502V7Fh81xECVj6fexyv1PBpng2R5rg3EsmFijYn3JGAKqNXm1uOpT0s+hZsLtzrN/AfuTHgKMAMvTiqEtlJLqoEQJVvm4/UUljeIJrwjiQlsHuOat6TPcazcrbWdrJNqCZ2RxxlvM9xirdrIhxcbNEWtWVw9nMvlP8o3HI9K5GvZLT4d+PbxEmfwzdhGwfnkRTj3BbNeZeJ/D+raBqMlvqmmXVkC7eX50ZUOoPVT0P4Vk5Rlsy4ppamRSqCzADqakji3ruMiL9SasWNuxuk3EbTk7l5pWHc29LiARFHQCuv0R2DooGQpBzjpXKW8bpt8srIP9k5/Sty1vV0yxe9kwZUGIlPRnPTPt3/Ct4WMJamlqogfxG1yFCvDEsLYORu6nj8QPwq9G7vcrbgZc4LDvnHTNYXh0R3HEzqJvvmQniRie9a0tzHYpNfuwBjGAc5JJrVfzC5erPfvhtqDXng20aU7mhLQ/QKSAPyxXRs+V4GOa89+BRnbwGstwMvNdSSH26cfpXcSTeWhLfhXyOJi3Xko92elD4ESsy4JPFVp2DLzziqr3UgZjj5fSljlEgY5wfSlKhOmrtFpor3tvBOpWeCORcfxLmuQkULvITyFGQWL9B2NdfeORDIwIyFNcdIQ8IB2kgfxdCfpWlDszopSqJe7sV/D/AMHfDaaxPctHKluW3R20r52e3Bzj0zXomn+BPDEAj/0KJ1TlRjgVtC2i1KNbq3G2VT6Yz7VoW4iChcHI7HiscZKfPzdGenSoRpxUexFDZWSQ/Z4rdBEQVKbRtIPBBHcYrxv4mfAi0vbp9T8FTRaTcyHMtm+RbPnuuATGfYAj2Fe6w+Vn7q/nTnZS3QflWNDE1KL91mVfDQq7o+YPB37POszaiZfEmoW1raH7yWrF5W9skAD68/SvffB/g/QfC1mLXQtLhtVwN8gXMkh9WY8mujVgDgA/lUgO4dM/jW1XG1ays3ZeRlTwtOn5sgeBivJrE8S+F9K8RaXNpusWkV1bSjBVxnHoQeoPuOR2rpApPGAKBC7MMdPpWMKkoO6ZpOCmuWR8E/FjwXdeA/FMumN+8s5CXtZHXJZPQn1H+B71z2juDO29EB28MK+sv2vfC6XngCHXFj/f2E6kvjojEKR+JK/9818jRhre6WRY2WNxwccfh+NfSYar7WCkzwq9L2cnE6KAL5oYjBHOR1qr4mu5bueC2RGZgOUiXlmP/wBbFSxzIliZ5flUDPI6gf4n+tYSXj/aUnIc7iS+1sHk9j2rs2RzxXU2rWcWSCS/dQ6/6u2Vu/qx/pXS6HbT3dsdX1VG+wwHesR4Nw5OBx+gFc7o+q6dbT+dHokV3L2V1aRgfUsTj9K7HSLvxT4nkltdLSNJ4YjKbeO281o1BxuGDwfqB1qotJ3YJSlJWR698H7maPw9LbtEok88yvGrgCMMBgfpXV3M8kqx74mjOMshIJH4ivMPgxa/2Nrl/YajNdR6nPCkhiulZGkUE/MoPYc9K9QuoxKd4wWHb1rwq7jDFNvY7KWkLNEDPyxPJPaolm2ENkfypwiuGyI4JSD2VT/Sll0rUxtMtjcRJ/eeMqPzIrpla2o0Q6jLts5SD1U8g5rk9xYcL35rodR0m4uWWztJ7VLyZgkULX8asxJ6BC3JP0rznVvEcMGVtVW5mVirLuCqT0wScfpXHSou2x6OFqQhB6nuPhbVrWeKOS1l3hyOp6GurmsY5GEoJBYcgeteR2ULeG/EcSbAbO8yo/2JANwx9QD+VelaLqAubffHJkp0XPJHp9a6FShVhzdDsjUcnbqX49qziDI8zbkL0Jx1xT5pUQZGPpmsC7abWb2Caykksb6yk3qs6/K4wQQceoP8q56Hxjq0N/qz+I7KKyNupeOKNlwEVSS2SeScVwVcLBaxNKnuxvck+IHxK8NeDWSPWLt3u5AGjtYF3ykeuDwo+pGe1c54Y+PXgrWNSSwuGvtJd3xHLdRr5WT6sGO38cAeor5v13WJda8S3Gq30kM93cTGRmKRtn0GMuMAYAHtUE2C5mijZVP39iYAP4RqB+ddccFFRs9zy/aucj9CNKimniDFGYEZVhzkfWrUr2cH+vvLSHb97zJ1XH1ya+aPAV/e6h4G0Vri8keRYtmZJSflWWVFXBf029q6/wAEW1vqC6slzHG5/s5JFLRhiu3AyOD/AD/OoWXQb1ZEsZKN9Dsfjc+h678N9S0eHXtINzPGDErXabWIYEfNnA5Ar5lsfhhr1xHHp0Ot6G0ZfzGt40lmlIIBG0rEeo9DXsEUkbfDV0C7JLe9AB+6cFXPByp/hB/AVW8DaibHxlYXtzdtCi+UDK7cKPlUk5z2z3ruo01RioxOOrU9o25HlWo/Cm3vJAl34m+zRRoGCW+mu2en/PRo+x/nWl4e+CegvanU5b/V9TtYpEVxGIYQCzFRn53I5B7V1OpGSXWLtrNC8csf7sxKfmAGBjhT2rX0rxHbaR4R1TSdVuDbXdwIDAtywXBV2LZ3scdRyMV0Nyepg7JaGH4S+H/hW/1SOy0zTNSiklYAyTagzZwGbqIcdhxkVc1DwrJBM2laRps1lqLIZJJYZJJJigcjOHlAKnAOQB14qfwJcS6N4is9TgFncGGZZUWK4jPmDPTcoyuVJr1XRteivTDe3ltEl/bboCd28SRnkJnABYADt6+tTUqcsbM6cJSdSV4u1jnLjw5caN4fstcVbcXsTfZZXls45Jo4S+AC5WQ5Oc8H8K5CHxTrPnCL+0rkcN80cqRjdtOPurGRziu18ZeO/DmjRrpuoyx3+lTJny4gsjQMfuqfY44PUY57V5dpV/od8rXFkt9H85RhJc8ZPYDHI/E1zqpC12jXEYacXdSvc9F+KFjcaULS6ttSviLmBifMvLhz9zcDy7Dnn+nevM7F4ZNRQyNFOSr/ADNsZc7Tx8yA9fet/WdaSS0t/t1qblYB5cXm3EjFRjHAJI6Dtiq1pr2jOoj+yw24H/Tun8wM/nTeISWkWzOGDrSjdI73xpN4bs9e8P3mjyWrPDdxSOLVgQqAgkbVyOo746nr2+cvHtvFZ+MdasTdsBDfzRjgE4V2A5r1PUtUNwqxWt0/kY+6jEKD9OlYd/Z2d3C0dzawzLk53KDzULFJvY6KWXyUbtnV/Gqe6i8OW17bSJE1teI4GOWPIH86b4U1a/XV1kjZ1e12faYyGVX3KG+64BHBBBxg1p+O7XRrtNP/AOEgmMekwTPPcKCQZNsblVBHIyf8O9eJ6x4gFx4kH/CBWV1Zwz7Y385zK0m3d8zMegAOMA9hU4NpUNTWrVdPE2SPcPikNakfS/Enga7jFy77Lq0Z1AnXHBO4jpj9a4DTvEza54xutK8VTx2Wpzj7ObZPugbSMbhwSQSeteS6x4q8Qtq0pa4KTRysoCkgKR6c8VUtrXWL7UpL+ZpftOTLvdsElRng+vFOdNW8jtpyVd8kU7iavoer6P4w/wCEcv1nadJwkQyxDgn5WXnGDVWPTLy51I2sNnLNMz7AioCxOcYHB5r1zR/ibZb7JPGXheO/udPl321yMedGVOQT07gZ5HuKuXPxN8OWMss/hbwt5eoysW+1XOMJnqVAJz3/AIh+NEak76xM4YTkm01c0YNO1D4e+GtE026ihmvXtmnuY2mIEJ3lsHap45A9yDWV468X694K8O6de3Flapc6srCG3MjuPIXB3OwK9SwwuOnXHSucvNd1O/vWvtauZ/trEqrvzwOwHYc8dh26V7b8f/Ai+N/h3BNYx51PSojPaKg5kUqN8eO5IVSPdcd6Sqcs1GT3DMMFCjRjKC957nzTcfFnxYxAt/sFquSQkcG4c8Hhy2ep61nXfjXxleSeYdXuID/0xZIMfQKB+VcrPG8UhRxgjiiON5CAvf3ruSR4DbO40PWrzWXFpq2tXMk6t9+a4LFx9SeapeKPDUyTG9tZkeN+fvDqOtc19kxgtJt9zQ7XAHlx3Erp7EgVpfTVGVne6Z2nhfx1qWip9nnf+HCOeSjDoa9gsvEMNn8PbG1tdZme7uEN2lyzg+VMxJdT7ZJ/PPavmMqQ2HJHvW3omq3UdnJo4k329w6kA/wEHPH1rKpHnR2YbEOhfzPY/BHw28beOfMfT9PkNjK4L3M7iJDk9QW+8B1+UGuitNEGhiTTVgIMMjJI27JLLwWH5e1e0fCzUZ9P8HaVZpwY7ZBj8BXj+v6tcr4l1CQIzo1zJgDoQXPP1rlxEFCKHSxEqsncyvEEhKrGqp1yc8+3NYwZX4AG3HBz0q/q+tWzTATR7TswdyYqC1e2uZR5JIBI4BztH40oaRPo8NaNJGjZsViRFDfKOe1SSOobbjB6mt6y8I3t7oy6jZ3NtHuyxR22ttGfw7etcvcyHKn+DdyRz71g4Pd9TGNWE21F7HovjbQm8Q6DJY71jfIeNj2Yf/rNZt/Z2nhjwhLcSxQRxWNtkADgtjH6n+ddekueNo/Kq+v6XHrWh3elzr8lxGVzjoeoP4EA15qntF7XKUFfm6nyDPby3l552GeSeTdn+Jizeg/HirlhpmqSaoLC1E0lwkm1PJy5J9AByT9K7m4+GfjSPUltI9KaWFV2JIkw2cdDnPTnPrzXq/hPwxoPgDSJ9c1Iw21wsQa7u5HyE9VTvgnsOTwPSvVqYmKS5dW9kXKNGh7yf6Hm+mfBTxtd2Rlnjs7dpDuYS3BEnX0UEA4z+dYF74C8R6NqX9n39hJFPcP5duUXdvJP8DA459z35re8VftF6gmomPwxpdv9kjbHm3oLNL77QRtH4n8K2vCH7SOlzyxR+KdCe0YEZuLRvMQH12nkfgTU8uKSu0n5HPDN6anqrnaeBPg9bWq2N74luFuJYY122ifcQhi2Gf8Aj5PsPwr2B1SKFijKuwZBA6YritG+J3w/1e1Wa28V6agIzsmlEbD6hsGuG+Mfxv8AD+i6NcWHhjUYdT1SZCiPAweOHP8AEWHHHp/SuBUa9WouZMjEY2NSLblc+bfiYtgPHWvW9jtW1h1C5WHb0CB2wB+QrmrVyJEAO35uTio3d3dpHYsznLMeppAxC4HHOc19CtDwG7l0vFvJZt5zwAOtWFQyKGfj0Qf1rMjcIcqOfU1ILhh1dj9OKu5DRbntvl5ApmmxiPUoHdtqLKpY+gzzURvGIxjj60sd0M8jFIVmfdGg30FvpEc0UivEsAZHU5BG3II9q8haVI2d3bMjOWYFskZzjGKyfgp4ye88Mah4WnuAt1FbsbF25yp4K9RnaTnHpn0rRGl6qvH2iwb/ALd3X+T1zYmMp2SNMPJU73MW8EUkrGT58tyS2eO1NsbOIyB44yrjkbeCfr2rTm8Oag2T51mpP91G/qTSRaNrVu5aN7KUE5+YsuP0NS4S5bJHurMKHLYctzexfuDI4TG04Y5Oe1RyTCIucdAoC+pJp8ljruSxgscnpiduv/fFQyabq5IZ7S3yDkkXB6/98+1YqlPqhRxlDe57jEcD7wFXI3G3O8VlxfdqUE7TzXiM60i5d6hZ2NnLeXl0kMES7ndjgCvnP4wa/f8Aj29WytZpYdNgcm3gUcu399x3Pt2H456n41Tzf2rY2/nSeSYyxj3HbnPXHTNYXhZE2O+xd2cZxzXu4LDwhDn3bPCxmJlOTj0R4jq+l3mmX7WdzGfMHTbyD9KntNGnlUF1ZSe1egfGBVUWMiqFfzSNwGD0rJ0Il4fnJb6816EI3OHmdjEHhaVkDfaVX2K5pY/CzgkvchgPRcV04+8B2zT8Da1a+zRHOzlLnTIoIXEzqqDqSKwbpIY2AikZ/XcuMV3d0AdpIBNcx4gVfNLbRuwOcc1nJWLizIjjL5wDx6DNTC3T+KXb/vKRVvRe9akgG0cDvUpDb1MQWcbAbZUb6NSmxI9av3EcflqfLXOPSktOCQKdhXZBpVzd6RqUGoWbkSwOGX0PsfY9K+jvDmsQ6xpEGoW/3JkztPVT3H4Gvn+RVx90flXpPwYZv7Kv03Has4IGeBlamSsDPRXmyQAnSlWQ/wB3FQp941MOlSA/cG6jI96bNsIHAXA7d6cAM9BVS9JATBx8w6VQj//Z
mmm = application/octet-stream
5283 bytes written
But I am not getting any of these log statements being printed:
if (error)
console.log('got error = ' + JSON.stringify(error) + '\n');
if (blobResult)
console.log('blobResult = ' + JSON.stringify(blobResult) + '\n');
if (response)
console.log('response = ' + JSON.stringify(response) + '\n');
So I am presuming it is hanging somewhere or I have not structured my code properly. Can anybody see what I am doing wrong ?
Cheers
Rob
My sources:
http://social.msdn.microsoft.com/Forums/en-US/wavirtualmachinesforlinux/thread/47bfe142-c459-4815-b09e-bd0a07ca18d5
Node.js base64 encode a downloaded image for use in data URI
Here's my simplified version. Since magic needs to operate on the whole file, there's no point in using the streaming blog API, instead this is written for the text api. body will be a buffer of the image, I suspect azure will be happy with it sans encoding, call toString('encoding') if it does need it.
var azure = require('azure')
, uuid = require('node-uuid')
, request = require('request')
, mmm = require('mmmagic')
, Magic = mmm.Magic
, stream = require('stream')
, bs = azure.createBlobService(config.storageAccount, config.storageAccessKey, config.blobHost);
function createVideoThumbnail(video, callback){
var sURL = video.Thumbnail;
request(sURL, {encoding:null}, function (err, res, body) {
// encoding:null makes request return a buffer, which is ideal to run magic.detect on
magic.detect(body, function (err, res) {
console.log(res);
var container = config.imageContainer;
var blob = uuid().replace(/-/gi, "").toLowerCase() + '.jpg';
var text = body; //might need to be converted into a string, I don't have azure setup to test
var options = {
contentType: res,
contentTypeHeader: res
};
bs.createBlockBlobFromText(container, blob, text, options, function(error, blobResult, response) {
if (error)
console.log('got error =', error);
// if you give console.log multiple arguments, it will format each of them,
// no need to manipulate objects into strings manually
if (blobResult)
console.log('blobResult =', blobResult);
if (response)
console.log('response =', response);
// now store in Azure blob storage
callback();
});
});
});
}
edit: temp file version
var azure = require('azure')
, uuid = require('node-uuid')
, request = require('request')
, mmm = require('mmmagic')
, Magic = mmm.Magic
, fs = require('fs')
, bs = azure.createBlobService(config.storageAccount, config.storageAccessKey, config.blobHost);
function createVideoThumbnail(video, callback){
var sURL = video.Thumbnail;
var name = uuid().replace(/-/gi, "").toLowerCase() + '.jpg';
var ws = fs.createWriteStream('./tmp/' + name);
request(sURL, {encoding:null})
.pipe(ws).on('close', function () {
console.log('downloaded');
magic.detectFile('./tmp/' + name, function (err, res) {
var container = config.imageContainer;
var blob = uuid().replace(/-/gi, "").toLowerCase() + '.jpg';
var options = {
contentType: res,
contentTypeHeader: res
};
bs.createBlockBlobFromFile(container, name, './tmp/' + name, function (err) {
callback();
});
});
});
}
Thanks to Valery Jacobs on msdn forums was able to come up with the answer. It's a nice clean solid solution using the stream, request and util object.
:)
http://social.msdn.microsoft.com/Forums/en-US/windowsazurepurchasing/thread/25c7705a-4ea0-4d9c-af09-cb48a031d06c
Related
I'm trying to populate a database with pictures paths & names using Node.js.
What I am trying to do is the following :
- A function send a list of pictures as Base64 string.
- Another function receive this list, loop through it, convert it into picture and get the path back.
I'm pretty new to node.js so I might be doing something really stupid.
Here is the reception code :
app.post('/chatBot/moreinfo/create', function (req, res) {
returnList = '';
res.set('Content-Type', 'application/json');
//IF LIST IS NOT EMPTY
if (req.body.imgList.length !== 0) {
const imagePath = '/var/lib/SMImageBank/';
const regex = /^data:image\/(.*);.*$/i;
const listePicture = req.body.imgList;
// LOOPING INTO THE LIST
req.body.imgList.map ( function (element) {
const file = element;
const filetype = file.match(regex)[1];
var picLink2 = '';
const base64data = file.replace(/^data:image\/.*;base64,/, "");
const latin1data = new Buffer(base64data, 'base64').toString('latin1');
const filename = new Date().getTime() + '' + new Date().getMilliseconds() + "." + filetype;
fs.mkdir(imagePath, () => {
fs.writeFile(imagePath + filename, latin1data, "latin1", function (err, content) {
if (err) {
routerLog(req, {'type': 'error', 'content': err} );
res.sendStatus(500);
}
else {
if (process.env.NODE_ENV === "production")
picLink2 = 'http://****.fr/image/' + filename;
else if(process.env.NODE_ENV === "test")
picLink2 = 'http://dev.****.fr:8010/image/' + filename;
else if(process.env.NODE_ENV === "master")
picLink2 = 'http://dev.****.fr:8008/image/' + filename;
else{
picLink2 = 'http://*****.com:8008/image/' + filename;
}
}
});
})
console.log(picLink2);
returnList = returnList + ";" + picLink2;
});
}
MoreInfo.create(req.body, function (ret) {
res.send(ret);
routerLog(req);
})
});
What I want to do is to be able to access the variable "picLink2" from outside the writeFile & mkdir function so I can populate my returnList at each iteration. Obviously as node.js is asynchronous I can't access to picLink2 content from outside fs.writeFile() function. I know there has been a ton of question about this and lot of the answers are to put the code inside the writeFile()/readFile() function but I don't see how I can do it here since the writeFile() function is inside a map that is iterating into a list.
I'm new to the asynchronous world and I don't see how I can solve this problem.
Use writeFileSync for a synchronous operation if that doesn't hurt performance.
I am new to nodejs. Need your help. From the nodejs terminal, i want to download an excel file and convert it to csv (say, mocha online.js). Note: i don't want to do this via a browser.
Below is a script i am working on to download and convert to csv. There is no error nor the expected result:
online.js
if (typeof require !== 'undefined') XLSX = require('xlsx');
var XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
/* set up XMLHttpRequest */
var url = "http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx";
var xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
xhr.responseType = "arraybuffer";
describe('suite', function () {
it('case', function () {
var arraybuffer = xhr.response;
/* convert data to binary string */
var data = new Uint8Array(arraybuffer);
var arr = new Array();
for (var i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
var bstr = arr.join("");
/* Call XLSX */
var sheetName = 'Database';
var workbook = XLSX.read(bstr, { type: "binary" });
var worksheet = workbook.Sheets[sheetName];
var csv = XLSX.utils.sheet_to_csv(worksheet);
console.log(csv);
xhr.send();
//.... perform validations here using the csv data
});
})}
I tried myself with this code, and it seems it is working, the only thing is that I spent 15 minutes trying to understand why my open office would not open the file, I eventually understood that they were sending a zip file ... here is the full code, the doc of the http get function is here http.get
You could have used the request module, but it isn't native, request is easier though.
enjoy!
const url = 'http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx'
const http = require('http')
const fs = require('fs')
http.get(url, (res) => {
debugger
const {
statusCode
} = res;
const contentType = res.headers['content-type'];
console.log(`The type of the file is : ${contentType}`)
let error;
if (statusCode !== 200) {
error = new Error(`Request Failed.\n` +
`Status Code: ${statusCode}`);
}
if (error) {
console.error(error.message);
// consume response data to free up memory
res.resume();
return;
}
res.setEncoding('binary');
let rawData = '';
res.on('data', (chunk) => {
rawData += chunk;
});
res.on('end', () => {
try {
const parsedData = xlsxToCSVFunction(rawData);
// And / Or just put it in a file
fs.writeFileSync('fileName.zip', rawData, 'binary')
// console.log(parsedData);
} catch (e) {
console.error(e.message);
}
});
}).on('error', (e) => {
console.error(`Got error: ${e.message}`);
});
function xlsxToCSVFunction(rawData) {
return rawData //you should return the csv file here whatever your tools are
}
I actually encountered the same problem 3 months ago : here is what I did!
I did not find any nodeJS module that was exactly as I wanted, so I used in2csv (a python shell program) to transform the data; the t option is to use tabulation as the delimiter
1) Step 1: transforming the xlsx file into csv using in2csv
This code takes all the xlsx files in the current directory, transform them into csv files and put them in another directory
var shelljs = require('shelljs/global')
var dir = pwd().stdout.split('/')
dir = dir[dir.length - 1].replace(/\s/g, '\\ ')
mkdir('../'+ dir + 'CSV')
ls('*.xlsx').forEach(function(file) {
// below are the two lines you need
let string = 'in2csv -t ' + file.replace(/\s/g, '\\ ') + ' > ../'+ dir + 'CSV/' + file.replace('xlsx','csv').replace(/\s/g, '\\ ')
exec(string, {silent:true}, function(code, stdout, stderr){
console.log('new file : ' + file.replace('xlsx','csv'))
if(stderr){
console.log(string)
console.log('Program stderr:', stderr)
}
})
});
Step 2: loading the data in a nodejs program:
my script is very long but the main two lines are :
const args = fileContent.split('\n')[0].split(',')
const content = fileContent.split('\n').slice(1).map(e => e.split(','))
And for the benefit of seekers like me...here is a solution using mocha, request and xlsx
var request = require('request');
var XLSX = require('xlsx');
describe('suite', function () {
it('case', function (done) {
var url = "http://oss.sheetjs.com/js-xlsx/test_files/formula_stress_test_ajax.xlsx";
var options = {
url: url,
headers: {
'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
},
encoding: null
};
request.get(options, function (err, res, body){
var arraybuffer = body;
/* convert data to binary string */
var data = arraybuffer;
//var data = new Uint8Array(arraybuffer);
var arr = new Array();
for (var i = 0; i != data.length; ++i) arr[i] = String.fromCharCode(data[i]);
var bstr = arr.join("");
/* Call XLSX */
var sheetName = 'Database';
var workbook = XLSX.read(bstr, { type: "binary" });
var worksheet = workbook.Sheets[sheetName];
var csv = XLSX.utils.sheet_to_csv(worksheet);
console.log(csv);
done();
});
});
});
I'm trying to download a Youtube video audio using the ytdl-core module (https://github.com/fent/node-ytdl-core).
I wrote an API using Express which lets me download an audio by its URL:
app.get('/api/downloadYoutubeVideo', function (req, res) {
res.set('Content-Type', 'audio/mpeg');
var videoUrl = req.query.videoUrl;
var videoName;
ytdl.getInfo(videoUrl, function(err, info){
videoName = info.title.replace('|','').toString('ascii');
res.set('Content-Disposition', 'attachment; filename=' + videoName + '.mp3');
});
var videoWritableStream = fs.createWriteStream('C:\\test' + '\\' + videoName); // some path on my computer (exists!)
var videoReadableStream = ytdl(videoUrl, { filter: 'audioonly'});
var stream = videoReadableStream.pipe(videoWritableStream);
});
The problem is that when I call this API I get a 504 error from my server.
I want to be able to save this downloaded audio on my local disk.
Help would be appreciated. Thank you
Well for some reason the videoName was undefined so it messed up my function...
Here is the correct code after a few changes and adding the destination path as query variable.
app.get('/api/downloadYoutubeVideo', function (req, res) {
var videoUrl = req.query.videoUrl;
var destDir = req.query.destDir;
var videoReadableStream = ytdl(videoUrl, { filter: 'audioonly'});
ytdl.getInfo(videoUrl, function(err, info){
var videoName = info.title.replace('|','').toString('ascii');
var videoWritableStream = fs.createWriteStream(destDir + '\\' + videoName + '.mp3');
var stream = videoReadableStream.pipe(videoWritableStream);
stream.on('finish', function() {
res.writeHead(204);
res.end();
});
});
});
app.getImage = function() {
var image = Meteor.http.get("https://turtlerock-discourse.global.ssl.fastly.net/user_avatar/talk.turtlerockstudios.com/takran/45/879.png", {
});
var prefix = "data:image/png;base64,";
var imagebase64 = new Buffer(image.content, 'binary').toString('base64');
imagebase64 = prefix + imagebase64;
console.log(imagebase64);
return imagebase64;
}
but I am not seeing results,
any help?
This is a dummy text for the error.
a pure Meteor solutions:
var response = HTTP.call('GET', url,{npmRequestOptions: { encoding: null }})
var data = "data:" + response.headers["content-type"] + ";base64," + new Buffer(response.content).toString('base64');
This is how I fixed this issue.
app.getImage = function(){
var myrequest = request.defaults({ encoding: null });
var fut = new Future();
var options = {
"url" : "https://any.domain.com/any.png"
}
myrequest.get(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
var data = "data:" + response.headers["content-type"] + ";base64," + new Buffer(body).toString('base64');
fut.return(data);
}
else
fut.return(null);
});
return fut.wait();
}
I was assuming this solution should have come with meteor code itself,
but it doesn't,
I had to use nodejs way to do it.
I will still be waiting for someone to post an answer based on pure meteor way.
Based on paparazzo.js lib, I'm trying to get base64 images from a MJPEG stream (streamed over TCP with GStreamer) in a Node.js server, and to send them to the clients via websockets.
I think I'm pretty close, but my images are corrupted. Here is the code I'm using :
var boundary = "----videoboundary";
var data = "";
var tcpServer = net.createServer(function (socket) {
socket.on('data', function(chunk) {
var boundaryIndex = chunk.toString().indexOf(boundary);
if (boundaryIndex !== -1) {
// Get the image's last piece of data :
data += chunk.toString().substring(0, boundaryIndex);
// Convert the data to a base64 image and broadcast it :
var image = new Buffer(data).toString('base64');
io.sockets.emit('image', image);
// Reset the data :
data = '';
// Get the remaining data (with new image's headers) :
var remaining = chunk.toString().substring(boundaryIndex);
// Remove the new image's headers and add the remaining data :
var contentTypeMatches = remaining.match(/Content-Type:\s+image\/jpeg\s+/);
var contentLengthMatches = remaining.match(/Content-Length:\s+(\d+)\s+/);
if(contentLengthMatches != null && contentLengthMatches.length > 1) {
var newImageBeginning = remaining.indexOf(contentLengthMatches[0]) + contentLengthMatches[0].length;
data += remaining.substring(newImageBeginning);
}
else if(contentTypeMatches != null) {
var newImageBeginning = remaining.indexOf(contentTypeMatches[0]) + contentTypeMatches[0].length;
data += remaining.substring(newImageBeginning);
}
else {
var newImageBeginning = boundaryIndex + boundary.length;
data += remaining.substring(newImageBeginning);
io.sockets.emit('error', { message: 'Could not find beginning of next image' });
}
}
else {
data += chunk;
}
});
});
Any idea ?
Thanks
chunk.toString() converts the binary Buffer to a utf8-encoded string (by default), so for binary image data that will probably cause you some problems.
Another option that might help simplify things for you is to use the dicer module. With that, your code may look like:
var Dicer = require('dicer');
var boundary = '----videoboundary';
var tcpServer = net.createServer(function(socket) {
var dice = new Dicer({ boundary: boundary });
dice.on('part', function(part) {
var frameEncoded = '';
part.setEncoding('base64');
part.on('header', function(header) {
// here you can verify content-type, content-length, or any other header
// values if you need to
}).on('data', function(data) {
frameEncoded += data;
}).on('end', function() {
io.sockets.emit('image', frameEncoded);
});
}).on('finish', function() {
console.log('End of parts');
});
socket.pipe(dice);
});