Can you please help me in create a Video URL Blob in NodeJS?
var xhr = new XMLHttpRequest();
xhr.responseType = 'blob';
xhr.onload = function() {
var reader = new FileReader();
reader.onloadend = function() {
var byteCharacters = atob(reader.result.slice(reader.result.indexOf(',') + 1));
var byteNumbers = new Array(byteCharacters.length);
for (var i = 0; i < byteCharacters.length; i++) {
byteNumbers[i] = byteCharacters.charCodeAt(i);
}
var byteArray = new Uint8Array(byteNumbers);
var blob = new Blob([ byteArray ], { type: 'video/ogg' });
var url = URL.createObjectURL(blob);
console.log(url);
};
reader.readAsDataURL(xhr.response);
};
xhr.open(
'GET',
'...Video URL...'
);
xhr.send();
Error Output:
throw new Error('cannot read as File: ' + JSON.stringify(file));
Error: cannot read as File: undefined
I have used the packages XMLHttpRequest, URL, FileReader and Blob
Please Help, Thanks
You can Use Azure Cloud for storing the videos and then us Azure BLOB
Node.JS does not have XMLHttpRequest, FileReader, Blob, etc. Instead, it uses the fs, http modules and other built-in classes, like Buffer
Trying to use these browser features on the server, even if using packages that try to emulate them, may not always work. In fact, many of those libraries which emulate those features do not include everything. For example, the xmlhttprequest node package does not use .response, but only the text response.
In Node.JS, there is no need for blob URLs (where would you even use that?), you just interact with the Buffers directly.
// making a http request to fetch something
const http = require("https");
const request = https.request({
method: "GET",
url: "https://example.com/video.ogg"
}, (res) => {
const chunks = [];
res.on("data", (chunk) => {
chunks.push(chunk);
});
res.on("end", () => {
// raw byte data
const buffer = Buffer.concat(chunks);
/*
* Interact with Buffer here
*/
});
});
request.end();
Related
I want to fetch image from azure blob storage and send it over to client without saving it locally. I am able to get image from blob storage and save it to local file but struggling to send it to client without saving locally. Please find code below
const containerClient = blobServiceClient.getContainerClient(containerName);
const blockBlobClient = containerClient.getBlobClient(blobName);
const downloadBlockBlobResponse = await blockBlobClient.download(0);
console.log('\nDownloaded blob content...');
let f = await streamToString(downloadBlockBlobResponse.readableStreamBody)
reply.type('image/jpg').send(f)
streamToString function is as follows
async function streamToString(readableStream) {
return new Promise((resolve, reject) => {
const chunks = [];
readableStream.on("data", (data) => {
chunks.push(data.toString());
});
readableStream.on("end", () => {
resolve(chunks.join(""));
});
readableStream.on("error", reject);
});
}
I get blank screen in browser when I run this code
If you want to fetch an image from azure blob storage and send it over to the client without saving it locally, the Node server sending a SAS token to the client and the client get this image from Azure storage directly would be a better solution I think. It also eases the pressure of the Node server: generate a SAS token and send it to the client is ok, no need to read data from Storage.
Try the code below to generate a SAS token:
var azure = require('azure-storage');
var connString = "your storage connection string";
var container ="your container name";
var blobName = "your image name"
var blobService = azure.createBlobService(connString);
// Create a SAS token that expires in an hour
// Set start time to five minutes ago to avoid clock skew.
var startDate = new Date();
startDate.setMinutes(startDate.getMinutes() - 5);
var expiryDate = new Date(startDate);
expiryDate.setMinutes(startDate.getMinutes() + 60);
var sharedAccessPolicy = {
AccessPolicy: {
Permissions: [azure.BlobUtilities.SharedAccessPermissions.READ], //grent read permission only
Start: startDate,
Expiry: expiryDate
}
};
var sasToken = blobService.generateSharedAccessSignature(container, blobName, sharedAccessPolicy);
var response = {};
response.image = blobService.getUrl(container,blobName,sasToken);
res.send(response);
Result:
Client-side could use this image URL to access this image directly from storage:
Try this to convert imageURL to base64 image content so that you can save/show images directly based on their content:
<html>
<body>
<img id="displayImg">
</body>
<script>
var nodeResult = {"image":"https://stantest1016.blob.core.windows.net/stantest/test.jpeg?st=2020-10-26T04%3A53%3A51Z&se=2020-10-26T05%3A53%3A51Z&sp=r&sv=2018-03-28&sr=b&sig=ZjY3LYbgvMn%2BSr4dAEUoqidVRT4YyH1FcW1DeKlpjYo%3D"}
function getImageData(nodeResult){
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
var reader = new FileReader();
reader.readAsDataURL(this.response);
reader.onload = function() {
document.getElementById('displayImg').src = reader.result;
}
};
};
xhr.open('GET', nodeResult.image);
xhr.responseType = 'blob';
xhr.send();
}
getImageData(nodeResult);
</script>
</html>
Result:
Details of tag:
I have an ajax endpoint where I generate a pdf based off the user's GET request, then send them back the pdf through a Buffer.
When I try to send them the buffer directly, the resulting downloaded pdf is corrupted, however, when I send them a base64 version then decode it on the client side it works properly.
AJAX endpoint:
module.exports = function(req, res) {
pdf.create('<h1>Title</h1>').toBuffer(function(err, buffer) {
res.setHeader('Content-Type', 'application/pdf');
res.send(buffer.toString('base64'));
});
};
Client:
$.get('/createPDF', (data) => {
var len = binary.length;
var buffer = new ArrayBuffer(len);
var view = new Uint8Array(buffer);
for (var i = 0; i < len; i++) {
view[i] = binary.charCodeAt(i);
}
const blob = new Blob([view], {type: "application/pdf"});
const url = window.URL.createObjectURL(blob);
const link = document.createElement('a');
link.href = url;
link.setAttribute('download', 'file.pdf');
document.body.appendChild(link);
link.click();
});
If I skip the toString('base64') on the server and the part above the Blob creation on the client it doesn't work, but if I include that it does. Does anyone know why this might be the case? Note I'm asking for this specific use-case of jquery's ajax.
I have a question about handling a gzip response on my client side application. I would like the client's browser to pop up an alert "how do you want to handle?" download prompt.
My Node.js server is compressing my files into a gzip format then sending it with a HTTP write response. My client receives a HTTP 200 status although the size of the response is very small compared to my file and nothing doesn't populate my web app. I have anticipated the browser to handle this sort of response to a server sending gzip. similar to how gmail handles downloading files. Can you help me to see if I have missed anything?
server.js
var server = http.createServer(function(request, response) {
if (request.url === '/download'){
let data_zip = retrievedata()
const scopedata_zip = ('./scopedata.txt.gz')
response.writeHead(200, { 'Content-Encoding': 'gzip' });
response.writeHead(200, { 'Content-Type': 'application/javascript' });
response.write(scopedata_zip);
}
})
var retrievedata = () =>{
const gzip = zlib.createGzip();
const inp = fs.createReadStream('scopedata.txt');
const out = fs.createWriteStream('scopedata.txt.gz');
inp.pipe(gzip).pipe(out);
return out
}
Client.js
var downloadData=()=>{
var xhr = new XMLHttpRequest();
xhr.open('POST', 'download', true);
//xhr.setRequestHeader("Accept-Encoding", "gzip")
xhr.setRequestHeader("Encoding", "null")
xhr.onload = function (){
if(this.status == 200){
let form = document.createElement("form");
let element1 = document.createElement("input");
document.body.appendChild(form);
let response = this.responseText
console.log(response)
document.getElementById("state").innerHTML = 'download'
document.getElementById("index").innerHTML = response;
// document.getElementById("state").appendChild(form)
}
}
xhr.onerror = function(err){
console.log("request error...",err)
}
xhr.send()
}
The client is just populating my index div the response to, but nothing is received.
my gzip file is 327mb.
Chrome inspector network says this request is only 170B so I am not receiving my file.
Note xhr.setRequestHeader("Accept-Encoding", "gzip") is commented out becuase I get this error: Refused to set unsafe header "Accept-Encoding". I have set it to null to allow the browser to handle this.
Any input on what I am doing wrong?
There were three things I was doing wrong. I managed to get the browser window by creating a new element, checking if the element has a download attribute and appending the XHR.Response as the location from the href. The second portion of my issue was not receiving the zip file with the appropriate request headers. Because my zip file was a larger size the browser handles the binary buffer stream as a blob. Read more about XHR response types XHR.response. The other issue was on my server side which was using fs.readFile to read the zip as a buffer. Because my zip was made up of multiple files fs.readFile it would stop reading as it hit the end of the first file.
so my client code looks like
var xhr = new XMLHttpRequest();
document.getElementById("state").innerHTML = ' '
document.getElementById("index").innerHTML = ' ';
xhr.open('POST', 'download', true);
xhr.setRequestHeader('Content-disposition', 'attachment')
xhr.setRequestHeader("Content-type","application/zip"); //content-type must be set
xhr.setRequestHeader("Encoding", "null") //unsure of why I need this but it doesnt work with out it for me
xhr.responseType = "blob"; // This must be set otherwise the browser was interpretting the buffer stream as string instead of binary
xhr.onload = function (){
if(this.status == 200){
let form = document.createElement("form");
let element1 = document.createElement("input");
document.body.appendChild(form);
let response = this.response // defined as blob above
document.getElementById("state").innerHTML = 'download'
document.getElementById("index").innerHTML = response;
var blob = new Blob([response], {type: "application/zip"});
var file = URL.createObjectURL(blob);
filename = 'Data.zip'
var a = document.createElement("a");
if ("download" in a) { //check if element can download
a.href = file;
a.download = filename;
document.body.appendChild(a);
a.click(); //automatically browser download
document.body.removeChild(a);
}
}
Server side
else if (request.url === '/download'){
archiveZip((data)=>{ // using archivezip and adding a callback function to insert my routes XHR response
response.setHeader('Content-Type', 'application/zip')
response.setHeader('Content-Length', data.length) // this is important header because without it the browser might truncate the entire response especially if there are end of file characters zipped up in the buffer stream
response.setHeader('Content-disposition', 'attachment; filename="Data.zip"');
response.end(data);
})
}
var archiveZip = (callback) =>{
var output = fs.createWriteStream(__dirname + '/Data.zip'); //output
var archive = archiver('zip', {zlib: { level: 9 }});
output.on('close', function() {
console.log(archive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
fs.readFile('./Data.zip', function (err, content) {
if (err) {
response.writeHead(400, {'Content-type':'text/html'})
console.log(err);
response.end("No such file");
} else {
callback(content);
}
});
});
output.on('end', function() {
console.log('Data has been drained');
});
archive.on('error', function(err) {
throw err;
});
archive.pipe(output);
// append a file
archive.file(data_files + '/parsed_scope.json', { name: 'parsed_scope.json' });
archive.file(data_files + '/scopedata_index.json', { name: 'scopedata_index.json' });
archive.file(data_files + '/scopedata.txt', { name: 'scopedata.txt' });
archive.finalize();
There are many zip libraries I was looking at ones that can handle zipping a directory with multiple files and went with archiver. I would have like to use the built in zlib that comes with node but only supports single small files.
I'm searching a way to get the base64 string representation of a PDFKit document. I cant' find the right way to do it...
Something like this would be extremely convenient.
var doc = new PDFDocument();
doc.addPage();
doc.outputBase64(function (err, pdfAsText) {
console.log('Base64 PDF representation', pdfAsText);
});
I already tried with blob-stream lib, but it doesn't work on a node server (It says that Blob doesn't exist).
Thanks for your help!
I was in a similar predicament, wanting to generate PDF on the fly without having temporary files lying around. My context is a NodeJS API layer (using Express) which is interacted with via a React frontend.
Ironically, a similar discussion for Meteor helped me get to where I needed. Based on that, my solution resembles:
const PDFDocument = require('pdfkit');
const { Base64Encode } = require('base64-stream');
// ...
var doc = new PDFDocument();
// write to PDF
var finalString = ''; // contains the base64 string
var stream = doc.pipe(new Base64Encode());
doc.end(); // will trigger the stream to end
stream.on('data', function(chunk) {
finalString += chunk;
});
stream.on('end', function() {
// the stream is at its end, so push the resulting base64 string to the response
res.json(finalString);
});
Synchronous option not (yet) present in the documentation
const doc = new PDFDocument();
doc.text("Sample text", 100, 100);
doc.end();
const data = doc.read();
console.log(data.toString("base64"));
I just made a module for this you could probably use. js-base64-file
const Base64File=require('js-base64-file');
const b64PDF=new Base64File;
const file='yourPDF.pdf';
const path=`${__dirname}/path/to/pdf/`;
const doc = new PDFDocument();
doc.addPage();
//save you PDF using the filename and path
//this will load and convert
const data=b64PDF.loadSync(path,file);
console.log('Base64 PDF representation', pdfAsText);
//you could also save a copy as base 64 if you wanted like so :
b64PDF.save(data,path,`copy-b64-${file}`);
It's a new module so my documentation isn't complete yet, but there is also an async method.
//this will load and convert if needed asynchriouniously
b64PDF.load(
path,
file,
function(err,base64){
if(err){
//handle error here
process.exit(1);
}
console.log('ASYNC: you could send this PDF via ws or http to the browser now\n');
//or as above save it here
b64PDF.save(base64,path,`copy-async-${file}`);
}
);
I suppose I could add in a convert from memory method too. If this doesn't suit your needs you could submit a request on the base64 file repo
Following Grant's answer, here is an alternative without using node response but a promise (to ease the call outside of a router):
const PDFDocument = require('pdfkit');
const {Base64Encode} = require('base64-stream');
const toBase64 = doc => {
return new Promise((resolve, reject) => {
try {
const stream = doc.pipe(new Base64Encode());
let base64Value = '';
stream.on('data', chunk => {
base64Value += chunk;
});
stream.on('end', () => {
resolve(base64Value);
});
} catch (e) {
reject(e);
}
});
};
The callee should use doc.end() before or after calling this async method.
I have a website with multiple pages, each page lists download links which I want to scrap and download.
I have few issues with it:
My script only downloads about 4-5 files and getting stuck.
I would like to concurrently download as much files as my CPU can.
I got stuck with maximum event emitters, I don't understand why is that so I just go
How to follow redirects purely using request module (without follow-redirects)?
How to download the file like the browser does without mentioning it's name? there is no content-disposition but I think the browser follow redirects and the redirected URL has the filename in it's path.
My current code looks like so:
var request = require('request');
var cheerio = require('cheerio');
var https = require('follow-redirects').https;
require('events').EventEmitter.prototype._maxListeners = 1000;
for(var i = 1; i <= 10000; i++) {
(function(i){
url = 'http://mywebsite.com/files?page=' + i;
request(url, gotHTML)
})(i);
}
function gotHTML(err, resp, html) {
var $ = cheerio.load(html);
$('.file-header').each(function() {
var data = $(this);
var fileLink = data.children().first().children().first().attr('href');
var fileName = fileLink.substring(10);
var downloadLink = 'https://mywebsite.com/api/download/' + fileName;
download(downloadLink, function() {
console.log('downloaded');
})
})
}
function download(url, cb) {
var request = https.get(url, function(response) {
var location = request.res.headers.location;
console.log(location);
location = location.split('/').pop();
console.log(location);
var file = fs.createWriteStream(location);
response.pipe(file);
file.on('finish', function() {
file.close(cb);
});
});
}
The default HTTP/HTTPS Agent only uses a maximum of 5 sockets (maxSockets) for requests to the same origin. So this could be causing some issues for you.
Try changing this:
var request = https.get(url, function(response) {
to this:
var options = require('url').parse(url);
options.agent = false; // or use a custom https.Agent with a higher `maxSockets`
var request = https.get(options, function(response) {