express res.download serving up 0 byte files - node.js

res.download is serving up the file but once downloaded it is 0 bytes?
Any ideas?
app.get('/download', function(req, res) {
console.log("download");
console.log(req.query.fileID);
fileDownload(req.query.fileID, function(rep){
if(rep.success){
console.log("Serving File to User, File: " + rep.data);
res.download(__dirname + "/" + rep.data, rep.data)
}else{
console.log(res);
}
});
})
A ls on the folder shows the file is there ready for download, the names are correct and all on download box that browser displays but download is always 0 bytes in size.
A check on the file from file file download shoes yes it is there and yes its all good.
ISSUE FOUND MAYBE:
I Think the issue is the file is not fully downloaded before its being served to client, see below,i will try adding a callback to the PIPE.....
function fileDownload(id, callback){
info(id, function(res){
if(!res.error){
info(id, function(res){
if(!res.error){
//console.log(res.data);
var d = JSON.parse(res.data);
//console.log(d['file_name']);
var url2 = baseurl + "/api/file/" + id ;
var r = request(url2);
r.on('response', function (res) {
res.pipe(fs.createWriteStream('./' + d['id'] + d['file_name']));
console.log("Download Done: " + './' + d['id'] + d['file_name']);
return callback({success:true, data:d['id'] + d['file_name']});
});
}else{
console.log("ERROR: " + res.data)
return callback({success:false, data: res.data});
}
});
}else{
console.log("ERROR: " + res.data)
return callback({success:false, data: res.data});
}
});
};

The issue was due to the file being served up before the stream was finished writing,
fix add .on('close') to the stream.
r.on('response', function (res) {
res.pipe(fs.createWriteStream('./' + d['id'] + d['file_name']).on('close', function() {
console.log('file done');
console.log("Download Done: " + './' + d['id'] + d['file_name']);
return callback({success:true, data:d['id'] + d['file_name']});
}));
console.log("You should not see this");
});

Related

Canva publish extension API : Endpoint never get call

I try to make a Canva App with a publish extension.
I just follow the Quick start (https://docs.developer.canva.com/apps/extensions/publish-extensions/quick-start) with Glitch and it work well on it.
But when I try to put in on my own public host name, with a other port (like http://mydomaine.com:3000) Canva NEVER call my endpoint. I just write a log file of every action on my app post and I never get a update on it, and when I try the app on Canva.com it just show me a error message.
//Copy from the Quick Start
app.post('/publish/resources/upload', async (request, response) => {
try{
writeLog("Uploading file");
await fs.ensureDir(path.join(__dirname, 'export'));
// Get the first asset from the "assets" array
const [asset] = request.body.assets;
// Download the asset
const image = await jimp.read(asset.url);
const filePath = path.join(__dirname, 'export', asset.name);
await image.writeAsync(filePath);
// Respond with the URL of the published design
response.send({
type: 'SUCCESS',
url: url.format({
protocol: request.protocol,
host: request.get('host'),
pathname: asset.name,
}),
});
} catch (err) {
writeLog("ERROR (app.post('/publish/resources/upload'): " + err);
}
});
//Just log on the log file
function writeLog(log){
// fs.appendFile(path.join(__dirname, '/log/' + `${month}/${date}/${year}` +'log.txt'), dateDisplay + "|" + log + "\n", (err) => {
// if (err) throw err;
// });
var today = new Date();
var time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
var date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
var dateTime = date + ' ' + time;
natifFS.appendFile('log.txt', dateTime + '| '+ log + "\n", (err) => {
if (err) throw err;
});
}
Last thing, when I try to call a post request on the same endpoint as Canva (/publish/resources/upload) with Postman, I get a update on my log.txt file
If anyone has idea, thank you.

I can't read file after checking if it exists in fs

So I'm trying to check if a file exists before I read that file but if I put the check before the read the read just becomes nothing even tho the data exists.
I've tried putting it below but then my file would just have an error from trying to read from a blank file.
bot.on('chat', function(username, message) {
var time = clock.zonedDateTime('SYSTEM').toString()
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username)) {
fs.mkdirSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username);
}
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/')) {
fs.mkdirSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/');
}
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/')) {
fs.mkdirSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/');
}
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/' + 'firstwords.txt')) {
fs.writeFile('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/' + 'lastwords.txt', 'sentat:' + time + ',<' + username + '>' + message, 'utf8', function(err) {
if (err) throw err;
});
}
if (message.startsWith('!lastwords ')) {
if (cooldown == 1) {
var lastwordsplit = message.toString().split(" ")
var lastwordperson = lastwordsplit[1]
if (fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)/players/' + lastwordperson + '/lirstwords/lastwords/lastwords.txt')) {
bot.chat(fs.readFileSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)/players/' + lastwordperson + '/lirstwords/lastwords/lastwords.txt', 'utf8'))
console.log('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)/players/' + lastwordperson + '/lirstwords/lastwords/lastwords.txt')
}
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)/players/' + lastwordperson + '/lirstwords/lastwords/lastwords.txt')) {
bot.chat(" does not have any documents that include the user " + lastwordperson)
if (!fs.existsSync('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/' + 'firstwords.txt')) {
fs.writeFile('C:/Users/La Fam/Desktop/kekbot_rewritten(tm)' + '/players/' + username + '/lirstwords/lastwords/' + 'lastwords.txt', 'sentat:' + time + ',<' + username + '>' + message, 'utf8', function(err) {
if (err) throw err;
});
}
}
}
}
});
You're not waiting for the fs.writeFile() to complete before you try to then read that data from the file in the lines of code below so probably all you will see is an empty file or maybe it won't even be created yet when you try to read it.
Then, you're mixing synchronous logic with asynchronous writes with no way to communicate success or errors. If this can use synchronous file I/O (which a server should never use except in startup code), then make it all synchronous file I/O.
If it all needs to be asynchronous, then it needs a complete rewrite and lots of this code can be refactored into a few shared functions.
It's not entirely clear to me everything that you're trying to accomplish in this function, but here's a simplified version of the logic that uses entirely asynchronous file I/O and attempts to DRY up the code a bit:
const mkdirp = require('mkdirp');
const fs = require('fs');
const path = require('path');
bot.on('chat', function(username, message) {
const time = clock.zonedDateTime('SYSTEM').toString();
const rootDir = 'C:/Users/La Fam/Desktop/kekbot_rewritten(tm)/players/';
const wordsDir = path.join(rootDir, username, 'lirstwords/lastwords');
const wordsFile = path.join(wordsDir, "lastwords.txt");
// do something useful when there's an error
// perhaps chat back that an unexpected error was encountered
function processError(err) {
console.log(err);
}
// make sure the desired directory exists (all the pieces of it)
mkdirp(wordsDir, function(err) {
if (err) return processError(err);
let data = 'sentat:' + time + ',<' + username + '>' + message;
fs.writeFile(wordsFile, data, {encoding: 'utf8', flag: 'wx'}, function(err) {
if (err) {
// we expect an error here if the file already exists, the wx flag will prevent overwriting existing file
// we do it this way to avoid race condition on checking if it exists and avoid extra system call
if (err.code !== 'EEXIST') {
processError(err);
}
if (message.startsWith('!lastwords ') && cooldown === 1) {
const lastwordperson = message.toString().split(" ")[1];
const lastWordFile = path.join(rootDir, lastwordperson, '/lirstwords/lastwords/lastwords.txt');
fs.readFile(lastWordFile, 'utf8', function(err, data) {
if (err) {
// treat file not found separately
if (err === 'ENOENT') {
bot.chat(" does not have any documents that include the user " + lastwordperson);
} else {
processError(err);
}
} else {
bot.chat(data);
}
});
}
});
});
});

Angular download file from byte array sent from node.js

I think I'm very close to what I want to do. I have the following api get method in node.js that is retrieving a file varbinary(MAX) from an SQL Server database. It was converted from a base64 encoded string before inserted so the Content Type information was stripped from the string.
node.js
router.get('/getFile', (req, res) => {
console.log("Calling getFile for file " + req.query.serialNumber + ".")
var serialNumber = req.query.serialNumber;
let request = new sql.Request(conn);
request.query('SELECT FileName + \'.\' + FileExtension AS \'File\', FileType, ContentType, SerialNumber, Chart ' +
'FROM dbo.ChangeFiles ' +
'WHERE SerialNumber = ' + serialNumber)
.then(function (recordset) {
log("Successfully retrieved file " + recordset[0].SerialNumber + " from database.");
log("Length of blob " + recordset[0].File + " is " + recordset[0].Chart.length)
res.status(200);
res.setHeader('Content-Type', recordset[0].ContentType);
res.setHeader('Content-Disposition', 'attachment;filename=' + recordset[0].File);
res.end(Buffer.from((recordset[0].Chart)));
}).catch(function (err) {
log(err);
res.status(500).send("Issue querying database!");
});
});
That works fine, but what to do in Angular to retrieve it and prompt for a download for the user has not been clear for me, nor has there been a lot as far as help/resources online. Here is what I have so far in my service class.
fileDownload.service.ts
downloadFile(serialNumber: string): Observable<any> {
return this.http.get(this.baseURL + '/getFile', { params: { serialNumber: serialNumber } })
.map(this.extractFile);
}
private extractFile(response: Response) {
const file = new Blob([response.blob]);
FileSaver.saveAs(file);
// const url = window.URL.createObjectURL(file);
// window.open(url);
return file;
}
As you can see I've tried a couple of approaches. The commented out portion of the extractFile method didn't work at all, and using the FileSaver.saveAs function produces a file download of an unknown type, so the headers sent from node.js didn't seem to affect the file itself.
Would someone be able to advise how to proceed in Angular with what is successfully being sent from node.js so that I can successfully download the file, regardless of type?
Thanks so much in advance.
I got it working afterall. I had to rework the api call so that it sent all of the file information separately so that the MIME type, and file name can be assigned to the file on the client side in the component class. For some reason when I tried to do so all in the api, it wouldn't work so that was my work around. So here is what works for me.
node.js api
router.get('/getFile', (req, res) => {
console.log("Calling getFile for file " + req.query.serialNumber + ".")
var serialNumber = req.query.serialNumber;
let request = new sql.Request(conn);
request.query('SELECT FileName + \'.\' + FileExtension AS \'File\', FileType, ContentType, SerialNumber, Chart ' +
'FROM dbo.ChangeFiles ' +
'WHERE SerialNumber = ' + serialNumber)
.then(function (recordset) {
log("Successfully retrieved file " + recordset[0].SerialNumber + " from database.");
log("Length of blob " + recordset[0].File + " is " + recordset[0].Chart.length)
res.send(recordset[0]);
}).catch(function (err) {
log(err);
res.status(500).send("Issue querying database!");
});
});
component class
downloadFile(serialNumber: string): void {
this.changeService.downloadFile(serialNumber).subscribe((res: any) => {
const ab = new ArrayBuffer(res.Chart.data.length);
const view = new Uint8Array(ab);
for (let i = 0; i < res.Chart.data.length; i++) {
view[i] = res.Chart.data[i];
}
const file = new Blob([ab], { type: res.ContentType });
FileSaver.saveAs(file, res.File);
console.log(res);
});
}
service class
downloadFile(serialNumber: string): Observable<any> {
return this.http.get(this.baseURL + '/getFile', { params: { serialNumber: serialNumber } })
.map(this.extractFile);
}
private extractFile(response: Response) {
// const file = new Blob([response.blob]);
// FileSaver.saveAs(file);
// const url = window.URL.createObjectURL(file);
// window.open(url);
const body = response.json();
return body || {};
}
Update your code to call subscribe instead of map

Upload a file from dart client to Node server

I'm stuck with this issue : I can't get my upload to work:
This is a node.js code taht works with a standard <form><input type="file" name="toUpload/>
router.post('/sp/file', function (req, res) {
// File to be uploaded
console.log("###" + req.files);
var fileToUpload = req.files.toUpload;
//console.log(fileToUpload);
var dir = __dirname + "/files";
/* var dir = __dirname + "/files/" + Date.now();
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
}*/
fileToUpload.mv( __dirname + "/files/" + fileToUpload.name, function (err) {
if (err) {
console.log("error: " + err);
} else
console.log("upload succeeded");
console.log(fileToUpload);
console.log(__dirname + "/files/" + fileToUpload.name);
uploadFilesStorj.uploadFile(__dirname + "/files/" + fileToUpload.name);
});
});
Now, when I try to upload a file through dart, I get stuck since the sent data is not in the same format:
class AppComponent {
void uploadFiles(dynamic files) {
if (files.length == 1) {
final file = files[0];
final reader = new FileReader();
//reader.onProgress.listen()
reader.onLoad.listen((e) {
sendData(reader.result);
});
reader.readAsDataUrl(file);
}
}
sendData(dynamic data) async {
final req = new HttpRequest();
req.onReadyStateChange.listen((Event e) {
if (req.readyState == HttpRequest.DONE &&
(req.status == 200 || req.status == 0)) {}
});
req.onProgress.listen((ProgressEvent prog) {
if (prog.lengthComputable)
print("advancement : " + (prog.total / prog.loaded).toString());
else
print("unable to compute advancement");
});
req.open("POST", "/sp/file");
req.send(data);
}
}
here's my dart angular front code
<input type="file" #upload (change)="uploadFiles(upload.files)"
(dragenter)="upload.style.setProperty('border', '3px solid green')"
(drop)="upload.style.setProperty('border', '2px dotted gray')" class="uploadDropZone" name="toUpload"/>
The data sent by this method is in the form :
Request payload:
data:text/html;base64,PGh0bWw+DQogICA8aGVhZD4NCiAgICAgIDx0aXRsZT5GaWxlIFVwbG9hZGluZyBGb3JtPC9
I passed a lot of time on it without success, can anybody help please
I finally found a way to post it as a multi-part form:
void uploadFiles() {
var formData = new FormData(querySelector("#fileForm"));
HttpRequest.request("/sp/file", method: "POST", sendData: formData).then((req) {
print("OK");
});
}
is used in conjunction with
<form id="fileForm">
<input type="file" #upload (change)="uploadFiles(upload.files)"
(dragenter)="upload.style.setProperty('border', '3px solid green')"
(drop)="upload.style.setProperty('border', '2px dotted gray')" class="uploadDropZone" name="toUpload"/>
</form>
You are writing the file content directly in the request body in the Dart variant. However the HTML form sends a request with multipart form encoding and embeds the file in there. That's also what your server expects.

Change the file name on the fly for downloading

I am saving user uploads by renaming it's original name to userID + '_' + new Date().getTime() + fileExt. I am storing the file properties in mongodb collections as :
{
name : String //generated name
, originalName : String //its original name
...
}
Now when the user requests for downloading of file I have to provide the file with its original name (which is persisted in db so no problem to grab it) to the user.
For the following request
GET /users/:userId/uploads/:fileId?type=download
I have this handler
//the mongoose query
UserUpload.findById(req.params.fileId).exec(function(err, doc){
var fileLocation = __dirname + '/public/uploads/users/' + req.params.userId + '/' + doc.name;
if(req.query.type && req.query.type == 'download') {
// I don't know what should I do now
// Downloader.download(fileLocation, newName);
}
});
I read wiki of node-static module but could not figure out how to do that job?
I found the answer here : Download a file from NodeJS Server using Express . Both using express and without using express.
It is too simple if you are using Express. Here is the documentation for res.download. I can't believe that the solution is just one line of code :
res.download('/path/to/file.ext', 'newname.ext');
Here what I use in one of my project, smil is a type of file I need to download, nevermind it.
In that project I have DOWNLOAD_DIR as global variable, which contain the full path to the download folder.
It may make a lot of people cringe (especcially the fileExistSync) but it s a start.
var DOWNLOAD_DIR = '/path/to/download',
url = require('url'),
http = require('http'),
/*
download
IN_:
file_url, url of the file to download
OUT:
file_name, full path to downloaded file, null if the file cound t be downloaded.
COM:
download a file, the file name will be the last part of the url (http://why.so/serious.txt => serious.txt).
WARNING: Do NOT follow redirections.
*/
function download(file_url, callback) {
var options = {host: url.parse(file_url).host, port: 80, path: url.parse(file_url).pathname},
file_name = url.parse(file_url).pathname.split('/').pop(),
//Creating the file
file = fs.createWriteStream(DOWNLOAD_DIR + file_name, {flags: 'w', encoding: 'binary'});
console.log('Downloading file from ' + file_url);
console.log('\tto ' + file_name);
http.get(options, function (res) {
res.pipe(file, {end: 'false'});
//When the file is complete
res.on('end', function () {
//Closing the file
file.end();
console.log('\t\tDownloaded '+ file_name);
callback(DOWNLOAD_DIR + file_name);
});
});
process.on('uncaughtException', function(err) {
console.log('Can t download ' + file_url + '\t(' + err + ')', false);
callback(null);
});
}
/*
download_smil
IN_:
file_url, url of the file to download
OUT:
file_name, full path to downloaded file
COM:
Follow http redirection and then call download function to download it.
You can modify the cad function to use custom names.
*/
function download_smil(file_url, callback) {
function cad(link, callback) {
//Does the file already exist?
var file = url.parse(link).pathname.substr(url.parse(link).pathname.lastIndexOf('/') + 1),
pkmn;
pkmn = fs.existsSync(DOWNLOAD_DIR + '/' + file);
if (pkmn) {
//YES: callback
console.log('File ' + file + ' already exist, skipping');
callback(DOWNLOAD_DIR + file, true);
} else {
//NO: Download it
console.log('Will download ' + link);
download(link, callback);
}
}
//GET the page
http.get(file_url, function (res) {
var link;
//console.log('res.statusCode = ' + res.statusCode + ' res.headers.location = ' + res.headers.location);
//Check if it is a redirect
if (res.statusCode > 300 && res.statusCode < 400 && res.headers.location) {
//console.log('redirect');
//Check if the hostname is in the location
if (url.parse(res.headers.location).hostname) {
//console.log('The link to the smil is res.headers.location');
link = res.headers.location;
cad(link, callback);
} else {
//console.log('The link to the smil is url_parse(file_url).hostname + res.headers.location = ' + url_parse(file_url).hostname + res.headers.location);
link = url_parse(file_url).hostname + res.headers.location;
cad(link, callback);
}
} else {
//console.log('The url is good : ' + file_url);
cad(file_url, callback);
}
});
}

Resources