Checking file after creating. FS + timeout? - node.js

After ajax request, I send options for Image Magick (child process) and create file.
If I send response to client immediately - I have error - img non exist. My solution, set Timeout and send response (hardcode).
I can set timeout and check file with fs module, but it is right way?
Or may be I can emit new event after file exist, but can you type sample for me?
Or this task have another solution?
fs.stat(currentImage, function(err, background) {
if (background && background.isFile) {
async.series([
function(callback) {
var magickOpts = memOptions;
var im = spawn('convert', magickOpts);
callback(null, 'done');
}
],
function(err, result) {
setTimeout(function() {
res.send({
status: '200',
src: '/images/mems/'+tempName+'.jpg',
tempName: tempName
});
}, 500);
});
} else {
res.send({
status: '404',
text: 'background image not found'
});
}
});

You should wait for the child process to exit (by listening for the close event) before you send the response to the client.
Also, async.series doesn't seem to be really used in this case, so I left it out:
fs.stat(currentImage, function (err, background) {
if (background && background.isFile) {
var magickOpts = memOptions;
var im = spawn('convert', magickOpts);
im.on('close', function() {
res.send({
status: '200',
src: '/images/mems/' + tempName + '.jpg',
tempName: tempName
});
});
} else {
res.send({
status: '404',
text: 'background image not found'
});
}
});
By the way, there's also the im package which wraps ImageMagick for use in Node.

Related

MEAN client-server connection timeout for downloading PDF

I am using MEAN Stack for our project. To generate pdf using pdfmake,
scenario goes like, At first have images from local server so every thing works fine. For scalibility have to move images to AWS and data from other server.
Now processs is like, first download images from aws then make buffer, convert it to base64 and give it to pdfmake. Now issue arises for client-server connection.
client makes http request for pdf, server process it but it takes too much time to download image from aws, so client connection breaks mean while
server processing the request and send response back but no one is there to listen.
/** client **/
function logicTriplogs() {
$rootScope.isLoading = true;
AssignsService.logicTriplogs({
driverId: vm.driver
}, {
_ids: vm.selectedTrips,
scheduleTime: vm.scheduleTime,
companyImage: vm.companyImage
},
function(response) {
console.log(response);
$rootScope.isLoading = false;
var Name = response.pdfName;
var data = response.content;
SaveFile.downloadURI(Name, data);
console.log('PDF Name:', Name);
},
function(err) {
console.log(err);
vm.error = err.data.message;
$rootScope.isLoading = false;
});
}
/** Server **/
getAssignedTripQuery(query, type)
.exec(function(err, assigns) {
if (err) {
console.log('Manifest');
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
}
if (assigns.length) {
logicMan(req.body, user, driver, assigns, function(docDefinition) {
var pdfName = `${moment_tz.tz(startDay, req.user.timeZone).format('MM-DD-YYYY')}.pdf`;
config.pdfBuffer(docDefinition, function(err, pdfDoc) {
console.log('EROROR PDF: ', err);
if (err) {
console.log('pdfmake function call error');
return res.status(400).send({
message: 'Error while generate pdf'
});
}
console.log('PDF Name:', pdfName);
return res.json({
message: 'Generated Susscessfuly',
pdfName: pdfName,
content: pdfDoc
});
});
});
} else {
return res.status(400).send({
message: 'Some thing went wrong. Please try later.'
});
}
});

Node js to wait till the response comes

copy_deliverable_script_tomaster(args.Software_name.value,function(state){
res.end("added")
}
)
function copy_deliverable_script_tomaster(software_name,callback){
client.scp('./Temporary_software_files/folder/', {
host: 'ip',
username: 'centos',
privateKey: String(require("fs").readFileSync('./foreman_keypairs/coe-
central.pem')),
path: '/home/centos/Software_with_files/'+software_name
}, function(err,response) {
if(err){
console.log(err)
}
else{
console.log("after copy in master")
return callback(response);
}
})
}
I have used the above code, to copy large files to the remote machine.
Copying file continues in the remote machine, but the response("no content")comes before copy completes.
console.log("after copy in master"), will be printed only after the copy is completed.
Unable to get the response.
Instead of res.end use res.send("added"); or res.write('Added'); res.end();,
Because you have ended response without writing anything.
copy_deliverable_script_tomaster(args.Software_name.value, function (state) {
res.send("added")
})
You are not handling the error case. If you are not doing anything inside callback of client.scp then just pass the callback.
copy_deliverable_script_tomaster(args.Software_name.value, function (err,state) {
if(err) return res.status(400).send(err);
return res.send("some response")
})
function copy_deliverable_script_tomaster(software_name, callback) {
client.scp('./Temporary_software_files/folder/', {
host: 'ip',
username: 'centos',
privateKey: String(require("fs").readFileSync('./foreman_keypairs/coe-central.pem')),
path: '/home/centos/Software_with_files/' + software_name
}, callback)
}

Nodejs is not receiving any code from Flask app.

I am really new in node js and a little bit more experienced in flaks. I am trying to connect a nodejs backend with a flask api. Basically I am sending a file that was uploaded in the nodejs app for processing (converting to another format) to my flask app.
For sending the data I am using request. In this way:
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path, this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data,
};
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
},
function(err, httpResponse, body) { //Upload Callback
if (err) {
return console.error('upload failed:', err);
}
res.redirect('/console');
});
});
Then I am receiving the file for processing in the flask app, like:
#app.route('/api/resource/converter', methods = ['POST','GET'])
def converter_csv():
if request.method == 'POST':
f = request.form['file']
if not f:
abort(400)
print('-----Converting-------')
file = open("temp/converting.txt","w")
file.write(f)
#....conversion process...
# Finish the process
return Response(converted_file,status=200)
In my console for the localhost of the flask app, I am getting:
127.0.0.1 - - [09/Aug/2017 15:47:59] "POST /api/resource/converter HTTP/1.1" 200 -
However my nodejs app did not receive any response. It just got frozen.
I appreciate any orientation anyone can give me. Thanks.
I think flow.exec is not in proper order
router.post('/converttest', uploader.single('file'), function(req, res) {
var filePath = req.file.path;
fs.readFile(filePath, 'utf8', function(err, data) { //change format reading as required
try {
formData = {file:data}
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
} catch(err) {
return console.error('upload failed:', err);
res.redirect('/console')
}
fs.unlink(filePath);}); });
I ended up using requestify. Seems like they make it a little bit easier for beginners like me:
var requestify = require('requestify');
app.get('/convertupload', function(req,res){
res.render('pages/convertupload');
});
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path,this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data
};
requestify.post('http://127.0.0.1:5000/api/resource/converter', {
form: formData
})
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
console.log(response)
response.getBody();
});
res.redirect('/login');
});
});

Node.js Streaming/Piping Error Handling (Change Response Status on Error)

I have millions of rows in my Cassandra db that I want to stream to the client in a zip file (don't want a potentially huge zip file in memory). I am using the stream() function from the Cassandra-Node driver, piping to a Transformer which extracts the one field from each row that I care about and appends a newline, and pipes to archive which pipes to the Express Response object. This seems to work fine but I can't figure out how to properly handle errors during streaming. I have to set the appropriate headers/status before streaming for the client, but if there is an error during the streaming, on the dbStream for example, I want to clean up all of the pipes and reset the response status to be something like 404. But If I try to reset the status after the headers are set and the streaming starts, I get Can't set headers after they are sent. I've looked all over and can't find how to properly handle errors in Node when piping/streaming to the Response object. How can the client tell if valid data was actually streamed if I can't send a proper response code on error? Can anyone help?
function streamNamesToWriteStream(query, res, options) {
return new Promise((resolve, reject) => {
let success = true;
const dbStream = db.client.stream(query);
const rowTransformer = new Transform({
objectMode: true,
transform(row, encoding, callback) {
try {
const vote = row.name + '\n';
callback(null, vote);
} catch (err) {
callback(null, err.message + '\n');
}
}
});
// Handle res events
res.on('error', (err) => {
logger.error(`res ${res} error`);
return reject(err);
});
dbStream.on('error', function(err) {
res.status(404).send() // Can't set headers after they are sent.
logger.debug(`dbStream error: ${err}`);
success = false;
//res.end();
//return reject(err);
});
res.writeHead(200, {
'Content-Type': 'application/zip',
'Content-disposition': 'attachment; filename=myFile.zip'
});
const archive = archiver.create('zip');
archive.on('error', function(err) { throw err; });
archive.on('end', function(err) {
logger.debug(`Archive done`);
//res.status(404).end()
});
archive.pipe(res, {
//end:false
});
archive.append(dbStream.pipe(rowTransformer), { name: 'file1.txt' });
archive.append(dbStream.pipe(rowTransformer), { name: 'file1.txt' });
archive.finalize();
});
}
Obviously it's too late to change the headers, so there's going to have to be application logic to detect a problem. Here's some ideas I have:
Write an unambiguous sentinel of some kind at the end of the stream when an error occurs. The consumer of the zip file will then need to look for that value to check for a problem.
Perhaps more simply, have the consumer execute a verification on the integrity of the zip archive. Presumably if the stream fails the zip will be corrupted.

box-node-sdk downloading a shared public file

I'm making a NodeJS app which should download a file given the public share URL that doesn't request login auth.
I have created the app inside the developer section and generated the proper tokens.
On the git documentation I've found this method that should be used to download a file. The first parameter is the fileId but I don't find written anywhere what this id is and where to find it.
https://rawgit.com/box/box-node-sdk/master/docs/jsdoc/Files.html#getReadStream
I've tried this code
var fs = require('fs'),
BoxSDK = require('box-node-sdk'),
sdk = new BoxSDK({
clientID: '...',
clientSecret: '...'
}),
client = sdk.getAnonymousClient(),
file_id = process.argv[2].replace(/\S*\/(\S+)$/,"$1");
client.files.getReadStream(file_id, null, function(error, stream)
{
if (error) {
throw error;
}
// write the file to disk
var output = fs.createWriteStream('./output/'+file_id+".zip"); //I know for sure there will only be zip files
stream.pipe(output);
});
But running it with this command
nodejs index.js https://adrime.box.com/s/s5w7lzm4xfifmxrod9r7eeniv9nhtenk
I get this error:
Error: Unexpected API Response [404 Not Found] (not_found: "Not Found")
at Object.module.exports.buildResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:57:23)
at Object.module.exports.buildUnexpectedResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:94:15)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/managers/files.js:148:20
at BoxClient._handleResponse (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:291:2)
at handleResponseCallback (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:233:9)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/api-request.js:285:3
at nextTickCallbackWith0Args (node.js:436:9)
at process._tickCallback (node.js:365:13)
Can anyone help me in programmatically downloading a public shared file from box.com?
Thank you in advance!
At the moment I've found this solution.
To me it works pretty well. I hope it will be useful to others as well.
var fs = require("fs"),
request = require("request");
function downloadFile(source, target, callback)
{
var wr = fs.createWriteStream(target);
wr.on("error", function (err)
{
callback(err);
});
wr.on("close", function (ex)
{
callback();
});
request
.get(source)
.on("error",function(err)
{
callback(err);
})
.on('response', function(response)
{
if((""+response.statusCode).charAt(0) != "2")
callback({
msg: "File not found: "+response.request.href,
path: response.request.href,
toString: function(){
return this.msg;
}
});
})
.on("end", function (ex)
{
console.log("request ended");
})
.pipe(wr);
}
function onDownloadComplete(err)
{
if(err)
throw err;
console.log("DOWNLOAD COMPLETE");
}
function init()
{
var static_url = process.argv[2].replace(/\/s\//,"/shared/static/") + ".zip";
console.log(static_url);
downloadFile(static_url, "./output/template.zip", onDownloadComplete);
}
init();

Resources