Im trying to download a file from S3 which doesn't exist on S3. I expect a error in this scenario and im also getting that error from aws-sdk i.e.
**/vagrant/node_modules/aws-sdk/lib/request.js:31
throw err;
^
NoSuchKey: The specified key does not exist.**
But the issue is, im not able to catch this error. If you check the code below, my listener request.on gets called and when i call reject in that the promise doesn't return from method downloadFontInfoFileFromS3 with reject.
Is there any way i can catch the error and gracefully reject the promise from downloadFontInfoFileFromS3 function?
downloadFontInfoFileFromS3(fileKey) {
return new Promise((resolve, reject) => {
const sThree = new awsSDK.S3();
const options = {
Bucket: awsConf.bucket,
Key: fileKey,
};
const downloadFilePath = SOME_PATH
const file = fs.createWriteStream(downloadFilePath);
const request = sThree.getObject(options);
const download = request
.createReadStream().pipe(file);
// called when error in aws-sdk
request.on('error', (error) => {
logger.error('Failed to download file from S3:', error.message);
reject(error);
});
download.on('error', (error) => {
reject(error);
});
download.on('finish', () => {
resolve(downloadFilePath);
});
// the synchronous code that we want to catch thrown errors on
});
}
I ran into this issue as well. Turns out I wasn't setting the error handler in the correct place.
Currently, you have:
// INCORRECT. Won't work
const download = request.createReadStream().pipe(file);
download.on('error', (error) => {
reject(error);
});
But you actually need to set the error handler on the stream, before you begin piping. Like this:
// Correct!
const download = request.createReadStream(); // notice the `.pipe()` has been removed
download.on('error', (error) => {
reject(error);
});
// now you can begin piping to the file
requestStream.pipe(file);
If you try setting the event handler after the pipe has begun, the error will "bypass" any try/catch or Promise .catch handlers you specify. Quite frustrating!
Check out this GitHub issue for another example.
const data = s3.getObject({ Bucket: process.env.AWS_BUCKET_NAME, Key: url }).createReadStream();
data.on("error", (err) => {
if (err)
return next(customError("Invalid Url", 400));
});
data.pipe(res);
Right Because Error is occourd when data in pipe.
Related
I am looking at a few, currently widely used request libraries and how I could use them to automate file downloads + make them reliable enough.
I stumbled over download (npm), but since its based on got (npm) I thought I would try got first directly.
Problem
One problem I could encounter while downloading a file, is that the source file (on the server) could be overwritten during download. When I try reproduce this behaviour with got, got just stops the download process without rising any errors.
What I have so far
The only solution I could come up with, was to use got.stream - piping the request into a FileWriter, and compare total with transferred after the request has ended.
const app = require('express')();
const fs = require('fs');
const stream = require('stream');
const { promisify } = require('util');
const got = require('got');
const pipeline = promisify(stream.pipeline);
app.use('/files', require('express').static('files'));
app.listen(8080);
(async () => {
try {
let progress = null;
// Setup Got Request + EventHandlers
const request = got.stream('http://localhost:8080/files/1gb.test')
.on('downloadProgress', (p) => { progress = p; })
.on('end', () => {
console.log("GOT END");
console.log(progress && progress.transferred === progress.total ? "COMPLETE" : "NOTCOMPLETE");
})
// this does not get fired when source file is overwritten
.on('error', (e) => {
console.log("GOT ERROR");
console.log(e.message);
});
// WriteStream + EventHandlers
const writer = fs.createWriteStream('./downloaded/1gb_downloaded.test')
.on('finish', () => {
console.log("WRITER FINISH");
})
.on('error', (error) => {
console.log("WRITER ERROR", error.message);
})
.on('end', () => {
console.log("WRITER END");
})
.on('close', () => {
console.log("WRITER CLOSE");
});
await pipeline(request, writer);
} catch (e) {
console.error(e.name, e.message);
}
})();
Where do the files come from
In the real world the files i am trying to download are coming from a server which I do not have access to, I don't own it. I don't have any information how this server is setup. However I added a simple local express server to the example code above to try things out.
const app = require('express')();
app.use('/files', require('express').static('files'));
app.listen(8080);
Question
Is this solution reliable enough to detect a "none-finished" download ( so for the case the source file gets overwritten during download ) ? Or are there any othere events I could listen to which I missed ?
The got Request stream emits a error event whenever something goes wrong.
const request = got('http://localhost:8080/files/1gb.test')
.on('downloadProgress', (p) => { progress = p; })
.on('end', (e) => {
console.log("GOT END");
})
.on('error', (err) => {
// Handle error here
});
Various properties in the error object are available here
progress.total will not be available unless the server explicity sets the Content-Length (Most servers do, but you might want to have a look out for that)
It seems there is no inbuilt way to safely check if a download has been completed 100% using got. I came to the conclusion that my best option for now would be to use the NodeJS Module http, which includes, on the ClientRequest Object, an aborted property. When the ReadStream emits an end event I can check whether aborted is true or false.
I tested this method in the case when the source file gets overwritten during download, it works !
const http = require('http');
const app = require('express')();
const fs = require('fs');
app.use('/files', require('express').static('files'));
app.listen(8080);
http.get('http://localhost:8080/files/1gb.test', function (response) {
// WriteStream + EventHandlers
const writer = fs.createWriteStream('./downloaded/1gb_downloaded.test')
.on('finish', () => {
console.log("WRITER FINISH");
})
.on('error', (error) => {
console.log("WRITER ERROR", error.message);
})
.on('end', () => {
console.log("WRITER END");
})
.on('close', () => {
console.log("WRITER CLOSE");
});
// ReadStream + EventHandlers
response
.on('error', (e) => {
console.log("READER ERROR", e.message)
})
.on('end', () => {
console.log("READER END")
console.log(response.aborted ? "DOWNLOAD NOT COMPLETE" : "DOWNLOAD COMPLETE")
})
.on('close', () => {
console.log("READER CLOSE")
})
response.pipe(writer);
});
On the upside, this gives me -1 on dependencies :) , since I don't need got.
On the downside, this just assures me, that a running download was not aborted due to the source file being overwritten. When using http module I need to include more error handling when for example the file was not found to begin with, which had been more convenient using a request library like axios or got.
UPDATE
Realizing that the ReadableStream from http has something like the aborted property made me wonder why none of the request wrapper libraries like got does offer something similar. So I tried axios again, with :
axios({
method: 'get',
url: 'http://localhost:8080/files/1gb.test',
responseType: 'stream'
}).then( function ( response ) {
});
Here the ReadableStream comes in response.data and it has the same aborted property ! 🎉 .
I have 2 calls to the YouTube v3 API in my NodeJS code: channels and PlaylistItems. They both return JSON and the response to the first call is parsed just fine, but parsing the response to the second call causes a syntax error. I am uncertain whether it's an error on my side or in the PlaylistItems API endpoint.
Here is my code (taken out irrelevant parts):
// At start of the bot, fetches the latest video which is compared to if an announcement needs to be sent
function setLatestVideo () {
fetchData().then((videoInfo) => {
if (videoInfo.error) return;
latestVideo = videoInfo.items[0].snippet.resourceId.videoId;
});
}
// Fetches data required to check if there is a new video release
async function fetchData () {
let path = `channels?part=contentDetails&id=${config.youtube.channel}&key=${config.youtube.APIkey}`;
const channelContent = await callAPI(path);
path = `playlistItems?part=snippet&maxResults=1&playlistId=${channelContent.items[0].contentDetails.relatedPlaylists.uploads}&key=${config.youtube.APIkey}`;
const videoInfo = await callAPI(path);
return videoInfo;
}
// Template HTTPS get function that interacts with the YouTube API, wrapped in a Promise
function callAPI (path) {
return new Promise((resolve) => {
const options = {
host: 'www.googleapis.com',
path: `/youtube/v3/${path}`
};
https.get(options, (res) => {
if (res.statusCode !== 200) return;
const rawData = [];
res.on('data', (chunk) => rawData.push(chunk));
res.on('end', () => {
try {
resolve(JSON.parse(rawData));
} catch (error) { console.error(`An error occurred parsing the YouTube API response to JSON, ${error}`); }
});
}).on('error', (error) => console.error(`Error occurred while polling YouTube API, ${error}`));
});
}
Examples of errors I'm getting: Unexpected token , in JSON and Unexpected number in JSON
Till ~2 weeks ago this code used to work just fine without throwing any errors, I have no clue what has changed and can't seem to figure it out. What could possibly be causing this?
10 minutes later I figure out the fix! The variable rawData holds a Buffer and looking more into that, I figured that I should probably use Buffer.concat() on rawData before calling JSON.parse() on it. Turns out, that's exactly what was needed.
I'm still unsure how this was causing problems only 6 months after writing this code, but that tends to happen.
Changed code:
res.on('end', () => {
try {
resolve(JSON.parse(Buffer.concat(rawData)));
} catch (error) { console.error(`An error occurred parsing the YouTube API response to JSON, ${error}`); }
});
I am currently writing a route which allows me to recieve information from a stored procudre I have in a database. I have written a request in AngularJS and a route in NodeJS but I am just recieving a pending request in the chrome Network developer window. I can see that the console.log in the NodeJs app has the data I require so it has retrieved it but there is nothing coming back in any of the console logs in the the AngularJS app.
Here is the code for the both the angularJS app and the Node App:
AnglaurJS:
checkForReplenishmentEmptyTrolley = async () => {
LIBRIS.extensions.openLoadingModal();
console.log('in checkForReplenishmentEmptyTrolley');
try {
const varPromise = await $http.get(`${LIBRIS.config.stockService}stockMovement/checkForUnattachedTrolley`)
.then((response) => {
console.log(response);
// Request completed successfully
}, (error) => {
// Request error
console.log(error);
});
console.log(varPromise.data);
// 1. check that there are no ghost replenish - lines 1-15
console.log('in try/catch');
console.log('promise', varPromise);
} catch (error) {
console.log(error);
}
},
NodeJS code:
app.get(`${ROUTE}/attachTrolley`, async function(req, res){
const newRequest = await DB.newRequest();
console.log('we have made it to the route');
try {
console.log('we have made it to the Try/Catch route');
newRequest.input();
const record = await newRequest.execute('dbo.usp_STK_CheckForUnattachedTrolley');
res.json(record)
console.log(record, 'record');
} catch (err){
handleError(res, err);
console.log(err);
}
});
The problem is that you are doing a .then on a awaited promises and not returning anything from that. You have two choice here
Either return response from then so when you try to access the value here console.log(varPromise.data); it works.
Or remove the .then alltogather as it is not required because you are awaiting it any ways.
Basically just do this
checkForReplenishmentEmptyTrolley = async () => {
LIBRIS.extensions.openLoadingModal();
console.log("in checkForReplenishmentEmptyTrolley");
try {
const varPromise = await $http.get(`${LIBRIS.config.stockService}stockMovement/checkForUnattachedTrolley`);
console.log(varPromise.data);
// 1. check that there are no ghost replenish - lines 1-15
console.log("in try/catch");
console.log("promise", varPromise);
} catch (error) {
console.log(error);
}
};
Hope this fixes your issue.
Solved it! I had no return statement in my route!
I am trying to call a rest API from Firebase function which servers as a fulfillment for Actions on Google.
I tried the following approach:
const { dialogflow } = require('actions-on-google');
const functions = require('firebase-functions');
const http = require('https');
const host = 'wwws.example.com';
const app = dialogflow({debug: true});
app.intent('my_intent_1', (conv, {param1}) => {
// Call the rate API
callApi(param1).then((output) => {
console.log(output);
conv.close(`I found ${output.length} items!`);
}).catch(() => {
conv.close('Error occurred while trying to get vehicles. Please try again later.');
});
});
function callApi (param1) {
return new Promise((resolve, reject) => {
// Create the path for the HTTP request to get the vehicle
let path = '/api/' + encodeURIComponent(param1);
console.log('API Request: ' + host + path);
// Make the HTTP request to get the vehicle
http.get({host: host, path: path}, (res) => {
let body = ''; // var to store the response chunks
res.on('data', (d) => { body += d; }); // store each response chunk
res.on('end', () => {
// After all the data has been received parse the JSON for desired data
let response = JSON.parse(body);
let output = {};
//copy required response attributes to output here
console.log(response.length.toString());
resolve(output);
});
res.on('error', (error) => {
console.log(`Error calling the API: ${error}`)
reject();
});
}); //http.get
}); //promise
}
exports.myFunction = functions.https.onRequest(app);
This is almost working. API is called and I get the data back. The problem is that without async/await, the function does not wait for the "callApi" to complete, and I get an error from Actions on Google that there was no response. After the error, I can see the console.log outputs in the Firebase log, so everything is working, it is just out of sync.
I tried using async/await but got an error which I think is because Firebase uses old version of node.js which does not support async.
How can I get around this?
Your function callApi returns a promise, but you don't return a promise in your intent handler. You should make sure you add the return so that the handler knows to wait for the response.
app.intent('my_intent_1', (conv, {param1}) => {
// Call the rate API
return callApi(param1).then((output) => {
console.log(output);
conv.close(`I found ${output.length} items!`);
}).catch(() => {
conv.close('Error occurred while trying to get vehicles. Please try again later.');
});
});
I have millions of rows in my Cassandra db that I want to stream to the client in a zip file (don't want a potentially huge zip file in memory). I am using the stream() function from the Cassandra-Node driver, piping to a Transformer which extracts the one field from each row that I care about and appends a newline, and pipes to archive which pipes to the Express Response object. This seems to work fine but I can't figure out how to properly handle errors during streaming. I have to set the appropriate headers/status before streaming for the client, but if there is an error during the streaming, on the dbStream for example, I want to clean up all of the pipes and reset the response status to be something like 404. But If I try to reset the status after the headers are set and the streaming starts, I get Can't set headers after they are sent. I've looked all over and can't find how to properly handle errors in Node when piping/streaming to the Response object. How can the client tell if valid data was actually streamed if I can't send a proper response code on error? Can anyone help?
function streamNamesToWriteStream(query, res, options) {
return new Promise((resolve, reject) => {
let success = true;
const dbStream = db.client.stream(query);
const rowTransformer = new Transform({
objectMode: true,
transform(row, encoding, callback) {
try {
const vote = row.name + '\n';
callback(null, vote);
} catch (err) {
callback(null, err.message + '\n');
}
}
});
// Handle res events
res.on('error', (err) => {
logger.error(`res ${res} error`);
return reject(err);
});
dbStream.on('error', function(err) {
res.status(404).send() // Can't set headers after they are sent.
logger.debug(`dbStream error: ${err}`);
success = false;
//res.end();
//return reject(err);
});
res.writeHead(200, {
'Content-Type': 'application/zip',
'Content-disposition': 'attachment; filename=myFile.zip'
});
const archive = archiver.create('zip');
archive.on('error', function(err) { throw err; });
archive.on('end', function(err) {
logger.debug(`Archive done`);
//res.status(404).end()
});
archive.pipe(res, {
//end:false
});
archive.append(dbStream.pipe(rowTransformer), { name: 'file1.txt' });
archive.append(dbStream.pipe(rowTransformer), { name: 'file1.txt' });
archive.finalize();
});
}
Obviously it's too late to change the headers, so there's going to have to be application logic to detect a problem. Here's some ideas I have:
Write an unambiguous sentinel of some kind at the end of the stream when an error occurs. The consumer of the zip file will then need to look for that value to check for a problem.
Perhaps more simply, have the consumer execute a verification on the integrity of the zip archive. Presumably if the stream fails the zip will be corrupted.