I'm using the library JSZip to unzip a file and I am able to get each of the file name. However, what I want to do is to take the content of the file to do some processing. I know that you are able to do this by doing zip.file(name).async(type).then() according to the API.
For some reason though, when I do that in my lambda function, it doesn't hit that function at all. I tried to do return that line of code, but I just got Promise <pending>... I tried to wrap it with a try/catch and that didn't do anything. I did an async/await but that didn't work either. A callback like:
zip.file(name).async('blob').then(function(content) {
//Do processing in here
});
doesnt seem to work either. What do I need to do to get the content of the specific file? Nothing I have done is work and I think it has to do with the promise pending. I'm stuck at this point and have no idea what to do.. Any help is greatly appreciated. For reference here is my code for how I'm doing it:
let zip = new JSZip();
zip.loadAsync(file).then(function(contents) {
Object.keys(contents.files).forEach(function(name) {
zip.file(name).async('blob').then(function(content) {
// Processing here
});
});
});
EDIT
Full code:
index.js
const manager = require("./manager.js");
exports.handler = async (event, context, callback) => {
return manager.manager(event, callback);
};
manager.js
exports.manager = async function (event, callback) {
const path = '/temp/' + fileName;
const file = fs.readFileSync(path);
let zip = new JSZip();
zip.loadAsync(file).then(function(contents) {
Object.keys(contents.files).forEach(function(name) {
zip.file(name).async('blob').then(function(content) {
// Processing here
});
});
});
}
Change to have a return for the function:
return zip.loadAsync(file).then(...)
Related
I'm doing a html to pdf function using the phantom-html-to-pdf on AWS Lambda with NodeJS. But I encountered a problem that did not go inside the function. The output only shows Start, Finish and Done is not shown which means it's not going in the function. What is the issue here?
var fs = require('fs')
var conversion = require("phantom-html-to-pdf")();
exports.handler = async (event) => {
console.log("Start")
conversion({ html: "<h1>Hello World</h1>" },
async(err, pdf) => {
var output = fs.createWriteStream('output.pdf')
console.log(pdf.logs);
console.log(pdf.numberOfPages);
// since pdf.stream is a node.js stream you can use it
// to save the pdf to a file (like in this example) or to
// respond an http request.
pdf.stream.pipe(output);
console.log("Done")
});
console.log("Finish")
};
The problem is you have marked your lambda function as async which means your function should return a promise. In your case you are not returning a promise. So you have two choices here
Convert the conversion function from callback to promise based.
OR instead of marking the function async, add callback and execute that. something like this
const fs = require("fs");
const conversion = require("phantom-html-to-pdf")();
exports.handler = (event, context, callback) => {
console.log("Start");
conversion({"html": "<h1>Hello World</h1>"},
// eslint-disable-next-line handle-callback-err
async (err, pdf) => {
const output = fs.createWriteStream("output.pdf");
console.log(pdf.logs);
console.log(pdf.numberOfPages);
// since pdf.stream is a node.js stream you can use it
// to save the pdf to a file (like in this example) or to
// respond an http request.
pdf.stream.pipe(output);
console.log("Done");
callback(null, "done");
});
};
I have the function, enter, that contains two fs.readFile functions inside and on the innermost readfile I'm checking to see whether a txt file contains a certain keyword, and if it does it should stop the entire enter function.
Her is what the enter function looks like:
async function enter(email, firstName, lastName){
fs.readFile(fileName, function(err, data){
parsedData = JSON.parse(data);
email = parsedData.email;
fs.readFile('./anotherfile.txt', function (err, data) {
if (err) throw err;
if(data.includes(email)){
console.log('Stopping function');
return;
}
});
});
console.log('Continuing with function');
}
The problem is that when anotherfile.txt contains the keyword, it does not stop the entire function, it continues and logs, "Continuing with function" as seen in the code above.
Any help would be appreciated!
fs promises are available Node v11.0.0
or You can can convert like this const readFile = util.promisify(fs.readFile);
const fsp = require('fs').promises;
async function enter(email, firstName, lastName) {
try {
let data = await fsp.readFile(fileName)
let parsedData = JSON.parse(data);
let email = parsedData.email;
data = await fsp.readFile('./anotherfile.txt')
if (data.includes(email)) {
console.log('Stopping function');
return;
}
console.log('Continuing with function');
} catch (err) {
throw err
}
}
This is because of two things.
You are using asynchronous File read, that is, the flow of code won't stop when this readFile is called. Instead, the program would keep executing in a normal fashion. And when the file read operation is completed, the callback function you supplied will be called with the corresponding error or data.
The return statement is inside of the callback function, hence it will only affect that function.
You need to use await when you are dealing with asynchronous functions. Look that up here
I'm trying to download a bunch of files. Let's say 1.jpg, 2.jpg, 3.jpg and so on. If 1.jpg exist, then I want to try and download 2.jpg. And if that exist I will try the next, and so on.
But the current "getFile" returns a promise, so I can't loop through it. I thought I had solved it by adding await in front of the http.get method. But it looks like it doesn't wait for the callback method to finish. Is there a more elegant way to solve this than to wrap the whole thing in a new async method?
// this returns a promise
var result = getFile(url, fileToDownload);
const getFile = async (url, saveName) => {
try {
const file = fs.createWriteStream(saveName);
const request = await http.get(url, function(response) {
const { statusCode } = response;
if (statusCode === 200) {
response.pipe(file);
return true;
}
else
return false;
});
} catch (e) {
console.log(e);
return false;
}
}
I don't think your getFile method is returning promise and also there is no point of awaiting a callback. You should split functionality in to two parts
- get file - which gets the file
- saving file which saves the file if get file returns something.
try the code like this
const getFile = url => {
return new Promise((resolve, reject) => {
http.get(url, response => {
const {statusCode} = response;
if (statusCode === 200) {
resolve(response);
}
reject(null);
});
});
};
async function save(saveName) {
const result = await getFile(url);
if (result) {
const file = fs.createWriteStream(saveName);
response.pipe(file);
}
}
What you are trying to do is getting / requesting images in some sync fashion.
Possible solutions :
You know the exact number of images you want to get, then go ahead with "request" or "http" module and use promoise chain.
You do not how the exact number of images, but will stop at image no. N-1 if N not found. then go ahed with sync-request module.
your getFile does return a promise, but only because it has async keyword before it, and it's not a kind of promise you want. http.get uses old callback style handling, luckily it's easy to convert it to Promise to suit your needs
const tryToGetFile = (url, saveName) => {
return new Promise((resolve) => {
http.get(url, response => {
if (response.statusCode === 200) {
const stream = fs.createWriteStream(saveName)
response.pipe(stream)
resolve(true);
} else {
// usually it is better to reject promise and propagate errors further
// but the function is called tryToGetFile as it expects that some file will not be available
// and this is not an error. Simply resolve to false
resolve(false);
}
})
})
}
const fileUrls = [
'somesite.file1.jpg',
'somesite.file2.jpg',
'somesite.file3.jpg',
'somesite.file4.jpg',
]
const downloadInSequence = async () => {
// using for..of instead of forEach to be able to pause
// downloadInSequence function execution while getting file
// can also use classic for
for (const fileUrl of fileUrls) {
const success = await tryToGetFile('http://' + fileUrl, fileUrl)
if (!success) {
// file with this name wasn't found
return;
}
}
}
This is a basic setup to show how to wrap http.get in a Promise and run it in sequence. Add error handling wherever you want. Also it's worth noting that it will proceed to the next file as soon as it has received a 200 status code and started downloading it rather than waiting for a full download before proceeding
I'm having some issues with how my functions flow in lambda. I'm trying to grab value stored in S3, increment it, and put it back. However, my program doesn't flow how I feel it should be. I'm using async waterfall to run the flow of my functions.
Here's my code:
let AWS = require('aws-sdk');
let async = require('async');
let bucket = "MY_BUCKET";
let key = "MY_FILE.txt";
exports.handler = async (event) => {
let s3 = new AWS.S3();
async.waterfall([
download,
increment,
upload
], function (err) {
if (err) {
console.error(err);
} else {
console.log("Increment successful");
}
console.log("test4");
return null;
}
);
console.log("test5");
function download(next) {
console.log("test");
s3.getObject({
Bucket: bucket,
Key: key
},
next);
}
function increment(response, next) {
console.log("test2");
console.log(response.Body);
let newID = parseInt(response.Body, 10) + 1;
next(response.ContentType, newID);
}
function upload(contentType, data, next) {
console.log("test3");
s3.putObject({
Bucket: bucket,
Key: key,
Body: data,
ContentType: contentType
},
next);
}
};
I'm only getting test and test5 on my log. I was under the impression that after the download function, increment should run if it was okay or the callback function at the end of the waterfall should run if there was an error. The program doesn't give an error on execution and it doesn't appear to go to either function.
Could someone guide me to what I'm missing in my understanding?
EDIT: So it seems my issue was related to my function declaration. The default template declared it as async(event). I thought this was odd as usually they are declared as (event, context, callback). Switching to the later (or even just (event) without the async) fixed this. It looks like my issue is with calling the function as asynchronous. This blocked the waterfall async calls?? Can anyone elaborate on this?
Your problem is that your handler is declared as an async function, which will create a promise for you automatically, but since you are not awaiting at all your function is essentially ending synchronously.
There are a couple of ways to solve this, all of which we'll go over.
Do not use promises, use callbacks as the async library is designed to use.
Do not use the async library or callbacks and instead use async/await.
Mix both together and make your own promise and resolve/reject it manually.
1. Do not use promises
In this solution, you would remove the async keyword and add the callback parameter lambda is passing to you. Simply calling it will end the lambda, passing it an error will signal that the function failed.
// Include the callback parameter ────┐
exports.handler = (event, context, callback) => {
const params =[
download,
increment,
upload
]
async.waterfall(params, (err) => {
// To end the lambda call the callback here ──────┐
if (err) return callback(err); // error case ──┤
callback({ ok: true }); // success case ──┘
});
};
2. Use async/await
The idea here is to not use callback style but to instead use the Promise based async/await keywords. If you return a promise lambda will use that promise to handle lambda completion instead of the callback.
If you have a function with the async keyword it will automatically return a promise that is transparent to your code.
To do this we need to modify your code to no longer use the async library and to make your other functions async as well.
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const Bucket = "MY_BUCKET";
const Key = "MY_FILE.txt";
async function download() {
const params = {
Bucket,
Key
}
return s3.getObject(params).promise(); // You can await or return a promise
}
function increment(response) {
// This function is synchronous, no need for promises or callbacks
const { ContentType: contentType, Body } = response;
const newId = parseInt(Body, 10) + 1;
return { contentType, newId };
}
async function upload({ contentType: ContentType, newId: Body }) {
const params = {
Bucket,
Key,
Body,
ContentType
};
return s3.putObject(params).promise();
}
exports.handler = async (event) => {
const obj = await download(); // await the promise completion
const data = increment(obj); // call synchronously without await
await upload(data)
// The handlers promise will be resolved after the above are
// all completed, the return result will be the lambdas return value.
return { ok: true };
};
3. Mix promises and callbacks
In this approach we are still using the async library which is callback based but our outer function is promised based. This is fine but in this scenario we need to make our own promise manually and resolve or reject it in the waterfall handler.
exports.handler = async (event) => {
// In an async function you can either use one or more `await`'s or
// return a promise, or both.
return new Promise((resolve, reject) => {
const steps = [
download,
increment,
upload
];
async.waterfall(steps, function (err) {
// Instead of a callback we are calling resolve or reject
// given to us by the promise we are running in.
if (err) return reject(err);
resolve({ ok: true });
});
});
};
Misc
In addition to the main problem of callbacks vs. promises you are encountering you have a few minor issues I noticed:
Misc 1
You should be using const rather than let most of the time. The only time you should use let is if you intend to reassign the variable, and most of the time you shouldn't do that. I would challenge you with ways to write code that never requires let, it will help improve your code in general.
Misc 2
You have an issue in one of your waterfall steps where you are returning response.ContentType as the first argument to next, this is a bug because it will interpret that as an error. The signature for the callback is next(err, result) so you should be doing this in your increment and upload functions:
function increment(response, next) {
const { ContentType: contentType, Body: body } = response;
const newId = parseInt(body, 10) + 1;
next(null, { contentType, newId }); // pass null for err
}
function upload(result, next) {
const { contentType, newId } = result;
s3.putObject({
Bucket: bucket,
Key: key,
Body: newId,
ContentType: contentType
},
next);
}
If you don't pass null or undefined for err when calling next async will interpret that as an error and will skip the rest of the waterfall and go right to the completion handler passing in that error.
Misc 3
What you need to know about context.callbackWaitsForEmptyEventLoop is that even if you complete the function correctly, in one of the ways discussed above your lambda may still hang open and eventually timeout rather than successfully complete. Based on your code sample here you won't need to worry about that probably but the reason why this can happen is if you happen to have something that isn't closed properly such as a persistent connection to a database or websocket or something like that. Setting this flag to false at the beginning of your lambda execution will cause the process to exit regardless of anything keeping the event loop alive, and will force them to close ungracefully.
In the case below your lambda can do the work successfully and even return a success result but it will hang open until it timesout and be reported as an error. It can even be re-invoked over and over depending on how it's triggered.
exports.handler = async (event) => {
const db = await connect()
await db.write(data)
// await db.close() // Whoops forgot to close my connection!
return { ok: true }
}
In that case simply calling db.close() would solve the issue but sometimes its not obvious what is hanging around in the event loop and you just need a sledge hammer type solution to close the lambda, which is what context.callbackWaitsForEmptyEventLoop = false is for!
exports.handler = async (event) => {
context.callbackWaitsForEmptyEventLoop = false
const db = await connect()
await db.write(data)
return { ok: true }
}
The above will complete the lambda as soon as the function returns, killing all connections or anything else living in the event loop still.
Your function terminates before the waterfall is resolved. That is, the asynchronous calls aren't executed at all. That is why you don't see any of the console.log calls you have within the waterfall functions, and only see the one that is called synchronously immediately after the call to async.waterfall.
Not sure how well async.waterfall is supported by AWS Lambda, but since promises are natively supported and perform the same functionality (with fewer loc), you could use promises instead. Your code would look something like this:
module.exports.handler = (event,context) =>
s3.getObject({
Bucket: bucket,
Key: key
}).promise()
.then(response => ({
Body: parseInt(response.Body, 10) + 1,
ContentType: response.contentType,
}))
.then(modifiedResponse => s3.putObject({
Bucket: bucket,
Key: key,
Body: modifiedResponse.data,
ContentType: modifiedResponse.contentType}).promise())
.catch(err => console.error(err));
I'm using unzip function to decompress the zipped response from API in typescript (angular2).
customers:any=[];
zlib.gunzip(Buffer.from(response), function(err, uncompressedMessage)
{
if(err)
{
console.log(err);
}
else
{
resultArray = JSON.parse(uncompressedMessage.toString());
consloe('response After Unzip within fun',resultArray);
this.customers = resultArray; // error undefined
}
});
console('response After Unzip outside fun',resultArray); // undefined
Here the resultArray is only accessible within function but not outside. I tried for declaring global variable and aceess it but not able access the value So, pls suggest me that how can I achieve this.
The issue is function(err, uncompressedMessage) {} is asynchronous. And you are trying to access resultArray before it is set.
What you can do is wrap the uncompress functionality in a function that returns a Promise as follows:
let uncompress = (response) => new Promise((resolve, reject) => {
zlib.gunzip(Buffer.from(response), function(err, uncompressedMessage) {
if(err) {
reject(err);
} else {
let resultArray = JSON.parse(uncompressedMessage.toString());
resolve(resultArray);
}
});
});
Once that is done, when you receive the response, you can get the customers using:
const customers = await uncompress(response); // not undefined anymore
Note that the method which uses await keyword will have have to be declared async. If you don't want to do that, you can anyways use plain Promise like this:
uncompress(response).then(function(customers){ /* logic */ })
Don't forget to use bind(this) to callback like this
uncompress(response).then(function(customers){ /* logic */ }.bind(this))