AWS Lambda Node.js - get output from async call (ec2.describeImages) - node.js

I am unable to get data from ec2.describeImages.
I have a function that wraps the call to ec2.describeImages:
async function GetLatestAMI (event, context, latestAMI) {
console.log("REQUEST RECEIVED:\n" + JSON.stringify(event));
var responseStatus = "FAILED";
var responseData = {};
var osBaseName = osNameToPattern[event.ResourceProperties.OSName];
console.log("OS: " + event.ResourceProperties.OSName + " -> " + osBaseName);
//var ec2 = new aws.EC2({region: event.ResourceProperties.Region});
var describeImagesParams = {
Filters: [{ Name: "name", Values: [osBaseName]}],
Owners: ["amazon"]
};
console.log( "Calling describeImages...");
// Get the available AMIs for the specified Windows version.
await ec2.describeImages(describeImagesParams, function(err, describeImagesResult) {
if (err) {
responseData = {Error: "DescribeImages call failed"};
console.log(responseData.Error + ":\n", err);
}
else {
console.log( "Got a response back from the server");
var images = describeImagesResult.Images;
console.log( "Got " + images.length + " images back" );
// Sort the images by descending creation date order so the
// most recent image is first in the array.
images.sort(function(x,y){
return x.CreationDate < y.CreationDate;
});
for (var imageIndex = 0; imageIndex < images.length; imageIndex++) {
responseStatus = "SUCCESS";
responseData["Id"] = images[imageIndex].ImageId;
latestAMI = responseData["Id"];
responseData["Name"] = images[imageIndex].Name;
console.log( "Found: " + images[imageIndex].Name + ", " + images[imageIndex].ImageId);
context.done(null, "success");
//break;
return latestAMI;
}
}
}).promise();}
I call this function on the Lambda handler:
exports.handler = async function(event, context, callback) {
console.log("Hello!")
var latestAMI = "";
var testResult = await GetLatestAMI(event, context, latestAMI);
console.log("Test result: " + latestAMI);
console.log("Test result: " + testResult);
}
I cannot get any output from GetLatestAMI(). I have tried using a return clause inside the function as you can see and even passing a variable as an argument and edit the value of that value inside the function with no luck.
Here is the output from the console:
Response:
"success"
Request ID:
"8f2dbba4-02d1-11e9-a504-83ed96cb467a"
Function Logs:
START RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a Version: $LATEST
2018-12-18T14:31:08.858Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Hello!
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a REQUEST RECEIVED:
{"ResourceProperties":{"OSName":"Windows Server 2016 (64-bit)"}}
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a OS: Windows Server 2016 (64-bit) -> Windows_Server-2016-English-Full-Base-*
2018-12-18T14:31:08.873Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Calling describeImages...
2018-12-18T14:31:10.556Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Got a response back from the server
2018-12-18T14:31:10.573Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Got 7 images back
2018-12-18T14:31:10.593Z 8f2dbba4-02d1-11e9-a504-83ed96cb467a Found: Windows_Server-2016-English-Full-Base-2018.12.12, ami-06a27ce600d784c71
END RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a
REPORT RequestId: 8f2dbba4-02d1-11e9-a504-83ed96cb467a Duration: 1776.51 ms Billed Duration: 1800 ms Memory Size: 128 MB Max Memory Used: 39 MB
I'm probably misunderstanding or missing something. Hopefully it would be obvious to someone with experience on AWS Lambda and Node.js!

I have written the below lambda for you to list down the ec2 instances. And have checked in nodejs8.10. It is working as expected.
Some of the issues your code has.
a) return statement inside loop.
b) context.done(null, "success"); done at wrong place that to inside loop
c) You are not resolving your promises anywhere.
Hope the Below code will give you an idea.
const AWS = require('aws-sdk');
module.exports.api = async (event, context, callback) => {
var response = await getInstances();
return {
statusCode: '200',
body: JSON.stringify({response}),
headers: {'Content-Type': 'application/json'}
}
}
async function getInstances() {
var ec2 = new AWS.EC2();
return new Promise(function (resolve, reject) {
ec2.describeInstances(function (err, data) {
if (err) reject(err);
else resolve(data);
});
});
}

Related

Unable to upload multiple images to AWS S3 if I don't first upload one image through a AWS NodeJS Lambda endpoint using Promises

I have the code below on AWS Lambda as an endpoint exposed through API Gateway. The point of this endpoint is to upload images to an S3 bucket. I've been experiencing an interesting bug and could use some help. This code is unable to upload multiple images to S3 if it does not first upload one image. I've listed the scenarios below. The reason I want to use Promises is because I intend to insert data into a mysql table in the same endpoint. Any advice or feedback will be greatly appreciated!
Code Successfully uploads multiple images:
Pass one image to the endpoint to upload to S3 first
Pass several images to the endpoint to upload to S3 after uploading one image first
Code fails to upload images:
Pass several images to the endpoint to upload to s3 first. A random amount of images might be uploaded, but it consistently fails to upload all of them. A 502 error code is returned because it failed to upload all images.
Code
const AWS = require('aws-sdk');
const s3 = new AWS.S3({});
function uploadAllImagesToS3(imageMap) {
console.log('in uploadAllImagesToS3')
return new Promise((resolve, reject) => {
awaitAll(imageMap, uploadToS3)
.then(results => {
console.log('awaitAllFinished. results: ' + results)
resolve(results)
})
.catch(e => {
console.log("awaitAllFinished error: " + e)
reject(e)
})
})
}
function awaitAll(imageMap, asyncFn) {
const promises = [];
imageMap.forEach((value, key) => {
promises.push(asyncFn(key, value));
})
console.log('promises length: ' + promises.length)
return Promise.all(promises)
}
function uploadToS3(key, value) {
return new Promise((resolve, reject) => {
console.log('Promise uploadToS3 | key: ' + key)
// [key, value] = [filePath, Image]
var params = {
"Body": value,
"Bucket": "userpicturebucket",
"Key": key
};
s3.upload(params, function (err, data) {
console.log('uploadToS3. s3.upload. data: ' + JSON.stringify(data))
if (err) {
console.log('error when uploading to s3 | error: ' + err)
reject(JSON.stringify(["Error when uploading data to S3", err]))
} else {
let response = {
"statusCode": 200,
"headers": {
"Access-Control-Allow-Origin": "http://localhost:3000"
},
"body": JSON.stringify(data),
"isBase64Encoded": false
};
resolve(JSON.stringify(["Successfully Uploaded data to S3", response]))
}
});
})
}
exports.handler = (event, context, callback) => {
if (event !== undefined) {
let jsonObject = JSON.parse(event.body)
let pictures = jsonObject.pictures
let location = jsonObject.pictureLocation
let imageMap = new Map()
for (let i = 0; i < pictures.length; i++) {
let base64Image = pictures[i].split('base64,', 2)
let decodedImage = Buffer.from(base64Image[1], 'base64'); // image string is after 'base64'
let base64Metadata = base64Image[0].split(';', 3) // data:image/jpeg,name=coffee.jpg,
let imageNameData = base64Metadata[1].split('=', 2)
let imageName = imageNameData[1]
var filePath = "test/" + imageName
imageMap.set(filePath, decodedImage)
}
const promises = [uploadAllImagesToS3(imageMap)]
Promise.all(promises)
.then(([uploadS3Response]) => {
console.log('return promise!! | uploadS3Response: ' + JSON.stringify([uploadS3Response]))
let res = {
body: JSON.stringify(uploadS3Response),
headers: {
"Access-Control-Allow-Origin": "http://localhost:3000"
}
};
callback(null, res);
})
.catch((err) => {
callback(err);
});
} else {
callback("No pictures were uploaded")
}
};
Reason for problem and solution :
After several hours of debugging this issue I realized what the error was! My Lambda endpoint was timing out early. The reason I was able to upload multiple images after first uploading one image was because my the lambda endpoint was being executed from a warm start - as it was already up and running. The scenario where I was unable to upload multiple images was actually only occurring when I would try to do so after not executing the endpoint in 10+ minutes - therefore a cold start. Therefore, the solution was to increase the Timeout from the default of 3 seconds. I increased it to 20 seconds, but might need to play around with that time.
How to increase the lambda timeout?
Open Lambda function
Scroll down to Basic Settings and select Edit
Increase time in Timeout
TLDR
This error was occurring because Lambda would timeout. Solution is to increase lambda timeout.

Lambda function taking >3 seconds to run + 5-10 secs warmup each time

I have a simple node.js function with 2 REST API calls and a socket connection output hosted in an AWS lambda. It takes 5-10 secs warmup time and >3+ secs execution time.
When the code is run locally it executes both requests, socket connection and completes in about ~1300ms. Why is AWS more then double the execution time? I have set-timeout to 120seconds and memory at 128mb (default).
I appreciate the code is not very tidy; I am working on cleaning it but needed something going for the time being.
The project simply gets info from ServiceM8 via API when called by a webhook subscription, then formats the info into ZPL strings and forwards them to a tcp server for printing via thermal printer.
My questions are:
Is it my code bottle necking?
Can it be optimized to run faster?
Do i simply need to employ a warming plugin for my function to allow hot starting?
My function:
'use strict';
//Require libraries
var request = require("request");
var net = require('net');
exports.handler = (event, context, callback) => {
if (event.eventName != 'webhook_subscription') {
callback(null, {});
}
//Global Variables
var strAssetUUID;
var strAssetURL;
var strFormUUID;
var strTestDate;
var strRetestDate;
var appliancePass = true;
var strAccessToken;
var strResponseUUID;
//Printer Access
const tcpUrl = 'example.com';
const tcpPort = 12345;
var client = new net.Socket();
//UUID of Appliance Test Form.
const strTestFormUUID = 'UUID_of_form';
//Begin function
/**
* Inspect the `eventArgs.entry` argument to get details of the change that caused the webhook
* to fire.
*/
strResponseUUID = event.eventArgs.entry[0].uuid;
strAccessToken = event.auth.accessToken;
console.log('Response UUID: ' + strResponseUUID);
console.log('Access Token: ' + strAccessToken);
//URL Options for FormResponse UUID query
const urlFormResponse = {
url: 'https://api.servicem8.com/api_1.0/formresponse.json?%24filter=uuid%20eq%20' + strResponseUUID,
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
};
//Query form Response UUID to get information required.
request.get(urlFormResponse, function(err, res, body) {
//Check response code from API query
if (res.statusCode != 200) {
// Unable to query form response records
callback(null, {err: "Unable to query form response records, received HTTP " + res.statusCode + "\n\n" + body});
return;
}
//If we do recieve a 200 status code, begin
var arrRecords = JSON.parse(body);
//Store the UUID of the form used for the form response.
strFormUUID = arrRecords[0].form_uuid;
console.log('Form UUID: ' + strFormUUID);
//Store the UUID of the asset the form response relates to.
strAssetUUID = arrRecords[0].asset_uuid;
console.log('Asset UUID: ' + strAssetUUID);
if (strFormUUID == strTestFormUUID){
//Get the edited date and parse it into a JSON date object.
var strEditDate = new Date(arrRecords[0].edit_date);
//Reassemble JSON date to dd-mm-yyyy.
strTestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
//Extract the response for retest period.
var strRetestAnswer = JSON.parse(arrRecords[0].field_data);
strRetestAnswer = strRetestAnswer[0].Response;
//Appropriate function based on retest response.
switch(strRetestAnswer) {
case '3 Months':
//Add x months to current test date object
strEditDate.setMonth(strEditDate.getMonth() + 3);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '6 Months':
strEditDate.setMonth(strEditDate.getMonth() + 6);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '12 Months':
strEditDate.setMonth(strEditDate.getMonth() + 12);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '2 Years':
strEditDate.setMonth(strEditDate.getMonth() + 24);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
case '5 Years':
strEditDate.setMonth(strEditDate.getMonth() + 60);
strRetestDate = strEditDate.getDate() + '/' + (strEditDate.getMonth() + 1) + '/' + strEditDate.getFullYear();
break;
default:
strRetestDate = "FAIL";
appliancePass = false;
}
console.log('Appliance Pass: ' + appliancePass);
console.log('Test Date: ' + strTestDate);
console.log('Retest Period: ' + strRetestAnswer);
console.log('Retest Date: ' + strRetestDate);
//URL Options for Asset UUID query
const urlAssetResponse = {
url: 'https://api.servicem8.com/api_1.0/asset/' + strAssetUUID + '.json',
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
};
//Query the api for the asset URL of the provided asset UUID.
request.get(urlAssetResponse, function(err, res, body) {
//Check response code from API query
if (res.statusCode != 200) {
// Unable to query asset records
callback(null, {err: "Unable to query asset records, received HTTP " + res.statusCode + "\n\n" + body});
return;
}
//If we do recieve a 200 status code, begin
var strAssetResponse = JSON.parse(body);
//Store the asset URL
strAssetURL = 'https://sm8.io/' + strAssetResponse.asset_code;
console.log('Asset URL: ' + strAssetURL);
//generate tag and send to printer
var strZPLPass = ('^XA....^XZ\n');
var strZPLFail = ('^XA....^XZ\n');
//Now that we have our ZPL generated from our dates and URLs
//Send the correct ZPL to the printer.
client.connect(tcpPort, tcpUrl, function() {
console.log('Connected');
//Send Appropriate ZPL
if (appliancePass) {
client.write(strZPLPass);
}else {
client.write(strZPLFail);
}
console.log('Tag Successfully Printed!');
//As the tcp server receiving the string does not return any communication
//there is no way to know when the data has been succesfully received in full.
//So we simply timeout the connection after 750ms which is generally long enough
//to ensure complete transmission.
setTimeout(function () {
console.log('Timeout, connection closing...');
client.destroy();
}, 750);
});
});
}
});
};
First of all, I would suggest you stop using the request module and switch to native. Everything can be done without a tons of lines these days. request is a module with 48 total dependencies; if you do the math, that's thousands of lines for a simple GET request.
You should always minimize the complexity of your dependencies. I use a Lambda to check the health of my sites, grabbing the whole request and checking the HTML on completely different servers. VPS is located in Frankfurt, AWS in Ireland. My ms/request is ranging between 100~150 ms.
Here's a simple promise request I'm using:
function request(obj, timeout) {
return new Promise(function(res, rej) {
if (typeof obj !== "object") {
rej("Argument must be a valid http request options object")
}
obj.timeout = timeout;
obj.rejectUnauthorized = false;
let request = http.get(obj, (response) => {
if (response.statusCode !== 200) {
rej("Connection error");
}
var body = '';
response.on('data', (chunk) => {
body += chunk;
});
response.on('end', () => {
res(body);
});
response.on('error', (error) => {
rej(error);
});
});
request.setTimeout(timeout);
request.on('error', (error) => {
rej(error);
})
request.on('timeout', () => {
request.abort();
rej("Timeout!")
})
});
}
Example
const reqOpts = {
hostname: 'www.example.com',
port: 443,
path: '/hello',
method: 'GET',
headers: {
handshake: "eXTNxFMxQL4pRrj6JfzQycn3obHL",
remoteIpAddress: event.sourceIp || "lambda"
}
}
try {
httpTestCall = await request(reqOpts, 250);
}
catch (e) {
console.error(e);
}
Now based on that change switch your handler to async using exports.handler = async(event, context, callback) => {} and use console to measure the execution time of every request using console.time() and console.timeEnd() for your request or anything. From there you could see what's stepping down your code using Cloudwatch logs. Here's another example based on your code:
let reqOpts = {
hostname: 'api.servicem8.com',
port: 443,
path: '/api_1.0/formresponse.json?%24filter=uuid%20eq%20' + strResponseUUID,
method: 'GET',
headers: {
// Use the temporary Access Token that was issued for this event
'Authorization': 'Bearer ' + strAccessToken
}
}
console.time("=========MEASURE_servicem8=========")
let error = null;
await request(reqOpts, 5555).catch((e)=>{
error = e;
})
console.timerEnd("=========MEASURE_servicem8=========")
if (error){
callback(null, {err: "Unable to query form response records, received HTTP" + error}); /* or anything similar */
}
References
https://docs.aws.amazon.com/lambda/latest/dg/best-practices.html
https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-handler.html
aws lambdas are not fast by nature (as of writing this answer). The startup time is not guaranteed, and it is known to be high.
If you need performance - you will not get it this way.

AWS Lambda unable to call external https endpoint

We are working on an Alexa skill and it will need to reach out to external REST API's to get data. I'm having a really hard time getting this to work in our lambda function for some reason. I'm also having a hard time determining if the problem is in my node.js code not using callback's correctly or if it's in the VPC settings for my function. Here is my code, I've stripped out the non-essential stuff.
/* eslint-disable func-names */
/* eslint quote-props: ["error", "consistent"]*/
/**
* This sample demonstrates a simple skill built with the Amazon Alexa Skills
* nodejs skill development kit.
* This sample supports multiple lauguages. (en-US, en-GB, de-DE).
* The Intent Schema, Custom Slots and Sample Utterances for this skill, as well
* as testing instructions are located at https://github.com/alexa/skill-sample-nodejs-fact
**/
'use strict';
const Alexa = require('alexa-sdk');
const APP_ID = undefined; // TODO replace with your app ID (OPTIONAL).
const https = require('https');
const handlers = {
'LaunchRequest': function () {
this.emit('GetFact');
},
'GetNewFactIntent': function () {
this.emit('GetFact');
},
'maintenanceIntent': function () {
console.log('inside maintenanceIntent');
var options = {
host: 'api.forismatic.com',
path: '/api/1.0/?method=getQuote&lang=en&format=text',
method: 'GET'
};
getQuote(options, function (quote){
if(quote === ''){
console.log("No quote");
//speechOutput = "Please try again later";
}
else{console.log(quote)}
//self.emit(':tellWithCard', speechOutput, SKILL_NAME, text);
});
// Create speech output
// Place holder
var randomFact = 'Test Fact';
const speechOutput = randomFact;
this.emit(':tellWithCard', speechOutput, 'test skill name', randomFact);
},
'AMAZON.HelpIntent': function () {
const speechOutput = this.t('HELP_MESSAGE');
const reprompt = this.t('HELP_MESSAGE');
this.emit(':ask', speechOutput, reprompt);
},
'AMAZON.CancelIntent': function () {
this.emit(':tell', this.t('STOP_MESSAGE'));
},
'AMAZON.StopIntent': function () {
this.emit(':tell', this.t('STOP_MESSAGE'));
},
};
exports.handler = function (event, context, callback) {
const alexa = Alexa.handler(event, context);
alexa.APP_ID = APP_ID;
// To enable string internationalization (i18n) features, set a resources object.
//alexa.resources = languageStrings;
alexa.registerHandlers(handlers);
alexa.execute();
};
function getQuote(options, callback){
var text = '';
console.log("in getquote");
https.get(options, function(res) {
console.error("Got response: " + res.statusCode);
res.on("data", function(chunk) {
console.error("BODY: " + chunk);
text = '' + chunk;
return callback(text);
});
}).on('error', function(e) {
text = 'error' + e.message;
console.error("Got error: " + e.message);
});
}
Now when I invoke the maintenanceIntent this is what I see in the logs.
{"timestamp":1508426249817,"message":"START RequestId: 9f66123e-b4e0-11e7-baac-1bfb01d2abc8 Version: $LATEST","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
{"timestamp":1508426250256,"message":"Warning: Application ID is not set","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
{"timestamp":1508426250256,"message":"inside maintenanceIntent","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
{"timestamp":1508426250256,"message":"in getquote","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
{"timestamp":1508426250256,"message":"END RequestId: 9f66123e-b4e0-11e7-baac-1bfb01d2abc8","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
{"timestamp":1508426250256,"message":"REPORT RequestId: 9f66123e-b4e0-11e7-baac-1bfb01d2abc8\tDuration: 378.28 ms\tBilled Duration: 400 ms \tMemory Size: 128 MB\tMax Memory Used: 33 MB\t","logStream":"2017/10/19/[$LATEST]0e048ab2fc5441cda8007e4a1963bf02","logGroup":"/aws/lambda/factDemo","requestID":"9f66123e-b4e0-11e7-baac-1bfb01d2abc8"}
So I can see it's actually calling the getQuote function. I'm not seeing any errors or success messages at all. I thought maybe I wasn't using callbacks correctly (node isn't my normal development language) but I've actually pulled code straight from an Amazon example on GitHub and I couldn't even get that to work. (This code is very similar to it except it's a little shorter.)
If I strip this down and run it locally through the node, it works fine.
As far as the networking stuff, I followed this guide: https://gist.github.com/reggi/dc5f2620b7b4f515e68e46255ac042a7
. I've also tried Amazon guides but at this point, I'm not even sure how I would check internet connectivity or if this is even the problem.
Any help to get on the right track would be greatly appreciated!
--EDIT--
I've changed my code as so. This comes straight from the alexa-cookbook at https://raw.githubusercontent.com/alexa/alexa-cookbook/master/external-calls/httpsGet/src/index.js
/* eslint-disable func-names */
/* eslint quote-props: ["error", "consistent"]*/
/**
* This sample demonstrates a simple skill built with the Amazon Alexa Skills
* nodejs skill development kit.
* This sample supports multiple lauguages. (en-US, en-GB, de-DE).
* The Intent Schema, Custom Slots and Sample Utterances for this skill, as well
* as testing instructions are located at https://github.com/alexa/skill-sample-nodejs-fact
**/
'use strict';
const Alexa = require('alexa-sdk');
const APP_ID = undefined; // TODO replace with your app ID (OPTIONAL).
const https = require('https');
const handlers = {
'LaunchRequest': function () {
this.emit('GetFact');
},
'GetNewFactIntent': function () {
this.emit('GetFact');
},
'maintenanceIntent': function () {
console.log('inside maintenanceIntent');
var myRequest = 'Florida';
httpsGet(myRequest, (myResult) => {
console.log("sent : " + myRequest);
console.log("received : " + myResult);
this.response.speak('The population of ' + myRequest + ' is ' + myResult);
this.emit(':responseReady');
}
);
// Create speech output
// Place holder
var randomFact = 'Test Fact';
const speechOutput = randomFact;
this.emit(':tellWithCard', speechOutput, 'test skill name', randomFact);
},
'AMAZON.HelpIntent': function () {
const speechOutput = this.t('HELP_MESSAGE');
const reprompt = this.t('HELP_MESSAGE');
this.emit(':ask', speechOutput, reprompt);
},
'AMAZON.CancelIntent': function () {
this.emit(':tell', this.t('STOP_MESSAGE'));
},
'AMAZON.StopIntent': function () {
this.emit(':tell', this.t('STOP_MESSAGE'));
},
};
exports.handler = function (event, context, callback) {
console.log("exports handler");
const alexa = Alexa.handler(event, context);
alexa.APP_ID = APP_ID;
// To enable string internationalization (i18n) features, set a resources object.
//alexa.resources = languageStrings;
alexa.registerHandlers(handlers);
alexa.execute();
console.log("post execute");
};
function httpsGet(myData, callback) {
// GET is a web service request that is fully defined by a URL string
// Try GET in your browser:
// https://cp6gckjt97.execute-api.us-east-1.amazonaws.com/prod/stateresource?usstate=New%20Jersey
console.log("in");
console.log(myData);
// Update these options with the details of the web service you would like to call
var options = {
host: 'cp6gckjt97.execute-api.us-east-1.amazonaws.com',
port: 443,
path: '/prod/stateresource?usstate=' + encodeURIComponent(myData),
method: 'GET',
// if x509 certs are required:
// key: fs.readFileSync('certs/my-key.pem'),
// cert: fs.readFileSync('certs/my-cert.pem')
};
var req = https.request(options, res => {
res.setEncoding('utf8');
var returnData = "";
console.log("request");
res.on('data', chunk => {
console.log("data");
returnData = returnData + chunk;
});
res.on('end', () => {
console.log("end");
// we have now received the raw return data in the returnData variable.
// We can see it in the log output via:
// console.log(JSON.stringify(returnData))
// we may need to parse through it to extract the needed data
var pop = JSON.parse(returnData).population;
callback(pop); // this will execute whatever function the caller defined, with one argument
});
});
console.log("req.end");
req.end();
}
Same idea but slightly different execution of getting the result from the endpoint. This is the log output.
{"timestamp":1508434982754,"message":"START RequestId: f4a39351-b4f4-11e7-a563-fbf7599fa72f Version: $LATEST","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434982887,"message":"exports handler","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434982887,"message":"Warning: Application ID is not set","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434982887,"message":"inside maintenanceIntent","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434982887,"message":"in","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434982887,"message":"Florida","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434983307,"message":"req.end","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434983309,"message":"post execute","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434983367,"message":"END RequestId: f4a39351-b4f4-11e7-a563-fbf7599fa72f","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
{"timestamp":1508434983367,"message":"REPORT RequestId: f4a39351-b4f4-11e7-a563-fbf7599fa72f\tDuration: 608.20 ms\tBilled Duration: 700 ms \tMemory Size: 128 MB\tMax Memory Used: 35 MB\t","logStream":"2017/10/19/[$LATEST]3252e394be9b4a229c3a0d042deffbf8","logGroup":"/aws/lambda/factDemo","requestID":"f4a39351-b4f4-11e7-a563-fbf7599fa72f"}
I've tried this in the VPC and out of the VPC with the same result.
So this is solved thanks to this question: Node JS callbacks with Alexa skill
Particurlarly, this line
The :tell function will call the lambda callback and terminate the execution of the lambda function.
Moving the this.emit outside the httpsGet into it like so fixed the issue.
httpsGet(myRequest, (myResult) => {
console.log("sent : " + myRequest);
console.log("received : " + myResult);
this.response.speak('The population of ' + myRequest + ' is ' + myResult);
this.emit(':responseReady');
// Create speech output
// Place holder
var randomFact = 'Test Fact';
const speechOutput = randomFact;
this.emit(':tellWithCard', speechOutput, 'test skill name', randomFact);
}
);

Multipart upload of zip file to AWS Glacier freezes midway

I'm trying to upload a 600mb .zip file to glacier using the multipartupload function of the node version of the aws-sdk. I figured out how to read the file as a buffer and start the upload using a script from the aws docs.
The script starts an upload for each part of the file but each one fails with a 400 error.
Uploading part 0 = bytes 0-2097151/*
Uploading part 2097152 = bytes 2097152-4194303/*
Uploading part 4194304 = bytes 4194304-6291455/*
Uploading part 6291456 = bytes 6291456-8388607/*
....
Uploading part 591396864 = bytes 591396864-591798963/*
//stops logging, then a couple seconds later, it starts returning an error message like this for each upload part:
{ [UnknownError: 400]
message: '400',
code: 'UnknownError',
statusCode: 400,
time: Tue Jan 10 2017 20:54:29 GMT-0500 (EST),
requestId: 'F16FEDE011D3039A',
retryable: false,
retryDelay: 91.54012566432357 }
Below is the upload script I'm using
var AWS = require('aws-sdk');
var creds = <path to creds>
var fs = require('fs');
var filePath = <path to file>;
var encoding = "utf8";
var myConfig = new AWS.Config({
accessKeyId: creds.AccessKeyID,
secretAccessKey: creds.SecretAccessKey,
region: 'us-west-1'
});
var glacier = new AWS.Glacier(myConfig)
var buffer = fs.readFileSync(filePath);
// var buffer = new Buffer(2.5 * 1024 * 1024); // 2.5MB buffer
var partSize = 1024 * 1024; // 1MB chunks,
var numPartsLeft = Math.ceil(buffer.length / partSize);
var startTime = new Date();
var params = {
accountId: '-',
vaultName: <vault name>
archiveDescription: '100media',
partSize: partSize.toString(),
};
// Compute the complete SHA-256 tree hash so we can pass it
// to completeMultipartUpload request at the end
var treeHash = glacier.computeChecksums(buffer).treeHash;
// Initiate the multipart upload
console.log('Initiating upload to', params.vaultName);
glacier.initiateMultipartUpload(params, function (mpErr, multipart) {
if (mpErr) { console.log('Error!', mpErr.stack); return; }
console.log("Got upload ID", multipart.uploadId);
// Grab each partSize chunk and upload it as a part
for (var i = 0; i < buffer.length; i += partSize) {
var end = Math.min(i + partSize, buffer.length),
partParams = {
vaultName: params.vaultName,
uploadId: multipart.uploadId,
range: 'bytes ' + i + '-' + (end-1) + '/*',
body: buffer.slice(i, end)
};
// Send a single part
console.log('Uploading part', i, '=', partParams.range);
glacier.uploadMultipartPart(partParams, function(multiErr, mData) {
if (multiErr) return;
console.log("Completed part", this.request.params.range);
if (--numPartsLeft > 0) return; // complete only when all parts uploaded
var doneParams = {
vaultName: params.vaultName,
uploadId: multipart.uploadId,
archiveSize: buffer.length.toString(),
checksum: treeHash // the computed tree hash
};
console.log("Completing upload...");
glacier.completeMultipartUpload(doneParams, function(err, data) {
if (err) {
console.log("An error occurred while uploading the archive");
console.log(err);
} else {
var delta = (new Date() - startTime) / 1000;
console.log('Completed upload in', delta, 'seconds');
console.log('Archive ID:', data.archiveId);
console.log('Checksum: ', data.checksum);
}
});
});
}
});
Any thoughts on where the 400 errors are coming from would be greatly appreciated! I have not worked with buffers or binary data before so I might be messing up the format for this. The other suspect is that I am just formatting the glacier request or params wrong.
Here is a script that I created which tries a multipart upload one at a time. "Could try to be reworked to be concurrent but it works as is, retrying if an upload fails:
var minm = require('minimist');
var argv = require('minimist')(process.argv.slice(2));
var AWS = require('aws-sdk');
var creds = <path to local json creds>
var fs = require('fs');
var encoding = "utf8";
var partSize = 1024 * 1024; // 1MB chunks,
var startTime = new Date();
var byteIncrementer = 0;
var MBcounter = 0;
var multipart;
//move these out to args
var filePath = argv.filepath;
var vaultName = argv.vaultname
var archiveDescription = argv.description
if (!filePath) {
throw "ERROR: must pass file path via --filepath <filepath>"
}
if (!archiveDescription) {
throw "ERROR: must pass description path via --description <description>"
}
var myConfig = new AWS.Config({
accessKeyId: creds.AccessKeyID,
secretAccessKey: creds.SecretAccessKey,
region: <region>
});
var params = {
accountId: '-',
vaultName: vaultName,
archiveDescription: archiveDescription,
partSize: partSize.toString(),
};
var buffer = fs.readFileSync(filePath);
var numPartsLeft = Math.ceil(buffer.length / partSize);
var glacier = new AWS.Glacier(myConfig)
var treeHash = glacier.computeChecksums(buffer).treeHash;
new Promise(function (resolve, reject) {
glacier.initiateMultipartUpload(params, function (mpErr, multi) {
if (mpErr) { console.log('Error!', mpErr.stack); return; }
console.log("Got upload ID", multi.uploadId);
multipart = multi
resolve();
});
}).then(function () {
console.log("total upload size: ", buffer.length);
recursivelyUploadPart(byteIncrementer)
}).catch(function (err) {console.log(err)});
function recursivelyUploadPart() {
var end = Math.min(byteIncrementer + partSize, buffer.length);
var partParams = {
accountId: '-',
uploadId: multipart.uploadId,
vaultName: params.vaultName,
range: 'bytes ' + byteIncrementer + '-' + (end-1) + '/*',
body: buffer.slice(byteIncrementer, end)
};
console.log('Uploading part', byteIncrementer, '=', partParams.range);
glacier.uploadMultipartPart(partParams, function(multiErr, mData) {
if (multiErr) {
console.log('part upload error: ', multiErr)
console.log('retrying')
return recursivelyUploadPart(byteIncrementer)
} else {
console.log("Completed part", this.request.params.range);
if (--numPartsLeft > 0) {
MBcounter++;
console.log("MB Uploaded: ", MBcounter);
byteIncrementer += partSize;
console.log('recursing');
return recursivelyUploadPart(byteIncrementer);
} else {
var doneParams = {
vaultName: params.vaultName,
uploadId: multipart.uploadId,
archiveSize: buffer.length.toString(),
checksum: treeHash // the computed tree hash
};
console.log("Completing upload...");
glacier.completeMultipartUpload(doneParams, function(err, data) {
if (err) {
console.log("An error occurred while uploading the archive: ", err);
} else {
var delta = (new Date() - startTime) / 1000;
console.log('Completed upload in', delta, 'seconds');
console.log('Archive ID:', data.archiveId);
console.log('Checksum: ', data.checksum);
console.log("==============================");
console.log('COMPLETED');
console.log("==============================");
}
});
}
}
});
};
As mentioned in the comment, it looks like i was opening up a ton of http connections and trying to do everything concurrently which won't work.

Synchronous http request in node js?

I'm looking for a simple way to perform synchronous http-requests in node.js, but it's still getting async responses ...
I've realised that node.js is recommended to async jobs, but in my case,
I need the synchronous response to call other functions that use this data, if it's null/undefined, I can't proceed with the flow...
What's the better way to do that?
Here's my code:
function callCellId(data) {
console.log("Data: " + data);
var towers = [],
rscp = [];
var request = require('sync-request');
for (var x = 0; x < data.length; x++) {
console.log("Request data: \n");
rscp[x] = data[x].rscp;
var res = request('POST', 'http://opencellid.org/cell/get?key=xxxxx&mcc=' + data[x].mcc + '&mnc=' + data[x].mnc + '&lac=' + data[x].LAC + '&cellid=' + data[x].cellID + '&format=json');
console.log("loop " + x);
data = res.getBody().toString();
console.log("rsp: " + data);
towers[x] = {
'latitude': data.lat,
'longitude': data.lon,
'rscp': rscp[x],
'signal': data.averageSignalStrength
};
}
console.log("Content for triangulation" + JSON.stringify(towers));
return towers;
}
Using async in a loop cloud be tricky.
I solved this without external libraries using generators:
LoopOperation: function() {
//define generator with the main loop
var loopIterator = function*() {
for (var index = 0; index < 10; index++) {
var result = yield asyncOperation( (res) => loopIterator.next(res)); //do something asyc and execute () => loopIterator.next() when done as callback
console.log(result);
}
}();
loopIterator.next(); //start loop
}
Since the nodejs nature is async, every time we need some sync call (like this nested request stack), we're able to use promises
"A Promise is an object representing the eventual completion or failure of an asynchronous operation"
reference
I.E:
const request = require('request-promise');
function callCellId(data) {
let towers = [];
let options = {
url: 'http://opencellid.org/cell/get',
method: 'POST',
json: true
};
data.forEach(location => {
options.body = {
key: 'YOUR_PRIVATE_KEY',
mcc: location.mcc,
mnc: location.mnc,
lac: location.lac,
cellId: location.cellID
}
request(options).then(cellInfo => {
towers.push({
latitude: cellInfo.lat,
longitude: cellInfo.lon,
rscp: location.rscp,
signal: cellInfo.averageSignalStrength
});
}).catch(err => {
console.log('Could not get cellId Info for',location);
console.log(err);
});
});
return towers;
}

Resources