Related
I have an azure function with this line of code.
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
But my code seems to just skip this part of code, with no errors. I am new to Azure and node.js so I might have missed some basic parts in setting this up.
Any ideas?
Edit:
Here is my full code
const https = require('https');
const querystring = require('querystring');
module.exports = async function (context, req) {
if (req.query.accessCode || (req.body && req.body.accessCode)) {
context.log('JavaScript HTTP trigger function processed a request.');
var options = {
host: 'httpbin.org',
port: 80,
path: '/post',
method: 'POST'
};
var postData = querystring.stringify({
client_id : '1234',
client_secret: 'xyz',
code: req.query.accessCode
});
var body = "";
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
context.log("help");
context.res = {
status: 200,
body: "Hello " + (body)
};
} else {
context.res = {
status: 400,
body: "Please pass a name on the query string or in the request body"
};
}
};
Ideally it should work. You can also try using request module like below
const request = require('request');
request('http://www.google.com', function (error, response, body) {
console.error('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
Try and see if it helps.
Solved by doing await properly. Used this as guide.
var https = require('https');
var util = require('util');
const querystring = require('querystring');
var request = require('request')
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
/*if (req.query.name || (req.body && req.body.name)) {*/
var getOptions = {
contentType: 'application/json',
headers: {
'Authorization': <bearer_token>
},
};
var postData = {
"key": "value"
};
var postOptions = {
method: 'post',
body: postData,
json: true,
url: <post_url>,
headers: {
'Authorization': <bearer_token>
},
};
try{
var httpPost = await HttpPostFunction(context, postOptions);
var httpGet = await HttpGetFunction(context, <get_url>, getOptions);
return {
res: httpPost
};
}catch(err){
//handle errr
console.log(err);
};
};
async function HttpPostFunction(context, options) {
context.log("Starting HTTP Post Call");
return new Promise((resolve, reject) => {
var data = '';
request(options, function (err, res, body) {
if (err) {
console.error('error posting json: ', err)
reject(err)
}
var headers = res.headers;
var statusCode = res.statusCode;
//context.log('headers: ', headers);
//context.log('statusCode: ', statusCode);
//context.log('body: ', body);
resolve(body);
})
});
};
async function HttpGetFunction(context, url, options) {
context.log("Starting HTTP Get Call");
return new Promise((resolve, reject) => {
var data = '';
https.get(url, options, (resp) => {
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
})
// The whole response has been received. Print out the result.
resp.on('end', () => {
resolve(JSON.parse(data));
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(err.message);
});
});
};
I have the following app to download productAds from amazons Sponsored Products API
Docs: https://advertising.amazon.com/API/docs/reference/reports
When I try to fun it, I get the response from requesting the report, and obtain a file in an s3 Bucket.
server is running at localhost:3000
{"reportId":"amzn1.clicPI.v1.p7.5B7EE.d141f-e5-4b-a8-6a4e712","status":"SUCCESS","statusDetails":"Report has been successfully generated","location":"https://advertising-api.amazon.com/v1/reports/amzn1.clsAPI.v1.p7.5B7EE.d151f-e5-4b-a8-699e712/download","fileSize":22}
Next I try to download the report
and the repose I get the following erro:
donwloading <?xml version="1.0" encoding="UTF-8"?>
<Error><Code>InvalidArgument</Code><Message>Only one auth mechanism allowed; only the X-Amz-Algorithm query parameter, Signature query string parameter or the Authorization header should be specified</Message><ArgumentName>Authorization</ArgumentName><ArgumentValue>bearer Atza|IwEBIJvCt20TUi122srkN4JCOdUxlBNuLJBrtbIGF9x5QbKG67f-K-0L4RkLzeyouXWy_U_-VscaCe1aFqOJK55X9Mu2X6nwWkAWRyhc6cCMfPjKpyyVjKtPqC8Plme84om1dqtmIqC93yUVc_clHimQqmnl262te2EXyUhYoVQg8hK2nlDG67Iw7xjsLK4rgl3E4RR36DHnZkEOnVQZtfjIkIbcYtsCSAdpRZRazF4FQfpS-jHvMlwuH8TZfY9tRpmBEx5fjJw1WZ14Dejqti23mZ7yt-MjNkUuD-DdPXs3fek1ZJePlHEVzcI2y_WzCnwJnoSVp5a5w1WgNco8YqEGEuLsT9S0dxQRluTiw8f4b4lx2FFFm9jz0K7pqo1Mvs6DZOVCDJzE-xJ_VLlWoE5QDMUAMor4AEQH44_0NWBjJDrYaVvn1vZLCER1uxW4jgr127W5yXaj4y1vj_vADwFq9a3330hAc73EWwL6FFSfoTQZyNc4Fh1d3DXfVHpXFk6cv0bHt4cV_OotGwGHat5fv75VHX5K3al3Xd5-QJv2cTiQ9srY5oqKsdbxptGaxAdrdMQaUlFHhyHEGbwED9xYoCw6-IauN15gvMAei9wz2kzRCA</ArgumentValue><RequestId>BDCE812C4363E9D2</RequestId><HostId>4ixH24mPtMBt+FDnI3rM9UJP95toaNBmmR1v0uQJ5XkyiXbtLEuZ8d+vDI0+gquwhn6/Fkz6/+o=</HostId></Error>
Here is the app. How can I download the report and pass the headers correctly? I found this repo in PHP that has the download function: https://github.com/dbrent-amazon/amazon-advertising-api-php/blob/master/AmazonAdvertisingApi/Client.php
const express = require('express')
const app = express();
const request = require('request');
const moment = require('moment');
const fs = require('fs')
const path = require('path')
const config = require('./config')
const auth = require('./helpers/auth');
const cron = require('./helpers/cron');
const con = require('./helpers/database');
const getProfiles = (headers) => {
return new Promise((resolve, reject) => {
request({
url: config.ad_url + '/v1/profiles',
method: "GET",
headers: headers
}, (err, httpResponse, body) => {
if (err) {
reject(err);
}
resolve(body);
});
})
}
const createReport = (params, recordType, headers) => {
return new Promise((resolve, reject) => {
request({
url: config.ad_url + '/v1/' + recordType + '/report',
method: "POST",
headers: headers,
json: params
}, (err, httpResponse, body) => {
if (err) {
reject(err);
}
resolve(body);
});
})
}
const getReport = (reportId, headers) => {
return new Promise((resolve, reject) => {
request({
url: config.ad_url + '/v1/reports/' + reportId,
method: "GET",
headers: headers
}, (err, httpResponse, body) => {
if (err) {
reject(err);
}
resolve(body);
});
})
}
function download(fileUrl, apiPath, callback) {
var url = require('url'),
http = require('http'),
p = url.parse(fileUrl),
timeout = 10000;
var file = fs.createWriteStream(apiPath);
var timeout_wrapper = function (req) {
return function () {
console.log('abort');
req.abort();
callback("File transfer timeout!");
};
};
console.log('before');
var request = http.get(fileUrl).on('response', function (res) {
console.log('in cb');
var len = parseInt(res.headers['content-length'], 10);
var downloaded = 0;
res.on('data', function (chunk) {
file.write(chunk);
downloaded += chunk.length;
process.stdout.write("Downloading " + (100.0 * downloaded / len).toFixed(2) + "% " + downloaded + " bytes" + isWin ? "\033[0G" : "\r");
// reset timeout
clearTimeout(timeoutId);
timeoutId = setTimeout(fn, timeout);
}).on('end', function () {
// clear timeout
clearTimeout(timeoutId);
file.end();
console.log(file_name + ' downloaded to: ' + apiPath);
callback(null);
}).on('error', function (err) {
// clear timeout
clearTimeout(timeoutId);
callback(err.message);
});
});
// generate timeout handler
var fn = timeout_wrapper(request);
// set initial timeout
var timeoutId = setTimeout(fn, timeout);
}
const recordType = 'productAds';
const campaignType = 'sponsoredProducts';
const reportDate = moment().format('YYYYMMDD');
const metrics = 'campaignId,sku,currency,attributedConversions1d';
const reportParam = {
campaignType: campaignType,
// segment: "query",
reportDate: reportDate,
metrics: metrics
}
auth().then(res => {
const headers = {
'Content-Type': 'application/json',
'Authorization': config.token_type + ' ' + res.access_token,
'Amazon-Advertising-API-Scope': '196354651614',
}
// getProfiles(headers).then(res => {})
const reportId = 'amzn1.clicksAI.1.p.5B057EE.d41f-e5-4a-a8-699e712';
getReport(reportId, headers).then(res => {
console.log(res)
const file = JSON.parse(res);
// console.log(file.location )
// download
request({
url: file.location,
method: 'GET',
headers: {
'Authorization': headers.Authorization,
'Amazon-Advertising-API-Scope': '196335474',
'Allow': 'GET, HEAD, PUT, DELETE'
}
}, (err, response, body) => {
console.log('donwloading', body)
})
})
createReport(reportParam, recordType, headers).then(res => {
const reportId = 'amzn1.clicksI.v1.p7.5B07EE.d1541f-e5-eb-a68-6996e712';
})
})
app.listen(3000, function(err) {
console.log("server is running at localhost:3000");
})
This is caused by sending the "Authorization" header to S3 and it doesn't want that. The reason is because the /download response from the API is a 307 redirect so the client you are using is redirecting with full API headers. To get around this, prevent your client from following redirects and parse the S3 location from the header of the response instead.
I'm trying to send a file to another node.js service. So for that i'm using http and form-data modules.
This is the code i wrote
function uintToString(uintArray) {
return String.fromCharCode.apply(null, new Uint8Array(uintArray));
}
function (file) {
var data = uintToString(file.buffer);
//
var crlf = "\r\n",
boundaryKey = Math.random().toString(16),
boundary = `--${boundaryKey}`;
delimeter = `${crlf}--${boundary}`,
preamble = "", // ignored. a good place for non-standard mime info
epilogue = "",
headers = [
'Content-Disposition: form-data; name="file"; filename="' + name + '"' + crlf
],
closeDelimeter = `${delimeter}--`,
multipartBody = Buffer.concat(
new Buffer(preamble + delimeter + crlf + headers.join('') + crlf),
data,
new Buffer(closeDelimeter + epilogue)
);
var options = {
host: 'localhost',
port: 3000,
method: 'POST',
path: '/data/get',
headers: {
'Content-Type': 'multipart/form-data; boundary=' + boundary,
'Content-Length': formData._valueLength
}
};
//make request
return httpsRequest(formData, options)
.then((result) => {
console.log(result);
}, (err) => {
console.log(err);
});
};
function httpsRequest(data, options) {
return new Promise(function (resolve, reject) {
// request object
var req = https.request(options, function (res) {
var result = '';
res.on('data', function (chunk) {
result += chunk;
});
res.on('end', function () {
console.log("https end result - " + result);
resolve(result);
});
res.on('error', function (err) {
reject(err);
})
});
// req error
req.on('error', function (err) {
reject(err);
});
//send request witht the postData form
req.write(data);
req.end();
});
}
It is giving ""list" argument must be an Array of Buffers" this error. It looks like something is wrong on the httpsRequest function.
Don't reinvent the wheel, needle/request can do this for you. If you want to promisify things, use bluebird
const Promise = require('bluebird')
const needle = Promise.promisifyAll(require('needle'))
function send (file) {
let url = 'https://localhost/data/get'
let data = {
zip_file: {
buffer : file.buffer,
filename : name,
content_type : 'application/octet-stream'
}
}
return needle.postAsync(url, data, { multipart: true })
}
I am writing a simple HTTP 'ping' function that is being periodically executed using AWS Lambda. It uses four asynchronous functions: http.get, S3.getObject, S3.putObject, and nodemailer.sendMail. Each seems to have a slightly different callback model.
After reading about promises, I spent way too much time trying to convert the following code to use Q promises and failed miserably.
For my own education and hopefully that of others, I was hoping someone could help me convert this to using promises (doesn't have to be Q):
'use strict';
var http = require('http');
var nodemailer = require('nodemailer');
var AWS = require('aws-sdk');
var s3 = new AWS.S3( { params: { Bucket: 'my-bucket' } } );
exports.handler = (event, context, callback) => {
var lastStatus;
var options = {
host: event.server.host,
port: event.server.port ? event.server.port : 80,
path: event.server.path ? event.server.path : '',
method: event.server.method ? event.server.method : 'HEAD',
timeout: 5000
};
var transporter = nodemailer.createTransport({
host: event.mail.host,
port: event.mail.port ? event.mail.port : 587,
auth: {
user: event.mail.user,
pass: event.mail.pass
}
});
var d = new Date();
var UTCstring = d.toUTCString();
// email templates
var downMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda DOWN alert: SITE (' + event.server.host + ') is DOWN',
text: 'LambdaAlert DOWN:\r\nSITE (' + event.server.host + ') is DOWN as at ' + UTCstring + '.'
};
var upMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda UP alert: SITE (' + event.server.host + ') is UP',
text: 'LambdaAlert UP:\r\nSITE (' + event.server.host + ') is UP as at ' + UTCstring + '.'
};
// Run async chain to ensure that S3 calls execute in proper order
s3.getObject( { Key: 'lastPingStatus' }, (err, data) => {
// get last status from S3
if (err) { lastStatus = "UP"; } else {
lastStatus = data.Body.toString();
console.log("Last observed status: " + lastStatus);
}
http_request(options, lastStatus);
});
function http_request(requestOptions, lastStatus) {
var req = http.request(requestOptions, function(res) {
if (res.statusCode == 200) {
if (lastStatus == "DOWN") {
console.log('Email up notice sending...');
transporter.sendMail(upMail, function(error, info) {
if (error) {
console.log("ERROR: " + error);
callback(null, "ERROR: " + error);
} else {
console.log('No further details available.');
callback(null, 'Up message sent');
}
});
}
s3.putObject({ Key: 'lastPingStatus', Body: 'UP', ContentType: 'text/plain' }, (error, data) => { console.log("Saved last state as UP"); });
callback(null, 'Website is OK.');
}
});
req.on('error', function(e) {
if (lastStatus == "UP") {
console.log('Email down notice sending...');
transporter.sendMail(downMail, function(error, info) {
if (error) {
console.log("ERROR: " + error);
callback(null, "ERROR: " + error);
} else {
console.log('No further details available.');
callback(null, 'Down message sent');
}
});
s3.putObject({ Key: 'lastPingStatus', Body: 'DOWN', ContentType: 'text/plain' }, (error, data) => { console.log("Saved last state as DOWN"); });
callback(null, 'Website is DOWN.');
}
});
req.end();
}
};
EDIT: First attempt at writing using promises:
'use strict';
var http = require('http');
var nodemailer = require('nodemailer');
var AWS = require('aws-sdk');
var s3 = new AWS.S3( { params: { Bucket: 'lambda-key-storage' } } );
exports.handler = (event, context, callback) => {
var lastStatus;
var options = {
host: event.server.host,
port: event.server.port ? event.server.port : 80,
path: event.server.path ? event.server.path : '',
method: event.server.method ? event.server.method : 'HEAD',
timeout: 5000
};
var transporter = nodemailer.createTransport({
host: event.mail.host,
port: event.mail.port ? event.mail.port : 587,
auth: {
user: event.mail.user,
pass: event.mail.pass
}
});
var d = new Date();
var UTCstring = d.toUTCString();
// email templates
var downMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda DOWN alert: SITE (' + event.server.host + ') is DOWN',
text: 'LambdaAlert DOWN:\r\nSITE (' + event.server.host + ') is DOWN as at ' + UTCstring + '.'
};
var upMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda UP alert: SITE (' + event.server.host + ') is UP',
text: 'LambdaAlert UP:\r\nSITE (' + event.server.host + ') is UP as at ' + UTCstring + '.'
};
var myProm = new Promise(function(resolve, reject) {
console.log("called 1");
s3.getObject( { Key: 'lastPingStatus' }, (err, data) => {
// get last status from S3
if (err) {
resolve("UP");
} else {
resolve(data.Body.toString());
}
});
})
.then(function(lastStatus) {
console.log("called 2");
console.log("Last observed status: " + lastStatus);
var req = http.request(options, function(res) {
resolve(res.statusCode);
});
req.on('error', function(e) {
reject(e);
});
req.end();
return "??";
})
.then(function(statusCode) {
console.log("called 3");
if (statusCode == 200) {
if (lastStatus == "DOWN") {
console.log('Email up notice sending...');
resolve("upTrigger");
} else {
resolve("upNoTrigger");
}
s3.putObject({ Key: 'lastPingStatus', Body: 'UP', ContentType: 'text/plain' }, (err, data) => { console.log("Saved last state as UP"); });
callback(null, 'Website is OK.');
}
})
.catch(function(err){
console.log("called 3 - error");
// Send mail notifying of error
if (lastStatus == "UP") {
console.log('Email down notice sending...');
resolve("downTrigger");
s3.putObject({ Key: 'lastPingStatus', Body: 'DOWN', ContentType: 'text/plain' }, (error, data) => { console.log("Saved last state as DOWN"); });
callback(null, 'Website is DOWN.');
return("downTrigger");
} else {
return "downNoTrigger";
}
})
.then(function(trigger) {
console.log("called 4");
if (trigger == "upTrigger") {
transporter.sendMail(upMail, (error, info) => {
if (error) {
console.log("ERROR: " + error);
callback(null, "ERROR: " + error);
} else {
console.log('Up message sent.');
callback(null, 'Up message sent');
}
});
} else if (trigger == "downTrigger") {
transporter.sendMail(downMail, (error, info) => {
if (error) {
console.log("ERROR: " + error);
callback(null, "ERROR: " + error);
} else {
console.log('Down message sent.');
callback(null, 'Down message sent');
}
});
}
console.log("Outcome of ping was: ", trigger);
});
};
This doesn't quite work. The result logs are:
called 1
called 2
Last observed status: UP
called 3
called 4
Outcome of ping was: undefined
ReferenceError: resolve is not defined
Converting your typical async function to a promise is pretty straight forward. I'd rather try and demonstrate how to convert it than write the code as you don't learn anything from that.
Usually with node you'll have something that looks similar to this:
doSomethingAsync(callback);
function doSomethingAsync(callback){
var err, result;
// Do some work
...
callback(err, result);
}
function callback(err, result){
if(err){
// Handle error
} else{
// Success so do something with result
}
}
A promise wrapping an async function generally looks something like this:
var myProm = new Promise(function(resolve, reject){
doSomethingAsync(function(err, result){
if(err){
reject(err);
} else{
resolve(result)
}
});
})
.then(function(result){
// Success so do something with result
console.log("Success:", result)
})
.catch(function(err){
// Handle error
console.log("Error: ", err);
})
.then(function(result){
// Where's my result? - result == undefined as we didn't return anything up the chain
console.log("I always execute but result is gone", result)
})
To pass the result down the chain to our "always then" method we need to return a promise or a value:
var myProm = new Promise(function(resolve, reject){
doSomethingAsync(function(err, result){
if(err){
reject(err);
} else{
resolve(result)
}
});
})
.then(function(result){
// Success so do something with result
console.log("Success:", result)
return result;
})
.catch(function(err){
// Handle error
console.log("Error: ", err);
return err;
})
.then(function(result){
// The err/result now gets passed down the chain :)
console.log("Oh there it is", result)
})
I think that using the above patterns should cater to most of the async methods and events in your code example if any particular ones are giving you trouble drop a comment in and I'll try to cover those specific examples.
Here's an attempt at converting it over to promises - I'm pretty tired so apologies about any mess or mistakes - also there's still plenty of cleanup that could be done.
Essentially what I've done is try to break down the code into tasks and wrap each of those tasks in a promise. That way we can resolve/reject and chain them as needed.
'use strict';
var http = require('http');
var nodemailer = require('nodemailer');
var AWS = require('aws-sdk');
var s3 = new AWS.S3( { params: { Bucket: 'my-bucket' } } );
exports.handler = function (event, context, callback) {
var lastStatus;
var options = {
host: event.server.host,
port: event.server.port ? event.server.port : 80,
path: event.server.path ? event.server.path : '',
method: event.server.method ? event.server.method : 'HEAD',
timeout: 5000
};
var transporter = nodemailer.createTransport({
host: event.mail.host,
port: event.mail.port ? event.mail.port : 587,
auth: {
user: event.mail.user,
pass: event.mail.pass
}
});
var d = new Date();
var UTCstring = d.toUTCString();
// email templates
var downMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda DOWN alert: SITE (' + event.server.host + ') is DOWN',
text: 'LambdaAlert DOWN:\r\nSITE (' + event.server.host + ') is DOWN as at ' + UTCstring + '.'
};
var upMail = {
from: event.mail.from,
to: event.mail.to,
subject: 'Lambda UP alert: SITE (' + event.server.host + ') is UP',
text: 'LambdaAlert UP:\r\nSITE (' + event.server.host + ') is UP as at ' + UTCstring + '.'
};
// Run async chain to ensure that S3 calls execute in proper order
function getLastPingStatus(){
return new Promise(function(resolve, reject){
s3.getObject( { Key: 'lastPingStatus' }, function(err, data) {
// get last status from S3
if (err) {
lastStatus = "UP";
reject(lastStatus)
} else {
lastStatus = data.Body.toString();
resolve(lastStatus);
console.log("Last observed status: " + lastStatus);
}
});
})
}
getLastPingStatus()
.then(httpRequest)
.catch(httpRequest); // Otherwise a reject will throw an error
function sendMail(mail, status){ // status = "up" or "down" -
return new Promise(function(resolve, reject){
transporter.sendMail(mail, function(error, info) {
if (error) {
console.log("ERROR: " + error);
reject(null, "ERROR: " + error);
} else {
console.log('No further details available.');
resolve(null, status + ' message sent');
}
});
});
}
function saveStatus(up) {
return new Promise(function (resolve, reject) {
var saveOptions,
message;
// I didn't bother refactoring these as promises at they do the same thing regardless of outcome
if(up){
saveOptions = [{ Key: 'lastPingStatus', Body: 'UP', ContentType: 'text/plain' }, function(error, data) { console.log("Saved last state as UP"); }];
message = 'Website is OK.';
} else{
saveOptions = [{ Key: 'lastPingStatus', Body: 'DOWN', ContentType: 'text/plain' }, function(error, data) { console.log("Saved last state as DOWN"); }];
message = 'Website is DOWN.';
}
s3.putObject.apply(this, saveOptions);
callback(null, message);
});
}
function httpRequest(lastStatus) {
var requestOptions = options;
return new Promise (function (resolve, reject){
var req = http.request(requestOptions, function(res) {
if (res.statusCode == 200) {
if (lastStatus == "DOWN") {
console.log('Email up notice sending...');
sendMail(upMail, "Up")
.then(resolve, reject)
.then(saveStatus(true))
.then(callback)
}
}
});
req.on('error', function(e) {
if (lastStatus == "UP") {
console.log('Email down notice sending...');
sendmail(downMail, "Down")
.then(resolve, reject)
.then(saveStatus(false))
.then(callback)
}
});
req.end();
});
}
};
The AWS-SDK supports native promises, for all services. Some need additional parameters to return properly, like Lambda.invoke().
You would essentially do
s3.putObject({ Key: 'key', Bucket: 'bucket' }).promise()
.then(data => {
// this is the same as the data callback parameter
})
.catch(error => {
// handle your error
})
Or, you could use async/await:
const file = await s3.getObject(params).promise()
// do things with the result
For quick access to the actual file (and not metadata):
const file = JSON.parse(await s3.getObject(params).promise().then(res => res.Body));
https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/using-promises.html
In order to "promisify" callback function, imho, the easiest and cleaner way is to use bluebird. You just don't want to write glue code in order to simplify your code, it's counter productive (and it's error prone).
From the doc :
var Promise = require("bluebird");
var readFile = Promise.promisify(require("fs").readFile);
readFile("myfile.js", "utf8").then(function(contents) {
return eval(contents);
}).then(function(result) {
console.log("The result of evaluating myfile.js", result);
}).catch(SyntaxError, function(e) {
console.log("File had syntax error", e);
//Catch any other error
}).catch(function(e) {
console.log("Error reading file", e);
});
After reading slaughtr's answer I decided to do it like this, for doing some data saving when pressing the AWS IoT button:
var AWS = require("aws-sdk");
var iot = new AWS.Iot();
exports.handler = (event, context, callback) => {
iot.listThings({
attributeName: 'dsn',
attributeValue: event.serialNumber,
maxResults: 1
})
.promise()
.then(response => {
return iot.listThingGroupsForThing({thingName: response.things[0].thingName}).promise();
})
.then(groupsList => insertRecordIntoDDB(date, serialNumber, groupsList.thingGroups[0].groupName))
.catch(err => console.log(err))
};
and shortly after I decided to compress it even further with async/await
exports.handler = async (event, context, callback) => {
var eventText = JSON.stringify(event, null, 2);
var thingsList = await iot.listThings({
attributeName: 'dsn',
attributeValue: event.serialNumber,
maxResults: 1
}).promise()
var groupsList = await iot.listThingGroupsForThing({
'thingName': thingsList.things[0].thingName
}).promise();
insertRecordIntoDDB(date, serialNumber, groupsList.thingGroups[0].groupName)
};
I'm still fairly new at this async programming stuff so I'm not sure what I should like the most. Promise chaining can get a bit spaghetti-like, while async await only helps mask all that into something that's easier to comprehend
Using node http promisified to call external api in aws lambda.
exports.handler = async (event) => {
return httprequest().then((data) => {
const response = {
statusCode: 200,
body: JSON.stringify(data),
};
return response;
});
};
function httprequest() {
return new Promise((resolve, reject) => {
const options = {
host: 'jsonplaceholder.typicode.com',
path: '/todos',
port: 443,
method: 'GET'
};
const req = http.request(options, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error('statusCode=' + res.statusCode));
}
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
} catch(e) {
reject(e);
}
resolve(body);
});
});
req.on('error', (e) => {
reject(e.message);
});
// send the request
req.end();
});
}
I have written a nice little error reporting middleware, that sits after all the GET and POST handling (after app.use(app.router); ). See below.
This works great for simple quick GET and POST that goes to the PostGIS database etc.
But I have one POST request that is designed to create a bunch of directories, a number of files, and then spawn 1 -> 8 child_processes tasks
childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root },
All that setup does not take much time (less than a second, and it is all async (I use the async library at one point to sequence 5 steps (see below).
My issue is error handling. Right now I return a response immediately before creating all the files and doing all the steps. This means that next(err) is not working as expected. What is a good paradigm for reporting back the errors? I am using WINSTON to log errors [logger.log() ], but should I just log the errors on the server, or should I also report it to the original request. here is the current post request (and remember, I would have to keep the rest, and req and next object around for quite a while to be able to call next(err).
exports.build = function (req, res, next) {
var config = global.app.settings.config;
var jobBatch = groupJobs(req.body.FrameList);
var ticket = tools.newGuid("", true);
var fileCount = req.body.FrameList.length * nitfMultiplier;
var ts = timespan.fromSeconds(fileCount / config.TileRate);
var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
res.set({ 'Content-Type': 'application/json; charset=utf-8' });
res.send({ ticket: ticket, maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
jobBatchRoot(req, res, jobBatch, config, ticket, next);
};
jobBatchRoot() (I will then go off and do a lot of processing, I did not include all that code.
exports.bugs = function (err, req, res, next) {
global.app.settings.stats.errors += 1;
if (err.code == undefined) {
err.code = 500;
err.message = "Server Error";
}
res.status(err.code);
logger.log('error', '%s url: %s status: %d \n', req.method, req.url, err.code, { query: req.query, body: req.body, message: err.message, stack: err.stack });
var desc = req.method + " " + req.url;
var body = util.format("%j", req.body);
var query = util.format("%j", req.query);
var stack = err.stack.split('\n');
res.format({
text: function () {
res.send(util.format("%j", { title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: err.stack, body: body}));
},
html: function () {
query = tools.pretty(req.query);
res.render('error', { title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: stack, body: body });
},
json: function () {
res.send({ title: err.message, code: err.code, desc: desc, query: query, message: err.message, stack: err.stack, body: body });
}
});
};
Perhaps I should be re-factoring this (maybe object oriented), anyway I thought I would post the full module here and all I am looking for is a few tips on structure, best practices.
var util = require('util');
var query = require("pg-query");
var timespan = require('timespan');
var _ = require('lodash');
var path = require('path');
var fs = require('fs');
var query = require("pg-query");
var async = require("async");
var childProcess = require("child_process");
var tools = require("../tools/tools");
var nitfMultiplier = 99;
var manifestVersionID = 5;
exports.setup = function (app) {
};
exports.estimate = function (req, res, next) {
var config = global.app.settings.config;
var fileCount = req.body.FrameList.length * nitfMultiplier;
var ts = timespan.fromSeconds(fileCount / config.TileRate);
var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
res.set({ 'Content-Type': 'application/json; charset=utf-8' });
res.send({ ticket: "Estimate", maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
};
exports.build = function (req, res, next) {
var config = global.app.settings.config;
var jobBatch = groupJobs(req.body.FrameList);
var ticket = tools.newGuid("", true);
var fileCount = req.body.FrameList.length * nitfMultiplier;
var ts = timespan.fromSeconds(fileCount / config.TileRate);
var estimate = ts.hours + ":" + tools.pad(ts.minutes, 2) + ":" + tools.pad(ts.seconds, 2);
res.set({ 'Content-Type': 'application/json; charset=utf-8' });
res.send({ ticket: ticket, maxTiles: fileCount, timeEstimate: estimate, tileRate: config.TileRate, wwwURL: config.WWWUrl });
jobBatchRoot(req, res, jobBatch, config, ticket, next);
};
function groupJobs(list) {
var jobBatch = {};
_.forEach(list, function (obj) {
if (jobBatch[obj.type] == undefined) {
jobBatch[obj.type] = [];
}
jobBatch[obj.type].push(obj);
});
return jobBatch;
};
function jobBatchRoot(req, res, jobBatch, config, ticket, next) {
var batchRoot = path.join(config.JobsPath, ticket);
fs.mkdir(batchRoot, function (err) {
if (err) return next(err);
var mapInfoFile = path.join(batchRoot, "MapInfo.json");
var mapInfo = {
Date: (new Date()).toISOString(),
Version: manifestVersionID,
Zoom: req.body.Zoom,
CenterLat: req.body.CenterLat,
CenterLon: req.body.CenterLon
};
fs.writeFile(mapInfoFile, tools.pretty(mapInfo), function (err) {
if (err) return next(err);
spawnJobs(req, res, batchRoot, mapInfo, config, ticket, jobBatch, next);
});
});
};
function spawnJobs(req, res, root, mapInfo, config, ticket, jobBatch, next) {
_.forEach(jobBatch, function (files, key) {
var job = {
req: req,
res: res,
type: key,
files: files,
batchRoot: root,
mapInfo: mapInfo,
config: config,
ticket: ticket,
missingFiles: [],
run: true,
next: next
};
setup(job);
});
};
function setup(job) {
job.root = path.join(job.batchRoot, job.type);
job.fmeParamFile = path.join(job.root, "fmeParameters.txt");
job.fmeWorkSpace = path.join(job.config.LibrarianPath, "TileBuilder.fmw");
job.fmeLogFile = path.join(job.root, "jobLog.log");
job.errorLog = path.join(job.root, "errorLog.log");
job.jobFile = path.join(job.root, "jobFile.json");
job.manifestFile = path.join(job.root, "manifest.json");
async.series({
one: function (callback) {
maxZoom(job, callback);
},
two: function (callback) {
fs.mkdir(job.root, function (err) {
if (err) return job.next(err);
callback(null, "Job Root Created");
});
},
three: function (callback) {
makeParamFile(job, callback);
},
four: function (callback) {
delete job.req;
delete job.res;
fs.writeFile(job.jobFile, tools.pretty(job), function (err) {
if (err) return job.next(err);
callback(null, "Wrote Job File");
});
},
five: function (callback) {
runJob(job, callback);
},
six: function (callback) {
tileList(job, callback);
},
seven: function (callback) {
finish(job, callback);
},
},
function (err, results) {
if (err) return job.next(err);
console.log(tools.pretty(results));
});
}
function maxZoom(job, callback) {
var qString = util.format('SELECT type, "maxZoom" FROM portal.m_type WHERE type=\'%s\'', job.type);
query(qString, function (err, rows, result) {
if (err) {
var err = new Error(queryName);
err.message = err.message + " - " + qString;
err.code = 400;
return job.next(err);
}
job.maxZoom = rows[0].maxZoom - 1; // kludge for 2x1 root layer in leaflet
return callback(null, "Got MaxZoom");
});
}
function makeParamFile(job, callback) {
var text = util.format("%s\n", job.fmeWorkSpace);
text += util.format("--OutputDir %s\n", job.root);
text += util.format("--LogFile %s\n", job.fmeLogFile);
var source = "";
_.forEach(job.files, function (file) {
var path = ('development' == process.env.NODE_ENV) ? file.path.replace(job.config.SourceRootRaw, job.config.SourceRoot) : file.path;
if (fs.existsSync(path)) {
source += wrap(path, '\\"') + " ";
}
else {
job.missingFiles.push(path);
}
});
source = wrap(wrap(source, '\\"'), '"');
text += "--Sources " + source;
if (job.missingFiles.length == job.files.length) job.run = false;
fs.writeFile(job.fmeParamFile, text, function (err) {
if (err) return job.next(err);
return callback(null, "Wrote Paramaters File");
})
};
function wrap(content, edge) {
return edge+content+edge;
}
function runJob(job, callback) {
if (!job.run) return callback(null, "Skipped JOB, no files");
childProcess.execFile(job.config.FMEPath, ["PARAMETER_FILE", job.fmeParamFile], { cwd: job.root },
function (err, stdout, stderr) {
if (err) return job.next(err);
job.stdout = stdout;
job.stderr = stderr;
var bar = "\n--------------------------------------------------------------------------------------------------------\n";
var results = util.format("%s STDOUT: \n %s%s STDERR: \n %s", bar, job.stdout, bar, job.stderr);
fs.appendFile(job.fmeLogFile, results, function (err) {
return callback(err, "FME JOB " + job.type + " run completed");
});
});
}
function tileList(job, callback) {
var tiles = [];
var byteCount = 0;
fs.readdir(job.root, function (err, files) {
if (err) {
logger.log('error', 'tileList directory read: %s \n', job.root, { message: err.message, stack: err.stack });
return job.next(err);
}
async.each(files, function (file, done) {
var fileName = file.split(".");
fs.lstat(job.root + "\\" + file, function (err, stats) {
if (!err && stats.isFile() && (fileName[1] == "png")) {
tiles.push({ id: fileName[0], size: stats.size });
byteCount += stats.size;
};
done(null);
});
}, function (err) {
job.tileList = tiles;
job.byteCount = byteCount;
return callback(null, "got tile list");}
);
});
}
function finish(job, callback) {
var manifest = {
Date: (new Date()).toISOString(),
Version: manifestVersionID,
MaxZoom: job.maxZoom,
Class: "OVERLAY",
FileCount: job.tileList.length,
Size: job.byteCount / (1024 * 1024), // Mbytes
files: job.tileList
};
fs.writeFile(job.manifestFile, tools.pretty(manifest), function (err) {
if (err) {
logger.log('error', 'manifest write: %s \n', job.manifestFile, { message: err.message, stack: err.stack });
return job.next(err);
}
return callback(null, "JOB " + job.type + " completed");
});
}
I went and re-factored this. I created a module, with module.exports = function(..) {...}
and then added lots of state and methods to create a class. That contains the Job definition. So now I create the top level directories, return a response, and spawn the sub jobs. They all run async after the express response. But they should not get errors, and if they do, then I use WINSTON to log them in the server, and also return a job done information to the user when all the builds are done.