I am having issues with exporting in node - node.js

I am trying to make an api endpoint for data coming from dynamoDB. I believe that I have everything connected but when I run postman to check the api (api/db) it doesn't recognize the functions from the db.js in the db.js (for routes). I have run a test on api/test and am getting the information back. Here is the code from both files:
1. This scans the database and I'm trying to export it to another file.
var AWS = require('aws-sdk');
var params = {
TableName : "iotbuttonsn",
//KeyConditionExpression: "serialNumber =:serialNumber",
//ExpressionAttributeValues: {
// ":serialNumber":"*"
//},
ScanIndexForward: false,
Limit: 3,
Select: 'ALL_ATTRIBUTES'
};
AWS.config.update({
region: "us-east-1",
endpoint: "https://dynamodb.us-east-1.amazonaws.com"
});
var docClient = new AWS.DynamoDB.DocumentClient();
var getDatabase = (function(){
return {
scanDB: function(){
docClient.scan(params, onScan);
var onScan = function(err, data){
if (err) {
console.log(err.message);
} else {
console.log('scan success');
len = data.Items.length;
for (n=0; n<len; n++) {
clickTypes[n] = data.Items[n].payload.clickType;
serialNums[n] = data.Items[n].serialNumber;
}
}
};
},
clickTypes: [],
serialNums: []
};
})();
module.exports = getDatabase;
2. This is where I'm trying to input but db.scanDB() isn't working:
var router = require('express').Router();
var db = require('../routes/db.js');
router.get('/', function(req, res){
db.scanDB();
buttons =
[
iot_buttonOne = {
serialNum: db.serialNum[0],
clickType: db.clickTypes[0]
},
iot_buttonTwo = {
serialNum: db.serialNum[1],
clickType: db.clickTypes[1]
}
]
.then(
function scanSuccess(data){
res.json(data);
},
function scanError(err){
res.send(500, err.message);
}
);
});
module.exports = router;

Change your db.scan() function to properly return an asynchronous result:
// db.js
module.exports = {
scanDB: function(cb){
docClient.scan(params, function(err, data) {
var clickTypes = [], serialNums = [];
if (err) {
console.log(err.message);
cb(err);
} else {
console.log('scan success');
len = data.Items.length;
for (n=0; n<len; n++) {
clickTypes[n] = data.Items[n].payload.clickType;
serialNums[n] = data.Items[n].serialNumber;
}
cb(null, {clickTypes, serialNums});
}
});
}
};
Then, when you use it:
var db = require('../routes/db.js');
db.scanDB(function(err, data) {
if (!err) {
// data.clickTypes
// data.serialNums
} else {
// process error
}
});
It really does not good to put the scanDB result on the DB object the way you were doing because there was no way for the caller to know when the asynchronous operation was done. So, since you have to provide some notification for the caller when the async operation is done (either via callback or promise), you may as well just pass the results there too.
Also, the .then() handler in your router.get(...) handler does not belong there. I don't know why it's there at all as there are no promises involved in the code you show. Perhaps a cut/paste error when creating the question?
Note, I removed the IIFE from your getDatabase() definition since there was no benefit to it other than a little more complicated code.

Related

node.js creating custom module with function to return a value

I've never had to do this before. I've created myModule.GetRecord() and I can see the recordset successfully has the expected record.
However, ret in router.get() is undefined.
I believe because I need to catch the returned value in a callback.
I don't know how to define this module/function for a callback (if that is what I am appropriately looking to do) or how to call this function for a callback.
I've tried a few different things i.e. the typical GetRecord(ret, function () { ... }); But didn't see anything that appeared to work. And I read a bunch on the line but didn't find what I believe I'm looking for.
I really don't care to much about how I get there, but all I'm really trying to do is have mm.GetRecord()'s returned value in some usable form in the router.get()
--- myModulefile.js ---
'use strict';
module.exports = {
GetRecord: function (id) {
var sql = require('mssql');
sql.connect({ user: 'sa', ... database: 'name' }, function (err) {
if (err) { console.log(err); return null; }
var cmd = new sql.Request();
cmd.query('select * from Records where id = ' + id, function (err, rs) {
if (err) { console.log(err); return null; }
if (rs.recordset.length > 0) {
console.log('rs[0]', rs.recordset[0]);
return rs.recordset[0];
} else {
return null;
}
});
});
}
};
--- myRouter.js ----
const express = require('express');
const router = express.Router();
const mm = require('../myModule');
router.get('/:id', function (req, res) {
var id = req.params.id;
var ret = mm.GetRecord(id)
console.log('ret', ret);
if (ret == null)
ret = JSON.stringify({ ID: -1, f1: 'unknown' });
res.send(ret);
});
module.exports = router;
Of course I find the answer after having placed the question on here.
GetRecord() has to be defined with a parameter that recieves the callback passed to it. And the callback paramater's variable fn is used in place of return
GetRecord: function (id, fn) {
var sql = require('mssql');
sql.connect({ user: 'sa', ... database: 'name' }, function (err) {
if (err) { console.log(err); fn(null); }
var cmd = new sql.Request();
cmd.query('select * from Records where id = ' + id, function (err, rs) {
if (err) { console.log(err); fn(null); }
if (rs.recordset.length > 0) {
console.log('rs[0]', rs.recordset[0]);
fn(rs.recordset[0]);
} else {
fn(null);
}
});
});
}
and
GetRecord(id, function(ret) {
console.log('ret', ret);
});

Gather multiple functions output on single route call and render to html in nodejs

Newbie to nodejs,trying to execute multiple functions output to html using nodejs,express and mysql as backend.Need to execute 20 functions on single routing call to combine the output of 20 functions and render as json to html.
My app.js function
var express = require('express');
var router = express.Router();
var path = require('path');
var app = express();
var todo = require('./modules/first');
var todo1 = require('./modules/second');
var connection = require('./connection');
connection.init();
app.get('/', function(req,res,next) {
Promise.all([todo.class1.getUsrCnt(),todo.class1.getTotlAmt(),todo.class1.getTotlOrdrCnt(),todo.class1.getTotlCntRcds(),todo.class1.getTotlScsRcds(),todo.class1.getTotlFailRcds(),todo.class1.getTotlAmtRcds()])
.then(function(allData) {
res.addHeader("Access-Control-Allow-Origin", "http://hostname:8183/");
res.json({ message3: allData });
});
res.send(send response to html);
})
app.get('/second', function(req,res,next) {
Promise.all([todo1.class2.getUsr........])
.then(function(allData) {
res.addHeader("Access-Control-Allow-Origin", "http://hostname:8183/");
res.json({ message3: allData });
});
res.send(send response to html);
})
var server = app.listen(8183, function(){
console.log('Server listening on port '+ server.address().port)
});
My todo.js is
var connection = require('../connection');
var data = {},obj={};
var d = new Date();
var month = d.getMonth() + 1;
var year = d.getFullYear();
obj.getUsrCnt = function getUsrCnt(callback) {
connection.acquire(function(err, con) {
con.query(query1, function(err, result) {
con.release();
data.usrs_cnt = result[0].some;
})
});
}
obj.getTotlAmt = function getTotlAmt(callback) {
connection.acquire(function(err, con) {
con.query(query2, function(err, result) {
con.release();
data.total_amt = result[0].some1;
})
});
}
obj.getTotlOrdrCnt = function getTotlOrdrCnt(callback) {
connection.acquire(function(err, con) {
con.query(query3, function(err, result) {
con.release();
data.total_orders = result[0].some2;
})
});
}
.
.
. functions go on
exports.class1 = obj;
Getting undefined in the promise all and unable to render to the html file.
Not sure about the code you wrote, but as I understand you want to call all the functions, get all the results and return back to the user?
so you can use many libraries that waits for several calls for example, promise based:
Promise.all([todo.getUsrCnt('dontcare'), todo.getTotlAmt('dontcate')])
.then(function(allData) {
// All data available here in the order it was called.
});
as for your updated code, you are not returning the data as promises, you assigning it to the local variable.
this is how your methods should look:
obj.getUsrCnt = function getUsrCnt(callback) {
var promise = new Promise(function(resolve, reject) {
connection.acquire(function(err, con) {
if(err) {
return reject(err);
}
con.query(query1, function(err, result) {
con.release();
resolve(result[0].some);
})
});
});
return promise;
}
as you can see here, I am creating a new promise and returning it in the main function.
Inside the new promise I have 2 methods: "resolve", "reject"
one is for the data and one is for errors.
so when you use the promise like this:
returnedPromise.then(function(data) {
//this data is what we got from resolve
}).catch(function(err) {
//this err is what we got from reject
});
you can see that a promise can or resolved or rejected,
do this to all the methods, and then you start seeing data

stub nodejs promise in chain to return error

I'm new to using promises in nodejs and also in testing them. I have managed to test the individual modules separately, but when it comes to testing the chain of promises, I am having some trouble. I tried following the examples found here and on the npm page for sinon-as-promised but don't seem to managed to control the flow and trigger the error in the first promise of the chain.
I am using mocha, chai and sinon for my tests with sinon-as-promised and chai-as-promised.
I am trying to test this module:
'use strict';
var mySQS = require('./modules/sqs/sqs-manager');
var sWebHook = require('./modules/webhooks/shopify/webhooks');
var main = {};
main.manageShopifyWebhook = function (params, callback) {
sWebHook.verify(params.srcHmac, params.rawBody, params.shopName.split('.myshopify.com')[0], params.productId)
.then(function(data) {
var body = {
"params": {
"productId": data.productId,
"shopName": data.shopName
},
"job": "call-update-item"
};
mySQS.create_Queue(body)
.then(mySQS.send_Message)
.then(function(result) {
callback(null, result);
})
.catch(function(error) {
callback(error, null);
});
});
};
module.exports = main;
This is the sWebHook module I want to trigger the reject callback in the main flow:
'use strict';
var crypto = require('crypto');
var nconf = require('../../../../config/nconfig');
var webHookManager = {};
webHookManager.verify = function (srcHmac, rawBody, shopName, productId) {
return new Promise(function (resolve, reject) {
rawBody = new Buffer(rawBody, 'base64');
var sharedSecret = nconf.get('SHOPIFY_CLIENT_SECRET');
var digest = crypto.createHmac('SHA256', sharedSecret).update(rawBody).digest('base64');
console.log('***** CALCULATED DIGEST *****');
console.log(digest);
console.log('***** HMAC FROM SHOPIFY *****');
console.log(srcHmac);
if (digest !== srcHmac) {
console.log('Hello');
var customError = new Error('Unauthorized: HMAC Not Verified');
reject(customError);
return false;
}
var newEvent = {
shopName: shopName,
productId: productId
};
console.log('!! WEBHOOK VERIFIED !!');
resolve(newEvent);
});
};
module.exports = webHookManager;
And these are my tests so far (which do not work):
'use strict';
var chai = require('chai');
var sinonChai = require("sinon-chai");
var expect = chai.expect;
var chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
var sinon = require('sinon');
chai.use(sinonChai);
var proxyquire = require('proxyquire').noCallThru();
var AWS = require('mock-aws');
describe('MAIN', function() {
require('sinon-as-promised');
var testedModule,
sWebHookStub,
sqsQueueStub,
sqsSendMsgStub,
callbackSpy,
fakeDataObj;
before(function() {
sWebHookStub = sinon.stub();
sqsQueueStub = sinon.stub();
sqsSendMsgStub = sinon.stub();
callbackSpy = sinon.spy();
fakeDataObj = {
srcHmac: '12345',
rawBody: 'helloworld',
shopName: 'mario-test.myshopify.com',
productId: '6789'
};
testedModule = proxyquire('../lib/main', {
'./modules/webhooks/shopify/webhooks': {
'verify': sWebHookStub
},
'./modules/sqs/sqs-manager': {
'create_Queue': sqsQueueStub,
'send_Message': sqsSendMsgStub
}
});
});
it('calling shopifyVeriWebhook returns an error', function() {
var fakeError = new Error('Error verifying webhook');
sWebHookStub.rejects(fakeError);
testedModule.manageShopifyWebhook(fakeDataObj, function() {
callbackSpy.apply(null, arguments);
});
expect(callbackSpy).has.been.called.and.calledWith(fakeError, null);
});
});
So, I ended up figuring out how to test chains of promises using sinon. For the following main module (Note: the other modules all return promises):
'use strict';
var mySQS = require('./modules/sqs/sqs-manager');
var sWebHook = require('./modules/webhooks/shopify/webhooks');
var main = {};
//#params {object} params
//#params {string} params.srcHmac
//#params {string} params.rawBody
//#params {string} params.shopName - <shop-name.myshopify.com>
//#params {string} params.productId
main.manageShopifyWebhook = function (params) {
return new Promise(function(resolve, reject) {
sWebHook.verify(params.srcHmac, params.rawBody, params.shopName.split('.myshopify.com')[0], params.productId)
.then(function(data) {
var body = {
"params": {
"productId": data.productId,
"shopName": data.shopName
},
"job": "call-update-item"
};
return mySQS.create_Queue(body);
})
.then(mySQS.send_Message)
.then(resolve)
.catch(function(err) {
reject(err);
});
});
};
module.exports = main;
The secret is to manually resolve or reject the promises and write the expectation within the callback functions of the then or catch methods (just as we would do if we were writing tests for async code using done). And we then trigger the method we want to test, saving its value to a variable. Like so:
'use strict';
var chai = require('chai');
var sinonChai = require("sinon-chai");
var expect = chai.expect;
var chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
require('sinon-as-promised');
var sinon = require('sinon');
chai.use(sinonChai);
var proxyquire = require('proxyquire').noCallThru();
describe('MAIN', function() {
require('sinon-as-promised');
var testedModule,
sWebHookStub,
sqsQueueStub,
sqsSendMsgStub,
callbackSpy,
fakeDataObj;
before(function() {
sWebHookStub = sinon.stub();
sqsQueueStub = sinon.stub();
sqsSendMsgStub = sinon.stub();
callbackSpy = sinon.spy();
fakeDataObj = {
srcHmac: '12345',
rawBody: 'helloworld',
shopName: 'mario-test.myshopify.com',
productId: '6789'
};
testedModule = proxyquire('../lib/main', {
'./modules/webhooks/shopify/webhooks': {
'verify': sWebHookStub
},
'./modules/sqs/sqs-manager': {
'create_Queue': sqsQueueStub,
'send_Message': sqsSendMsgStub
}
});
});
it('calling shopifyVeriWebhook returns an error when trying to VERIFY WEBHOOK', function() {
var fakeError = new Error('Error verifying webhook');
sWebHookStub.rejects(fakeError)().catch(function(error) {
expect(shopifyWebhook).to.eventually.equal(error);
});
var shopifyWebhook = testedModule.manageShopifyWebhook(fakeDataObj);
});
it('calling shopifyVeriWebhook returns an error when trying to CREATE SQS QUEUE', function() {
var fakeBody = {
"params": {
"productId": '1234',
"shopName": 'name'
},
"job": "call-update-item"
};
var fakeError = new Error('Error creating sqs queue');
sWebHookStub.resolves(fakeBody)().then(function(result) {
sqsQueueStub.rejects(fakeError)().catch(function(error) {
expect(shopifyWebhook).to.eventually.equal(error);
});
});
var shopifyWebhook = testedModule.manageShopifyWebhook(fakeDataObj);
});
it('calling shopifyVeriWebhook returns an error when trying to SEND SQS MESSAGE', function() {
var fakeData = {
queueUrl: '5678',
payLoad: '{"message": "Hello World"'
};
var fakeBody = {
"params": {
"productId": '1234',
"shopName": 'name'
},
"job": "call-update-item"
};
var fakeError = new Error('Error sending sqs message');
sWebHookStub.resolves(fakeBody)().then(function(result) {
sqsQueueStub.resolves(fakeData)().then(function(result) {
sqsSendMsgStub.rejects(fakeError)().catch(function(error) {
expect(shopifyWebhook).to.eventually.equal(error);
});
});
});
var shopifyWebhook = testedModule.manageShopifyWebhook(fakeDataObj);
});
it('calling shopifyVeriWebhook is SUCCESSFUL', function() {
var fakeData = {
queueUrl: '5678',
payLoad: '{"message": "Hello World"'
};
var fakeBody = {
"params": {
"productId": '1234',
"shopName": 'name'
},
"job": "call-update-item"
};
var fakeResponse = {
'message': 'success'
};
sWebHookStub.resolves(fakeBody)().then(function(result) {
sqsQueueStub.resolves(fakeData)().then(function(result) {
sqsSendMsgStub.resolves(fakeResponse)().then(function(result) {
expect(shopifyWebhook).to.eventually.equal(result);
});
});
});
var shopifyWebhook = testedModule.manageShopifyWebhook(fakeDataObj);
});
});
Bonus sample - I needed to run my code on aws lambda, and therefore needed to have a final callback. So I had the main entry point to my code in a file called lambda.js:
'use strict';
var main = require('./lib/main');
//Verifies shopify webhooks
//#params {object} event
//#params {string} event.srcHmac
//#params {string} event.rawBody
//#params {string} event.shopName - <shop-name.myshopify.com>
//#params {string} event.productId
exports.shopifyVerifyWebHook = function (event, context, callback) {
console.log('---- EVENT ----');
console.log(event);
main.manageShopifyWebhook(event)
.then(function(result) {
callback(null, result);
})
.catch(function(err) {
callback(err, null);
});
};
And for this I needed to control the result of the promises and make sure the callback was called with either an error or a success message.
The premiss is the same.
describe('LAMBDA', function() {
var testedModule,
mainShopStub,
callbackSpy,
mainModule,
fakeEvent;
before(function() {
callbackSpy = sinon.spy();
fakeEvent = {
srcHmac: '12345',
rawBody: 'helloworld',
shopName: 'mario-test.myshopify.com',
productId: '6789'
};
testedModule = require('../lambda');
mainModule = require('../lib/main');
mainShopStub = sinon.stub(mainModule, 'manageShopifyWebhook');
});
after(function() {
mainShopStub.restore();
});
it('calling shopifyVerifyWebHook returns an error', function() {
var fakeError = new Error('Error running lambda');
mainShopStub.rejects(fakeError);
mainShopStub().catch(function (error) {
expect(callbackSpy).has.been.called.and.calledWith(error, null);
});
testedModule.shopifyVerifyWebHook(fakeEvent, {}, function() {
callbackSpy.apply(null, arguments);
});
});
it('calling shopifyVerifyWebHook return a data object', function() {
var fakeObj = {message: 'success'};
mainShopStub.resolves(fakeObj);
mainShopStub().then(function (result) {
expect(callbackSpy).has.been.called.and.calledWith(null, result);
});
testedModule.shopifyVerifyWebHook(fakeEvent, {}, function() {
expected.resolves(fakeObj);
callbackSpy.apply(null, arguments);
});
});
});
Before getting into how to test multiple promises and validating errors, there is a much larger problem with your code.
manageShopifyWebhook() is constructed using an anti-pattern of promises which is, you're using a callback structure to return your promise value instead of returning your promise directly. If you do this, you're taking away a large benefit of promises, direct chain for error handling. Furthermore, you won't be able to use sinon-as-promised and chai-as-promised since they expect a Promise/thenable to be returned.
However, this is a rather quick fix in your code, by simply returning the promise created by sWebHook.verify():
main.manageShopifyWebhook = function (params) {
// Return the promise directly
// the final return will be returned to the original caller of manageShopifyWebhook
return sWebHook.verify(params.srcHmac, params.rawBody, params.shopName.split('.myshopify.com')[0], params.productId)
.then(function(data) {
var body = {
"params": {
"productId": data.productId,
"shopName": data.shopName
},
"job": "call-update-item"
};
return mySQS.create_Queue(body);
})
.then(mySQS.send_Message)
.then(function(result) {
return result;
})
.catch(function(err) {
// In reality you can let error propagate out here
// if you don't need to do anything special with it and let
// the promise just return the error directly
// I've only done this so we can return 'Error Verifying Webhook' as an error from the promise returned by manageShopifyWebhook()
return Promise.reject(new Error('Error verifying webook'));
});
});
};
Now that manageShopfiyWebhook() is returning a promise, you can use the two as-promised test libraries.
For chai-as-promised you need to convert your expect() to look for a promise using the chain eventually and then you can use rejectedWith() to validate the Error/Error Message.
To validate multiple promises tests you can use Promise.all() and pass in all your promise returning assertions and return the outcome of Promise.all() to your mocha it().
I don't use sinon but the above should've given you enough direction to figure out how to use this pattern with sinon-as-promised as well since it will work for any Promise returning testing library.
it('calling shopifyVeriWebhook returns an error', function() {
var fakeError = new Error('Error verifying webhook');
let shopifyWebhook = testedModule.manageShopifyWebhook(fakeDataObj);
return Promise.all([
expect(shopifyWebhook).to.eventually.be.rejectedWith(fakeError);
]);
});

Is it a good idea to program in a non-event driven way in nodejs?

In order to separate the business logic and data access layer , I have separated the files and folders in a non-event driven way. Here is an example of the User signup process I have taken , Here is my Business logic layer file.
var userDA = require('./../DataAccess/UserDA');
module.exports = {
signUpUser: function(userGiven)
{
//Some bookeeping
userGiven.userType = "admin";
return userDA.save(tutorGiven);
}
}
and here is my data access file
"use strict";
var mongoose = require('mongoose');
if(!mongoose.connection)
mongoose.connect('mongodb://localhost/test');
var User = require('./../models/User');
module.exports ={
save : function(UserGiven){
var pass = true;
var user = new User(UserGiven);
user.save(function (err) {
if(err) {
console.log(err);
pass = false;
}
});
return pass;
},
getUser: function (email) {
var user = null;
user.findOne({email:email},function(err,foundUser){
if(err)
console.log(err);
else
user = foundUser;
});
return user;
}
}
This a for a real project of medium-large size , and as a newcomer in nodejs I was wondering and in need of an expert advice if It would be any problem if I take this kind of designing approach?
You are running into a classic pitfall of those new to asynchronous programming in node. For example, consider this code:
setTimeout(function () {
console.log('timer');
}, 200);
console.log('done');
which will output the following:
done
timer
The reason for this is that setTimeout is scheduling asynchronous work for later. What actually happens is the setTimeout function returns immediately, console.log('done') is called, then ~200 ms later the callback function is called and console.log('timer') is called.
A more proper approach would be to use callbacks to indicate when your work is complete. Something like this:
"use strict";
var mongoose = require('mongoose');
if(!mongoose.connection)
mongoose.connect('mongodb://localhost/test');
var User = require('./../models/User');
module.exports ={
save : function(UserGiven, callback){
var pass = true;
var user = new User(UserGiven);
user.save(function (err) {
if(err) {
console.log(err);
return callback(err);
}
return callback();
});
},
getUser: function (email, callback) {
user.findOne({email:email},function(err,foundUser){
if(err) {
console.log(err);
return callback(err);
}
return callback(null, foundUser);
});
}
}
This follows the convention of error-first callbacks, where a callback's first parameter is an error, the second is the data. Your first code segment would then look like this:
var userDA = require('./../DataAccess/UserDA');
module.exports = {
signUpUser: function(userGiven, callback) {
//Some bookeeping
userGiven.userType = "admin";
userDA.save(tutorGiven, callback);
}
}
Now, you would call signUpUser like so:
var userRepo = require('<PATH TO FILE>');
var someUser = {};
userRepo.signUpUser(someUser, function(err, user) {
if (err) {
// oops!
}
// do whatever you need to accomplish after the user is signed up
});

Node.js & Amazon S3: How to iterate through all files in a bucket?

Is there any Amazon S3 client library for Node.js that allows listing of all files in S3 bucket?
The most known aws2js and knox don't seem to have this functionality.
Using the official aws-sdk:
var allKeys = [];
function listAllKeys(marker, cb)
{
s3.listObjects({Bucket: s3bucket, Marker: marker}, function(err, data){
allKeys.push(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextMarker, cb);
else
cb();
});
}
see s3.listObjects
Edit 2017:
Same basic idea, but listObjectsV2( ... ) is now recommended and uses a ContinuationToken (see s3.listObjectsV2):
var allKeys = [];
function listAllKeys(token, cb)
{
var opts = { Bucket: s3bucket };
if(token) opts.ContinuationToken = token;
s3.listObjectsV2(opts, function(err, data){
allKeys = allKeys.concat(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextContinuationToken, cb);
else
cb();
});
}
Using AWS-SDK v3 and Typescript
import {
paginateListObjectsV2,
S3Client,
S3ClientConfig,
} from '#aws-sdk/client-s3';
/* // For Deno
import {
paginateListObjectsV2,
S3Client,
S3ClientConfig,
} from "https://deno.land/x/aws_sdk#v3.32.0-1/client-s3/mod.ts"; */
const s3Config: S3ClientConfig = {
credentials: {
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
},
region: 'us-east-1',
};
const getAllS3Files = async (client: S3Client, s3Opts) => {
const totalFiles = [];
for await (const data of paginateListObjectsV2({ client }, s3Opts)) {
totalFiles.push(...(data.Contents ?? []));
}
return totalFiles;
};
const main = async () => {
const client = new S3Client(s3Config);
const s3Opts = { Bucket: 'bucket-xyz' };
console.log(await getAllS3Files(client, s3Opts));
};
main();
For AWS-SDK v2 Using Async Generator
Import S3
const { S3 } = require('aws-sdk');
const s3 = new S3();
create a generator function to retrieve all the files list
async function* listAllKeys(opts) {
opts = { ...opts };
do {
const data = await s3.listObjectsV2(opts).promise();
opts.ContinuationToken = data.NextContinuationToken;
yield data;
} while (opts.ContinuationToken);
}
Prepare aws parameter, based on api docs
const opts = {
Bucket: 'bucket-xyz' /* required */,
// ContinuationToken: 'STRING_VALUE',
// Delimiter: 'STRING_VALUE',
// EncodingType: url,
// FetchOwner: true || false,
// MaxKeys: 'NUMBER_VALUE',
// Prefix: 'STRING_VALUE',
// RequestPayer: requester,
// StartAfter: 'STRING_VALUE'
};
Use generator
async function main() {
// using for of await loop
for await (const data of listAllKeys(opts)) {
console.log(data.Contents);
}
}
main();
thats it
Or Lazy Load
async function main() {
const keys = listAllKeys(opts);
console.log(await keys.next());
// {value: {…}, done: false}
console.log(await keys.next());
// {value: {…}, done: false}
console.log(await keys.next());
// {value: undefined, done: true}
}
main();
Or Use generator to make Observable function
const lister = (opts) => (o$) => {
let needMore = true;
const process = async () => {
for await (const data of listAllKeys(opts)) {
o$.next(data);
if (!needMore) break;
}
o$.complete();
};
process();
return () => (needMore = false);
};
use this observable function with RXJS
// Using Rxjs
const { Observable } = require('rxjs');
const { flatMap } = require('rxjs/operators');
function listAll() {
return Observable.create(lister(opts))
.pipe(flatMap((v) => v.Contents))
.subscribe(console.log);
}
listAll();
or use this observable function with Nodejs EventEmitter
const EventEmitter = require('events');
const _eve = new EventEmitter();
async function onData(data) {
// will be called for each set of data
console.log(data);
}
async function onError(error) {
// will be called if any error
console.log(error);
}
async function onComplete() {
// will be called when data completely received
}
_eve.on('next', onData);
_eve.on('error', onError);
_eve.on('complete', onComplete);
const stop = lister(opts)({
next: (v) => _eve.emit('next', v),
error: (e) => _eve.emit('error', e),
complete: (v) => _eve.emit('complete', v),
});
Here's Node code I wrote to assemble the S3 objects from truncated lists.
var params = {
Bucket: <yourbucket>,
Prefix: <yourprefix>,
};
var s3DataContents = []; // Single array of all combined S3 data.Contents
function s3Print() {
if (program.al) {
// --al: Print all objects
console.log(JSON.stringify(s3DataContents, null, " "));
} else {
// --b: Print key only, otherwise also print index
var i;
for (i = 0; i < s3DataContents.length; i++) {
var head = !program.b ? (i+1) + ': ' : '';
console.log(head + s3DataContents[i].Key);
}
}
}
function s3ListObjects(params, cb) {
s3.listObjects(params, function(err, data) {
if (err) {
console.log("listS3Objects Error:", err);
} else {
var contents = data.Contents;
s3DataContents = s3DataContents.concat(contents);
if (data.IsTruncated) {
// Set Marker to last returned key
params.Marker = contents[contents.length-1].Key;
s3ListObjects(params, cb);
} else {
cb();
}
}
});
}
s3ListObjects(params, s3Print);
Pay attention to listObject's documentation of NextMarker, which is NOT always present in the returned data object, so I don't use it at all in the above code ...
NextMarker — (String) When response is truncated (the IsTruncated
element value in the response is true), you can use the key name in
this field as marker in the subsequent request to get next set of
objects. Amazon S3 lists objects in alphabetical order Note: This
element is returned only if you have delimiter request parameter
specified. If response does not include the NextMarker and it is
truncated, you can use the value of the last Key in the response as
the marker in the subsequent request to get the next set of object
keys.
The entire program has now been pushed to https://github.com/kenklin/s3list.
In fact aws2js supports listing of objects in a bucket on a low level via s3.get() method call. To do it one has to pass prefix parameter which is documented on Amazon S3 REST API page:
var s3 = require('aws2js').load('s3', awsAccessKeyId, awsSecretAccessKey);
s3.setBucket(bucketName);
var folder = encodeURI('some/path/to/S3/folder');
var url = '?prefix=' + folder;
s3.get(url, 'xml', function (error, data) {
console.log(error);
console.log(data);
});
The data variable in the above snippet contains a list of all objects in the bucketName bucket.
Published knox-copy when I couldn't find a good existing solution. Wraps all the pagination details of the Rest API into a familiar node stream:
var knoxCopy = require('knox-copy');
var client = knoxCopy.createClient({
key: '<api-key-here>',
secret: '<secret-here>',
bucket: 'mrbucket'
});
client.streamKeys({
// omit the prefix to list the whole bucket
prefix: 'buckets/of/fun'
}).on('data', function(key) {
console.log(key);
});
If you're listing fewer than 1000 files a single page will work:
client.listPageOfKeys({
prefix: 'smaller/bucket/o/fun'
}, function(err, page) {
console.log(page.Contents); // <- Here's your list of files
});
Meekohi provided a very good answer, but the (new) documentation states that NextMarker can be undefined. When this is the case, you should use the last key as the marker.
So his codesample can be changed into:
var allKeys = [];
function listAllKeys(marker, cb) {
s3.listObjects({Bucket: s3bucket, Marker: marker}, function(err, data){
allKeys.push(data.Contents);
if(data.IsTruncated)
listAllKeys(data.NextMarker || data.Contents[data.Contents.length-1].Key, cb);
else
cb();
});
}
Couldn't comment on the original answer since I don't have the required reputation. Apologies for the bad mark-up btw.
I am using this version with async/await.
This function will return the content in an array.
I'm also using the NextContinuationToken instead of the Marker.
async function getFilesRecursivelySub(param) {
// Call the function to get list of items from S3.
let result = await s3.listObjectsV2(param).promise();
if(!result.IsTruncated) {
// Recursive terminating condition.
return result.Contents;
} else {
// Recurse it if results are truncated.
param.ContinuationToken = result.NextContinuationToken;
return result.Contents.concat(await getFilesRecursivelySub(param));
}
}
async function getFilesRecursively() {
let param = {
Bucket: 'YOUR_BUCKET_NAME'
// Can add more parameters here.
};
return await getFilesRecursivelySub(param);
}
This is an old question and I guess the AWS JS SDK has changed a lot since it was asked. Here's yet another way to do it these days:
s3.listObjects({Bucket:'mybucket', Prefix:'some-pfx'}).
on('success', function handlePage(r) {
//... handle page of contents r.data.Contents
if(r.hasNextPage()) {
// There's another page; handle it
r.nextPage().on('success', handlePage).send();
} else {
// Finished!
}
}).
on('error', function(r) {
// Error!
}).
send();
If you want to get list of keys only within specific folder inside a S3 Bucket then this will be useful.
Basically, listObjects function will start searching from the Marker we set and it will search until maxKeys: 1000 as limit. so it will search one by one folder and get you first 1000 keys it find from different folder in a bucket.
Consider i have many folders inside my bucket with prefix as prod/some date/, Ex: prod/2017/05/12/ ,prod/2017/05/13/,etc.
I want to fetch list of objects (file names) only within prod/2017/05/12/ folder then i will specify prod/2017/05/12/ as my start and prod/2017/05/13/ [your next folder name] as my end and in code i'm breaking the loop when i encounter the end.
Each Keyin data.Contents will look like this.
{ Key: 'prod/2017/05/13/4bf2c675-a417-4c1f-a0b4-22fc45f99207.jpg',
LastModified: 2017-05-13T00:59:02.000Z,
ETag: '"630b2sdfsdfs49ef392bcc16c833004f94ae850"',
Size: 134236366,
StorageClass: 'STANDARD',
Owner: { }
}
Code:
var list = [];
function listAllKeys(s3bucket, start, end) {
s3.listObjects({
Bucket: s3bucket,
Marker: start,
MaxKeys: 1000,
}, function(err, data) {
if (data.Contents) {
for (var i = 0; i < data.Contents.length; i++) {
var key = data.Contents[i].Key; //See above code for the structure of data.Contents
if (key.substring(0, 19) != end) {
list.push(key);
} else {
break; // break the loop if end arrived
}
}
console.log(list);
console.log('Total - ', list.length);
}
});
}
listAllKeys('BucketName', 'prod/2017/05/12/', 'prod/2017/05/13/');
Output:
[ 'prod/2017/05/12/05/4bf2c675-a417-4c1f-a0b4-22fc45f99207.jpg',
'prod/2017/05/12/05/a36528b9-e071-4b83-a7e6-9b32d6bce6d8.jpg',
'prod/2017/05/12/05/bc4d6d4b-4455-48b3-a548-7a714c489060.jpg',
'prod/2017/05/12/05/f4b8d599-80d0-46fa-a996-e73b8fd0cd6d.jpg',
... 689 more items ]
Total - 692
I ended up building a wrapper function around ListObjectsV2, works the same way and takes the same parameters but works recursively until IsTruncated=false and returns all the keys found as an array in the second parameter of the callback function
const AWS = require('aws-sdk')
const s3 = new AWS.S3()
function listAllKeys(params, cb)
{
var keys = []
if(params.data){
keys = keys.concat(params.data)
}
delete params['data']
s3.listObjectsV2(params, function(err, data){
if(err){
cb(err)
} else if (data.IsTruncated) {
params['ContinuationToken'] = data.NextContinuationToken
params['data'] = data.Contents
listAllKeys(params, cb)
} else {
keys = keys.concat(data.Contents)
cb(null,keys)
}
})
}
Here's what I came up with based on the other answers.
You can await listAllKeys() without having to use callbacks.
const listAllKeys = () =>
new Promise((resolve, reject) => {
let allKeys = [];
const list = marker => {
s3.listObjects({ Marker: marker }, (err, data) => {
if (err) {
reject(err);
} else if (data.IsTruncated) {
allKeys.push(data.Contents);
list(data.NextMarker || data.Contents[data.Contents.length - 1].Key);
} else {
allKeys.push(data.Contents);
resolve(allKeys);
}
});
};
list();
});
This assumes you've initialized the s3 variable like so
const s3 = new aws.S3({
apiVersion: API_VERSION,
params: { Bucket: BUCKET_NAME }
});
I made it as simple as possible. You can iterate uploading objects using for loop, it is quite simple, neat and easy to understand.
package required: fs, express-fileupload
server.js :-
router.post('/upload', function(req, res){
if(req.files){
var file = req.files.filename;
test(file);
res.render('test');
}
} );
test function () :-
function test(file){
// upload all
if(file.length){
for(var i =0; i < file.length; i++){
fileUP(file[i]);
}
}else{
fileUP(file);
}
// call fileUP() to upload 1 at once
function fileUP(fyl){
var filename = fyl.name;
var tempPath = './temp'+filename;
fyl.mv(tempPath, function(err){
fs.readFile(tempPath, function(err, data){
var params = {
Bucket: 'BUCKET_NAME',
Body: data,
Key: Date.now()+filename
};
s3.upload(params, function (err, data) {
if (data) {
fs.unlink(tempPath, (err) => {
if (err) {
console.error(err)
return
}
else{
console.log("file removed from temp loaction");
}
});
console.log("Uploaded in:", data.Location);
}
});
});
});
}
}
This should work,
var listAllKeys = async function (token) {
if(token) params.ContinuationToken = token;
return new Promise((resolve, reject) => {
s3.listObjectsV2(params, function (err, data) {
if (err){
reject(err)
}
resolve(data)
});
});
}
var collect_all_files = async function () {
var allkeys = []
conti = true
token = null
while (conti) {
data = await listAllKeys(token)
allkeys = allkeys.concat(data.Contents);
token = data.NextContinuationToken
conti = data.IsTruncated
}
return allkeys
};
Using the new API s3.listObjectsV2 the recursive solution will be:
S3Dataset.prototype.listFiles = function(params,callback) {
var self=this;
var options = {
};
for (var attrname in params) { options[attrname] = params[attrname]; }
var results=[];
var s3=self.s3Store.GetInstance();
function listAllKeys(token, callback) {
var opt={ Bucket: self._options.s3.Bucket, Prefix: self._options.s3.Key, MaxKeys: 1000 };
if(token) opt.ContinuationToken = token;
s3.listObjectsV2(opt, (error, data) => {
if (error) {
if(self.logger) this.logger.error("listFiles error:", error);
return callback(error);
} else {
for (var index in data.Contents) {
var bucket = data.Contents[index];
if(self.logger) self.logger.debug("listFiles Key: %s LastModified: %s Size: %s", bucket.Key, bucket.LastModified, bucket.Size);
if(bucket.Size>0) {
var Bucket=self._options.s3.Bucket;
var Key=bucket.Key;
var components=bucket.Key.split('/');
var name=components[components.length-1];
results.push({
name: name,
path: bucket.Key,
mtime: bucket.LastModified,
size: bucket.Size,
sizehr: formatSizeUnits(bucket.Size)
});
}
}
if( data.IsTruncated ) { // truncated page
return listAllKeys(data.NextContinuationToken, callback);
} else {
return callback(null,results);
}
}
});
}
return listAllKeys.apply(this,['',callback]);
};
where
function formatSizeUnits(bytes){
if (bytes>=1099511627776) {bytes=(bytes/1099511627776).toFixed(4)+' PB';}
else if (bytes>=1073741824) {bytes=(bytes/1073741824).toFixed(4)+' GB';}
else if (bytes>=1048576) {bytes=(bytes/1048576).toFixed(4)+' MB';}
else if (bytes>=1024) {bytes=(bytes/1024).toFixed(4)+' KB';}
else if (bytes>1) {bytes=bytes+' bytes';}
else if (bytes==1) {bytes=bytes+' byte';}
else {bytes='0 byte';}
return bytes;
}//formatSizeUnits
Although #Meekohi's answer does technically work, I've had enough heartache with the S3 portion of the AWS SDK for NodeJS. After all the previous struggling with modules such as aws-sdk, s3, knox, I decided to install s3cmd via the OS package manager and shell-out to it using child_process
Something like:
var s3cmd = new cmd_exec('s3cmd', ['ls', filepath, 's3://'+inputBucket],
function (me, data) {me.stdout += data.toString();},
function (me) {me.exit = 1;}
);
response.send(s3cmd.stdout);
(Using the cmd_exec implementation from this question)
This approach just works really well - including for other problematic things like file upload.
The cleanest way to do it for me was through execution of s3cmd from my node script like this (The example here is to delete files recursively):
var exec = require('child_process').exec;
var child;
var bucket = "myBucket";
var prefix = "myPrefix"; // this parameter is optional
var command = "s3cmd del -r s3://" + bucket + "/" + prefix;
child = exec(command, {maxBuffer: 5000 * 1024}, function (error, stdout, stderr) { // the maxBuffer is here to avoid the maxBuffer node process error
console.log('stdout: ' + stdout);
if (error !== null) {
console.log('exec error: ' + error);
}
});

Resources