I need to loop through a json and make multiple getitem calls to dynamo db. My issue is that node js flies through the code not waiting for the function return so I can't create an xml that has a header, multiple calculated lines, and then a footer.
In the sample below, the !make footer would be written long before the function taxrate finishes. How can I force the script to wait for the taxrate function to finish?
!make xml header
for(i=0; i<linelength; i++)
{
business_unit = '100'
invoice_line = 1
total = 100
taxrate(business_unit, invoice_line, total);
!write line xml
}
!make xml footer
function taxrate(business_unit, i, gross_total) {
const params = {
Key: {
"tax_rate": {
S: business_unit
}
},
TableName:"tax_table"
};
dynamodb.getItem(params,function(err, data){
if(err) {
console.log("call error");
console.log(err);
} else {
console.log(data.Item.tax.N);
return(data.Item.tax.N);
}
});
There are several patterns for solving this problem; the first one is one you've already got in your code, which is the callback function. The function you are passing to dynamodb.GetItem() is a callback that executes after some other code is run, so that the result (data) can be accessed.
A more modern way is to use Promises, which take a bit of time to understand. It's worth doing this though, so that you can understand what's happening behind the scenes when you use the newest way, which is async and await, which would look like this:
exports.lambdaHandler = async (event, context) => {
!make xml header
for(i=0; i<linelength; i++)
{
business_unit = '100'
invoice_line = 1
total = 100
await taxrate(business_unit, invoice_line, total);
!write line xml
}
!make xml footer
}
async function taxrate(business_unit, i, gross_total) {
const params = {
Key: {
"tax_rate": {
S: business_unit
}
},
TableName:"tax_table"
};
try {
let data = await dynamodb.getItem(params).promise();
return data.Item.tax.N;
} catch(err) {
console.log("call error");
console.log(err);
}
}
Notice that both lambdaHandler() and taxrate() have been turned into async functions. In the for loop, we've now awaited the result of taxrate(). The callback to getItem has been replaced with a method that returns a promise.
Related
I need some advice on how to structure this function as at the moment it is not happening in the correct order due to node being asynchronous.
This is the flow I want to achieve; I don't need help with the code itself but with the order to achieve the end results and any suggestions on how to make it efficient
Node routes a GET request to my controller.
Controller reads a .csv file on local system and opens a read stream using fs module
Then use csv-parse module to convert that to an array line by line (many 100,000's of lines)
Start a try/catch block
With the current row from the csv, take a value and try to find it in a MongoDB
If found, take the ID and store the line from the CSV and this id as a foreign ID in a separate database
If not found, create an entry into the DB and take the new ID and then do 6.
Print out to terminal the row number being worked on (ideally at some point I would like to be able to send this value to the page and have it update like a progress bar as the rows are completed)
Here is a small part of the code structure that I am currently using;
const fs = require('fs');
const parse = require('csv-parse');
function addDataOne(req, id) {
const modelOneInstance = new InstanceOne({ ...code });
const resultOne = modelOneInstance.save();
return resultOne;
}
function addDataTwo(req, id) {
const modelTwoInstance = new InstanceTwo({ ...code });
const resultTwo = modelTwoInstance.save();
return resultTwo;
}
exports.add_data = (req, res) => {
const fileSys = 'public/data/';
const parsedData = [];
let i = 0;
fs.createReadStream(`${fileSys}${req.query.file}`)
.pipe(parse({}))
.on('data', (dataRow) => {
let RowObj = {
one: dataRow[0],
two: dataRow[1],
three: dataRow[2],
etc,
etc
};
try {
ModelOne.find(
{ propertyone: RowObj.one, propertytwo: RowObj.two },
'_id, foreign_id'
).exec((err, searchProp) => {
if (err) {
console.log(err);
} else {
if (searchProp.length > 1) {
console.log('too many returned from find function');
}
if (searchProp.length === 1) {
addDataOne(RowObj, searchProp[0]).then((result) => {
searchProp[0].foreign_id.push(result._id);
searchProp[0].save();
});
}
if (searchProp.length === 0) {
let resultAddProp = null;
addDataTwo(RowObj).then((result) => {
resultAddProp = result;
addDataOne(req, resultAddProp._id).then((result) => {
resultAddProp.foreign_id.push(result._id);
resultAddProp.save();
});
});
}
}
});
} catch (error) {
console.log(error);
}
i++;
let iString = i.toString();
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(iString);
})
.on('end', () => {
res.send('added');
});
};
I have tried to make the functions use async/await but it seems to conflict with the fs.openReadStream or csv parse functionality, probably due to my inexperience and lack of correct use of code...
I appreciate that this is a long question about the fundamentals of the code but just some tips/advice/pointers on how to get this going would be appreciated. I had it working when the data was sent one at a time via a post request from postman but can't implement the next stage which is to read from the csv file which contains many records
First of all you can make the following checks into one query:
if (searchProp.length === 1) {
if (searchProp.length === 0) {
Use upsert option in mongodb findOneAndUpdate query to update or upsert.
Secondly don't do this in main thread. Use a queue mechanism it will be much more efficient.
Queue which I personally use is Bull Queue.
https://github.com/OptimalBits/bull#basic-usage
This also provides the functionality you need of showing progress.
Also regarding using Async Await with ReadStream, a lot of example can be found on net such as : https://humanwhocodes.com/snippets/2019/05/nodejs-read-stream-promise/
I want to extract all child-Folders and child-Docs querying a node-js Service, which every time it is called, returns an array of such items. I do not know the depth fo the folders-tree so I want to recursively call a function that in the end will return an array that will contain all child-folders and child-docs, starting from a list of root-Folders. Each folder is identified by a folder id.
So I have made a "recPromise(fId)" which returns a promise. Inside, this function calls recursively the recFun(folderId).I start invoking the "recPromise(fId)" from a rootFolder so once all root-promises are resolved I can go on.
rootFolders.map( folderOfRootlevel =>{
var folderContentPromise = recPromise(folderOfRootlevel.id);
folderContentPromises.push(folderContentPromise);
})
$q.all(folderContentPromises)
.then(function(folderContent) {
// Do stuff with results.
}
function recPromise(fId){
return new Promise((resolve, reject) => {
var items = [];
function recFun( folderId) { // asynchronous recursive function
function handleFolderContent( asyncResult) { // process async result and decide what to do
items.push(asyncResult);
//Now I am in a leaf-node, no child-Folders exist so I return
if (asyncResult.data.childFolders.length === 0){
return items;
}
else {
//child_folders exist. So call again recFun() for every child-Folder
for(var item of asyncResult.data.childFolders) {
return recFun(item._id);
}
}
}
// This is the service that returns the array of child-Folders and child-Docs
return NodeJSService.ListFolders(folderId).then(handleFolderContent);
}
resolve(recFun(fId));
})
}
It works almost as expected except the loop inside else, where I call again recFun().
The NodeJSService will return an array of sub-Folders so I wish to call recfun() for every sub-Folder.
Now, I only get the result of the 1st sub-Folder of the loop,
which makes sense since I have a return statement there.
If I remove the return statement and call like this "recFun(item._id);"
then it breaks the $q.all().
Finally, I decided to remove the Promise wrapper function and make use of async-await.
var items = [];
(async() => {
for(var item of rootFolders) {
await recFun(item.id)
}
// Do stuff with items
// go on form here..
})()
function listFolders(folderId) {
return new Promise( function( resolve, reject) {
resolve(FolderService.ListFolders(folderId));
})
}
async function recFun(folderId) {
var foldersResponse= await listFolders(folderId);
items.push(foldersResponse);
if (foldersResponse.data.childFolders.length === 0){
return items ;
}
else {
for(var item of foldersResponse.data.childFolders) {
await recFun(item._id);
}
}
}
My removeObjects function has me stummped.The function is suppose to syncronoulsy get a list of objects in an S3 bucket then asyncronously removes the objects. Repeat if the list was truncated, until the there are no more objects to remove. (AWS doesn't provide the total count of objects in the bucket and listObjects pages the results.)
What am I doing wrong / why doesn't my function work? The solution should exploit single thread and async nature of JS. For the bounty I am hoping for an answer specific to the module. The git repo is public if you want to see the entire module.
export function removeObjects(params: IS3NukeRequest): Promise<S3.Types.DeleteObjectsOutput> {
const requests: Array<Promise<S3.Types.DeleteObjectsOutput>> = [];
let isMore;
do {
listObjectsSync(params)
.then((objectList: S3.Types.ListObjectsV2Output) => {
isMore = objectList.ContinuationToken = objectList.IsTruncated ? objectList.NextContinuationToken : null;
requests.push(params.Client.deleteObjects(listObjectsV2Output2deleteObjectsRequest(objectList)).promise());
})
.catch((err: Error) => { Promise.reject(err); });
} while (isMore);
return Promise.all(requests);
}
export async function listObjectsSync(params: IS3NukeRequest): Promise<S3.Types.ListObjectsV2Output> {
try {
return await params.Client.listObjectsV2(s3nukeRequest2listObjectsRequest(params)).promise();
} catch (err) {
return Promise.reject(err);
}
}
Thanks.
The thing is that listObjectsSync function returns a Promise, so you need to treat it as an async function and can't just use a loop with it. What you need to do is to create a chain of promises while your isMore is true, I've done it using a recursive approach (I'm not pro in TS, so please check the code before using it). I also haven't tried the code live, but logically it should work :)
const requests: Array<Promise<S3.Types.DeleteObjectsOutput>> = [];
function recursive(recursiveParams) {
return listObjectsSync(recursiveParams).then((objectList: S3.Types.ListObjectsV2Output) => {
let isMore = objectList.ContinuationToken = objectList.IsTruncated ? objectList.NextContinuationToken : null;
requests.push(params.Client.deleteObjects(listObjectsV2Output2deleteObjectsRequest(objectList)).promise());
if (isMore) {
//do we need to change params here?
return recursive(recursiveParams)
}
//this is not necessary, just to indicate that we get out of the loop
return true;
});
}
return recursive(params).then(() => {
//we will have all requests here
return Promise.all(requests);
});
Is there a way to make Node.js stream as coroutine.
Example
a Fibonacci numbers stream.
fibonacci.on('data', cb);
//The callback (cb) is like
function cb(data)
{
//something done with data here ...
}
Expectation
function* fibonacciGenerator()
{
fibonacci.on('data', cb);
//Don't know what has to be done further...
};
var fibGen = fibonacciGenerator();
fibGen.next().value(cb);
fibGen.next().value(cb);
fibGen.next().value(cb);
.
.
.
Take desired numbers from the generator. Here Fibonacci number series is just an example, in reality the stream could be of anything a file, mongodb query result, etc.
Maybe something like this
Make the 'stream.on' function as a generator.
Place yield inside the callback function.
Obtain generator object.
Call next and take the next value in stream.
Is it at-least possible if yes how and if not why? Maybe a dumb question :)
If you don't want to use a transpiler (e.g. Babel) or wait until async/await make it to Node.js, you can implement it yourself using generators and promises.
The downside is that your code must live inside a generator.
First, you can make a helper that receives a stream and returns a function that, when called, returns a promise for the next "event" of the stream (e.g. data).
function streamToPromises(stream) {
return function() {
if (stream.isPaused()) {
stream.resume();
}
return new Promise(function(resolve) {
stream.once('data', function() {
resolve.apply(stream, arguments);
stream.pause();
});
});
}
}
It pauses the stream when you're not using it, and resumes it when you ask it the next value.
Next, you have a helper that receives a generator as an argument, and every time it yields a promise, it resolves it and passes its result back to the generator.
function run(fn) {
var gen = fn();
var promise = gen.next().value;
var tick = function() {
promise.then(function() {
promise = gen.next.apply(gen, arguments).value;
}).catch(function(err) {
// TODO: Handle error.
}).then(function() {
tick();
});
}
tick();
}
Finally, you would do your own logic inside a generator, and run it with the run helper, like this:
run(function*() {
var nextFib = streamToPromises(fibonacci);
var n;
n = yield nextFib();
console.log(n);
n = yield nextFib();
console.log(n);
});
Your own generator will yield promises, pausing its execution and passing the control to the run function.
The run function will resolve the promise and pass its value back to your own generator.
That's the gist of it. You'd need to modify streamToPromises to check for other events as well (e.g. end or error).
class FibonacciGeneratorReader extends Readable {
_isDone = false;
_fibCount = null;
_gen = function *() {
let prev = 0, curr = 1, count = 1;
while (this._fibCount === -1 || count++ < this._fibCount) {
yield curr;
[prev, curr] = [curr, prev + curr];
}
return curr;
}.bind(this)();
constructor(fibCount) {
super({
objectMode: true,
read: size => {
if (this._isDone) {
this.push(null);
} else {
let fib = this._gen.next();
this._isDone = fib.done;
this.push(fib.value.toString() + '\n');
}
}
});
this._fibCount = fibCount || -1;
}
}
new FibonacciGeneratorReader(10).pipe(process.stdout);
Output should be:
1
1
2
3
5
8
13
21
34
55
I want to fill each object of the result of a query, with other querys, and I want to do all in asynchronously way
Here is an example of the way how I do actually
var q = knex.select().from('sector');
q.then(function (sectores) {
var i = -1;
(function getDetalles(sectores) {
i++;
if(i < sectores.length){
knex.select().from('sector_detalle')
.where('sector_id', sectores[i].id)
.then(function (detalles) {
// this what i want to do asynchronously
sectores[i].sector_detalles = detalles;
console.log(sectores[i]);
getDetalles(sectores);
});
} else {
res.send({sucess: true, rows: sectores});
}
})(sectores);
});
I do some reserch and found this wait for all promises to finish in nodejs with bluebird
is close to what I want but don't know how to implement
I think you're looking for the map method that works on a promise for an array, and will invoke an asynchronous (promise-returning) callback for each of the items in it:
knex.select().from('sector').map(function(sector) {
return knex.select().from('sector_detalle')
.where('sector_id', sector.id)
.then(function(detalles) {
sector.sector_detalles = detalles;
// console.log(sector);
return sector;
});
}).then(function(sectores) {
res.send({sucess: true, rows: sectores});
});