Ima rookie using async/await but must now to use Redis-om. NN_walkd walks through a Redis database looking for loop-chains and does this by recursion. So the 2 questions I have is:
Am I calling the inner recursive NN_walkd calls correctly via async/await?
At runtime, the compSearchM proc is called first and seems to work (it gets 5 entries so it has to call NN_walkd 5 times). A NN_walkd is then recursively called, and then when it loops the 1st time it then calls compSearchK where the problems are. It seems to sit on the first Redis call in compSearchK (.search). Yet the code for compSearchK and compSearchM look basically identical.
main call
NN_walk = async function(req, db, cnode, pnode, chain, cb) {
var vegas, sneaker;
req.session.walk = [];
await NN_walkd(req, cnode, pnode, [], 1);
req.session.walk = null;
console.log('~~~~~~~~~~~~ Out of Walk ~~~~~~~~~~~~~~~');
cb();
};
redis.mjs
export class RedisDB {
constructor() {
...
this._companyRepo = ...
}
compSearchK(ckey) { // doesn't matter if I have a async or not here
return new Promise(async (resolve) => {
const sckey = await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
if (sckey.length) {
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
resolve(ttrr.toJSON());
} else
resolve(null);
});
}
compSearchM(mkey) {
var tArr=[];
return new Promise(async (resolve) => {
const smkey = await this._companyRepo.search()
.where('MASTERKEY').equals(mkey)
.and('TBLNUM').equals(10)
.return.all();
if (smkey.length) {
for (var spot in smkey) {
const ttrr = await this._companyRepo.fetch(smkey[spot].entityId);
tArr.push(ttrr.toJSON());
}
resolve(tArr);
} else {
resolve(null);
}
});
}
walk.js
NN_walkd = async function(req, cnode, pnode, chain, lvl) {
...
if (cnode[1]) {
const sObj = await req.app.get('redis').compSearchK(cnode[1]);
if (sObj) {
int1 = (sObj.TBLNUM==1) ? null : sObj.CLIENTKEY;
(async () => await NN_walkd(req, [sObj.COMPANYKEY,int1], cnode, Array.from(chain), tlvl))()
}
} else {
const sArr = await req.app.get('redis').compSearchM(cnode[0]);
if (sArr.length) {
for (sneaker in sArr) {
(async () => await NN_walkd(req, [sArr[sneaker].COMPANYKEY,sArr[sneaker].CLIENTKEY], cnode, Array.from(chain), tlvl))()
}
} else {
console.log('no more links on this chain: ',cnode);
}
}
}
"doesn't matter if i have async or not here"
compSearchK(ckey) { // doesn't matter if I have a async or not here
return new Promise(async (resolve) => {
const sckey = await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
if (sckey.length) {
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
resolve(ttrr.toJSON());
} else
resolve(null);
});
}
Of course it doesn't matter, because you're not using await inside of compSearchK!
You are using the explicit promise contructor anti-pattern. You should avoid it as it demonstrates lack of understanding. Here is compSearchK rewritten without the anti-pattern -
async compSearchK(ckey) {
// await key
const sckey =
await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
// return null if key is not found
if (sckey.length == 0) return null;
// otherwise get ttrr
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
// return ttrr as json
return ttrr.toJSON();
}
Related
This is my controller:
const rssService = require('../services/rss.service');
async function parser() {
const result = await rssService.rssParser('someurl');
return result;
};
const parse = async function (req, res) {
const p = new Promise((resolve, reject) => {
const t = parser();
if (t === undefined) {
resolve(t);
} else {
// eslint-disable-next-line prefer-promise-reject-errors
reject('something bad happened');
}
});
p.then((result) => res.send(result)).catch((message) => console.log(`ERROR ${message}`));
};
module.exports = {
parse,
};
in the function : parser() above, I am trying to call my rss.service.js file which I have the logic. This file is a rss parser which tries to parse the feed and do some calculations (which needs promises and async) and then return the json object.
Here is how my rss.service look :
const rssParser = async function parseRssFeed(url) {
const parser = new Parser();
const appRoot = process.env.PWD;
const downloadDir = `${appRoot}/downloads/`;
if (!fs.existsSync(downloadDir)) {
fs.mkdirSync(downloadDir);
}
try {
const feed = await parser.parseURL('someurl');
const processedFeedItems = await Promise.all(feed.items.map(async (currentItem) => {
const {
fileUrl,
downloadPath,
} = await downloadFile(currentItem);
const hashValue = calculateHash(downloadPath)
return {
title: currentItem.title,
hash: hashValue,
url: mp3FileUrl,
};
}));
return (JSON.stringify(processedFeedItems));
} catch (error) {
console.error(error);
return 'error';
}
};
when I debug my code I can verify that Json object has been created with correct data, but the result does not return to the callee(controller).
I'll go in a little deeper since you mentioned you're new:
const rssService = require('../services/rss.service');
// This is an async function (always returns a promise)
async function parser() {
const result = await rssService.rssParser('someurl');
return result;
};
const parse = async function (req, res, next) {
// In await/async, you should use try/catch/throw instead of .then and .catch
// It does the same thing, but it's the preferred syntax and is "easier" to read IMO
// Out in "the community" people will complain if you mix await/async with promises like that
try {
// Adding await to ensure left-assign works.
// This is necessary because parser is an async function (so returns a promise)
const result = await parser();
// here, since you used `await` you get the value instead of the promise
if (result === undefined) throw new Error('something bad happened')
return res.send(result)
} catch (error) {
console.log(`ERROR ${error.message}`;
// Do you need to do anything else with this error? I assume something like:
return next(error);
}
};
module.exports = {
parse,
};
In a fast look, It seems you have forgot to wait for resolve the parser promise.
...
const p = new Promise(async(resolve, reject) => {
const t = await parser();
...
I am performing the useMutation operation in the innermost loop and want to check the remaining cost upon every mutation. But it gets checked after all the mutations which is a problem because for some reason even if all the mutations get done(When the cost is under limits), It calls the .then() part for cost-checking and waiting for unknown reason.
Edit: I also noticed that even though the program is waiting again and again, the network status of chrome shows that all the mutations have happened and only the query of handleDiscountMore i.e. fetchMore is pending
const { loading, error, data, fetchMore, extensions, refetch } = useQuery(GET_COLLECTION, {
variables: { "id": coll.collection.id }
});
const [updatePrice] = useMutation(UPDATE_PRICE);
const redirectToModify = async (data, totalProducts) => {
wait(20000);
var cursor, fetchCount;
fetchCount = data.collection.products.edges.length;
totalProducts -= fetchCount;
data.collection.products.edges.map(async(product) => {
const results = await Promise.all(product.node.variants.edges.map(variant => {
if (selected == 'curr_price') {
//do stuff
}
else {
//do stuff
}
const productVariableInput = {
//Object
};
updatePrice({ variables: { input: productVariableInput } }).then(({ data, extensions }) => {
console.log("Remaining", extensions.cost.throttleStatus.currentlyAvailable)
console.log(data)
if (extensions.cost.throttleStatus.currentlyAvailable < 100) {
console.log("WAITING")
wait(18000);
}
}).catch(e => {
console.log(e)
})
console.log("AFTER")
return 0;
}))
})
if (totalProducts > 0) {
console.log("Calling")
wait(15000);
handleDiscountMore(data, cursor, totalProducts)
}
};
//Below function is Just for reference. It gets called before checking the throttleStatus above. afaik there's no problem with this
const handleDiscountMore = (data, cursor, pc) => {
console.log("Call received")
fetchMore({
variables: {
"id": data.collection.id,
"cursor": cursor
},
updateQuery: (
previousResult,
{ fetchMoreResult }
) => {
console.log("adding", fetchMoreResult);
redirectToModify(fetchMoreResult, pc);
// return fetchMoreResult;
}
})
}
Your map of maps is evaluating all promises at exactly the same time. Here's a cleaned up example that uses a nested for loop instead, which will wait for each request to finish before starting the next (note: I couldn't run it to test, so there's probably some bugs, but the idea is there):
const id = coll.collection.id;
const { loading, error, data, fetchMore, extensions, refetch } = useQuery(GET_COLLECTION, {
variables: { id }
});
const [updatePrice] = useMutation(UPDATE_PRICE);
// Given a product, returns a promise that resolves when all variants are processed
async function process_product(product){
const variants = product.node.variants.edges;
for (let i = 0; i < variants.length; i++){
await process_variant(variants[i]);
}
}
// Given a variant, returns a promise after the product is processed
async function process_variant(variant){
if (variant) {
console.log('doing stuff')
}
else {
console.log('doing other stuff')
}
const productVariableInput = {};
const variables = { input: productVariableInput };
try {
const {data, extensions} = await updatePrice({ variables });
const remaining_throttle = extensions.cost.throttleStatus.currentlyAvailable;
console.log("Remaining", remaining_throttle)
console.log(data)
// Change to a while loop to make sure you actually wait until resources are available
if (remaining_throttle < 100) {
console.log("WAITING")
await wait(18000);
}
} catch (e) {
console.log('error:', e);
}
console.log("AFTER")
return 0;
}
const redirectToModify = async (data, totalProducts) => {
await wait(20000);
let cursor;
const products = data.collection.product.edges;
totalProducts = totalProducts - products.length;
// Wait for all products to finish processing
for (var i = 0; i < products.length; i++){
await process_product(products[i]);
}
if (totalProducts > 0) {
console.log("Calling")
await wait(15000);
handleDiscountMore(data, cursor, totalProducts)
}
};
function updateQuery(previousResult, { fetchMoreResult }){
console.log("adding", fetchMoreResult);
redirectToModify(fetchMoreResult, pc);
return fetchMoreResult;
}
//Below function is Just for reference. It gets called before checking the throttleStatus above. afaik there's no problem with this
function handleDiscountMore(data, cursor, pc) {
console.log("Call received")
const variables = { id: data.collection.id, cursor };
fetchMore({ variables, updateQuery })
}
I used async-lock module for my typescript program of concurrency.
As I use this, I want to return the result in lock.acquire(...) {...}, but it's not working well.
How can I return the value? I'll be grateful for any advice about this. Thanks!
public async getValue(key: string): Promise<any> {
const base = this;
lock.acquire(key, async function (done) {
base.logger.info(`${key} lock enter`);
if (!await base.myRepository.checkDBTable(key)) {
const valueFromNetwork: number = await base.getValueFromNetwork(key);
const initResult: MyEntity = await base.myRepository.initNonce(key, valueFromNetwork);
if (!initResult) {
throw new Error('initValue failed...');
}
base.logger.debug(JSON.stringify(initResult, null, 4));
}
const valueFromDB: number = await base.myRepository.getValueFromDB(key);
if (valueFromDB === -1 || valueFromDB === undefined) {
throw new Error('getValueFromDB failed...');
} else {
const updateResult: MyEntity = await base.myRepository.updateValue(key, valueFromDB);
if (!updateResult) {
throw new Error('updateValue failed...');
}
base.logger.info(`${valueFromDB}`);
base.logger.info(`${key} lock done`);
done();
}
// I'd like to return valueFromDB above.
});
}
You have two options:
You can implement a wrapper around lock.aquire (Recommended as it is a little easier to read and can handle acquisition errors):
public async getValue(key: string): Promise<any> {
const base = this;
const done = await this.aquireLock(key);
base.logger.info(`${key} lock enter`);
if (!await base.myRepository.checkDBTable(key)) {
const valueFromNetwork: number = await base.getValueFromNetwork(key);
const initResult: MyEntity = await base.myRepository.initNonce(key, valueFromNetwork);
if (!initResult) {
throw new Error('initValue failed...');
}
base.logger.debug(JSON.stringify(initResult, null, 4));
}
const valueFromDB: number = await base.myRepository.getValueFromDB(key);
if (valueFromDB === -1 || valueFromDB === undefined) {
throw new Error('getValueFromDB failed...');
} else {
const updateResult: MyEntity = await base.myRepository.updateValue(key, valueFromDB);
if (!updateResult) {
throw new Error('updateValue failed...');
}
base.logger.info(`${valueFromDB}`);
base.logger.info(`${key} lock done`);
done();
}
return valueFromDB;
}
private async aquireLock(key: string): Promise<() => void> {
return new Promise((resolve, reject) => {
lock.acquire(key, done => {
resolve(done);
}, (err)=>{ // in case our aquire fails(times out, etc.)
if(err){
reject(err);
}
})
})
}
Playground
Or, you can use a function constructor (even if it is an anti-pattern):
getValue(key: string): Promise<any> {
return new Promise((resolve, reject) => {
const base = this;
lock.acquire(key, async function (done) {
base.logger.info(`${key} lock enter`);
if (!await base.myRepository.checkDBTable(key)) {
const valueFromNetwork: number = await base.getValueFromNetwork(key);
const initResult: MyEntity = await base.myRepository.initNonce(key, valueFromNetwork);
if (!initResult) {
reject(new Error('initValue failed...'));
}
base.logger.debug(JSON.stringify(initResult, null, 4));
}
const valueFromDB: number = await base.myRepository.getValueFromDB(key);
if (valueFromDB === -1 || valueFromDB === undefined) {
reject(new Error('getValueFromDB failed...'));
} else {
const updateResult: MyEntity = await base.myRepository.updateValue(key, valueFromDB);
if (!updateResult) {
reject(new Error('updateValue failed...'));
}
base.logger.info(`${valueFromDB}`);
base.logger.info(`${key} lock done`);
done();
}
resolve(valueFromDB)
});
})
}
Playground (Many things stubbed out, but it gets the general idea across)
I want to supply plugin to my services that will measure each call to redis and send metric with the response time.
how can I wrap ioredis to do this?
I thought to use Proxy but it's work to me only for async (promise) methods.
let handler = {
get: (target, name, receiver) => {
const startTime = Date.now();
return (...args) => new Promise((resolve, reject) => {
const apiMethod = Reflect.get(target, name, receiver);
const boundApiMethod = apiMethod.bind(target);
boundApiMethod(...args).then((data) => {
const duration = Date.now() - startTime;
metric({duration});
resolve(data)
}, (err) => {
rawMetering.reportError(err);
reject(err);
});
});
}
};
return new Proxy(redis, handler);
We were facing a very similar requirement recently, what we ended up doing is creating a proxy-class around ioredis as follows:
class RedisClient {
constructor() {
this.initClient(); // init your ioredis here
}
#MeasureTime()
async get(key) {
return this._ioredis.get(key);
}
// other methods like set here
}
As you can see we've defined a decorator MeasureTime which is really just a higher order function wrapping the call and performing the measurement:
function MeasureTime = () => (target, _propertyKey, descriptor) => {
const originalMethod = descriptor.value;
descriptor.value = async function (...args) {
const start = moment.now();
const result = await originalMethod.apply(this, args);
const duration = moment.now() - start;
// do something with the duration here
return result;
};
return descriptor;
}
In one file called SearchBuilder.js i have this function ( wrote in this way as pure test )
self.validateInputs = async params => {
const validateDates = async () => {
const pick_up_date = moment(`${params.pick_up_date} ${params.pick_up_time}`),
drop_off_date = moment(`${params.drop_off_date} ${params.drop_off_date}`);
return true;
};
const validatePickUpId = async () => {
return true;
};
const validateDropOffId = async () => {
throw { 'code': 'NOT_FOUND'};
};
return Promise.all([
validateDates,
validatePickUpId,
validateDropOffId,
]);
};
In another file called searchEngine.js i have this:
self.run = async (type, search_inputs, account) => {
const searchBuilder = self.getSearchBuilder(type);
try {
const isValid = await searchBuilder.validateInputs(search_inputs);
console.log(isValid);
} catch (e) {
console.log('error input validation');
}
return search;
};
I erroneously thought that if any of the promises in Promise.all would fail, so my code will enter the catch block. Instead, even if the promise fails code inside the catch is never executed and inside isValid i get this.
[ [AsyncFunction: validateDates],
[AsyncFunction: validatePickUpId],
[AsyncFunction: validateDropOffId] ]