Proper async/await function - node.js

I am attempting to run a bot that scrapes Amazon (using amazon-buddy) for certain products (using array of ASINs) and checks the price. If the price is not 0, it should be sending a message on discord. I currently have this set to run every 30 seconds and it's working, but there are times where it seems like each element is not waiting for the previous one to get a response in the forEach loop and my function doesn't seem to be correct (I'm still trying to understand async/await functions properly).
Is there a better way to run this so that each element waits for the previous element to get scraped before moving on to the next one and THEN run the loop again after 30 seconds?
(function() {
var c = 0;
var timeout = setInterval(function() {
const checkStock = (async () => {
config.items.itemGroup.forEach(element => {
console.log('Checking stock on ' + element)
try {
const product_by_asin = await amazonScraper.asin({ asin: element });
console.log(product_by_asin)
const price = product_by_asin.result[0].price.current_price
const symbol = product_by_asin.result[0].price.symbol
const asin = product_by_asin.result[0].asin
const title = product_by_asin.result[0].title
const url = product_by_asin.result[0].url
const image = product_by_asin.result[0].main_image
if (price != 0) {
const inStockResponse = {
color: 0x008000,
title: title + ' is in stock!',
url: url,
author: {
name: config.botName,
icon_url: config.botImg,
url: config.botUrl
},
description: '<#767456705306165298>, click the tite to go purchase!\n\n' +
'Price: ' + symbol + price,
thumbnail: {
url: image
},
timestamp: new Date()
}
message.channel.send({embed: inStockResponse });
console.log(title + ' (' + asin + ') IS available!')
} else {
console.log(title + ' (' + asin + ') IS NOT available!')
}
} catch (error) {
console.log(error);
}
});
checkStock()
});
console.log('Counter: ' + c)
c++;
}, 30000);
})();

You could use a for...of loop which can wait for each iteration to finish:
async function checkItems(items) {
// Check all items, wait for each to complete.
for (const item of items) {
try {
const product_by_asin = await amazonScraper.asin({ asin: item });
console.log(product_by_asin);
const price = product_by_asin.result[0].price.current_price;
const symbol = product_by_asin.result[0].price.symbol;
const asin = product_by_asin.result[0].asin;
const title = product_by_asin.result[0].title;
const url = product_by_asin.result[0].url;
const image = product_by_asin.result[0].main_image;
if (price != 0) {
const inStockResponse = {
color: 0x008000,
title: title + " is in stock!",
url: url,
author: {
name: config.botName,
icon_url: config.botImg,
url: config.botUrl,
},
description:
"<#767456705306165298>, click the tite to go purchase!\n\n" +
"Price: " +
symbol +
price,
thumbnail: {
url: image,
},
timestamp: new Date(),
};
// NOTE: you might want to wait for this too, the error
// currently isn't being handled like this either.
message.channel.send({ embed: inStockResponse });
console.log(title + " (" + asin + ") IS available!");
} else {
console.log(title + " (" + asin + ") IS NOT available!");
}
} catch (err) {
console.log(err);
}
}
// Wait 30s and check again.
setTimeout(() => checkItems(items), 30000);
}
checkItems(config.items.itemGroup);

Related

Using node.js for-loop index in a coinbase-api callback function

I am new to node.js and i am trying to make a simple script that will connect to the coinbase-api and get the current price of whatever markets are defined in the MARKET array.
The problem i am having is that the for-loop that iterates through the array is asynchronous and the callback function is not getting the correct index value for the array.
The two main solutions i have found are to use promises or force the loop to wait. I think i need to be using promises rather than forcing the for loop to wait but honestly i have failed to implement a solution either way. I have found may example of promises but i just cant seem to figure out how to implement them into my script. I would appreciate any help.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
let tickerCount = null;
const getCallback = (error, response, data) =>
{
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
}
}
setInterval(() =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
publicClient.getProductTicker(MARKET[tickerCount], getCallback);
}
}, 10000);
I was able to figure out how to use promises with trial and error from the helpful examples on the Mozilla Developer Network. I am sure i am making some mistakes but at least it is working now. Another little bonus is that i was able to remove a global.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
function getProductTicker(tickerCount) {
return new Promise(resolve => {
publicClient.getProductTicker(MARKET[tickerCount],function callback(error, response, data){
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
resolve();
}
});
});
}
setInterval( async () =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (var tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
await getProductTicker(tickerCount);
}
}, 10000);

MongoDB aggregation returns empty array with NodeJS

I want to create a script which is taking the average of the Volume for last 7(for example) days.
I'm stuck with aggregation stages since first stage I need to take Date for last 7 days and in second stage calculate Average of Volume
Package list:
Node-schedule - */1 * * * * (Runs the script every minute)
Binance API - Taking data from them.
Screenshot for showcasing how the document looks like in MongoDB.
Aggregation part of the Code.
const average = await dbo.collection(symbol).aggregate([{
'$match': {
'Date': { '$gte': new Date((new Date().getTime() - (7 * 24 * 60 * 60 * 1000))) }
},
},
{
'$group': {
_id: null,
'Volume': { '$avg': '$Volume' }
},
}
]).toArray();
This code returns me an empty array in terminal like this > []
Full Code here.
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
const fetch = require("node-fetch");
const symbols = ["ADABTC", "AEBTC", "AIONBTC", "ALGOBTC", "ARDRBTC"];
//a descriptive name helps your future self and others understand code easier
const getBTCData = async symbol => { //make this function accept the current symbol
//async/await lets us write this much nicer and with less nested indents
let data = await fetch(`https://api.binance.com/api/v3/klines?symbol=${symbol}&interval=30m&limit=1`).then(res => res.json());
const btcusdtdata = data.map(d => {
return {
Open: parseFloat(d[1]),
High: parseFloat(d[2]),
Low: parseFloat(d[3]),
Close: parseFloat(d[4]),
Volume: parseFloat(d[5]),
Timespan: 30,
}
});
console.log(btcusdtdata);
saveToDatebase(symbol, btcusdtdata);
//recursive functions are complicated, we can get rid of it here
//by moving the responsibility to the caller
};
//helper function for an awaitable timeout
const sleep = ms => new Promise(res => setTimeout(res, ms));
const j = schedule.scheduleJob('*/1 * * * *', async() => {
//expand this function to be responsible for looping the data
for (let symbol of symbols) {
//we can pass symbol to getBTCData instead of making it
//responsible for figuring out which symbol it should get
await getBTCData(symbol);
await sleep(8000);
}
});
//make this a helper function so `saveToDatabase()` isn't also responsible for it
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return date + ' ' + time;
};
const saveToDatebase = async(symbol, BTCdata) => {
try {
const url = 'mongodb://username:password#ipadress:port/dbname';
let dateTime = getDateTime();
let db = await MongoClient.connect(url, { useUnifiedTopology: true });
const dbo = db.db('Crypto');
const myobj = Object.assign({ Name: symbol, Date: dateTime }, BTCdata[0]);
await dbo.collection(symbol).insertOne(myobj);
const average = await dbo.collection(symbol).aggregate([{
'$match': {
'Date': { '$gte': new Date((new Date().getTime() - (7 * 24 * 60 * 60 * 1000))) }
},
},
{
'$group': {
_id: null,
'Volume': { '$avg': '$Volume' }
},
}
]).toArray();
console.log('1 document inserted');
console.log(average);
db.close();
} catch (e) {
console.error(e)
}
};
EDIT1
If I delete $match part my script is working and I receive average of Volume.
Screenshot of terminal after success try without $match
EDIT2
According to the last answer I understand that I need to change Date format from string to object, but I really can't get how I can do it in this part?
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return date + ' ' + time;
};
EDIT3
After editing the Date format I receive a Document in MongoDB in strange Date format like - Date:2020-07-20T13:24:02.390+00:00
Code here:
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return new Date();
};
The problem is on the Date field format.
The getDateTime function returns a string so Mongo is managing the field as a string not as a Date object so the $gte check will compare string not dates.
You should change the function to getDateTime = () => new Date(). Mongo will manage the date correctly storing in UTF Timezone.
Tring to query a date-string in the $match field would be really difficult.
Edit:
To update the typing just:
const getDateTime = () => {
return new Date();
};

Nodejs - Google API Send email doesn't group by thread

As part of small email CRM project, i have created a Nodejs app. While i send email through gmail send api, the messages are not grouped.
const sendObject = {}
output.data.messages.map(each => {
sendObject.threadId = each.threadId // ThreadID is unique.
each.payload.headers.map(head => {
if (head.name === 'From') {
sendObject.replyTo = head.value
} else if (head.name === 'Subject') {
if (head.value.indexOf('Re:') >= 0) {
sendObject.subject = head.value
} else {
sendObject.subject = 'Re: ' + head.value
}
} else if (head.name === 'Message-Id') {
sendObject.inReplyTo = head.value // The last thread messageId is inserted into In-Reply-To tag
sendObject.reference.push(head.value) // All the messageId are appended as part of References tag
}
})
})
const email_lines = []
email_lines.push('References:' + sendObject.reference.join(' '))
email_lines.push('In-Reply-To:' + sendObject.inReplyTo)
email_lines.push('Subject: ' + sendObject.subject)
email_lines.push('To:' + sendObject.replyTo)
email_lines.push('')
email_lines.push(req.body.body)
const email = email_lines.join('\r\n').trim()
let base64EncodedEmail = new Buffer(email).toString('base64')
base64EncodedEmail = base64EncodedEmail.replace(/\+/g, '-').replace(/\//g, '_')
emailConfig.gmail.users.messages.send({
userId: 'me',
threadId: sendObject.threadId,
resource: {
raw: base64EncodedEmail
}
}, async (err) => {
if (err) {
console.log('The Threads API returned an error: ' + err)
}
})
When i login to gmail through browser, i can see 2 different emails instead of one thread.
You should put threadId inside of the resource.
const response = await this.gmail.users.messages.send({
auth: this.oAuth2Client,
userId: this.gmailAddress,
uploadType: 'multipart',
resource: {
threadId: email.threadId,
raw: raw
}
})
For more details, you can check my Gmail-APIs-wrapper here

Node.js call back function on termination of a user defined function

I have a node.js app consisting of a timer calling a user defined function made up of a bunch of functions in node language. The calling script has a timer calling function mybuy() every 10 seconds; mybuy() buys crypto currencies using Binance api according trigger prices contained in a mySQL table (alarms). I would like to start mysell() (not shown , but similar to myBuy()) right after mybuy() has run its course.
How to make mysell() the callback function of mybuy()?
This the calling script:
var fs = require('fs');
var sl = require('alberto/buy');
var loop = 0;
setImmediate(() => {
// start the log
fs.appendFile('./log.txt', "\n Loop-> " + loop + "\n", function (err) {
if (err) { console.log(err); }
})
//execute the function
sl.mybuy(); // USD function; everything happens here.Can take long to finish
var myInt = setInterval(function () {
loop++;
fs.appendFile('./log.txt', "Loop-> " + loop + "\n", function (err) {
if (err) { console.log(err); }
})
//execute every 10 secs
sl.mybuy();
if (loop > 5) { clearInterval(myInt); } // max 6 loops for testing
}, 10000);
});
the UDF id here
exports.mybuy = function () {
var fs = require('fs'); // I keep a log.txt
process.stdout.write("\u001b[2J\u001b[0;0H");// clear screen
aww = (new Date()).toJSON().slice(0, 19).replace(/[-T]/, '-');
aww = aww.replace(/T/, ' ');
console.log(aww, '\n\n'); // practicing with dates
var mysql = require('mysql');
var con = mysql.createConnection({
host: "www.photobangkok.com",
user: "photoban_user",
password: "xxxxxxxx",
database: "photoban_datab"
});
// 'added' is for never processed entries in alarms table.It will change to BOUGHT or SOLD
sql = "SELECT rec, id,coin,buy,amount_b,stat FROM alarms where stat='added' AND buy>0 order by coin";
var cnt = 0; // not used, perhaps an idea to emit an event when cnt reaches the number of rows
con.query(sql, function (err, result) {
if (err) throw err;
str = "";
for (var index in result) {
str = result[index].rec + "-" + result[index].id + "-" + result[index].coin + "-" + result[index].buy + "-" + result[index].amount_b + "-" + result[index].stat;
// set up variables
coin = result[index].coin;
buy = result[index].buy;
rec = result[index].rec;
id = result[index].id;
amount = result[index].amount_b;
console.log('\x1b[36m%s\x1b[0m', str); // set color green. Display str
checkprice(coin, buy, rec, id, amount); //check Binance today price for the coin.The function will execute sometimes
} // end of loop
console.log('\x1b[36m%s\x1b[0m', str); // set color green. Display str
});
//check single coin price using binance api
function checkprice(coin, buy, rec, id, amount) {
const binance = require('node-binance-api')().options({
APIKEY: '<key>',
APISECRET: '<secret>',
useServerTime: true,
test: true //sandbox does not work
});
binance.prices(coin, (error, ticker) => {
act = "Nothing"; // default value
pricenow = ticker[coin]; // note ticker[coin]
if (pricenow < buy) {
show(id, rec, coin, buy, amount, pricenow);// Display sometimes then call book()
} else { console.log(coin, pricenow, buy, act, '\n'); }
});
}
function show(id, rec, coin, buy, amount, pricenow) {
delta = buy - pricenow; // posted trigger - today price
delta = delta.toFixed(8);
console.log('\x1b[31m%s\x1b[0m', coin, buy, amount, id, rec, ">BUY", delta); //display entries from alarms higher that today price
book(id, rec, coin, buy, amount, pricenow);
}
// dummy function to be replaced with a buy api order
function book(id, rec, coin, buy, amount, pricenow) {
const binance = require('node-binance-api')().options({
APIKEY: '<key>',
APISECRET: '<secret>',
useServerTime: true,
test: true //sandbox
});
console.log("Order:buy what??", coin, "amount:", amount, '\n');
/* binance.prices(coin, (error, ticker) => {
console.log("booking",coin, ticker[coin]);
update(id,rec);
}); */
update(id, rec, amount); // update mySql table. Slow but sure
}
function update(id, rec, amount) {
var sql = "UPDATE alarms SET stat = 'BOUGHT' ,today =
CONVERT_TZ(now(), '+00:00', '+7:00') WHERE id = "+id+" AND rec = "+rec;
con.query(sql, function (err, result) {
if (err) throw err;
console.log(result.affectedRows + " record updated");
// keep a log.tx
fs.appendFile('./log.txt', aww + " bought " + id + "-" + rec + "-" + amount + "\n",
function (err) {
if (err) { console.log(err); }
})
});
}
// I could check if all rows are done and raise an event? (how to do it)
} // end
To make mySell as the callback method of myBuy, invoke the myBuy method using the following structure.
myBuy(() => {
// operation of mySell method
});
And your myBuy method should return the callback, after perform its own operation.
exports.myBuy = function(cb) {
// operation of myBuy method
return cb; // return to the mySell method
}

Batch requests placed via nodejs request.get using rxjs

I am currently using the following function to create a Promise from the result of calling request.get:
function dlPromiseForMeta(meta) {
return new Promise(function (resolve, reject) {
meta.error = false;
var fileStream = fs.createWriteStream(meta.filePath);
fileStream.on('error', function (error) {
meta.error = true;
console.log('filesystem ' + meta.localFileName + ' ERROR: ' + error);
console.log('record: ' + JSON.stringify(meta));
reject(meta);
});
fileStream.on('close', function () {
resolve(meta);
});
request.get({
uri: meta.url,
rejectUnauthorized: false,
followAllRedirects: true,
pool: {
maxSockets: 1000
},
timeout: 10000,
agent: false
})
.on('socket', function () {
console.log('request ' + meta.localFileName + ' made');
})
.on('error', function (error) {
meta.error = true;
console.log('request ' + meta.localFileName + ' ERROR: ' + error);
console.log('record: ' + JSON.stringify(meta));
reject(meta);
})
.on('end', function () {
console.log('request ' + meta.localFileName + ' finished');
fileStream.close();
})
.pipe(fileStream);
});
}
This works fine except when I am trying to call it too many times, as in the example below, where imagesForKeywords returns an rxjs Observable:
imagesForKeywords(keywords, numberOfResults)
.mergeMap(function (meta) {
meta.fileName = path.basename(url.parse(meta.url).pathname);
meta.localFileName = timestamp + '_' + count++ + '_' + meta.keyword + '_' + meta.source + path.extname(meta.fileName);
meta.filePath = path.join(imagesFolder, meta.localFileName);
return rxjs.Observable.fromPromise(dlPromiseForMeta(meta))(meta);
});
I start getting ESOCKETTIMEDOUT errors when the source observable becomes sufficiently large.
So what I would like to do is somehow batch what happens in mergeMap for every, say, 100 entries... so I do those 100 in parallel, and each batch serially, and then merge them at the end.
How can I accomplish this using rxjs?
I think the simplest thing to use is bufferTime() which triggers after a certain number of ms but also has a parameter at the end for count.
Using a timeout seems useful, in case there's a stream pattern that does not reach the batch limit in a reasonable time.
If that does not fit your use-case, comment me with some more details and I will adjust accordingly.
Your code will look like this,
bufferTime as described above
forkjoin - run the buffer contents in parallel and emit when all return
mergeMap - coalesce the results
imagesForKeywords(keywords, numberOfResults)
.mergeMap(function (meta) {
meta.fileName = path.basename(url.parse(meta.url).pathname);
meta.localFileName = timestamp + '_' + count++ + '_' + meta.keyword + '_' + meta.source + path.extname(meta.fileName);
meta.filePath = path.join(imagesFolder, meta.localFileName);
return meta;
})
.bufferTime(maxTimeout, null, maxBatch)
.mergeMap(items => rxjs.Observable.forkJoin(items.map(dlPromiseForMeta)))
.mergeMap(arr => rxjs.Observable.from(arr))
Here's a runnable mockup to show it working. Have commented out the last mergeMap to show the buffering.
I have assumed a couple of things,
imagesForKeywords breaks keywords into observable stream of keyword
there is one keyword per dlPromiseForMeta call
// Some mocking
const imagesForKeywords = (keywords, numberOfResults) => {
return Rx.Observable.from(keywords.map(keyword => { return {keyword} }))
}
const dlPromiseForMeta = (meta) => {
return Promise.resolve(meta.keyword + '_image')
}
// Compose meta - looks like it can run at scale, since is just string manipulations.
const composeMeta = meta => {
// meta.fileName = path.basename(url.parse(meta.url).pathname);
// meta.localFileName = timestamp + '_' + count++ + '_' + meta.keyword + '_' + meta.source + path.extname(meta.fileName);
// meta.filePath = path.join(imagesFolder, meta.localFileName);
return meta;
}
const maxBatch = 3
const maxTimeout = 50 //ms
const bufferedPromises = (keywords, numberOfResults) =>
imagesForKeywords(keywords, numberOfResults)
.map(composeMeta)
.bufferTime(maxTimeout, null, maxBatch)
.mergeMap(items => Rx.Observable.forkJoin(items.map(dlPromiseForMeta)))
//.mergeMap(arr => Rx.Observable.from(arr))
const keywords = ['keyw1', 'keyw2', 'keyw3', 'keyw4', 'keyw5', 'keyw6', 'keyw7'];
const numberOfResults = 1;
bufferedPromises(keywords, numberOfResults)
.subscribe(console.log)
<script src="https://cdnjs.cloudflare.com/ajax/libs/rxjs/5.5.6/Rx.js"></script>

Resources