MongoDB aggregation returns empty array with NodeJS - node.js

I want to create a script which is taking the average of the Volume for last 7(for example) days.
I'm stuck with aggregation stages since first stage I need to take Date for last 7 days and in second stage calculate Average of Volume
Package list:
Node-schedule - */1 * * * * (Runs the script every minute)
Binance API - Taking data from them.
Screenshot for showcasing how the document looks like in MongoDB.
Aggregation part of the Code.
const average = await dbo.collection(symbol).aggregate([{
'$match': {
'Date': { '$gte': new Date((new Date().getTime() - (7 * 24 * 60 * 60 * 1000))) }
},
},
{
'$group': {
_id: null,
'Volume': { '$avg': '$Volume' }
},
}
]).toArray();
This code returns me an empty array in terminal like this > []
Full Code here.
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
const fetch = require("node-fetch");
const symbols = ["ADABTC", "AEBTC", "AIONBTC", "ALGOBTC", "ARDRBTC"];
//a descriptive name helps your future self and others understand code easier
const getBTCData = async symbol => { //make this function accept the current symbol
//async/await lets us write this much nicer and with less nested indents
let data = await fetch(`https://api.binance.com/api/v3/klines?symbol=${symbol}&interval=30m&limit=1`).then(res => res.json());
const btcusdtdata = data.map(d => {
return {
Open: parseFloat(d[1]),
High: parseFloat(d[2]),
Low: parseFloat(d[3]),
Close: parseFloat(d[4]),
Volume: parseFloat(d[5]),
Timespan: 30,
}
});
console.log(btcusdtdata);
saveToDatebase(symbol, btcusdtdata);
//recursive functions are complicated, we can get rid of it here
//by moving the responsibility to the caller
};
//helper function for an awaitable timeout
const sleep = ms => new Promise(res => setTimeout(res, ms));
const j = schedule.scheduleJob('*/1 * * * *', async() => {
//expand this function to be responsible for looping the data
for (let symbol of symbols) {
//we can pass symbol to getBTCData instead of making it
//responsible for figuring out which symbol it should get
await getBTCData(symbol);
await sleep(8000);
}
});
//make this a helper function so `saveToDatabase()` isn't also responsible for it
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return date + ' ' + time;
};
const saveToDatebase = async(symbol, BTCdata) => {
try {
const url = 'mongodb://username:password#ipadress:port/dbname';
let dateTime = getDateTime();
let db = await MongoClient.connect(url, { useUnifiedTopology: true });
const dbo = db.db('Crypto');
const myobj = Object.assign({ Name: symbol, Date: dateTime }, BTCdata[0]);
await dbo.collection(symbol).insertOne(myobj);
const average = await dbo.collection(symbol).aggregate([{
'$match': {
'Date': { '$gte': new Date((new Date().getTime() - (7 * 24 * 60 * 60 * 1000))) }
},
},
{
'$group': {
_id: null,
'Volume': { '$avg': '$Volume' }
},
}
]).toArray();
console.log('1 document inserted');
console.log(average);
db.close();
} catch (e) {
console.error(e)
}
};
EDIT1
If I delete $match part my script is working and I receive average of Volume.
Screenshot of terminal after success try without $match
EDIT2
According to the last answer I understand that I need to change Date format from string to object, but I really can't get how I can do it in this part?
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return date + ' ' + time;
};
EDIT3
After editing the Date format I receive a Document in MongoDB in strange Date format like - Date:2020-07-20T13:24:02.390+00:00
Code here:
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return new Date();
};

The problem is on the Date field format.
The getDateTime function returns a string so Mongo is managing the field as a string not as a Date object so the $gte check will compare string not dates.
You should change the function to getDateTime = () => new Date(). Mongo will manage the date correctly storing in UTF Timezone.
Tring to query a date-string in the $match field would be really difficult.
Edit:
To update the typing just:
const getDateTime = () => {
return new Date();
};

Related

Calculate difference between two timestamps

I'm using the twitch api and TMI.JS
I'm trying to fetch the timestamp of followed_at. The timestamp I got was 2021-12-25T15:49:57Z, how would I go about getting the current timestamp, and then calculating the difference. So in this instance it would return followername has been following streamername for 20 days 19 hours (at the time of writing it's been 20 days and 19 hours since the followed_at timestamp.)
if (message.toLowerCase() === '!followage') {
client.say(channel, `#${tags.username}, does not work yet `)
console.log(`${channel.id}`)
console.log(`${tags['user-id']}`)
async function getData() {
const res = await fetch(`https://api.twitch.tv/helix/users/follows?to_id=226395001&from_id=${tags['user-id']}`, { method: 'get', headers: {
'Client-Id': 'cut out for a reason',
'Authorization': 'Bearer cut out for a reason'
}});
const data = await res.json();
console.log(data)
if (data.data[0]) {
client.say(channel, `#${tags.username}, followed at ${data.data[0].followed_at}`)
} else {
client.say(channel, `You aren't followed!`)
}
}
getData()
}
Above is my code for fetching it, and then sending it to the channel.
Here's how you can do it:
const followedAt = new Date('2021-12-25T15:49:57Z');
const currentDate = new Date();
const diffTime = Math.abs(currentDate - followedAt);
const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24));
console.log(diffTime + " milliseconds");
console.log(diffDays + " days");
And that's the way to get days and hours:
const followedAt = new Date('2021-12-25T15:49:57Z');
const currentDate = new Date();
const diffTime = Math.abs(currentDate - followedAt);
const diffTotalHours = Math.floor(diffTime / (1000 * 60 * 60));
const diffDays = Math.floor(diffTotalHours / 24);
const diffHoursWithoutDays = diffTotalHours % 24;
console.log(`${diffDays} days and ${diffHoursWithoutDays} hours`);

Set polygon.io incoming socket messages to redis in every 5 minutes and post to lambda using node js

I am using polygon.io websocket for receiving stock trades. The logic is to store the incoming messages to redis , one message with expiry time 5 minutes. So that , as I am start receiving socket message , I used map function to process the messages and passed them to redis and checked whether currently the stock has any value in redis. If no, then set the value with 5minute expiry .At that time I need to post the value to lambda. Then after the expire time , need to set another incoming value. So the purpose is to post to lambda at every 5 minutes based on redis set value with expiry time. I have used following code to recieve socket message and passed them to redis function.
ws.on('message', (data: string) => {
let polygonMessage = JSON.parse(data)
polygonMessage.map((msg: PolygonResponse) => {
if (msg.ev === 'status') {
socketLog.info('Status Update:', msg.message)
return console.log('Status Update:', msg.message)
}
if ((msg.hasOwnProperty('sym')) && (msg.hasOwnProperty('p'))) {
redis.stockDetails(msg.sym, msg.p).then((res:any) => {
if (res == true) {
let currentDateTime = this.checkTime();
socketLog.info('Set to Redis :' + msg.sym + ': ' + msg.p + ' ,DateTime: ' + currentDateTime)
axios.post('https://********', {
stock: msg.sym,
price: msg.p
}).then((res: any) => {
console.log(' statusCode : ' + res.status )
})
.catch((error: any) => {
console.log('Lambda Error : ' + error)
})
} else {
socketLog.info('delay :' + msg.sym + ': ' + msg.p)
}
})
}
})
})
CheckTime Function
public checkTime() {
const today = new Date();
today.setDate(today.getDate());
const date = (today.getDate() < 10) ? '0' + today.getDate() : today.getDate();
const month = (today.getMonth() < 10) ? '0' + (today.getMonth() + 1) : today.getMonth() + 1;
const currentDate = today.getFullYear() + '-' + month + '-' + date;
const hours = (today.getHours() < 10) ? '0' + today.getHours() : today.getHours();
const minutes = (today.getMinutes() < 10) ? '0' + today.getMinutes() : today.getMinutes();
const seconds = (today.getSeconds() < 10) ? '0' + today.getSeconds() : today.getSeconds();
let currentdatetime: any = currentDate + ' ' + hours + ':' + minutes + ':' + seconds;
return currentdatetime;
}
The redis function is as follows,
public async stockDetails(symbol: any, price: any) {
const getAsync = promisify(this.client.get).bind(this.client);
const setAsync = promisify(this.client.set).bind(this.client);
const expireAsync = promisify(this.client.expire).bind(this.client);
let getSymbol = await getAsync(symbol);
console.log("getSymbol:" + getSymbol)
if (getSymbol == null) {
await setAsync(symbol, price)
await expireAsync(symbol, 300)
console.log("Get current value : " + getAsync(symbol))
return true;
} else {
console.log("set not empty")
return false;
}
}
My desired output is like this during every 5 minutes.
Set to Redis : AAPL : 131, DateTime: 2021-02-19 10:10:10
Set to Redis : AAPL : 132, DateTime: 2021-02-19 10:15:10
But now it is like,setting to redis in same minute itself
Set to Redis : AAPL : 131, DateTime: 2021-02-19 10:10:10
Set to Redis : AAPL : 132, DateTime: 2021-02-19 10:10:10
This will result in multiple lambda post at same time. I need to post to lambda only once in every 5 minute.

Proper async/await function

I am attempting to run a bot that scrapes Amazon (using amazon-buddy) for certain products (using array of ASINs) and checks the price. If the price is not 0, it should be sending a message on discord. I currently have this set to run every 30 seconds and it's working, but there are times where it seems like each element is not waiting for the previous one to get a response in the forEach loop and my function doesn't seem to be correct (I'm still trying to understand async/await functions properly).
Is there a better way to run this so that each element waits for the previous element to get scraped before moving on to the next one and THEN run the loop again after 30 seconds?
(function() {
var c = 0;
var timeout = setInterval(function() {
const checkStock = (async () => {
config.items.itemGroup.forEach(element => {
console.log('Checking stock on ' + element)
try {
const product_by_asin = await amazonScraper.asin({ asin: element });
console.log(product_by_asin)
const price = product_by_asin.result[0].price.current_price
const symbol = product_by_asin.result[0].price.symbol
const asin = product_by_asin.result[0].asin
const title = product_by_asin.result[0].title
const url = product_by_asin.result[0].url
const image = product_by_asin.result[0].main_image
if (price != 0) {
const inStockResponse = {
color: 0x008000,
title: title + ' is in stock!',
url: url,
author: {
name: config.botName,
icon_url: config.botImg,
url: config.botUrl
},
description: '<#767456705306165298>, click the tite to go purchase!\n\n' +
'Price: ' + symbol + price,
thumbnail: {
url: image
},
timestamp: new Date()
}
message.channel.send({embed: inStockResponse });
console.log(title + ' (' + asin + ') IS available!')
} else {
console.log(title + ' (' + asin + ') IS NOT available!')
}
} catch (error) {
console.log(error);
}
});
checkStock()
});
console.log('Counter: ' + c)
c++;
}, 30000);
})();
You could use a for...of loop which can wait for each iteration to finish:
async function checkItems(items) {
// Check all items, wait for each to complete.
for (const item of items) {
try {
const product_by_asin = await amazonScraper.asin({ asin: item });
console.log(product_by_asin);
const price = product_by_asin.result[0].price.current_price;
const symbol = product_by_asin.result[0].price.symbol;
const asin = product_by_asin.result[0].asin;
const title = product_by_asin.result[0].title;
const url = product_by_asin.result[0].url;
const image = product_by_asin.result[0].main_image;
if (price != 0) {
const inStockResponse = {
color: 0x008000,
title: title + " is in stock!",
url: url,
author: {
name: config.botName,
icon_url: config.botImg,
url: config.botUrl,
},
description:
"<#767456705306165298>, click the tite to go purchase!\n\n" +
"Price: " +
symbol +
price,
thumbnail: {
url: image,
},
timestamp: new Date(),
};
// NOTE: you might want to wait for this too, the error
// currently isn't being handled like this either.
message.channel.send({ embed: inStockResponse });
console.log(title + " (" + asin + ") IS available!");
} else {
console.log(title + " (" + asin + ") IS NOT available!");
}
} catch (err) {
console.log(err);
}
}
// Wait 30s and check again.
setTimeout(() => checkItems(items), 30000);
}
checkItems(config.items.itemGroup);

Using node.js for-loop index in a coinbase-api callback function

I am new to node.js and i am trying to make a simple script that will connect to the coinbase-api and get the current price of whatever markets are defined in the MARKET array.
The problem i am having is that the for-loop that iterates through the array is asynchronous and the callback function is not getting the correct index value for the array.
The two main solutions i have found are to use promises or force the loop to wait. I think i need to be using promises rather than forcing the for loop to wait but honestly i have failed to implement a solution either way. I have found may example of promises but i just cant seem to figure out how to implement them into my script. I would appreciate any help.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
let tickerCount = null;
const getCallback = (error, response, data) =>
{
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
}
}
setInterval(() =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
publicClient.getProductTicker(MARKET[tickerCount], getCallback);
}
}, 10000);
I was able to figure out how to use promises with trial and error from the helpful examples on the Mozilla Developer Network. I am sure i am making some mistakes but at least it is working now. Another little bonus is that i was able to remove a global.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
function getProductTicker(tickerCount) {
return new Promise(resolve => {
publicClient.getProductTicker(MARKET[tickerCount],function callback(error, response, data){
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
resolve();
}
});
});
}
setInterval( async () =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (var tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
await getProductTicker(tickerCount);
}
}, 10000);

MongoDB misses saving data in a long period of time

Good afternoon, I have a script which is receiving data from Binance API every 30 minutes and saves it in MongoDB specific database in different collections.
My MongoDB is installed on VPS. I'm connecting to it from my local computer.
Problem is that I leave my code to work constantly for ± 3 days and I receive that sometimes data did not save on specific collection or missed, etc. How I can configure my code or what can I do to create a perfect connection to save all data correctly.
Problem Explanation:
I have an array with
symbols=["ada","ae","kava","eth","etc","zrx","xzc","faq","gas","vfg","req"];
And when I leave the code for let's say 25 hours. I suppose to see 50 documents on every collection name.
But I receive that some collections got 48 documents saved instead of 50, some of them got 49, some of them got 50 like sometimes collections didn't save d properly.
FULLCODE HERE:
const { MongoClient } = require('mongodb');
const schedule = require('node-schedule');
const fetch = require("node-fetch");
const symbols=["ada","ae","kava","eth","etc","zrx","xzc","faq","gas","vfg","req"];
//a descriptive name helps your future self and others understand code easier
const getBTCData = async symbol => { //make this function accept the current symbol
//async/await lets us write this much nicer and with less nested indents
let data = await fetch(`https://api.binance.com/api/v3/klines?symbol=${symbol}&interval=30m&limit=1`).then(res => res.json());
const btcusdtdata = data.map(d => {
return {
Open: parseFloat(d[1]),
High: parseFloat(d[2]),
Low: parseFloat(d[3]),
Close: parseFloat(d[4]),
Volume: parseFloat(d[5]),
Timespan: 30,
}
});
console.log(btcusdtdata);
saveToDatebase(symbol, btcusdtdata);
//recursive functions are complicated, we can get rid of it here
//by moving the responsibility to the caller
};
//helper function for an awaitable timeout
const sleep = ms => new Promise(res => setTimeout(res, ms));
const j = schedule.scheduleJob('*/30 * * * *', async() => {
//expand this function to be responsible for looping the data
for (let symbol of symbols) {
//we can pass symbol to getBTCData instead of making it
//responsible for figuring out which symbol it should get
await getBTCData(symbol);
await sleep(8000);
}
});
//make this a helper function so `saveToDatabase()` isn't also responsible for it
const getDateTime = () => {
let today = new Date();
let date = today.getFullYear() + '-' + (today.getMonth() + 1) + '-' + today.getDate();
let time = today.getHours() + ":" + today.getMinutes() + ":" + today.getSeconds();
return date + ' ' + time;
};
const saveToDatebase = async(symbol, BTCdata) => {
const url = 'mongodb://username:password#server:port/dbname?retryWrites=true&w=majority';
let dateTime = getDateTime();
//use await here and below to vastly simplify this function
let db = await MongoClient.connect(url, { useUnifiedTopology: true });
const dbo = db.db('Crypto');
const myobj = { Name: symbol, Array: BTCdata, Date: dateTime };
await dbo.collection(symbol).insertOne(myobj);
console.log('1 document inserted');
db.close();
};
Goal: Solve the saving issue, I think it also can be possible something with VPS response itself. How can I maybe update my code that it looks cleaner and faster if possible?
Any suggestions or help really appreciated.

Resources