I'm using batch write in order to avoid 500 maximum writes limit. Even if I'm using batchArray in order to process more than 500 writes, I get this error:
Exception from a finished function: Error: 3 INVALID_ARGUMENT: maximum 500 writes allowed per request
Code: https://controlc.com/3c3a59be
exports.checkSchedine2 = functions.pubsub
.schedule('15,45 * * * *')
.onRun(async (context) => {
const batch = firestore.batch();
var currentTime = Math.floor(Date.now() / 1000);
const risultati = await firestore.collection("Risultati").doc("risultati").get();
const ref1 = await firestore.collection("Schedine").get();
let batchArray = [];
batchArray.push(batch);
let operationCounter = 0;
let batchIndex = 0;
let _notcommit = false;
await Promise.all(ref1.docs.map(async (doc) => {
const ref2 = await firestore.collection("Schedine").doc(doc.id).collection("in corso").get();
for (const doc2 of ref2.docs) {
const documentData = doc2.data();
for (matchId in doc2.data()["Match"]) {
try {
for (id in risultati.data()) {
if (matchId == id && risultati.data()[id]["status"] != 0 && doc2.data()["Match"][matchId]["status"] == 0) {
if ((doc2.data()["Match"][matchId]["Bet"] == 1 || doc2.data()["Match"][matchId]["Bet"] == 2 || doc2.data()["Match"][matchId]["Bet"] == 3) && doc2.data()["Match"][matchId]["Bet"] == risultati.data()[id].ris) {
documentData["Match"][matchId]["status"] = 1;
} else if (doc2.data()["Match"][matchId]["Bet"] == 4 && risultati.data()[id].goal == 1) {
documentData["Match"][matchId]["status"] = 1;
} else if (doc2.data()["Match"][matchId]["Bet"] == 5 && risultati.data()[id].goal == 0) {
documentData["Match"][matchId]["status"] = 1;
} else if (doc2.data()["Match"][matchId]["Bet"] == 6 && risultati.data()[id].over == 1) {
documentData["Match"][matchId]["status"] = 1;
} else if (doc2.data()["Match"][matchId]["Bet"] == 7 && risultati.data()[id].over == 0) {
documentData["Match"][matchId]["status"] = 1;
} else {
documentData["Match"][matchId]["status"] = 2;
}
}
}
} catch (e) {}
}
if (_notcommit == false) {
await batchArray[batchIndex].update(doc2.ref, documentData);
operationCounter++;
if (operationCounter > 100) {
batchArray.push(batch);
batchIndex++;
operationCounter = 0;
}
}
};
}));
Promise.all(batchArray.map(batch => batch.commit()));
});
It seems that the code launches too many commits, isn't it?
You are pushing the same batch in that batchArray in every iteration. You should be creating a new batch for every 500 operations:
let batchArray = [firestore.batch()];
// ...
if (operationCounter > 100) {
const newBatch = firestore.batch();
batchArray.push(newBatch);
batchIndex++;
operationCounter = 0;
}
Related
This is my current code:
function leaderboardembed() {
const filtered = client.points.filter(p => p.guild === message.guild.id).array();
let orilent;
const sorted = filtered.sort((a, b) => b.vouches - a.vouches );
let embeds = [];
let j = 0;
let first = (10)
let maxnum = 50;
orilent = sorted.length;
if(isNaN(maxnum)) {
console.log("maximum_leaderboard NOT A NUMBER")
maxnum = 50;}
if (maxnum > sorted.length)
maxnum = sorted.length + (10 - Number(String(sorted.length/10).slice(2)));
if (maxnum < 10) maxnum = 10;
for (let i = 10; i <= maxnum; i += 10) {
const top = sorted.splice(0, 10);
const embed = new Discord.MessageEmbed()
.setTitle(`\`${message.guild.name}\` | Leaderboard`)
.setTimestamp()
.setDescription(`Top ${i<orilent?i:orilent}/${orilent} Ranking:`)
.setColor(embedcolor);
for (const data of top) {
j++;
try {
embed.addField(`**${j}**. \`${data.usertag}\``, ` | **Vouches:** \`${data.vouch}\``);
} catch {
embed.addField(`**${j}**. \`${data.usertag}\``, ` | **Vouches:** \`${data.vouch}\``);
}
}
embeds.push(embed);
}
return embeds;
}
async function leaderboard() {
let currentPage = 0;
const embeds = leaderboardembed();
if (embeds.length == 1)
return message.channel.send(embeds[0]).catch(e=>console.log("ranking: " + e))
const lbembed = await message.channel.send(
`**Current Page - ${currentPage + 1}/${embeds.length}**`,
embeds[currentPage]).catch(e=>console.log("ranking: " + e));
try {
await lbembed.react("⏪");
await lbembed.react("⏹");
await lbembed.react("⏩");
} catch (error) {
console.error(error);
}
const filter = (reaction, user) => ["⏪", "⏹", "⏩"].includes(reaction.emoji.name) && message.author.id === user.id;
const collector = lbembed.createReactionCollector(filter, {
time: 60000
});
collector.on("collect", async (reaction, user) => {
try {
if (reaction.emoji.name === "⏩") {
if (currentPage < embeds.length - 1) {
currentPage++;
lbembed.edit(`**Current Page - ${currentPage + 1}/${embeds.length}**`, embeds[currentPage]);
}
} else if (reaction.emoji.name === "⏪") {
if (currentPage !== 0) {
--currentPage;
lbembed.edit(`**Current Page - ${currentPage + 1}/${embeds.length}**`, embeds[currentPage]);
}
} else {
collector.stop();
reaction.message.reactions.removeAll();
}
await reaction.users.remove(message.author.id);
} catch (error) {
console.error(error);
}
});
}
This is how it displays when doing the command
The problem is that there's about 1000 members and some don't have the "vouches" and id rather just display the people that have the highest to lowest.
Here's an example of what I'm aiming it to be like:
Firstly, to sort all of them, you do need them all in an array. Otherwise, it will be excessively difficult. Here is a simple sort function that JavaScript provides for us:
var ranks = [7, 2, 9, 5, 10, 110]
ranks.sort() //don’t use this, this would put 10, 110, 2 etc since it’s alphabetical, you need to specify how to sort it
ranks.sort((a, b)=> a - b) //2, 5, 7, 9, 10, 110
//to reverse that
ranks.sort((a, b) => b-a)
I have a Discord BOT written with Node.JS using Discord.JS
There i have a function that starts when the bot becomes online
It consists in getting an array from an API: https://jacob.rede-epic.com/api/upcomingEvents
With the timestamp, it is supposed to send a message 10, 5 and 1 minute before it happens, when it happens (Exact hour + 15 minutes, e.g: 08:15/09:15/10:15 etc), and when it finishes (20 minutes later, e.g: 08:35/09:35/10:35 etc)
How can i make it to reset the whole system ?
Code:
const { MessageEmbed } = require('discord.js');
const axios = require('axios');
class Jacob {
constructor(client) {
this.client = client;
}
async start() {
const jacobContestsChannel = this.client.channels.cache.get('852596632591269968');
let nextEventObject = null;
let sentNotification = false;
let sentNotification2 = false;
let sentNotification3 = false;
let sentNotificationActive = false;
let sentNotificationFinished = false;
const cropNames = [
'Cactus',
'Carrot',
'Cocoa beans',
'Melon',
'Mushroom',
'Nether wart',
'Potato',
'Pumpkin',
'Sugar cane',
'Wheat',
];
setInterval(async () => {
let upcomingEvents = [];
let response = await axios.get('https://jacob.rede-epic.com/api/upcomingEvents');
let request = response.data;
for (let i in request) {
upcomingEvents.push(request[i]);
}
if (nextEventObject == null) {
nextEventObject = upcomingEvents[0];
} else {
let diff = nextEventObject.timestamp - Date.now();
let active = diff < 0;
setInterval(() => {
try {
diff = nextEventObject.timestamp - Date.now();
} catch {
nextEventObject = null;
}
}, 1000);
if (diff < -20 * 60 * 1000) {
sentNotification = false;
sentNotification2 = false;
sentNotification3 = false;
sentNotificationActive = false;
nextEventObject == null;
if (!sentNotificationFinished) {
jacobContestsChannel.send(new MessageEmbed()
.setColor(this.client.embedColor)
.setTitle('The current contest has ended!')
);
sentNotificationFinished = true;
}
} else if (!active && diff < 10 * 60 * 1000 && !sentNotification) {
jacobContestsChannel.send(new MessageEmbed()
.setColor(this.client.embedColor)
.setTitle('A contest is starting in 10 minutes')
.setDescription(`Crops: ${nextEventObject.crops.map(crop => cropNames[crop]).join(', ')}`)
);
sentNotification = true;
} else if (!active && diff < 5 * 60 * 1000 && sentNotification && !sentNotification2) {
jacobContestsChannel.send(new MessageEmbed()
.setColor(this.client.embedColor)
.setTitle('A contest is starting in 5 minutes')
.setDescription(`Crops: ${nextEventObject.crops.map(crop => cropNames[crop]).join(', ')}`)
);
sentNotification2 = true;
} else if (!active && diff < 60 * 1000 && sentNotification && sentNotification2 && !sentNotification3) {
jacobContestsChannel.send(new MessageEmbed()
.setColor(this.client.embedColor)
.setTitle('A contest is starting in 1 minute')
.setDescription(`Crops: ${nextEventObject.crops.map(crop => cropNames[crop]).join(', ')}`)
);
sentNotification3 = true;
} else if (active && !sentNotificationActive) {
jacobContestsChannel.send(new MessageEmbed()
.setColor(this.client.embedColor)
.setTitle('A contest has started!')
.setDescription(`Crops: ${nextEventObject.crops.map(crop => cropNames[crop]).join(', ')}`)
);
sentNotificationActive = true;
}
console.clear();
console.log(nextEventObject);
console.log(`Diff: ${diff}`);
console.log(`Active: ${active}`);
console.log(`Notification: ${sentNotification}`);
console.log(`Notification Active: ${sentNotificationActive}`);
console.log(`Notification Finished: ${sentNotificationFinished}`);
}
}, 1000);
}
async requestUpcomingEvents() {
let response = await axios.get('https://jacob.rede-epic.com/api/upcomingEvents');
return response.data;
}
}
module.exports = Jacob;
I am trying to implement a basic matrix multiplication using worker threads with a limit of 10 threads. But the threads which are not worker threads are not responding at all.
Matrix A and B are filled with random number generator.
if (isMainThread) {
const M = 100;
const N = 100;
const matrixA = [];
for (let i = 1; i <= M; i++) {
// random numbers for each element of mat A
}
const matrixB = [];
for (let i = 1; i <= M; i++) {
// random numbers for each element of mat B
}
let matCCount = 0;
const totalCount = (M * N);
const matrixC = [];
const time1 = new Date();
let threadCount = 0;
const calculateResult = (matrixC) => {
const time2 = new Date();
console.log(`\nTime taken: ${(time2 - time1) / 1000}s`);
}
const mainService = function(getResult) {
for (let i = 0; i < M; i++) {
for (let j = 0; j < N; j++) {
let success = false;
do {
if (threadCount < 11) {
const worker = new Worker(__filename, { workerData: { i, j, matrixA, matrixB } });
threadCount ++;
worker.on('message', (message) => {
getResult(message);
success = true;
threadCount --;
});
worker.on('online', () => { console.log('online'); })
worker.on('error', (err) => {
console.log(err);
});
worker.on('exit', (code) => { console.log(code); })
}
} while (!success);
}
}
}
mainService((res) => {
const { i, j, result } = res;
if (matrixC[i] === undefined) {
matrixC[i] = [];
}
matrixC[i][j] = result;
matCCount += 1;
if (matCCount === totalCount) {
calculateResult(matrixC);
}
});
} else {
const { i, j, matrixA, matrixB } = workerData;
const x = multiplyOperation(i, j, matrixA, matrixB);
parentPort.postMessage({ i, j, result: x, threadId });
}
neither 'online' event is fired nor this console.log runs
} else {
console.log(threadId);
const { i, j, matrixA, matrixB } = workerData;
const x = multiplyOperation(i, j, matrixA, matrixB);
parentPort.postMessage({ i, j, result: x, threadId });
}
I am trying to use multiple functions within a for loop but, one is returning undefined (or not waiting?)
Code is as below:
var arrNew = [];
var id = [];
var get1 = [];
var get2 = [];
var get3 = [];
var res = [];
var get4 = [];
for (j = 0; j < arrNew.length; j++) {
id[j] = arrNew[j];
get1[j] = await get1Fun(context, id[j]);
if (get1[j] !== false) {
get2[j] = await get2Fun(context, get1[j]);
context.log(get2[j]);
if (get2[j].length) {
get3[j] = await get3Fun(context, get2[j]);
if (get3[j] !== false && get3[j].status == 200) {
res[j] = get3[j].result;
get4[j] = await get4Fun(context, id[j], res[j]);
context.log(get4[j])
}
}
}
}
async function get1Fun(context, id) {
const querySpec = {
query: `SELECT * FROM r where r.id = "${id}"`,
};
const { resources } = await client.database(databaseId).container(containerId).items.query(querySpec).fetchAll();
if (resources.length == 1) {
return resources[0];
} else if (!resources.length) {
return false;
}
}
function get2Fun(context, getInt) {
return new Promise((resolve, reject) => {
var msg = []
// do for loop stuff with getInt and push stuff into msg
context.log(msg)
resolve(msg);
});
}
The code works up until this point where get2Fun either returns undefined or if (get2[j].length) does not wait for await get2Fun.
Can someone clarify what is going wrong please?
I have tried with async function and without the promise.
The reason I have the arrays as below is because if I structured the function as var get1 = await get1Fun(context, id) then the for loop would stop after 1 or 2 items and not do every item in arrNew.
Thanks in advance.
EDIT:
Also tried the below but the for loop stops after 1 loop (presumably an issue with each loop redefining a variable?):
for (j = 0; j < arrNew.length; j++) {
var id = arrNew[j];
var get1 = await get1Fun(context, id);
if (get1 !== false) {
var get2 = await get2Fun(context, get1);
context.log(get2);
if (get2.length) {
var get3 = await get3Fun(context, get2);
if (get3 !== false && get3.status == 200) {
var res = get3.result;
var get4 = await get4Fun(context, id, res);
context.log(get4)
}
}
}
}
async function get1Fun(context, id) {
const querySpec = {
query: `SELECT * FROM r where r.id = "${id}"`,
};
const { resources } = await client.database(databaseId).container(containerId).items.query(querySpec).fetchAll();
if (resources.length == 1) {
return resources[0];
} else if (!resources.length) {
return false;
}
}
async function get2Fun(context, getInt) {
var msg = []
// do for loop stuff with getInt and push stuff into msg
context.log(msg)
return msg;
}
This code is from https://medium.com/#Trott/using-worker-threads-in-node-js-80494136dbb6;
I have modified it to write it in Typescript, but I have a compilation error on:
test_workers_numbers.ts:50:18 - error TS2304: Cannot find name
'workerData'. 50 generatePrimes(workerData.start,
workerData.range)
What is the problem?
'use strict';
import {Worker, isMainThread, parentPort} from 'worker_threads';
const min = 2;
let primes:number[] = [];
function generatePrimes(start: number, range: number) {
let isPrime = true;
let end = start + range;
for (let i = start; i < end; i++) {
for (let j = min; j < Math.sqrt(end); j++) {
if (i !== j && i%j === 0) {
isPrime = false;
break;
}
}
if (isPrime) {
primes.push(i);
}
isPrime = true;
}
}
if (isMainThread) {
const max = 1e7;
const threadCount = +process.argv[2] || 2;
const threads = new Set<Worker>();;
console.log(`Running with ${threadCount} threads...`);
const range = Math.ceil((max - min) / threadCount);
let start = min;
for (let i = 0; i < threadCount - 1; i++) {
const myStart = start;
threads.add(new Worker(__filename, { workerData: { start: myStart, range }}));
start += range;
}
threads.add(new Worker(__filename, { workerData: { start, range: range + ((max - min + 1) % threadCount)}}));
for (let worker of Array.from(threads)) {
worker.on('error', (err) => { throw err; });
worker.on('exit', () => {
threads.delete(worker);
console.log(`Thread exiting, ${threads.size} running...`);
if (threads.size === 0) {
console.log(primes.join('\n'));
}
})
worker.on('message', (msg) => {
primes = primes.concat(msg);
});
}
} else {
generatePrimes(workerData.start, workerData.range);
parentPort.postMessage(primes);
}
worker.workerData is an arbitrary JavaScript value that contains a clone of the data passed to this thread’s Worker constructor so it needs to be imported. More info HERE