I have a simple for loop containing an async function, my problem is that the loop goes through all the values, then only executes the async function inside.
for(key in splitedDataArray){
calculatePower(splitedDataArray[key]);
var octopusMAC = getOctopusIDFromRaw(splitedDataArray[key]);
var channelID = getChannelIDFromRaw(splitedDataArray[key]);
console.log("outside")
getChannelCTSize(octopusMAC, channelID, function (result) {
console.log("inside")
var timestamp = convertDateTimeToInt(transmission.date);
var adcTicks = getAdcTicksFromRaw(splitedDataArray[key]);
var power = convertAdcTicksToAmps(parseInt(adcTicks, 16), result);
console.log("channelID: " + channelID + " ADC ticks HEX: " + adcTicks + " timestamp: " + timestamp + " power for " + result + "A CT Size: " + power);
});
}
So it prints outside for every element in the array then it only prints inside after all of the outised's, I want to have an outside then and inside and so on. I have looked at https://www.promisejs.org/ but I am not sure on how to implement this with my code.
You can use lodash.eachSeries but it is not best solution. If i were you i would try to move everything inside of for loop into a function so you will have all variables visible inside.
More like this:
function calculate(value){
calculatePower(value);
var octopusMAC = getOctopusIDFromRaw(value);
var channelID = getChannelIDFromRaw(value);
console.log("outside")
getChannelCTSize(octopusMAC, channelID, function (result) {
console.log("inside")
var timestamp = convertDateTimeToInt(transmission.date);
var adcTicks = getAdcTicksFromRaw(value);
var power = convertAdcTicksToAmps(parseInt(adcTicks, 16), result);
console.log("channelID: " + channelID + " ADC ticks HEX: " + adcTicks + " timestamp: " + timestamp + " power for " + result + "A CT Size: " + power);
});
}
for(key in splitedDataArray){
calc(splitedDataArray[key]);
}
It will not print out: out/in/out/in, but will get good values in each loop.
If you require that order /out/in/out/in i would go for eachSeries from lodash.
Related
This is shown in the lab environment:
check the manual that corresponds to your MySQL server version for the right syntax to use near '0, intensity = 0 WHERE id = 1' at line 1.
I don't know why it keeps showing me this error, no matter what I do to try to "update my device"
below is the table:
below is the devices that can't be updated, other devices that I have can be updated, for these that can't it keep showing the same error, whether or not it has anything to do with intensity,
when edit device is click, will be redirected to the update page
but, when i click edit this device on this page, the error will show.
app.post("/backend-update", function (req, res) {
let sqlquery = "UPDATE myDevices SET status = '" + req.body.status + "', state = '" + req.body.state + "', temperature = " + req.body.temperature + ", volume = " + req.body.volume + ", speed = " + req.body.speed + ", intensity = " + req.body.intensity + " WHERE id = " + req.body.id;
const html = "<html><body><br><form action='/showDeviceStatus'><input type='submit' value='Back'></form></body></html>"
db.query(sqlquery, (err, update) => {
if (err) {
// return res.send("<html><p><b>ERROR!!!</b> Click Back to try again.</p></html>" + html);
return console.error(err.message)
}
else {
return res.send("<html><p>Your device have been successfully <b>updated</b>! Click Back to continue.</p></html>" + html);
}
});
});
I am new to Firebase cloud functions. I would like to have a function in index.js that trigger when a node in database is created - 1. it will take the params of the context 2. and find a value of a node, 3. get value from another database, 4. adding that value to 3 diff places.
exports.runningNumber = functions.database.ref("Main/Enemy/{enem_Id}/{event_Id}"}
.onCreate((snapshot, context) => {
var enemid = context.params.enem_Id;
var eventid = context.params.event_Id;
return admin.database().ref("Main/Enemy/" + enemid + "/SequenceNumber").once('value', (snapshot) => {
var newSeqNum = snapshot.val() + 1;
//then, get var userid through database().ref("Main/Enemy/" + enemid + "/" + eventid + "/user_Id")
//and then, use newSeqNum to:
//1. Replacing "Main/Enemy/" + enemid + "/SequenceNumber"
//2. Adding as string into "Main/Enemy/" + enemid + "/" + eventid + "/SeqNum"
//3. Adding as string into "Main/Users/" + userid + "/" + eventid + "/SeqNum"
});
});
Hope somebody can help on how to do this...
I've created a bot which goes in Google Spreadsheet getting some datas before sending them by DM to 50 guild members into Discord.
However, due to high requests of datas, I've got an error message saying that I've exceeded the quota for group 'ReadGroup' and limit 'USER-100s'.
To avoid getting this error, I've created a buffer function however it still doesn't work, does anyone know how to avoid getting this limit error?
Here is the main code which is launched when I type a specific commande in Discord :
const client1 = new google.auth.JWT(keys.client_email, null, keys.private_key, ['https://www.googleapis.com/auth/spreadsheets']);
const client2 = new discord.Client();
.
.
.
let valeur1 = await liste(client1);
await sleep(100000);
console.log("End of first buffering time (100s)");
for (i = 0; i < valeur1.length; i++){
if (valeur1[i] != undefined){
try{
let valeur2 = await envoi(client1, valeur1[i]);
const user = client2.users.get(String(valeur1[i])) || await client2.fetchUser(String(valeur1[i]));
console.log("Ready to send to : " + user.id);
await user.send("The character you need to improve is : " + valeur2[0] + "\n 1. " + valeur2[1] + " = " + valeur2[2] + " >>> " + valeur2[3] + "\n 2. " + valeur2[4] + " = " + valeur2[5] + " >>> " + valeur2[6] + "\n 3. " + valeur2[7] + " = " + valeur2[8] + " >>> " + valeur2[9]);
console.log("Message sent for : " + user.id);
}
catch(err){
console.error(err);
console.log("Error detected for : " + valeur1[i]);
break;
}
}
}
Here is the first function called ("liste") which return the list of the 50 members id :
async function liste(client){
const gsapi = google.sheets({version:'v4',auth: client});
let data1 = new Array();
for (i = 0; i < 50; i++) {
const opt1 = {spreadsheetId: 'XXXXXX', range: 'Serveur!C' + (3+i)};
let data2 = await gsapi.spreadsheets.values.get(opt1);
data1.push(data2.data.values);
}
return data1;
}
And here is the second function called ("envoi") which is supposed to send the DM to the 50 different members of the guild :
async function envoi(client, id){
const gsapi = google.sheets({version:'v4',auth: client});
for (i = 0; i < 50; i++){
const opt1 = {spreadsheetId: 'XXXXXX', range: 'Discord!A' + (3+i)};
let data1 = await gsapi.spreadsheets.values.get(opt1);
if (parseInt(id) === parseInt(data1.data.values)){
const opt2 = {spreadsheetId: 'XXXXXX', range: 'Discord!C' + (3+i)};
let data2 = await gsapi.spreadsheets.values.get(opt2);
const opt3 = {spreadsheetId: 'XXXXXX', range: 'Discord!D' + (3+i)};
let data3 = await gsapi.spreadsheets.values.get(opt3);
.
.
.
const opt10 = {spreadsheetId: 'XXXXXX', range: 'Discord!K' + (3+i)};
let data10 = await gsapi.spreadsheets.values.get(opt10);
const opt11 = {spreadsheetId: 'XXXXXX', range: 'Discord!L' + (3+i)};
let data11 = await gsapi.spreadsheets.values.get(opt11);
var stats = [data2.data.values,data3.data.values,data4.data.values,data5.data.values,data6.data.values,data7.data.values,data8.data.values,data9.data.values,data10.data.values,data11.data.values];
await sleep(10000);
console.log("Extraction done for " + parseInt(id));
return stats;
}
}
console.log("Member not found");
return "erreur";
}
As a result, I would like to get all the members to get their DM. However after the 18th member, an error appear, even though I put some buffering time.
In the console.log, I get :
End of first buffering time (100s)
Extraction done for 408575708424699900
Ready to send to : 408575708424699925
Message sent for : 408575708424699925
.
.
.
Extraction done for 438420776652374000
Ready to send to : 438420776652374036
Message sent for : 438420776652374036
Error: Quota exceeded for quota group 'ReadGroup' and limit 'USER-100s' of service 'sheets.googleapis.com'
.
.
.
Error detected for : 493854774446391296
This is even more strange that the error concerns a member who already have received his DM (he is one the the first 10 members in the list)
Thanks to Tanaike, I updated my code using spreadsheets.values.batchGet() method. In that way instead of extraction values by values, I extracted a batch of values.
And then I made my formula. Now I don't have any issues anymore and even better, my script is way much quicker :)
I am doing some api to crawler from many sources (with millions of records) but I have the problem involve in out of memory. I googled and found some resource but it doesn't solve my problem.
similar question that did not solve my problem
here are my sample code:
function getContent() {
let d = q.defer();
let urls = [];
array.forEach(function(mang, index) {
for (let i = 1; i <= 600000; i++) {
urls.push(function (callback) {
setTimeout(function () {
let link = 'http://something.com/' + i;
let x = link;
let options = {
url: link,
headers: {
'User-Agent': 'something'
}
};
function callback1(error, response, html) {
if (!error) {
let $ = cheerio.load(html);
let tag_name = $('h1').text();
tag_name = tag_name.trim();
let tag_content = $('#content-holder').find('div').text();
let tagObject = new Object();
tagObject.tag_name = tag_name;
tagObject.tag_content = tag_content;
tagObject.tag_number = i;
tagArray.push(tagObject);
for (let v = 0; v < tagArray.length; v++) {
//console.log("INSERT INTO `tags` (tag_name, content, story_id, tag_number) SELECT * FROM (SELECT " + "'" + tagArray[v].tag_name + "'" + "," + "'" + tagArray[v].tag_content + "','" + array[c].story_id + "','" + tagArray[v].tag_number + "' as ChapName) AS tmp WHERE NOT EXISTS (SELECT `tag_name` FROM `tags` WHERE `tag_name`=" + "'" + tagArray[v].tag_name + "'" + ") LIMIT 1");
db.query("INSERT INTO `tags` (tag_name, content) " +
"SELECT * FROM (SELECT " + "'" + tagArray[v].tag_name + "'" + "," + "'" + tagArray[v].tag_content + "','" + "' as TagName) AS tmp " +
"WHERE NOT EXISTS (SELECT `tag_name` FROM `tags` WHERE `tag_name`=" + "'" + tagArray[v].tag_name + "'" + ") " +
"LIMIT 1", (err) => {
if (err) {
console.log(err);
}
});
}
urls = null;
global.gc();
console.log("Program is using " + heapUsed + " bytes of Heap.")
}
}
request(options, callback1);
callback(null, x);
}, 15000);
});
}
});
d.resolve(urls);
return d.promise;
}
getContent()
.then(function (data) {
let tasks = data;
console.log("start data");
async.parallelLimit(tasks, 10, () => {
console.log("DONE ");
});
})
I tried to use global.gc() function but it seems to be not effectively
Ah, I see your problem now. You're trying to do it all in memory in one loop. That way lies madness for any nontrivial amount of work as each of those anonymous functions you're creating is added to the heap. Plus, it's not very robust. What happens if you get a network outage on the 450,000th crawl? Do you lose it all and start over?
Look to having a job that runs in smaller batches. I've used task managers like Kue for this before but frankly all you need to do is start by populating your array of URLs by some reasonable number like 10 or 25. One way is to have a table with all the URLs in it and either a flag that they've been successfully crawled, or a last crawled date if you plan to do them again I've time.
Then query for all URLs that haven't been crawled (or have a last crawled earlier than some date like a week ago) and limit the results to 10 or 25 or whatever. Crawl and store those first, I'd probably use something like async.js#map or Promise.all to do that rather than the loop you're currently using.
If all the URLs are hitting the same domain, you probably want to have a short timeout between each request just to be respectful of those resources.
After a batch is done, query your DB for the next batch and repeat.
Depending on your architecture, it might be better to have this program be more simple, doing nothing but getting the one batch and resolving the crawl for the one batch. You can then run it on a cron job or as a Windows service to run every 5 minutes or every 15 minutes or whatever.
On mobile right now, but I'll try and get on a laptop later to give you a code example if you need it.
I am using nodejs and mysql client in nodejs
I am trying to use pool function od the mysql module.
"use strict";
var mysqlClient = require('mysql')
, dbPool = mysqlClient.createPool(require('../config/database'));
function MyModel(params) {
this.tbl = params.tbl;
this.primary_key = params.primary_key;
this.primary_name = params.primary_name;
this.dbPool = dbPool;
}
module.exports = MyModel;
//primary key
MyModel.prototype.fromPK = function fromPK(pk, callback) {
this.dbPool.getConnection(function (err, connection) {
var query = "SELECT * FROM " + this.tbl + " WHERE " + this.primary_key + " = " + connection.escape(pk);
connection.query(query, callback);
});
};
I know I cannot access this inside getConnection and I could simply set var t = this outside it and access it with var t, but is there any other way to access this var in this condition. Should I define var t = this in each prototype function I write?
I have following detailed gist
https://gist.github.com/yalamber/6bd1df0cc27849eb09b3
You can determine what this is in a function using .bind().
The elaborate syntax is:
var newfunction = oldfunction.bind(THIS_OBJECT);
That would make THIS_OBJECT be the this object inside the oldfunction. As you can see, .bind() returns a new (bound) function.
You don't need that elaborate syntax though, it works on anonymous functions as well:
var newfunction = function() { ... }.bind(THIS_OBJECT);
In your case, you could use this:
this.dbPool.getConnection(function (err, connection) {
var query = "SELECT * FROM " + this.tbl + " WHERE " + this.primary_name + " = " + connection.escape(pn);
connection.query(query, callback);
}.bind(this));
This makes the this inside the callback function the same as the this on the outside.