I'm running on Amazon ECS a cluster of Selenium Standalone Server w/ Firefox attached to another container for my API.
I'm querying my API to get a response and a screenshot of a long running webpage. 90% of the time, I'm getting a NoSessionIdError or a RuntimeError
{ Error: A session id is required for this command but wasn't found in the response payload
at execute(<Function>) - selectorExecute.js:58:65
at execute("return !!document.evaluate;") - ensureClientSideSelectorSupport.js:23:17
type: 'NoSessionIdError',
message: 'A session id is required for this command but wasn\'t found in the response payload' }
{ status: -1,
type: 'ESOCKETTIMEDOUT',
message: 'Couldn\'t connect to selenium server',
orgStatusMessage: 'ESOCKETTIMEDOUT' } }
Here is my code :
var webdriverio = require('webdriverio');
var options = {
desiredCapabilities: {
browserName: 'firefox'
},
host: 'selenium'
};
var WebRefresh = function () {
this.client = webdriverio.remote(options);
this.statusMessage = "";
this.errorMessage = "";
};
WebRefresh.prototype.refreshURL = function (url, cb) {
var self = this;
self.client
.init()
.url(url)
.pause(10000)
.element('iframe').then(function(res)
{
console.log("FRAME 1");
return self.client.frame(res.value).waitUntil(
function () {
return self.client.getHTML('.statusMessageHolder=Success.');
}, 300000, "TIMED OUT", 1000);
}).catch(function (err) {
self.errorMessage += "WaitUntil-" + err.type + "| ";
console.error(err);
})
.getHTML('.statusMessageHolder', false).then(function (text) {
self.statusMessage = text;
}).catch(function (err) {
self.errorMessage += "GetStatus-" + err.type + "| ";
})
.getHTML('.error-title', false).then(function (text) {
self.errorMessage = text;
}).catch(function (err) {
console.error(err);
self.errorMessage += "GetError-" + err.type + "| ";
}).screenshot().then(
function (data) {
cb(self.statusMessage, self.errorMessage, data.value);
}).catch(function (err) {
console.error(err);
self.errorMessage += "GetScreenshot-" + err.type + "| ";
cb(self.statusMessage, self.errorMessage, undefined);
})
.end();
self.client.on('error', function(err) {
console.error(err);
});
};
module.exports = WebRefresh;
I'm sure there is something wrong with that, I'm not used to promise. Could you help me ?
EDIT
I have 1 Selenium instance linked to 1 API instance, 10 instances in parallel behind an ELB. I have 500 request I need to send, with 10 request per batch. I'm waiting to get a response before sending another request.
I switched to Chrome, results are a bit better but still not what I expect. First requests have better results than other ones.
I added .manage().deleteAllCookies().close().end(); at the end but it's not making a huge difference.
Related
I have a NodeJS API running which uses Node Schedule to call a function every month.
This function gets a list of clients from my MYSQL db.
The clients are looped through and then each is charged for SMS's they have sent this month.
My issue is that although it is running, most clients were charged between 2 and 5 times (none only once).
This leads me to think I have issues with my stripe call or perhaps the NodeJS loop... or the Async/Await properties.
App.js:
schedule.scheduleJob("0 0 1 * *",async () => {
console.log("SMS Charging Schedule Running");
await stripePayments.chargeCustomers();
})
stripePayments.js:
module.exports = {
chargeCustomers : async function chargeCustomers() {
{
var stripeCusID;
var success = false;
try{
//I get the list of clients here. I have checked this query and it is 100% correct, with only one record per client.
var sqlString = "CALL chemPayments();";
var countCharged = 0;
authConnection.query(sqlString,async(err,rows,fields)=>{
let chemRows = rows[0];
if(!err){
success = true;
for (let i=0; i < chemRows.length; i++){
stripeCusID = chemRows[i].stripeCusID;
chargeValue = chemRows[i].chargeValue;
chemistName = chemRows[i].chemistName;
chemistID = chemRows[i].chemistID;
countSMS = chemRows[i].countSMS;
//console.log(stripeCusID);
try{
await chargeSMS(stripeCusID,chargeValue,chemistName,chemistID,countSMS);
countCharged++;
}catch (e) {
// this catches any exeption in this scope or await rejection
console.log(e);
}
}
if (countCharged == 0){
console.log("No SMS Charges to perform - " + Date(Date.now()).toString());
}
}
else
{
console.log("Error calling DB");
}
})
} catch (e) {
// this catches any exeption in this scope or await rejection
console.log(e);
//return res.status(500).json({ Result: e });
}
}
}
}
I then charge each client.
async function chargeSMS(stripeCusID,chargeValue,chemistName,chemistID,countSMS){
let customerObject;
let payMethodID;
let secondaryPayMethodID;
let payMethodToUse;
customerObject = await stripe.customers.retrieve(
stripeCusID
);
payMethodID = customerObject.invoice_settings.default_payment_method;
secondaryPayMethodID = customerObject.default_source;
if(payMethodID != null){
payMethodToUse = payMethodID;
}else{
payMethodToUse = secondaryPayMethodID;
}
await stripe.paymentIntents.create({
amount: chargeValue,
currency: 'aud',
customer: stripeCusID,
description: `SMS Charges: ${countSMS} sent - ${chemistName}`,
payment_method: payMethodToUse,
confirm: true
},
function(err, response) {
if (err) {
console.log(chemistName + ": " + err.message + " " + stripeCusID + " " + chargeValue );
return;
}else{
console.log(`Successful SMS Charge: ${response.id} ${chemistName}`);
chemCharged(chemistID);
}
})
}
This final step then updates teh database and tags each sms as "charged" = true, thus they are no longer on the initial select query.
async function chemCharged(chemistID){
try{
var sqlString = "SET #chemistID = ?;CALL chemCharged(#chemistID);";
authConnection.query(sqlString,chemistID,async(err,rows,fields)=>{
//console.log(rows);
if(err){
console.error("Error marking chemist as charged");
}else{
console.log(chemistID + "updated on SMS DB");
}
})
} catch (e) {
// this catches any exeption in this scope or await rejection
console.log(e);
//return res.status(500).json({ Result: e });
}
}
My largest issue is that when I copy this code and run it with the stripe TEST key... I can't replicate the problem!! The code runs fine and each client is only charged once, but when I leave my code for the cron to run at the start of each month.. I get heaps of charges per client. Sometimes 1 or 2, but up to 5 of the same charge goes through!
I am new to nodejs.
I have an array of string that consists of around 30000+ values, which has the below format
tickerArray = ["A","AA","AAA", ..........., "C"]
I want to loop through these and need to sent each value to an external polygon.io API. But the Polygo.io free plan only allows 5 API Call per minute. Below is my code.
await tickerArray.reduce((key:any, tickerSymbol:any) =>
key.then(async () => await stockTickeDetails.runTask(tickerSymbol)),
starterPromise
);
}).catch(function (error: any) {
console.log("Error:" + error);
});
My runTask function is below :
public runTask(tickerSymbol:any) {
return axios.get('https://api.polygon.io/v1/meta/symbols/' + tickerSymbol + '/company?apiKey=' + process.env.API_KEY).then(
function (response: any) {
console.log("Logo : " + response.data.logo + 'TICKER :' + tickerSymbol);
let logo = response.data.logo;
let updateLogo = stockTickers.updateOne({ ticker: tickerSymbol }, { $set: { "logo": logo } })
}).catch(function (error: any) {
console.log("Error from symbol service file : " + error);
});
}
Here what I need is, if I pass the 0th index value ie, "A" to runTask method, it should process the API and should return the result and from the result I need to update the database collection accordingly.
Then should go back to the 1code and need to fetch the 1st index value ie "AA" and repeat the process.
Here after executing 2 APIs I am getting the following error
Request failed with status code 429. You've exceeded the maximum requests per minute.
I guess this is because it is not waiting till the request to get processed with each value. How can I resolve it by adding a set time out which delays 5 API call per minute.
You can easily achieve this using Promise pattern, here is your solution:
var tickerArray = ["A","AA","AAA", ..........., "C"]
let requests = tickerArray.map(tickerSymbol => {
//generate a promise for each API call
return new Promise((resolve, reject) => {
request({
uri: https://api.polygon.io/v1/meta/symbols/' + tickerSymbol + '/company?apiKey=' + process.env.API_KEY,
method: 'GET'
},
(err, res, body) => {
if (err) { reject(err) }
//call to resolve method which is passed to the caller
//method passed to the promise
resolve( { response : body, request: tickerSymbol })
})
})
})
Promise.all(requests).then((objArray) => {
//this is expected when all the promises have been resolved/rejected.
objArray.forEach(obj => {
if (obj) {
console.log("Logo : " + obj.response.data.logo + 'TICKER :' + obj.request);
let logo = obj.response.data.logo;
let updateLogo = stockTickers.updateOne({ ticker: obj.request }, { $set: { "logo": logo } })
}
})
}).catch(error => console.log("Error from symbol service file : " + error);)
I'm trying to run an async function that send post request to the server and logs the response in the console. Whenever I run a for loop nothing is happening. I have checked a different similar questions here on stackoverflow but nothing is working in my case.
Note this code is for ionic + angular for android.
Below is the code for the function.
postAPI(suffixUrl, queryString, data, header: any = {}) {
// console.log(suffixUrl + "\n=>", queryString + "\n=>", data + "\n=>", header);
this.isDisabled = true;
data = (header.is == 'yes') ? JSON.parse(data) : data;
let type = this.fileNameSubStringFn(suffixUrl, '?');
return new Promise(resolve => {
this.http.post(this.con.baseUrl + suffixUrl + queryString, data)
.subscribe(res => {
// res = res.json();
// res = JSON.parse(res.data);
console.log("RES:", res);
this.hideLoader();
resolve(this.data);
return;
}, (err) => {
this.logFn("ERROR:" + JSON.stringify(err));
this.isDisabled = false;
// this.isConnected = false;
});
});
}
Below is the code that is accessing the above function in-order to submit the data.
var i;
for (i = 0; i < summeriesArray.length; i++) {
this.postAPI('service_summery', '', JSON.stringify(summeriesArray[i]), { is: 'yes', method: 'post' }).then(data => {
this.hideLoader();
this.toast("Successfully saved");
console.log(data);
});
}
Code inside this.postAPI is not executing which means the data is not being posted and I'm getting no errors so I don't know what is going on here.
I'm expecting the this.postAPI to run successfully in a for lop.
Thank you, posted with Love.
You can use the await keyword, so you wait for the next iteration until the completion of the 'postAPI'.
for (i = 0; i < summeriesArray.length; i++) {
const data = await this.postAPI('service_summery', '', JSON.stringify(summeriesArray[i]), { is: 'yes', method: 'post' })
this.hideLoader();
this.toast("Successfully saved");
console.log(data);
}
I am trying to test a function that looks like:
function sendVerifySms(patientId, data) {
var smsNumber = data.smsNumber;
var verifyCode = ((Math.random() * 1000000) | 0).toString();
var sql = "UPDATE [patient]";
sql += " SET phone_sms_verify_code = '" + verifyCode + "',";
// verification must take place within a one hour period
sql += " phone_sms_verify_code_expire = '" + moment.utc().add(1, 'hour').formatSqlDatetime() + "',";
sql += " phone_sms_verified = 0,"
sql += " phone_sms = '" + escapeSql(smsNumber) + "'";
sql += " WHERE id = " + escapeSql(patientId.toString());
return sqlServer.query(sql).then(function(result) {
twilioClient.sendMessage({
to: smsNumber,
from: twilioUser.verifyNumber,
body: verifyCode
}).then(function(res) {
console.log('sent verification');
return verifyCode;
}).fail(function(err) {
console.log('error sending verification');
console.log(err);
return err;
});
}).fail(function(err) {
console.log(err);
return err;
});
}
Easy enough, right? Well, first I need to decide what EXACTLY I'm testing for.
That the sql command matches the format given some data
That the twilioClient.sendMessage has been called.
Here is what I have for my test so far:
var should = require('should');
var methods;
var mockery = require('mockery');
var sinon = require('sinon');
describe('RPC Methods', function() {
before(function() {
mockery.enable();
mockery.registerMock('msnodesql', {
open: function() {
return true;
}
});
mockery.registerMock('../../db/sqlserver', {
query: function() {
return {
then: function() {
return true;
}
}
}
});
methods = require('../../../rpc/methods');
});
it('should send a verify SMS', function() {
var data = {
}
methods.sendVerifySms(1, data);
should(1).equal(1);
});
});
So now I'm a bit lost from here. I have a ton of require and since I want to isolate my individual functions for testing, I figured something like mockery would work. Where do I go from here?
If anything is unclear, please post a comment and I'll clarify.
Thanks!
With mocha tests you have an optional done callback that makes testing async functions easier, like this:
it('should send a verify SMS', function(done) {
var data = {};
var code = 1;
methods.sendVerifySms(code, data)
.then(function(actualCode) {
should(actualCode).equal(code);
done();
});
});
I would also have some feedback to offer on the async function you're testing. First I'd say you don't have to nest promise chains the way you are, and in this case its probably better that you didn't. If you simply return a promise from within a promise callback, you can chain it at the same level, like this:
return sqlServer.query(sql).then(function(result) {
return twilioClient.sendMessage({
to: smsNumber,
from: twilioUser.verifyNumber,
body: verifyCode
});
}).then(function(res) {
console.log('sent verification');
return verifyCode;
}).fail(function(err) {
console.log(err);
throw err;
});
Second, in the error handler you had before you were simply returning the err. This is probably a bad idea because it tells the consumer of your promise that everything is hunky dory and that they should proceed as such. Throwing an error on the other hand will allow that consumer to handle the error however they want in their own .fail block.
I tried to copy several file with node js.
Here is my an example of what i'm trying to do :
var request = require('request');
va photos [{ 'url': 'http://xxxx.com/im1', 'name' : 'name1' }, { 'url': 'http://xxxx.com/im12', 'name' : 'name2' },
for (var i = 0; i < photos.length; i++) {
request(photos[i].source).pipe(fs.createWriteStream(photos[i].name));
}
After maybe 1000 call i have a socket hang out error.
Following #Timothy Strimple advice i decided to use async module.
My code is now something like this :
async.whilst(function () { return !stop; },
function (callback) {
console.log("get next 20 image");
JM.api('/' + album.id + '/photos', { after: next }, function (resf) {
if (!resf || resf.error) {
console.log(!resf ? 'error occurred' : resf.error);
}
console.log("albums" + album.id + " " + resf.data.length + " dir" + dir);
async.eachSeries(resf.data, function (photo, done) {
request(photo.source).pipe(fs.createWriteStream(dir + "/" +photo.name));
console.log("copy of image " + img_basename);
}, function (err) {
if (err) {
console.log('An images failed to copy');
} else {
console.log('All 20 image have been copied successfully');
}
if (resf.paging && resf.paging.cursors) {
console.log("suite de l'album à venir");
next = resf.paging.cursors.after;
setTimeout(function () { callback(); }, 5000);
}
else {
console.log("Fin de l'album");
stop = true;
setTimeout(function () { callback(); }, 5000);
}
});
});
},
function (err) {
if (err) {
console.log('An images failed to process');
albumcallback();
} else {
console.log('All images in this group have been processed successfully');
albumcallback();
}
}
);// end while
I still having a crash after maybe 1 00 file copied. I'm sure that async.whilst and async.eachSeries are weel because my log show that each call is on series. But i have a crash. I temporary solved the proble by ading a wait after each copy like this :
request(photo.source).pipe(fs.createWriteStream(dir + "/" + img_basename));
console.log("copy of image " + img_basename);
setTimeout(function () { done(); }, 5000);
Is it a limit of request module ? How to change this fea line to make sure that each connection are closed before continung the program ?
You probably need to move to an asynchronous loop. Something like eachLimit from the async module would probably be ideal.
async.eachLimit(photos, 10, function(photo, done) {
var r = request(photos[i].source).pipe(fs.createWriteStream(photos[i].name));
r.on('finish', done);
}, function(err) {
// All images done or there was an error
});
Now it will process all the items in your photos list, but it will only process 10 of them concurrently. This will prevent it from spinning up hundreds or thousands of concurrent outgoing connections.
The request call and pipe call are asyncrhon. So i have to rewrite this line : request(photos[i].source).pipe(fs.createWriteStream(photos[i].name));
See here :
Downloading N number of remote files using Node.js synchronously