global.gc() doesn't reduce memory after each loop? - node.js

I have one api to crawler, I try use global().gc to reduce heap memory after each loop but it doesn't work. Please point what wrong did I do?
It also leads to heap out of memory problem
app.get('/test2', (req, res) => {
res.json({mes: 'is getting data'});
array = [...]; //array contains about 1000 element as link
function something() {
let d = q.defer();
let urls = [];
array.forEach(function (mang, index) {
let tagArray = [];
tagArray = null;
tagArray = [];
//use this function to reduce the memory heap after looping each element of array
global.gc();
for (let i = 1; i <= 4000; i++) {
urls.push(function (callback) {
setTimeout(function () {
let link = 'http://something' + mang.link + '/tag-' + i;
//we will have about 4000 links due to i
let x = link;
let options = {
url: link,
headers: {
'User-Agent': 'MY IPHONE 7s'
}
};
function callback1(error, response, html) {
if (!error) {
let $ = whacko.load(html);
let tag_name = $('h1').text();
tag_name = tag_name.trim();
console.log(tag_name);
let tag_content = $('#content').find('div').contents();
tag_content = tag_content.toString();
if (tag_name !== "" && tag_content !== "") {
let tagObject = new Object();
tagObject.tag_name = tag_name;
tagObject.tag_content = tag_content;
tagObject.tag_number = i;
tagArray.push(tagObject);
if (tagArray.length == 4000) {
tagArray.sort(function (a, b) {
return parseInt(a.tag_number) - parseInt(b.tag_number);
});
for (let v = 0; v < tagArray.length; v++) {
db.query("INSERT INTO `tags` (tag_name, content, tag_number) " +
"SELECT * FROM (SELECT " + "'" + tagArray[v].tag_name + "'" + "," + "'" + tagArray[v].tag_content + "','" + tagArray[v].tag_number + "' as ChapName) AS tmp " +
"WHERE NOT EXISTS (SELECT `tag_name` FROM `tags` WHERE `tag_name`=" + "'" + tagArray[v].tag_name + "'" + ") " +
"LIMIT 1", (err) => {
if (err) {
console.log(err);
}
});
}
urls = null;
}
}
}
}
request(options, callback1);
callback(null, x);
}, 12000);
});
}
});
d.resolve(urls);
return d.promise;
}
something()
.then(function (data) {
let tasks = data;
console.log("start data");
async.parallelLimit(tasks, 40, () => {
console.log("DONE ");
});
})

Related

Code runs before request is completed on NodeJS

I have the following code: (the code runs without issues)
function getValoresAcao(acoes) {
acoes.forEach(acao => {
getValorAcao(acao)
});
console.log("the end")
}
function getValorAcao(acao) {
url = 'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&parse_mode=HTML&symbol='+acao+'&apikey='+API_KEY
request(url, { json: true }, (err, res, body) => {
if (err) { return console.log(err); }
let contador = 0;
let valorDeDoisDias = [];
const dailyJsonObject = body["Time Series (Daily)"]
var objectKeysArray = Object.keys(dailyJsonObject)
objectKeysArray.forEach(function(objKey) {
var objValue = dailyJsonObject[objKey]
if (contador < NUMERO_DE_ELEMENTOS )
{
//console.log(dailyJsonObject);
const valorFechamento = objValue["4. close"]
console.log(valorFechamento);
contador = contador + 1
valorDeDoisDias.push(valorFechamento);
}
});
const textoDiferenca = getDiferencaEmPorcentagem(valorDeDoisDias);
let bigString = "" + acao + " | " + valorDeDoisDias[0] + " | " + textoDiferenca
request(urlTelegram + bigString, { json: true }, (err, res, bodyTelegram) => {
console.log(bodyTelegram);
})
});
}
function getDiferencaEmPorcentagem(valprDeDoisDias) {
let myString = ""
const valorDiaAnterior = valprDeDoisDias[0]
const valorDiaMaisRecente = valprDeDoisDias[1]
myString = (valorDiaAnterior - valorDiaMaisRecente ) / valorDiaMaisRecente * 100
myString = myString.toFixed(2)
console.log(myString)
return myString + "%"
}
But the code console.log("the end") is supposed to run after the request is completed, but it runs when I start the code, it didn't wait the request to be finished.
How can I make the "the end" part of the code wait the request be executed?
probably something like that could help:
async function getValoresAcao(acoes) {
await Promise.all(acoes.map(getValorAcao))
console.log("the end")
}
async function getValorAcao(acao) {
const url = 'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&parse_mode=HTML&symbol=' + acao + '&apikey=' + API_KEY
return new Promise ((resolve, reject)=>{
try{
request(url, { json: true }, (err, res, body) => {
if (err) {
throw err
}
let contador = 0;
let valorDeDoisDias = [];
const dailyJsonObject = body["Time Series (Daily)"]
var objectKeysArray = Object.keys(dailyJsonObject)
objectKeysArray.forEach(function (objKey) {
var objValue = dailyJsonObject[objKey]
if (contador < NUMERO_DE_ELEMENTOS) {
//console.log(dailyJsonObject);
const valorFechamento = objValue["4. close"]
console.log(valorFechamento);
contador = contador + 1
valorDeDoisDias.push(valorFechamento);
}
});
const textoDiferenca = getDiferencaEmPorcentagem(valorDeDoisDias);
let bigString = "" + acao + " | " + valorDeDoisDias[0] + " | " + textoDiferenca
request(urlTelegram + bigString, { json: true }, (err, res, bodyTelegram) => {
if(err){
throw err
}
resolve(bodyTelegram)
})
});
} catch (e) {
reject(e)
}
})
}
function getDiferencaEmPorcentagem(valprDeDoisDias) {
let myString = ""
const valorDiaAnterior = valprDeDoisDias[0]
const valorDiaMaisRecente = valprDeDoisDias[1]
myString = (valorDiaAnterior - valorDiaMaisRecente) / valorDiaMaisRecente * 100
myString = myString.toFixed(2)
console.log(myString)
return myString + "%"
}

Array push gives empty result node js

I am creating an API for listing trip data with image and pdf base url,
All things are working fine but I can not access the last result array data_to_send out of for loop.
app.js
app.get("/getChallanList/:userId/:role", (req, res) => {
const userData = req.params;
let site_source = "";
let site_destination = "";
var site_from_name = "";
const data_to_send = [];
if (userData.role == "D") {
db.select("trip", "*", `driver_id = '${req.params.userId}'`, (data) => {
for (let i = 0; i < data.data.length; i++) {
site_source = data.data[i].site_from;
site_destination = data.data[i].site_to;
db.select(
"site",
"*",
`id in ('${site_source}','${site_destination}')`,
(data_site) => {
data.data[i].site_from = data_site.data[0].name;
data.data[i].site_to = data_site.data[1].name;
if (data.data[i].truck_challan_pdf != "") {
data.data[i].truck_challan_pdf =
base_url + "truckchallan/" + data.data[i].truck_challan_pdf;
}
if (data.data[i].driver_challan_pdf != "") {
data.data[i].driver_challan_pdf =
base_url + "driverchallan/" + data.data[i].driver_challan_pdf;
}
if (data.data[i].preparer_img != "") {
data.data[i].preparer_img = base_url + data.data[i].preparer_img;
}
if (data.data[i].driver_img != "") {
data.data[i].driver_img = base_url + data.data[i].driver_img;
}
data_to_send.push(data.data);
// console.log(data_to_send); // working
}
);
}
console.log(data_to_send); // empty
});
}
}
db.select
let select = (table, column, condition, callback) => {
try {
let sql = "SELECT " + column + " FROM " + table + " WHERE " + condition;
conn.query(sql, (err, results) => {
if (err) {
let data = {
status: 0,
data: sql,
message: "Something went wrong!",
};
callback(data);
} else {
let data = {
status: 1,
data: results,
message: "Success",
};
callback(data);
}
});
} catch (err) {
let data = {
status: 0,
data: err,
message: "In catch",
};
callback(data);
}
};
async await
app.get("/getChallanList/:userId/:role", async (req, res) => {
const userData = req.params;
let site_source = "";
let site_destination = "";
var site_from_name = "";
const data_to_send = [];
if (userData.role == "D") {
await db.select(
"trip",
"*",
`driver_id = '${req.params.userId}'`,
async (data) => {
// const data_to_send_ = [];
for (let i = 0; i < data.data.length; i++) {
site_source = data.data[i].site_from;
site_destination = data.data[i].site_to;
await db.select(
"site",
"*",
`id in ('${site_source}','${site_destination}')`,
(data_site) => {
data.data[i].site_from = data_site.data[0].name;
data.data[i].site_to = data_site.data[1].name;
if (data.data[i].truck_challan_pdf != "") {
data.data[i].truck_challan_pdf =
base_url + "truckchallan/" + data.data[i].truck_challan_pdf;
}
if (data.data[i].driver_challan_pdf != "") {
data.data[i].driver_challan_pdf =
base_url + "driverchallan/" + data.data[i].driver_challan_pdf;
}
if (data.data[i].preparer_img != "") {
data.data[i].preparer_img =
base_url + data.data[i].preparer_img;
}
if (data.data[i].driver_img != "") {
data.data[i].driver_img = base_url + data.data[i].driver_img;
}
data_to_send.push(data.data);
// console.log(data_to_send); // working
}
);
// data_to_send_.push(data_to_send);
}
console.log(data_to_send); // empty
}
);
}
}
this is because of the asynchronous behavior of NodeJs, so you have to plan things accordingly i.e
console.log(1)
db.select(
"trip",
"*",
`driver_id = '${req.params.userId}'`,
async (data) => {
console.log(2)
})
console.log(3)
The output of the above code would be 1 then 3 and then 2 and this is how NodeJs works it does not wait for I/O calls i.e DB query in your case.
Please check how promises work in NodeJs for more details.
Here is how you can accomplish your task:
const challanList = (userData) => {
return new Promise((resolve, reject) => {
const data_to_send = [];
db.select("trip", "*", `driver_id = '${req.params.userId}'`, data => {
for (let i = 0; i < data.data.length; i++) {
const site_source = data.data[i].site_from;
const site_destination = data.data[i].site_to;
db.select("site", "*", `id in ('${site_source}','${site_destination}')`, data_site => {
data.data[i].site_from = data_site.data[0].name;
data.data[i].site_to = data_site.data[1].name;
if (data.data[i].truck_challan_pdf != "") {
data.data[i].truck_challan_pdf = base_url + "truckchallan/" + data.data[i].truck_challan_pdf;
}
if (data.data[i].driver_challan_pdf != "") {
data.data[i].driver_challan_pdf = base_url + "driverchallan/" + data.data[i].driver_challan_pdf;
}
if (data.data[i].preparer_img != "") {
data.data[i].preparer_img = base_url + data.data[i].preparer_img;
}
if (data.data[i].driver_img != "") {
data.data[i].driver_img = base_url + data.data[i].driver_img;
}
data_to_send.push(data.data);
// console.log(data_to_send); // working
});
}
resolve(data_to_send);
});
});
};
app.get("/getChallanList/:userId/:role", async (req, res) => {
const userData = req.params;
const challanListResult =await challanList(userData);
console.log(challanListResult);
resp.json(challanListResult);
});
Without knowing what database or ORM you are using it is difficult to answer, but my suspicion is that db.select is an asynchronous method, i.e. it is returning a Promise. If so, the second console log is still seeing the "old" data_to_send.
Try adding an await in front of the first db.select call. (Don't forget the async in front of the callback in second argument of app.get.
Your database is asynchronous so console.log(data_to_send) gets called before the query finished executing. Try adding async before (req, res) in line 1 then await before db.select.
This works for me
app.get("/getChallanList/:userId/:role", async (req, res) => {
const userData = req.params;
let site_source = "";
let site_destination = "";
var site_from_name = "";
const data_to_send = [];
if (userData.role == "D") {
const data = await db.query(
`SELECT * FROM trip WHERE driver_id = '${req.params.userId}'`
);
// console.log(data.length);
// const data_to_send_ = [];
for (let i = 0; i < data.length; i++) {
site_source = data[i].site_from;
site_destination = data[i].site_to;
// cons
const site_data = await db.query(
`SELECT * FROM site WHERE id in ('${site_source}','${site_destination}')`
);
// console.log(site_data);
db.select(
"site",
"*",
`id in ('${site_source}','${site_destination}')`,
(data_site) => {
data[i].site_from = data_site.data[0].name;
data[i].site_to = data_site.data[1].name;
if (data[i].truck_challan_pdf != "") {
data[i].truck_challan_pdf =
base_url + "truckchallan/" + data[i].truck_challan_pdf;
}
if (data[i].driver_challan_pdf != "") {
data[i].driver_challan_pdf =
base_url + "driverchallan/" + data[i].driver_challan_pdf;
}
if (data[i].preparer_img != "") {
data[i].preparer_img = base_url + data[i].preparer_img;
}
if (data[i].driver_img != "") {
data[i].driver_img = base_url + data[i].driver_img;
}
data_to_send.push(data);
// console.log(data.data);
// console.log(data_to_send); // working
}
);
// data_to_send_.push(data_to_send);
}
// console.log(data_to_send);
// console.log(data_to_send);
res.send({ success: 1, data: data, message: "" });
}

return value from callback function to send client response

I have a middleware that it should send the request to a function, wait for it to finish and send back to the client what that function returns. the problem is I dont figure it out how to return the value that I'm getting inside a cb, hence the response is always an empty array.
const insertMultiDoc = async function (req, res) {
const documents = await uploadMultipleFiles(userData, documentsList, index);
res.send(documents);
}
here is the middleware func simplified, so documents is always an empty array. I dont return correctly from the below function
const uploadMultipleFiles = function (userData, documentsList, index) {
let savedDoc;
let savedDocuments = [];
if (pathHelper.ensureDirectoryExistence(randomFilePath)) {
mv(documentsList[index].path, randomFilePath, async function (error) {
if ((userData.email !== undefined) && (userData.component !== undefined) && (userData.email != "") && (userData.component != "")) {
var tempFileFolderPath = pathHelper.createSpecificPath(userData.component, userData, uploadTime);
var randomFilePath = path.posix.normalize(tempFileFolderPath + "/" + generatedFileName);
var inputPath = randomFilePath + '.' + fileExt;
var outputFileFFmpeg720 = tempFileFolderPath + '/720p' + generatedFileName + '.' + fileExt;
var encodedFilePath = pathHelper.createDBPath(userData.component, userData) + "/720p" + generatedFileName + "." + fileExt;
var accessFilePath = pathHelper.createDBPath(userData.component, userData) + "/720p" + generatedFileName + "." + fileExt;
const output = await fileCompresser.compressFile(inputPath, outputFileFFmpeg720);
fileSize = await getSize(output);
newDoc = createNewDocumentObject(userData.senderId, userData.senderUsername, userData.email, userData.component, userData.sessionId, fileName, fileType, accessFilePath, encodedFilePath, null, fileSize, userData.parentComponent);
savedDoc = await documentController.createLocalDocument(newDoc);
fs.unlinkSync(inputPath);
index = index + 1;
console.log(savedDoc);
savedDocuments.push(savedDoc);
}
})
}
return savedDocuments;
}
for this situation I use q module
const q= require('q');
const uploadMultipleFiles = async function (userData, documentsList, index) {
let defer = q.defer(); // create a defer
try {
let savedDoc;
let savedDocuments = [];
if (pathHelper.ensureDirectoryExistence(randomFilePath)) {
mv(documentsList[index].path, randomFilePath, async function (error) {
if ((userData.email !== undefined) && (userData.component !== undefined) && (userData.email != "") && (userData.component != "")) {
var tempFileFolderPath = pathHelper.createSpecificPath(userData.component, userData, uploadTime);
var randomFilePath = path.posix.normalize(tempFileFolderPath + "/" + generatedFileName);
var inputPath = randomFilePath + '.' + fileExt;
var outputFileFFmpeg720 = tempFileFolderPath + '/720p' + generatedFileName + '.' + fileExt;
var encodedFilePath = pathHelper.createDBPath(userData.component, userData) + "/720p" + generatedFileName + "." + fileExt;
var accessFilePath = pathHelper.createDBPath(userData.component, userData) + "/720p" + generatedFileName + "." + fileExt;
const output = await fileCompresser.compressFile(inputPath, outputFileFFmpeg720);
fileSize = await getSize(output);
newDoc = createNewDocumentObject(userData.senderId, userData.senderUsername, userData.email, userData.component, userData.sessionId, fileName, fileType, accessFilePath, encodedFilePath, null, fileSize, userData.parentComponent);
savedDoc = await documentController.createLocalDocument(newDoc);
fs.unlinkSync(inputPath);
index = index + 1;
console.log(savedDoc);
savedDocuments.push(savedDoc);
}
})
}
defer.resolve(savedDocuments); // return result
} catch (error) {
defer.reject(error) // return error
}
return defer.promise; //return a promise
}
you can check the q documentation

how to create multiple it statements with in one For loop protractor

I have multiple it statements in one describe {}. In one it statement I am reading data from excel and doing for loop to execute test scripts for number of users in the sheet. I need to create page model and so for each page thinking on creating one it statement. How can we do that. Here is my actual script
var login = require('./login');
var utility = require('./utility');
var exRead = require('./readExcel');
describe('MSIX Smoke', function () {
var userList = [];
beforeAll(function(done) {
browser.get('https://url'); //Dev url
expect(browser.getTitle()).toEqual('test');
exRead().then(function(excelData) {
userList = prepData(excelData);
done();
});
});
it('should login users', function() {
var loopCount = userList.length
loopCount=1;
for (var i = 0; i<loopCount; i++) {
var data = userList[i];
loginScript(data[0], data[1], data[2]);
login.clickSignOut();
}
})
it('search for a student', function () {
console.log(data[3]);
});
it ('click on my account', function () {
//some code
})
});
function prepData(data) {
var formattedData = [];
var counter = data.length / 5;
for (var i = 0; i < counter; i++) {
formattedData.push(data.splice(0,5))
}
return formattedData;
}
function loginScript(username, password, userType) {
var loginName = element(by.css('.nameP'));
console.log(username, password);
login.fillUsername(username);
login.fillPassword(password);
login.clickSignup();
login.clickPrivacy();
console.log("User " + username + " with user type "+userType+" logged in successfully");
loginName.isPresent;
utility.getStringText("Logged in user name is: ", loginName);
return loginName;
};
Give code examples with two approaches:
describe('xxx', function() {
browser.get('https://www.npmjs.com');
var datas = [1, 2, 3];
// Option 1, use for loop and javascript closure
for (var i = 0, len = datas.length; i < len; i++) {
(function(i) {
return it('yyy ' + i, function() {
console.log('data[' + i + '] = ' + datas[i]);
browser.getTitle().then(function(title) {
console.log('title[' + i + '] = ' + title);
});
});
})(i);
}
// option 2, use Array.forEach()
datas.forEach(function(data, i) {
it('yyy ' + i, function() {
console.log('data[' + i + '] = ' + data);
browser.getTitle().then(function(title) {
console.log('title[' + i + '] = ' + title);
});
});
})
});

TypeError: Cannot read property 'emails' of null

Here is line 54, where I am getting the error:
if (docr.emails) {
And here is the rest of my original code:
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
var config = require('./config'),
xlsx = require('./xlsx'),
utils = require('./utils'),
_ = require('lodash'),
url = config.DB_URL;
var meetings = [];
function findNumberOfNotesByMeeting(db, meeting, callback) {
var meetingId = meeting._id.toString(),
meetingName = meeting.name.displayValue,
attendees = meeting.attendees;
host = meeting.host;
var count = 1, pending = 0, accepted = 0;
console.log("==== Meeting: " + meetingName + '====');
_.each(attendees, function(item) {
console.log(count++ + ': ' + item.email + ' (' + item.invitationStatus + ')');
if (item.invitationStatus == 'pending') { pending++; }
else if (item.invitationStatus == 'accepted') { accepted++; }
});
console.log("*** " + attendees.length + ", " + pending + "," + accepted);
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, doc) {
var emails = [];
if (doc.emails) {
doc.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
db.collection('notes').find({ 'meetingId': meetingId }).count(function(err, count) {
if (count != 0) {
console.log(meetingName + ': ' + count + ',' + attendees.length + ' (' + email + ')');
meetings.push([ meetingName, count, email, attendees.length, pending, accepted ]);
}
callback();
});
}
});
}
function findMeetings(db, meeting, callback) {
var host = meeting.host;
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, docr) {
var emails = [];
if (docr.emails) {
docr.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
var cursor = db.collection('meetings').find({
'email': {'$regex': 'abc', '$options': 'i' }
});
}
cursor.count(function(err, count) {
console.log('count: ' + count);
var cnt = 0;
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
findNumberOfNotesByMeeting(db, doc, function() {
cnt++;
if (cnt >= count) { callback(); }
});
}
});
});
});
};
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
findMeetings(db, function() {
var newMeetings = meetings.sort(function(m1, m2) { return m2[1] - m1[1]; });
newMeetings.splice(0, 0, [ 'Meeting Name', 'Number of Notes', 'Emails' ]);
xlsx.writeXLSX(newMeetings, config.xlsxFileNameMeetings);
db.close();
});
});
Try the following:
function findMeetings(db, meeting, callback) {
var host = meeting.host;
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, docr) {
var emails = [];
if (!err && docr && docr.emails) {
docr.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
...

Resources