I am new Node.JS developer.
I have a question, when I call function the parameter with Dictionary format. Then function that had been called can't be executed, why?
function testFunction(allObjects, test, response) {
console.log("go in testFunction");
var dic = {a: "aaa", b:"bbb"};
testFunctionTo(dic, response);
}
function testFunctionTo(dic, response) {
console.log("go in testFunctionTo");
response.send("success");
}
result:
go in testFunction;
the original code like below:
//generate and mapping all master data from this import
function prepareMasterRequireData(allObjects, response) {
var singleCustomer = {}; //id: ParseObjectId
var singleCustomerData = {}; //id: syncLog
var singleMaterial = {}; //id: ParseObjectId
var singleMaterialData = {}; // id: syncLog
var singleUom = {}; //id: ParseObjectId
var singleUomData = {}; // id: syncLog
var singleOrderHeader = {}; //id: ParseObjectId
var singleOrderHeaderData = {}; //id: syncLog
//prepare single data
for (var i=allObjects.length-1; i>=0; i--) {
var syncLog = allObjects[i];
if (syncLog.get("TypeFlag") == "A") {
//prepare for customer
var customerId = syncLog.get("Customer_SN");
if (singleCustomer[customerId] == null) {
singleCustomer[customerId] = "";
singleCustomerData[customerId] = syncLog;
}
//prepare for material
var materialId = syncLog.get("Product_SN");
if (singleMaterial[materialId] == null) {
singleMaterial[materialId] = "";
singleMaterialData[materialId] = syncLog;
}
//prepare for uom
var UomId = syncLog.get("Product_Unit");
if (singleUom[UomId] == null) {
singleUom[UomId] = "";
singleUomData[UomId] = syncLog;
}
//prepare for Orderheader
var orderId = syncLog.get("Order_SN_Master");
if (singleOrderHeader[orderId] = null) {
singleOrderHeader[orderId] = "";
singleOrderHeaderData[orderId] = syncLog;
}
} else {
delete allObjects[i];
}
}
var customerCount = Object.keys(singleCustomer).length;
var materialCount = Object.keys(singleMaterial).length;
var UomCount = Object.keys(singleUom).length;
var currentCustomerCount = 0;
var currentMaterialCount = 0;
var currentUomCount = 0;
for (key in singleCustomer) {
getCustomerID(key, singleCustomerData[key], {
success: function(key, objectId) {
singleCustomer[key] = objectId;
currentCustomerCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("customer succ " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go customer succ");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
},
error: function(key, error) {
currentCustomerCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("customer fail " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go customer fail");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
}
});
}
for (key in singleMaterial) {
getMaterialId(key, singleMaterialData[key], {
success: function(key, objectId) {
singleMaterial[key] = objectId;
currentMaterialCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("material succ " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go material succ");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
},
error: function(key, error) {
currentMaterialCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("material fail " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go material fail");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
}
});
}
for (key in singleUom) {
getUOMId(key, singleUomData[key], {
success: function(key, objectId) {
singleUom[key] = objectId;
currentUomCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("uom succ " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
//testFunctionTo(singelCustomer, response);
console.log("go uom succ");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
},
error: function(key, error) {
currentUomCount ++;
var stringValue = customerCount + " " + materialCount + " " + UomCount + " " +currentCustomerCount + " " +currentMaterialCount + " " + currentUomCount;
console.log("uom fail " + stringValue);
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go uom fail");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
}
});
}
//response.send(resultString);
}
//update syncLog for this import
function updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response) {
console.log("yes in update");
......
}
result:
customer succ 1 1 1 1 0 0
material succ 1 1 1 1 1 0
uom succ 1 1 1 1 1 1
go uom succ
You call the function within the same block of "console.log("go material succ");" and that "go material succ" thing wasn't printed out, that means it didn't enter the if condition...
if ((currentCustomerCount == customerCount) &&
(currentMaterialCount == materialCount) &&
(currentUomCount == UomCount)) {
console.log("go material succ");
updateImportMasterData(allObjects, singelCustomer, singleMaterial, singleUom, singleOrderHeader, response);
}
Related
I have one api to crawler, I try use global().gc to reduce heap memory after each loop but it doesn't work. Please point what wrong did I do?
It also leads to heap out of memory problem
app.get('/test2', (req, res) => {
res.json({mes: 'is getting data'});
array = [...]; //array contains about 1000 element as link
function something() {
let d = q.defer();
let urls = [];
array.forEach(function (mang, index) {
let tagArray = [];
tagArray = null;
tagArray = [];
//use this function to reduce the memory heap after looping each element of array
global.gc();
for (let i = 1; i <= 4000; i++) {
urls.push(function (callback) {
setTimeout(function () {
let link = 'http://something' + mang.link + '/tag-' + i;
//we will have about 4000 links due to i
let x = link;
let options = {
url: link,
headers: {
'User-Agent': 'MY IPHONE 7s'
}
};
function callback1(error, response, html) {
if (!error) {
let $ = whacko.load(html);
let tag_name = $('h1').text();
tag_name = tag_name.trim();
console.log(tag_name);
let tag_content = $('#content').find('div').contents();
tag_content = tag_content.toString();
if (tag_name !== "" && tag_content !== "") {
let tagObject = new Object();
tagObject.tag_name = tag_name;
tagObject.tag_content = tag_content;
tagObject.tag_number = i;
tagArray.push(tagObject);
if (tagArray.length == 4000) {
tagArray.sort(function (a, b) {
return parseInt(a.tag_number) - parseInt(b.tag_number);
});
for (let v = 0; v < tagArray.length; v++) {
db.query("INSERT INTO `tags` (tag_name, content, tag_number) " +
"SELECT * FROM (SELECT " + "'" + tagArray[v].tag_name + "'" + "," + "'" + tagArray[v].tag_content + "','" + tagArray[v].tag_number + "' as ChapName) AS tmp " +
"WHERE NOT EXISTS (SELECT `tag_name` FROM `tags` WHERE `tag_name`=" + "'" + tagArray[v].tag_name + "'" + ") " +
"LIMIT 1", (err) => {
if (err) {
console.log(err);
}
});
}
urls = null;
}
}
}
}
request(options, callback1);
callback(null, x);
}, 12000);
});
}
});
d.resolve(urls);
return d.promise;
}
something()
.then(function (data) {
let tasks = data;
console.log("start data");
async.parallelLimit(tasks, 40, () => {
console.log("DONE ");
});
})
Update student
function update_student(req, student, next, res) {
console.log('----',student);
if (Object.keys(student).length > 0) {
var query = updateQuery('student', req, student, '_id');
var update_data = [];
Object.keys(student).forEach(function(key) {
update_data.push(student[key])
});
db.query(req, update_data).then(function(result) {
return res.send('Updated');
})
}
else {
return res.send('nothing to update');
}
};
Update query
function updateQuery(tablename, id, data, condition) {
var query = ['UPDATE'];
query.push(tablename)
query.push('SET');
var set = [];
Object.keys(data).forEach(function(key, i) {
if ((key == 'university_id') || (key == 'name') || (key == 'branch') || (key == 'age') || (key == 'email')) {
set.push('"' + key + '"' + ' = $' + (i + 1));
} else {
set.push(key + ' = $' + (i + 1));
}
});
query.push(set.join(', '));
query.push('WHERE ' + condition + ' = ' + id);
return query.join(' ');
}
What's wrong in the code? I am trying to write the update query in postgreSQL using node.js. I'm not able to update the student table using this query.
Here is line 54, where I am getting the error:
if (docr.emails) {
And here is the rest of my original code:
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var ObjectId = require('mongodb').ObjectID;
var config = require('./config'),
xlsx = require('./xlsx'),
utils = require('./utils'),
_ = require('lodash'),
url = config.DB_URL;
var meetings = [];
function findNumberOfNotesByMeeting(db, meeting, callback) {
var meetingId = meeting._id.toString(),
meetingName = meeting.name.displayValue,
attendees = meeting.attendees;
host = meeting.host;
var count = 1, pending = 0, accepted = 0;
console.log("==== Meeting: " + meetingName + '====');
_.each(attendees, function(item) {
console.log(count++ + ': ' + item.email + ' (' + item.invitationStatus + ')');
if (item.invitationStatus == 'pending') { pending++; }
else if (item.invitationStatus == 'accepted') { accepted++; }
});
console.log("*** " + attendees.length + ", " + pending + "," + accepted);
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, doc) {
var emails = [];
if (doc.emails) {
doc.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
db.collection('notes').find({ 'meetingId': meetingId }).count(function(err, count) {
if (count != 0) {
console.log(meetingName + ': ' + count + ',' + attendees.length + ' (' + email + ')');
meetings.push([ meetingName, count, email, attendees.length, pending, accepted ]);
}
callback();
});
}
});
}
function findMeetings(db, meeting, callback) {
var host = meeting.host;
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, docr) {
var emails = [];
if (docr.emails) {
docr.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
var email = emails.join(', ');
if (utils.toSkipEmail(email)) {
callback();
} else {
var cursor = db.collection('meetings').find({
'email': {'$regex': 'abc', '$options': 'i' }
});
}
cursor.count(function(err, count) {
console.log('count: ' + count);
var cnt = 0;
cursor.each(function(err, doc) {
assert.equal(err, null);
if (doc != null) {
findNumberOfNotesByMeeting(db, doc, function() {
cnt++;
if (cnt >= count) { callback(); }
});
}
});
});
});
};
MongoClient.connect(url, function(err, db) {
assert.equal(null, err);
findMeetings(db, function() {
var newMeetings = meetings.sort(function(m1, m2) { return m2[1] - m1[1]; });
newMeetings.splice(0, 0, [ 'Meeting Name', 'Number of Notes', 'Emails' ]);
xlsx.writeXLSX(newMeetings, config.xlsxFileNameMeetings);
db.close();
});
});
Try the following:
function findMeetings(db, meeting, callback) {
var host = meeting.host;
db.collection('users').findOne({'_id': new ObjectId(host)}, function(err, docr) {
var emails = [];
if (!err && docr && docr.emails) {
docr.emails.forEach(function(e) {
emails.push(e.email + (e.primary ? '(P)' : ''));
});
}
...
In my local setup of a node.js project I have a php page test.php. Now when in browser I put url http://localhost:8080/test.php I get the following error in console.
[12/28/2015 12:40:15] user: p1 IP: ::ffff:127.0.0.1 URL: /test.ph
Error requesting PHP backend (GET): Error: connect ECONNREFUSED.
Should I install some module of node js or it's a issue for my ubuntu version?
this is the whole code :
var http = require("http"),
https = require("https"),
mustache = require("mustache"),
fs = require("fs"),
pg = require("pg"),
url = require("url"),
crypto = require("crypto"),
serveStatic = require("serve-static"),
docxtemplater = require("docxtemplater"),
querystring = require("querystring"),
callstream = require("./callstream");
pg.defaults.poolSize = 25;
function log(text, req, stateObj) {
var u = (stateObj?stateObj.u:undefined),
ip = (req?req.connection.remoteAddress:undefined),
url = (req?req.url:undefined);
var d = new Date();
process.stderr.write("===============\n[" + (d.getMonth() + 1) + "/" + d.getDate() + "/" + d.getFullYear() + " " + d.getHours() + ":" + d.getMinutes() + ":" + d.getSeconds() + "] "
+ "user: " + (u||"(none)") + " IP: " + (ip||"(none)") + " URL: " + (url||"")
+ "\n" + text + "\n===============\n\n"
);
};
function validateToken(req) {
var rawCookie = req.headers.cookie;
if (!rawCookie) {return false;}
if (!rawCookie.length) {return false;}
var rawCookie = rawCookie.match(/t\=[A-Za-z0-9\+\/\=]+\.[A-Za-z0-9\+\/\=]+/);
if (!rawCookie) {return false;}
var parsedCookie = rawCookie[0].replace("t=", "").split(".");
if (parsedCookie[1] !== (crypto.createHash("sha256").update("sECr6eTt" + parsedCookie[0]).digest("base64"))) {return false;}
try {
var parsedPayload = JSON.parse((new Buffer(parsedCookie[0], "base64")).toString());
}
catch (e) {
return false;
}
if (!parsedPayload) {return false;}
var u = parseInt(parsedPayload.u),
m = (parseInt(parsedPayload.m)||0),
a = (parseInt(parsedPayload.a)||0);
if (!u) {return false;}
//if (!m) {return false;}
//if (!a) {return false;}
if ((new Date(parseInt(parsedPayload.t))).getDate() !== (new Date()).getDate()) {return false;}
if (parsedPayload.s !== crypto.createHash("sha256").update("true" + req.connection.remoteAddress + "s4CcrEEtT").digest("hex")) {return false;}
return {u:u, m:m, a:a};
};
function generateToken(req, u, m, a) {
var u = parseInt(u),
m = (parseInt(m)||0),
a = (parseInt(a)||0);
//if (!(u && m && a)) {log("token generation error");return "t=x";}
var rawPayload = {u:u, m:m, a:a, t:Date.now(), s:(crypto.createHash("sha256").update("true" + req.connection.remoteAddress + "s4CcrEEtT").digest("hex"))};
var encodedPayload = (new Buffer(JSON.stringify(rawPayload))).toString("base64");
var sig = crypto.createHash("sha256").update("sECr6eTt" + encodedPayload).digest("base64");
return encodedPayload + "." + sig;
};
function dbError(e, req, res, stateObj) {
log("DB Error: " + e, req, stateObj);
res.writeHead(500);
res.end();
};
function noRoute(req, res, stateObj) {
log("Bad Route: " + req.url, req, stateObj);
res.writeHead(404);
res.end();
};
function badParam(req, res, stateObj, pName, pVal) {
log("Bad GET Param: " + pName + " = " + pVal, req, stateObj);
res.writeHead(500);
res.end();
};
var static = serveStatic("../html/static");
var staticPub = serveStatic("../html/pub");
function serve(req, res) {
try {
if (req.method !== "GET" && req.method !== "POST") {
res.end();
return;
}
var parsedUrl = url.parse(req.url, true),
path = parsedUrl.pathname,
queryParams = parsedUrl.query,
stateObj = validateToken(req);
if (path == "/favicon.ico") {
res.writeHead(200, {'Content-Type': 'image/x-icon'});
res.write(favicon);
res.end();
return;
}
if (stateObj) {
// test.php url
var cReq = http.request({hostname:"localhost", port:8080, path:parsedUrl.path}, function(cRes) {
res.writeHead(200, cRes.headers);
cRes.pipe(res);
});
cReq.on("error", function(e) {
log("Error requesting PHP backend (GET): " + e, req, stateObj);
res.writeHead(500);
res.end();
});
cReq.end();
}
}
catch (e) {
log("Exception in primary request callback: " + e + "\n" + e.stack, req, stateObj)
res.writeHead(500);
res.end();
return;
}
};
http.createServer(serve).listen(8080);
I have a problem with my nodejs git branches test. This is the code:
var exec = require('child_process').exec;
var request = require('request');
var cheerio = require('cheerio');
var async = require('async');
var currentDir = ""; // dir
var getBranches = "git for-each-ref --sort=-committerdate refs/remotes/origin/ --format='%(refname:short)'";
var getAuthors = "git for-each-ref --sort=-committerdate refs/remotes/origin/ --format='%(committername)'";
var branches = [];
var errorBranches = [];
var authors = [];
var manager = [];
var result = [];
var fs = require('fs');
var openedBranches = "";
var outFile = 'BranchesTest-results.xml';
var failuresFound = false;
function eliminateDuplicates(arr) {
'use strict';
var i,
out = [],
obj = {};
for (i = 0; i < arr.length; i++)
obj[arr[i]] = 0;
for (i in obj) {
if (obj.hasOwnProperty(i))
out.push(i);
}
return out;
}
function showBranchesStats(branches, cb) {
'use strict';
console.log("in showBranchesStats method\n");
var branchesName = [];
var committers = [];
var temp = [];
var j = 0,
k = 0;
for (var i = 0; i < branches.length; i++) {
var filter = branches[i].match(/(md|dr)[_\-](\d+)/i);
if (filter !== null) {
filter = filter[1] + '-' + filter[2];
temp[j] = authors[i];
branchesName[j++] = filter;
}
else {
if (!branches[i].match(/(develop|master|HEAD)/ig))
errorBranches[k++] = branches[i];
}
}
var obj = {};
for (i = 0; i < branchesName.length; i++)
obj[branchesName[i]] = 0;
for (i in obj) {
if (obj.hasOwnProperty(i))
committers.push(temp[branchesName.indexOf(i)]);
}
branchesName = eliminateDuplicates(branchesName);
errorBranches = eliminateDuplicates(errorBranches);
var closedBranches = [];
var tasks = [];
j = 0;
for (i = 0; i < branchesName.length; i++) {
tasks.push(getTask(i));
}
tasks[0]();
var endedCounter = 0;
function onEnd() {
if (++endedCounter < branchesName.length) {
console.log(endedCounter + '/' + branchesName.length);
tasks[endedCounter]();
return;
}
console.log("All requests finished\n");
for (var i = 0; i < manager.length; i++) {
result[i] = [];
result[i] = closedBranches[i].slice(0, -1).split("\n");
}
TestResult(cb);
}
function getTask(i) {
return function () {
console.log("Request #" + i + "\n");
request('SERVERNAME' + branchesName[i] + '?page=com.atlassian.jira.plugin.system.issuetabpanels%3Aall-tabpanel#issue-tabs', function (error, response, html) {
if (!error && response.statusCode === 200) {
var $ = cheerio.load(html);
try {
var data = $('.actionContainer:contains("Closed")');
if (data.length > 0) {
//var _manager = ('not(.twixi-wrap)', data).last().children().eq(0).children().children().eq(0).text();
var _manager = committers[i];
var _hasClosed = ('not(.twixi-wrap)', data).last().find(".activity-new-val").text().indexOf("Closed");
if (_manager && _hasClosed) {
var flag = false;
for (var s = 0; s < manager.length; s++)
if (manager[s] === _manager)
flag = true;
if (_hasClosed !== -1) {
if (j >= 0 && flag === false) {
manager[j++] = _manager;
closedBranches[manager.indexOf(_manager)] = "";
}
closedBranches[manager.indexOf(_manager)] += "SERVERNAME" + $('title').text().substring(2, 9) + "\n";
}
else
openedBranches += "SERVERNAME" + $('title').text().substring(2, 9) + " --- OPENED [OK] ----\n";
}
}
else
openedBranches += "SERVERNAME" + $('title').text().substring(2, 9) + " --- OPENED [OK] ----\n";
}
catch (ex) {
console.log("Something wrong happened with your request. Details: " + ex);
}
onEnd();
}
else {
console.log("Something wrong happened with your request. Details: " + error);
failuresFound = true;
cb();
}
});
};
}
}
function TestResult(cb) {
'use strict';
console.log("Results file constructing started\n");
outFile = fs.openSync(outFile, 'w');
fs.writeSync(outFile, "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n");
fs.writeSync(outFile, "<testsuites>\n");
for (var i = 0; i < manager.length; i++) {
fs.writeSync(outFile, "\t<testsuite name=\"" + manager[i] + "\">\n");
for (var j = 0; j < result[i].length; j++) {
fs.writeSync(outFile, "\t\t<testcase name=\"" + result[i][j] + "\">\n");
fs.writeSync(outFile, "\t\t\t<failure message=\"Branch Test Failure\">\n");
fs.writeSync(outFile, "\t\t\t\tError! Found closed branch: " + result[i][j] + "\n");
fs.writeSync(outFile, "\t\t\t</failure>\n");
fs.writeSync(outFile, "\t\t</testcase>\n");
failuresFound = true;
}
fs.writeSync(outFile, "\t</testsuite>\n");
}
console.log("\nOpened branches: \n\n" + openedBranches);
if (!failuresFound) {
fs.writeSync(outFile, "\t<testsuite name=\"BranchTest\">\n");
fs.writeSync(outFile, "\t\t<testcase name=\"BranchTest\"/>\n");
fs.writeSync(outFile, "\t</testsuite>\n");
}
fs.writeSync(outFile, "</testsuites>");
fs.closeSync(outFile);
console.log("\nError branches: \n");
for (var k = 0; k < errorBranches.length; k++)
console.log(errorBranches[k]);
console.log('Test result : ' + (failuresFound ? '' : 'no ') + 'failures found');
cb();
}
this.check = function () {
'use strict';
console.log("Execution started\n");
var funcs = [
function (cb) {
console.log('started 1');
exec(getAuthors, {cwd: currentDir}, function (error, stdout, stderr) {
console.log('started 1.1');
if (stderr) {
console.log('stderr: ' + stderr + "\n");
failuresFound = true;
cb();
}
else {
authors = stdout.split('\n');
console.log("Authors has been received\n");
cb();
}
if (error !== null) {
console.log('Execution command error: ' + error + "\n");
failuresFound = true;
cb();
}
});
},
function (cb) {
console.log('started 2');
exec(getBranches, {cwd: currentDir}, function (error, stdout, stderr) {
console.log("started 2.1");
if (stderr) {
console.log('stderr: ' + stderr + "\n");
failuresFound = true;
cb();
}
else {
branches = stdout.split('\n');
console.log("Going to showBranchesStats method\n");
showBranchesStats(branches, cb);
}
if (error !== null) {
console.log('Execution command error: ' + error + "\n");
failuresFound = true;
cb();
}
});
}
];
console.log('async.waterfall');
async.waterfall(funcs, function () {
console.log("Execution finished\n");
return !failuresFound;
});
};
it' all works just perfect locally, but when i'm trying to run it in command line or on Jenkins like this:
grunt test_Branches
something wrong happens & i get this log:
+ npm i
+ grunt test_Branches
[4mRunning "test_Branches" task[24m
Execution started
async.waterfall
Done, without errors.
Help, pls!
I resolved the problem. The solution is here.
You need rightly use asynchronous functions.