how to loop curl/request when each request is done? like php/blocking.
its currently using loop foreach async-foreach and request.
my code:
forEach(array_lines, function(page_url, index, arr) {
request(page_url, function (err, resp, body) {
if (err) {
console.log("Error!: " + err + " using ("+j+")" + page_url);
throw err;
}
var $ = cheerio.load(body,{ decodeEntities: false,xmlMode: true });
console.log('page_url: ',page_url);
build_json.items[j] = {};
var id = $("#xx").val();
console.log('id: ',id);
build_json.items[j].id = id;
build_json.items[j].source_url = page_url;
var title = $("h1.title").text();
console.log('title: ',title);
build_json.items[j].title = title;
});
});
Related
I am having trouble passing 2 ids via request response function. I can pass 1 id/argument without any issue but having trouble passing 2 arguments below is a snippet of my code:
server:
var http = require("http");
var org = require("../controllers/org");
var school = require("../controllers/school");
var academicSession = require("../controllers/academicSession");
var term = require("../controllers/term");
var gradingperiod = require("../controllers/gradingperiod");
var course = require("../controllers/course");
var stu = require("../controllers/student");
var cls = require("../controllers/class");
var user = require("../controllers/user");
var demo = require("../controllers/demographic");
var enroll = require("../controllers/enrollment");
var teach = require("../controllers/teacher");
var crscls = require("../controllers/schoolcs");
var enr = require("../controllers/enrollqrys");
var settings = require("../settings");
var httpMsgs = require("../core/httpMsgs");
http.createServer(function (req, resp) {
switch (req.method) {
case "GET":
if (req.url === "/") {
httpMsgs.showHome(req, resp);
}
else if (req.url === "/orgs") {
org.getOrgs(req, resp);
}
else if (req.url.match("/orgs/[0-9]+$")) {
var idorg = "[0-9]+";
var patt = new RegExp("/orgs/" + idorg);
if (patt.test(req.url)) {
patt = new RegExp(idorg);
var id = patt.exec(req.url);
org.getOrg(req, resp, id);
}
else {
httpMsgs.show404(req, resp);
}
}
***else if (req.url.match("/schools/[0-9]+[A-Za-z0-9\-\_]+/classes/[0-9]+[A-Za-z0-9\-\_]+/enrollments$")) {
var idcl = "[0-9]+[A-Za-z0-9\-\_]+";
var idsc = "[0-9]+[A-Za-z0-9\-\_]+";
var str = idcl + "[0-9]+[A-Za-z0-9\-\_]+" + idsc;
var patt = new RegExp("/schools/" + idcl + "/classes/" + idsc + "/enrollments");
if (patt.test(req.url)) {
patt = new RegExp(str);
var id = patt.exec(req.url);
enr.getEnrollmentsForClassInSchools(req, resp, id, arg2);
}
else {
httpMsgs.show404(req, resp);
}
}***
enrollqrys.js:
var db = require("../core/db");
var httpMsgs = require("../core/httpMsgs");
exports.getEnrollmentsForClassInSchools = function (req, resp, id, arg2) {
db.executeSql("EXEC dbo.getEnrollmentsForClassInSchools #clid = '" + id + "'" + ", #scid = '" + arg2 + "'", function (data, err) {
if (err) {
httpMsgs.show500(req, resp, err);
}
else {
//resp.writeHead(200, { "Content-Type": "application/json" });
//resp.write(JSON.stringify(data));
//resp.end();
httpMsgs.sendJson(req, resp, data);
}
});
};
exports.getStudentsForClassInSchool = function (req, resp, id, id) {
db.executeSql("EXEC dbo.getEnrollmentsForClassInSchools #clid = '" + clid + "'" + ", #scid =" + scid, function (data, err) {
if (err) {
httpMsgs.show500(req, resp, err);
}
else {
//resp.writeHead(200, { "Content-Type": "application/json" });
//resp.write(JSON.stringify(data));
//resp.end();
httpMsgs.sendJson(req, resp, data);
}
});
};
I get a not referenced error when testing this GET. Any help would be greatly appreciated. I will add that I am new to both javascript and nodejs
Error below:
Started listening at: 9000
C:\Users\THOMMA02\source\repos\NodejsConsoleApp1\NodejsConsoleApp1\core\server.js:281
enr.getEnrollmentsForClassInSchools(req, resp, id, arg2);
^
ReferenceError: arg2 is not defined
at Server. (C:\Users\THOMMA02\source\repos\NodejsConsoleApp1\NodejsConsoleApp1\core\server.js:281:72)
at emitTwo (events.js:126:13)
at Server.emit (events.js:214:7)
at parserOnIncoming (_http_server.js:619:12)
at HTTPParser.parserOnHeadersComplete (_http_common.js:112:17)
Waiting for the debugger to disconnect...
I'm very new to node, and I'm trying to pull a list of IDs from an API, iterate through that list for each ID saving the output, and ultimately rename each file generated. The code below is the closest I've come, and while it works sometimes, it frequently fails as I believe one function isn't waiting for the other to complete (e.g. tries to read before a write), but I'm sure I have other issues going on.
const apiKey = inputData.apiKey
var https = require('https');
var sync = require('sync');
var fs = require('fs');
var JSONfileloc = "./pdfs/file.json"
var queryurl = 'https://intakeq.com/api/v1/intakes/summary?startDate=2018-01-01'
var authHeaders = { 'X-Auth-Key': apiKey }
var queryOpts = { method: 'GET', headers: authHeaders}
function handleFile (error, file)
{
if (error) return console.error('Ran into a problem here', error)
}
fetch(queryurl, queryOpts)
.then
(function findAPI(res, err)
{
if( err )
{ console.log('I cant find the API '+err) }
return res.json()
{console.log('found the API!')}
}
)
.then (function itID(res, err)
{
if( err )
{ console.log('I cant iterate the API '+err) }
for(var i = 0; i < res.length; i++)
{
var intakeID=res[i].Id;
var APIoptions={ host:"intakeq.com", path:"/api/v1/intakes/"+ intakeID, headers: authHeaders };
var PDFoptions={ host:"intakeq.com", path:"/api/v1/intakes/"+ intakeID+'/pdf', headers: authHeaders };
console.log('Working on ID:'+intakeID)
var JSONrequest = https.get(APIoptions, writeJSON)
}})
//READ JSON FUNCTION
function readJSON (err, data)
{
if (err) throw err;
if(data.indexOf('New Patient Forms') >= 0)
var contents = fs.readFileSync(JSONfileloc, handleFile);
var jsonContent = JSON.parse(contents)
//pull PT Name
pName = (jsonContent.ClientName);
console.log('The Patient Name Is ' + jsonContent.ClientName)
//pull PT DOB
pDob = (jsonContent.Questions[3].Answer)
console.log('Patient DOB Is ' + jsonContent.Questions[3].Answer)
//pull Form Type
pForm = (jsonContent.QuestionnaireName)
console.log('The Form Submitted is ' + jsonContent.QuestionnaireName)
//rename and move JSON
fs.rename("./pdfs/file.json", './JSONLogs/'+pName+' '+pForm+' '+Date.now()+'.json', function(err) {
if ( err ) console.log('Problem renaming! ' + err)
else console.log('Copying & Renaming JSON File!');
})
};
//WRITE JSON FUNCTION
function writeJSON(response, err)
{
var JSONfile = fs.createWriteStream(JSONfileloc, handleFile);
if (err) throw err;
response.pipe(JSONfile);
console.log('JSON Created')
fs.readFile(JSONfileloc, readJSON)
}
The research I've done leads me to believe that async.forEach is probably the right approach here, but I've been having a hard time getting that to work properly. Thanks in advance and any suggestions are much appreciated.
const apiKey = inputData.apiKey
var https = require('https');
var sync = require('sync');
var fs = require('fs');
var JSONfileloc = "./pdfs/file.json"
var queryurl = 'https://intakeq.com/api/v1/intakes/summary?startDate=2018-01-01'
var authHeaders = {
'X-Auth-Key': apiKey
}
var queryOpts = {
method: 'GET',
headers: authHeaders
}
function handleFile(error, file) {
if (error) return console.error('Ran into a problem here', error)
}
fetch(queryurl, queryOpts)
.then(function findAPI(res) {
return res.json();
})
.then(function itID(res) {
const JSONRequests = [];
for (var i = 0; i < res.length; i++) {
var intakeID = res[i].Id;
var APIoptions = {
host: "intakeq.com",
path: "/api/v1/intakes/" + intakeID,
headers: authHeaders
};
var PDFoptions = {
host: "intakeq.com",
path: "/api/v1/intakes/" + intakeID + '/pdf',
headers: authHeaders
};
// https.get has response as a stream and not a promise
// This `httpsGet` function converts it to a promise
JSONRequests.push(httpsGet(APIoptions, i));
}
return Promise.all(JSONRequests);
})
function httpsGet(options, filename) {
return new Promise((resolve, reject) => {
https.get(options, (response) => {
// The WriteJSON function, just for brewity
// Otherwise pass resolve to the seperate writeJSON and call it in there
var JSONfile = fs.createWriteStream(filename + ".json");
response.pipe(JSONfile);
JSONfile.on('close', () => {
readJSON(filename + ".json").then(() => {
resolve();
})
})
})
})
}
//READ JSON FUNCTION
function readJSON(filename) {
// if (err) throw err;
var contents = fs.readFileSync(filename, 'utf-8'); // removed handleFile as readFileSync does not allow callbacks, added format
var jsonContent = JSON.parse(contents)
// Make your conditional checks here with the jsonContents
//pull PT Name
pName = (jsonContent.ClientName);
console.log('The Patient Name Is ' + jsonContent.ClientName)
//pull PT DOB
pDob = (jsonContent.Questions[3].Answer)
console.log('Patient DOB Is ' + jsonContent.Questions[3].Answer)
//pull Form Type
pForm = (jsonContent.QuestionnaireName)
console.log('The Form Submitted is ' + jsonContent.QuestionnaireName)
//rename and move JSON
return new Promise((resolve, reject) => {
fs.rename("./pdfs/file.json", './JSONLogs/' + pName + ' ' + pForm + ' ' + Date.now() + '.json', function (err) {
if (err) {
console.log('Problem renaming! ' + err);
reject(err);
} else {
console.log('Copying & Renaming JSON File!');
resolve();
}
})
})
};
Updated to convert https.get response stream to return a Promise which can be handled much better.
Being new to nodejs ans async following is the code that I came across.
app = express();
/*
other express use calls like - app.use(bodyParser.json());
*/
var async = require("async");
var server;
app.post('/callType/call', function(req, res) {
var startTime = Date.now();
server = req.body.server;
//async.map asynchronuously call enrollStep1 for every element in the req.body.nodes array
//HOW DOES THIS WORK??!! - WHERE IS THE CALLBACK DEFINED OR SOURCED FROM???
//******************************************************
async.map(req.body.nodes, function(node, callback) {
someFunc(node.property1,node.property2,callback)
},
//This function is called when every task triggered by async.map has called its callback.
function(err, results) {
var response = {};
if (err) {
response.success = false;
response.error = err;
console.log("ERROR returned: " + JSON.stringify(response));
res.json(response);
} else {
var returnResults = [];
//Results is an array of array - flatten it
var flattenedResults = [].concat.apply([], results);
//then remove duplicates
for (var i = 0; i < flattenedResults.length; i++){
var obj = flattenedResults[i];
var isDup = returnResults.some(function(element) {
return element.tid === obj.tid;
});
if (!isDup) {
returnResults.push(obj);
}
}
response.success = true;
response.results = returnResults;
res.json(response);
}
});
});
function someFunc(property1, property2, callback) {
var url = '/'+callTypes +'/'+ call +'/'+ property1 +'/'+ property2
urClient
.get(server + url)
.header('Content-Type', 'application/json')
.end(
function(response) {
if (response.code !== 200) {
callback("Error " + ". Code: " + response.code + " Response: " + JSON.stringify(response));
} else {
callback("Success " + ". Code: " + response.code + " Response: " + JSON.stringify(response));
}
}
);
}
The iteratee function for async.map has a definition starting function(node, callback) { but the callback function is never assigned. How does the callback work over here.
Isn't it supposed to be assigned somewhere like callback = myCallbackFunction;
The async.map takes 3 arguments, the array/object, the function to map the data and the callback function, so your code should be:
async.map(req.body.nodes, someFunc , function(err, results) {
if (err) return console.log(err);
console.log(results);
});
And your someFunc should be:
function someFunc(item, callback) {
// do something with item
// it's each item in the original array/object
callback('The results');
}
This is a basic example: http://code.runnable.com/UyR-6c2DZZ4SmfSh/async-map-example-for-node-js
I am using node js script to sync data(for multiple users) from GeoTab server APIs to my local db and using below code
var syncUsers = function () {
for (var i = 0; i < geotabUsers.length; i++) {
if (apiInstanse[geotabUsers[i].userId] == undefined)
{
apiInstanse[geotabUsers[i].userId] = new API(geotabUsers[i].apiUsername, geotabUsers[i].apiPassword, geotabUsers[i].apiDatabase, js_lang.GEOTAB_SERVER);
}
syncStatusData(apiInstanse[geotabUsers[i].userId], i, userInfo[geotabUsers[i].userId].statusDataFromVersion, geotabUsers[i].userId, userInfo[geotabUsers[i].userId].currentCompany, geotabUsers[i].apiUsername, geotabUsers[i].apiPassword, geotabUsers[i].apiDatabase);
}
var syncStatusData = function (api, i, fromVersion, userId, currentCompany, apiUsername, apiPassword, apiDatabase){
try {
api.call(js_lang.GEOTAB_GETFEED_METHOD, {
typeName: js_lang.GEOTAB_STATUS_DATA,
resultsLimit: js_lang.GEOTAB_API_LIMIT,
fromVersion: fromVersion
}, function (err, data) {
if (err) {
console.log('api Call Error:', userId);
console.log('apiUsername:', apiUsername);
console.log('apiPassword:', apiPassword);
console.log('apiDatabase:', apiDatabase);
console.log('Error', err);
apiInstanse[userId] = new API(apiUsername, apiPassword, apiDatabase, js_lang.GEOTAB_SERVER);
return;
}
var insertStatus = [];
var sql = "INSERT INTO " + js_lang.TABLE_STATUS_DATA + " (companyId,dateTime,deviceId ,diagnosticId,value,version,uniqueId,userId,unitOfMeasure ) VALUES ?";
//iterate data
if (data.data != undefined)
{
for (var key in data.data) {
if (diagnosticList[data.data[key].diagnostic.id] == undefined)
{
continue;
}
var thisDate = moment(new Date(data.data[key].dateTime));
var thisDayDate = thisDate.format("YYYY-MM-DD");
//prepare data to insert
var insertRow = [
currentCompany,
thisDate.format("YYYY-MM-DD HH:MM:ss"),
data.data[key].device.id,
data.data[key].diagnostic.id,
data.data[key].data,
data.data[key].version,
data.data[key].id,
userId,
diagnosticList[data.data[key].diagnostic.id].unitOfMeasure
];
insertStatus.push(insertRow);
var todayDate = moment(new Date());
var todayDayDate = todayDate.format("YYYY-MM-DD");
//send alert in case of current day data
if (todayDayDate == thisDayDate)
{
//send mails in case of high pressure
if (diagnosticList[data.data[key].diagnostic.id].unitOfMeasure == js_lang.GEOTAB_PRESSURE_UNIT_STR &&
data.data[key].data > js_lang.MAX_PRESSURE &&
alertTemplates[userId] != undefined)
{
console.log('alert time');
if (alertTemplates[userId] != undefined)
{
for (var templateIndex = 0; templateIndex < alertTemplates[userId].length; templateIndex++) {
var template = alertTemplates[userId][templateIndex];
var res = template.devices.split(",");
var index = FUNCTION_CLASS.contains.call(res, data.data[key].device.id)
if (index)
{
var emailSubject = 'High Pressure Alert';
if (userDevices[userId][data.data[key].device.id].name != undefined)
{
var emailText = 'Vehicle:' + userDevices[userId][data.data[key].device.id].name;
}
else
{
var emailText = '';
}
var toEmails = template.emails;
var emailHtml = 'Vehicle:' + userDevices[userId][data.data[key].device.id].name + '<br>' + diagnosticList[data.data[key].diagnostic.id].name + ' <br> value:' + data.data[key].data + ' ' + js_lang.GEOTAB_PRESSURE_UNIT_PA + '<br>\
' + js_lang.DATE_TIME + ':' + thisDate.format("YYYY-MM-DD HH:MM:ss");
//call mail function
sendEmail(toEmails, emailSubject, emailText, emailHtml);
}
}
}
}
}
}
}
if (insertStatus.length > 0)
{
connection.query(sql, [insertStatus], function (err) {
if (err)
throw err;
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET statusDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
});
}
else {
console.log('update user:', userId);
connection.query('UPDATE ' + js_lang.TABLE_USER + ' SET statusDataFromVersion = ? WHERE id = ?',
[data.toVersion, userId]);
}
if ((geotabUsers.length - 1) == i)
{
console.log('loop ended');
continueSync();
}
});
}
catch (e) {
continueSync();
}
}
var continueSync = function () {
setTimeout(function () {
connection.end();
geotabUsers = [];
userDevices = [];
diagnosticList = [];
userInfo = [];
alertTemplates = {};
eventEmitter.emit('getUsers');
}, 60000);
}
Its work fine for few iteration but getting random errors like
events.js:85
throw er; // Unhandled 'error' event
^
Error: Quit inactivity timeout
Or
events.js:85
throw er; // Unhandled 'error' event
^
Error: Connection lost: The server closed the connection.
I was getting this error while executing app on local, To remove problem I have used pm2 node module for process management.Now its working fine.
I am new to node.js I wrote a scraper as below and result it produces is not fine. All entries are not being written and incomplete broken data is being added to file, though individual data extraction if fine in console log.
The original file is complex sample from all code parts I have added to show my logic please tell what is being done wrong.
var request = require('request');
var cheerio = require('cheerio');
var url = 'http://example.com/index.html';
request(url, function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body);
var categoryname = $('#mcat span').html();
var subcategoryname = $('span.arrow').html();
$('.listing').each(function() {
var companyname = $(this).find('.company-name > span').html();
var compwebsite = $(this).find('.company-link > a').html();
var phonelumber = "+91-" + $(this).find('span[itemprop="telephone"]').html();
var data = categoryname + ", " + subcategoryname + ", " + companyname + ", " + phonelumber;
var fs = require('fs');
fs.writeFile("data.txt", data, function(err) {
if(err) {
console.log("Error: "+err);
} else {
console.log("Success!");
}
});
});
});
.each is called synchronously, hence it is blocking. But the fs.writeFile is called asynchronously so it makes your data to shuffle, but no way it is going to be incomplete.
Solutions:
Use Callback
request(url, function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body);
var categoryname = $('#mcat span').html();
var subcategoryname = $('span.arrow').html();
var count = 0;
var len = $('.listing').length;
var data = '';
$('.listing').each(function() {
count++;
var companyname = $(this).find('.company-name > span').html();
var compwebsite = $(this).find('.company-link > a').html();
var phonelumber = "+91-" + $(this).find('span[itemprop="telephone"]').html();
data += categoryname + ", " + subcategoryname + ", " + companyname + ", " + phonelumber + "\r\n";
if(count == len)
writeData(data);
});
});
function writeData(data) {
var fs = require('fs');
fs.writeFile("data.txt", data, function(err) {
if (err) {
console.log("Error: " + err);
} else {
console.log("Success!");
}
});
}
Use async module. It has various usable functions to apply callback and get the necessary result.
I think you could also do it easier (just call the writedata function after the each loop (because cherio's each() is synchronous, so there will be no problem)
request(url, function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body);
var categoryname = $('#mcat span').html();
var subcategoryname = $('span.arrow').html();
var data = '';
$('.listing').each(function() {
var companyname = $(this).find('.company-name > span').html();
var compwebsite = $(this).find('.company-link > a').html();
var phonelumber = "+91-" + $(this).find('span[itemprop="telephone"]').html();
data += categoryname + ", " + subcategoryname + ", " + companyname + ", " + phonelumber + "\r\n";
});
writeData(data);
});
function writeData(data) {
var fs = require('fs');
fs.writeFile("data.txt", data, function(err) {
if (err) {
console.log("Error: " + err);
} else {
console.log("Success!");
}
});
}