I am unable to get proper output when trying to read data from redis and then send it out to user in json format.
This is my code
//API to get slot for a particular date range
app.get('/listcapacity/:ticketid/:fromdate/:todate', function(req, res) {
var id = req.params.ticketid;
var fromdate = req.params.fromdate;
var todate = req.params.todate;
var result = {};
var data_output = [];
var asyncTasks = [];
var currentDate = new Date(fromdate);
var between = [];
var end = new Date(todate);
while (currentDate <= end) {
var tempdate = new Date(currentDate).toISOString();
var dump = tempdate.toString().split("T");
between.push(dump[0]);
currentDate.setDate(currentDate.getDate() + 1);
}
between.forEach(function(entry) {
asyncTasks.push(function(callback) {
client.exists(id+entry, function (err, reply) {
if (reply === 1) {
console.log("Found");
client.get(id+entry, function (err, reply) {
var output = JSON.parse(reply);
data_output = data_output.concat(output);
});
}
callback();
});
});
});
async.parallel(asyncTasks, function(){
// All tasks are done now
result['data'] = data_output;
result['response'] = 1;
result['message'] = 'vacancies list fetched successfully!';
res.json(result);
});
});
I am always getting the output {"data":[],"response":1,"message":"vacancies list fetched successfully!"}
Even though I have checked that the keys are there in redis and it is always being Found by the application too.
For some reason the redis part is getting executed after the data has been sent to user. Why is this happening? I know redis works in async hence used the async.parallel function to take care of this but even then I endup with the same issue that I would have if I wouldnt have used async. What is the reason for this?
The client.get is an async operation and you have to callback in there. Also, you need an else statement and you need a callback in there as well:
//API to get slot for a particular date range
app.get('/listcapacity/:ticketid/:fromdate/:todate', function (req, res) {
var id = req.params.ticketid;
var fromdate = req.params.fromdate;
var todate = req.params.todate;
var result = {};
var data_output = [];
var asyncTasks = [];
var currentDate = new Date(fromdate);
var between = [];
var end = new Date(todate);
while (currentDate <= end) {
var tempdate = new Date(currentDate).toISOString();
var dump = tempdate.toString().split("T");
between.push(dump[0]);
currentDate.setDate(currentDate.getDate() + 1);
}
between.forEach(function (entry) {
asyncTasks.push(function (callback) {
client.exists(id + entry, function (err, reply) {
if (reply === 1) {
console.log("Found");
client.get(id + entry, function (err, reply) {
var output = JSON.parse(reply);
data_output = data_output.concat(output);
return callback();
});
} else {
return callback();
}
});
});
});
async.parallel(asyncTasks, function () {
// All tasks are done now
result['data'] = data_output;
result['response'] = 1;
result['message'] = 'vacancies list fetched successfully!';
res.json(result);
});
});
Related
I am posting value But I am getting empty array . I know its node asynchronous problem . But I don't know how do i solve this. I have refer this following link:
How do I return the response from an asynchronous call?
But I could not able to understand . Kindly help me to understand promises and how do i use that in my code.
router.post('/inspection_list', function (req, res) {
var id = req.body.project_id;
console.log(id)
// res.send("ok")
db.inspection.findOne({'_id':id},(err,response)=>{
if(err){
console.log("error");
}
else{
console.log("Data")
var inspection = [];
var data = response.inspection_data;
var f = data.map(function (item) {
var fielduser = item.fielduser_id
db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)},(err,user)=>{
console.log(user.owner_name);
console.log(item.inspection_name)
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
})
});
console.log(inspection) // Here am getting empty value
// setTimeout(function(){ console.log(inspection) }, 5000); my timeout code
}
})
});
router.post('/inspection_list', async function (req, res) {
var id = req.body.project_id;
try{
var response = await db.inspection.findOne({'_id':id})
var inspection = [];
var data = response.inspection_data;
for ( var i = 0; i<data.length; i++){
var item = data[i]
var fielduser = item.fielduser_id
var user = await db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)})
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
}
}
catch(err){
throw err
}
})
This uses async and await, you can use it if you are using node version >=7.6
Also note the following:
router.post('/inspection_list', async function (req, res)
Handling each error seperately
router.post('/inspection_list', async function (req, res) {
var id = req.body.project_id;
try{
var response = await db.inspection.findOne({'_id':id})
}
catch(err){
// handle error here
throw err
}
var inspection = [];
var data = response.inspection_data;
for ( var i = 0; i<data.length; var item = data[i]
var fielduser = item.fielduser_id
try{
var user = await db.fielduser.findOne({'_id': mongoose.Types.ObjectId(fielduser)})
}
catch(err){
// handle error
}
inspection.push({inspection_name:item.inspection_name,field_user_name : user.owner_name})
}
})
Using mongoose would be the easy way out, it returns Promises for all query and save functions, so you'd simply do:
YourModel.findOne({params}).then(() => {...})
If you're unable to do that, in your case, a 'promisified' example would be:
var findAndFillArray = (project_id) => new Promise((resolve) => {
.... your previous code here ....
inspection.push({inspection_name:item.inspection_name,field_user_name :
user.owner_name})
if (data.length === inspection.length){ // Or some other preferred condition
resolve(inspection);
}
})
Then you'd call this function after you get the id, like any other function:
var id = req.body.project_id;
findAndFillArray(id).then((inspection_array) => {
res.send(inspection_array) // Or whatever
})
Now, map and all list functions are synchronous in JS, are you sure the error is due to that?
This is a continuation from another question I asked earlier Node.js Printing info from JSON file using a function from another JS file
In my previous question I had problems in calling a function from my data-service.js file that printed all the items in my JSON array, and had it resolved, but now I'm struggling in doing something similar in printing only the employees from my JSON array that I specify through the url. For example http://localhost:8080/employeesstatus=5 would print only the employee with a status of 5 however nothing is getting printed
SERVER.JS
var HTTP_PORT = process.env.PORT || 8080;
var express = require('express');
var data = require('./data-service');
var fs = require('fs');
var app = express();
var object = require('./data-service');
console.log("Express http server listening on 8080");
//app.get('/employees', function(req,res){
// return object.getAllEmployees()
// .then((response) => res.send(response))
//}); //QUESION FROM PREVIOUS POST WHICH WAS RESOLVED
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
DATA SERVICE.JS
var employees = [];
var departments = [];
var error = 0;
var fs = require("fs");
function initialize(){
employees = fs.readFileSync("./data/employees.json", 'utf8', function(err, data){
if(err){
error = 1;
}
employees = JSON.parse(data);
});
departments = fs.readFileSync("./data/department.json", 'utf8',
function(err, data){
if(err){
error = 1;
}
departments = JSON.parse(data);
});
}
function check() {
return new Promise(function(resolve,reject){
if (error === 0){
resolve("Success");
}
else if(error === 1){
reject("unable to read file");
}
})
};
var getAllEmployees = function(){
return check().then(function(x){
console.log(x);
console.log(employees);
return employees;
}).catch(function(x){
console.log("No results returned");
});
}
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return console.log(employees2[i]);
}
}
}).catch(function(){
console.log("no results returned");
})
}
module.exports.getAllEmployees = getAllEmployees;
module.exports.getEmployeesByStatus = getEmployeesByStatus;
The 2 functions in question
app.get('/employees:?status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
var getEmployeesByStatus = function (status){
return check().then(function(x){
var employees2 = JSON.parse(employees);
for (var i = 0; i<employees2.length; i++){
if (employees2[i].status == status){
return employees2[i];
}
}
}).catch(function(){
console.log("no results returned");
})
}
1) You should replace /employees route with the following
app.get('/employees/status=:value', function(req,res){
return object.getEmployeesByStatus(req.params.value)
.then((response) => res.send(response));
});
You are able to access using http://localhost:8080/employees/status=5
2) Return employees2[i] instead of console.log(employees2[i]).
I have written below .js file to call below defined function.
objectRepositoryLoader.readObjectRepository() returns me a hashmap from where i have to use values in enterUserName(), enterPassword(), clickLoginButton() functions.
var path = require('path');
var elementRepoMap = {}
var LandingPage = function(){
var fileName = path.basename(module.filename, path.extname(module.filename))
objectRepositoryLoader.readObjectRepository(fileName+'.xml' , function(elementRepo){
console.log(elementRepo) //values are being printed here
this.elementRepoMap = elementRepo
});
this.enterUserName = function(value){
console.log(elementRepoMap) //values are not being printed here
//Some Code
};
this.enterPassword = function(value){
//Some Code
};
this.clickLoginButton = function(){
//Some Code
};
};
module.exports = new LandingPage();
The objectRepositoryLoader.readObjectRepository() function defined in another file is as below:
var ObjectRepositoryLoader = function() {
this.readObjectRepository = function(fileName, callback) {
var filePath = './elementRepository/'+fileName;
this.loadedMap = this.objectRepoLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.objectRepoLoader = function(filePath, callback){
if (filePath.includes(".xml")) {
this.xmlObjectRepositoryLoader(filePath, function(loadedMap){
return callback(loadedMap);
});
}
this.xmlObjectRepositoryLoader = function (xmlPath, callback){
var innerMap = {};
var elementName;
fs.readFile(xmlPath, "utf-8",function(err, data) {
if(err){
console.log('File not found!!')
}
else{
var doc = domparser.parseFromString(data,"text/xml");
var elements = doc.getElementsByTagName("A1");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("AS");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
loadedMap[elementName] = innerMap;
innerMap={};
};
}
return callback(loadedMap);
});
};
How can I call enterUserName(), enterPassword(), clickLoginButton() function from spec.js file and is there any way I can avoid using callback and use async.js and call enterUserName(), enterPassword(), clickLoginButton() from spec.js file ?
EDIT
I have modified my file like below:
this.xmlObjectRepositoryLoader = function (xmlPath){
var innerMap = {};
var elementName;
var filePath = xmlPath+'.xml'
var self = this
return new Promise(
function(resolve, reject){
console.log("In xmlObjectRepositoryLoader : "+filePath)
self.readFilePromisified(filePath)
.then(text => {
var doc = domparser.parseFromString(text,"text/xml");
var elements = doc.getElementsByTagName("Element");
for(var i =0 ; i< elements.length;i++){
var elm = elements[i];
elementName = elm.getAttribute("name");
var params = elm.getElementsByTagName("param");
innerMap = {};
for(var j =0 ; j< params.length;j++){
var param = params[j];
var locatorType = param.getAttribute("type");
var locatorValue = param.getAttribute("value");
innerMap[locatorType] = locatorValue;
}
map[elementName] = innerMap;
innerMap={};
}
console.log(map) // prints the map
resolve(text)
})
.catch(error => {
reject(error)
});
});
}
this.readFilePromisified = function(filename) {
console.log("In readFilePromisified : "+filename)
return new Promise(
function (resolve, reject) {
fs.readFile(filename, { encoding: 'utf8' },
(error, data) => {
if (error) {
reject(error);
} else {
resolve(data);
}
})
})
}
I am calling above function from another file as below:
objectRepositoryLoader.readObjectRepository(fileName)
.then(text => {
console.log(text);
})
.catch(error => {
console.log(error);
});
But it gives me error as
.then(text => { ^
TypeError: Cannot read property 'then' of undefined
In this case how can I use promise to call another promise function and then use the returned value in one more promise function and return calculated value to calling function where I can use the value in other functions. I sound a bit confused. Please help
You can use async.waterfall and async.parallel to perform this task
see the reference
I just tried your code to make it working, I explained the way of implementation in comment.
async.waterfall([
function(next){
objectRepositoryLoader.readObjectRepository(fileName+'.xml' ,next)//pass this next as parameter in this function defination and after manipulation return result with callback like this(null,result)
}
],function(err,result){
if(!err){
//Do wahtever you want with result
async.parallel([
function(callback){
this.enterUserName = function(value){
console.log(elementRepoMap)
//Some Code
};
},
function(callback){
this.enterPassword = function(value){
//Some Code
};
},
function(callback){
this.clickLoginButton = function(){
//Some Code
};
}
], function(err, results) {
// optional callback
};
}
})
I am trying to write some items I pushed into an array into a JSON file in node.js but I can't figure out how to wait for the array to contain the items before writing the JSON file. As a result the file is always empty. Do i need to have a callback? If so, how? NB:I'm still new to node.js
This is the code below:
var getLinks = require('./news_archive/news_links.js');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var saveNews = './news_archive/news.json';
var jsonObj = [];
var i;
var number_of_links = getLinks.links.length;
for(i=0; i<number_of_links; i++){
//GET ARTICLE LINK FROM link.js
var url = "http://www.times.co.sz/"+getLinks.links[i];
request(url, function(err, resp, body){
var $ = cheerio.load(body);
//GET ARTICLE HEADLINE
var storyHeadline = $('#article_holder h1');
var storyHeadlineText = storyHeadline.text();
//GET DATE POSTED
var datePosted = $('.metadata_time');
var datePostedText = datePosted.text();
//GET ARTICLE REPORTER'S NAME
var reporterName = $('.article_metadata a');
var reporterNameText = reporterName.text();
//GET ARTICLE SUMMARY
var fullStory = $('#article_body span');
var fullStoryText = fullStory.text();
//PUSH ITEMS TO jsonObj ARRAY
jsonObj.push({
id: i,
storyHeadline: storyHeadlineText,
datePosted: datePostedText,
reporterName: reporterNameText,
fullStory: fullStoryText
})
});
} //END for LOOP
//WRITE TO news.json file
fs.writeFile(saveNews, JSON.stringify(jsonObj, null, 4), function(err) {
if(err) {
console.log(err);
} else {
console.log("JSON saved to " + saveNews);
}
});
The issue is that request is asyncronous and you cannot use syncronous loop to iterate through. You can use async lib for that
var getLinks = require('./news_archive/news_links.js');
var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');
var saveNews = './news_archive/news.json';
var number_of_links = getLinks.links.length;
var async = require('async');
async.times(number_of_links, function (i, next) {
var url = "http://www.times.co.sz/"+getLinks.links[i];
request(url, function(err, resp, body){
var $ = cheerio.load(body);
//GET ARTICLE HEADLINE
var storyHeadline = $('#article_holder h1');
var storyHeadlineText = storyHeadline.text();
//GET DATE POSTED
var datePosted = $('.metadata_time');
var datePostedText = datePosted.text();
//GET ARTICLE REPORTER'S NAME
var reporterName = $('.article_metadata a');
var reporterNameText = reporterName.text();
//GET ARTICLE SUMMARY
var fullStory = $('#article_body span');
var fullStoryText = fullStory.text();
//PUSH ITEMS TO jsonObj ARRAY
next(err, {
id: i,
storyHeadline: storyHeadlineText,
datePosted: datePostedText,
reporterName: reporterNameText,
fullStory: fullStoryText
});
});
}, function (err, res) {
// do not forget to handle error
fs.writeFile(saveNews, JSON.stringify(res, null, 4), function(err) {
if(err) {
console.log(err);
} else {
console.log("JSON saved to " + saveNews);
}
});
})
I'm trying to run a function and once that function is complete, then run another function. The first function reads a CSV file, makes a GET request, and builds an object. The second function uses that newly created object to create a new CSV file.
The problem I'm having is that the new CSV file is being created prior to the GET requests finishing.
I'm using async.parallel to set the flow, but not able to get the logic right.
I'd love to know what I'm doing wrong and better understand how node thinks about these tasks.
// Require
var request = require('request');
var fs = require('fs');
var json2csv = require('json2csv');
var csv = require('csv');
var async = require('async');
// Params
var emailHunter_apiKey = '0000';
var emails = [];
var fields = ['email'];
var i = 0;
// Start
async.parallel([
function(callback){
setTimeout(function(){
var file = fs.readFileSync('file.csv');
csv.parse(file, {delimiter: ','}, function (err, data) {
for (var key in data) {
if (i < 5) {
if (data.hasOwnProperty(key)) {
var h = data[key];
if (h[5] != '') {
var url = h[5];
url = url.replace('//', '');
url = url.replace('www.', '');
request('https://api.emailhunter.co/v1/search?domain=' + url + '&api_key=' + emailHunter_apiKey + '', function (error, response, body) {
if (!error && response.statusCode == 200) {
var json = JSON.parse(body);
for (var subObj in json) {
if (json.hasOwnProperty(subObj) && subObj == 'emails') {
var emailObj = json[subObj];
for (var key in emailObj) {
var email = {
'email': emailObj[key]['value']
};
emails.push(email);
}
}
}
}
});
}
}
}
i++;
}
});
callback(null, emails);
}, 200);
console.log(emails);
}
],
function(err, results){
json2csv({data: results, fields: fields}, function (err, csv) {
if (err) console.log(err);
fs.writeFile('export.csv', csv, function (err) {
if (err) throw err;
console.log('file saved');
});
});
console.log(results);
});
As laggingreflex mentioned, you're using async incorrectly.
First you should build a an array of functions that you want to execute in parallel. And then use async to execute them.
Furthermore, your callback was getting executed immediately because csv.parse() is an async function. Therefore node fires it immediately and then executes callback(). You need to move the callback inside of parse().
Try this...
// Params
var emailHunter_apiKey = '0000';
var emails = [];
var fields = ['email'];
var i = 0;
var functionsToRunAsync = [];
var file = fs.readFileSync('file.csv');
csv.parse(file, {delimiter: ','}, function (err, data) {
for (var key in data) {
if (i < 5) {
if (data.hasOwnProperty(key)) {
var h = data[key];
if (h[5] != '') {
var url = h[5];
url = url.replace('//', '');
url = url.replace('www.', '');
// add a new function to an array, to be executed later
functionsToRunAsync.push(function(callback) {
request('https://api.emailhunter.co/v1/search?domain=' + url + '&api_key=' + emailHunter_apiKey + '', function (error, response, body) {
if (!error && response.statusCode == 200) {
var json = JSON.parse(body);
for (var subObj in json) {
if (json.hasOwnProperty(subObj) && subObj == 'emails') {
var emailObj = json[subObj];
for (var key in emailObj) {
var email = {
'email': emailObj[key]['value']
};
emails.push(email);
// callback to tell async this function is complete
callback()
}
}
}
} else {
// callback to tell async this function is complete
callback
}
});
});
}
}
}
i++;
}
// now that we have all of the functions in an array, we run them in parallel
async.parallel(
functionsToRunAsync,
function(err, results) { // all async functions complete
json2csv({data: results, fields: fields}, function (err, csv) {
if (err) console.log(err);
fs.writeFile('export.csv', csv, function (err) {
if (err) throw err;
console.log('file saved');
});
});
console.log(results);
});
});