Hello and thank you for taking a look at this.
I have a scheduled function that contacts an API and is supposed to save the data to a local DB. It works several times and then stops saving. I have confirmed that the functions are executing per their scheduled times. Mongoose does not save the data and does not give me an error.
2 part request:
I would love help trying to catch and log the error.
-Maybe my logger(s) are in the wrong location?
Also, if you have any insight as to why it works when I start the
server and then eventually stops working that would be appreciated as
well.
-Maybe I am not 'returning' the functions properly?
Get the data from the API:
function updateWeather(){
const weatherurl = 'https://api.aerisapi.com/observations/pendleton,or?&format=json&filter=allstations&limit=1&client_id=bRZUxQc8j4z41CrH7SM6u&client_secret=GaEA7lMyLKnQwRsUcxv1mnNhTUkx6KUtVsrIXVcR';
axios.get(weatherurl)
.then(response=>{
// console.log(response.data);
let condition = response.data.response.ob.weather,
temp = response.data.response.ob.tempF,
windDir = response.data.response.ob.windDir,
windSpd = response.data.response.ob.windSpeedMPH,
windGust = response.data.response.ob.windGustMPH,
windChill = response.data.response.ob.windchillF,
humidity = response.data.response.ob.humidity,
icon = response.data.response.ob.icon,
date = moment().utcOffset(-8).format('YYYY-MM-DD'),
updated = moment().utcOffset(-8).format('MM/DD/YY HH:mm');
if(temp <= 35){
var tempIcon = 'https://cdn.aerisapi.com/wxicons/v2/cold.png';
}else if(temp >= 80){
var tempIcon = 'https://cdn.aerisapi.com/wxicons/v2/hot.png';
}else{
var tempIcon = '';
}
currentWeather.push({date:date, condition:condition, temp:temp, windChill:windChill, windDir:windDir, windSpd:windSpd, windGust:windGust, humidity:humidity, icon:icon, tempIcon:tempIcon, updated:updated});
return currentWeather[0]
}).then(currentWeather=>{
// console.log(currentWeather);
saveWeather(currentWeather);
}).catch(error=>{
logger.error(error);
});
return currentWeather;
}
Save data to DB:
function saveWeather(weather){
logger.info('saveWeather connection status: '+mongoose.connection.readyState);
if(mongoose.connection.readyState !== 1){
mongoose.connect("mongodb://localhost:27017/fdData", {useNewUrlParser: true});
}
Weather.updateOne({date: weather.date}, {date:weather.date, condition:weather.condition, temp:weather.temp, windChill:weather.windChill, windDir:weather.windDir, windSpd:weather.windSpd, windGust:weather.windGust, humidity:weather.humidity, icon:weather.icon, tempIcon:weather.tempIcon, updated:weather.updated}, {upsert:true}, error=> {
if (error){
logger.error(error);
} else{
logger.info('Weather Saved to DB')
}
})
return weather;
}
Thanks again!
-Adam
Related
I want to get the result from the closure function. I did try doing some callback functions too but still I'm getting the same issue. I was able to console.log the result inside the closure/callback but can't return the response into a variable. I tried some of the solutions posted already but weren't able to succeed.
Here's my code:
var fcmTokens = [input.fcmToken];
switch(topicType) {
case 'post':
const username = (input.username).toLowerCase();
const userPrefix = Config.get(`setting.topic.user.prefix`);
fcmTokens = Topic.get(`${userPrefix}-${username}`, {AttributesToGet : ['fcmToken']}, function(err,foundTopic) {
var result = foundTopic.attrs.fcmToken;
console.log(result) //Able to log my expected result
return result;
})
console.log(fcmTokens) //undefined
break;
}
I was able to resolve my own issue by installing a library called promisified-vogels as my Model is using a DynamoDB.
From my current code above to this:
let userTopic = await Topic
.getAsync(`${userPrefix}-${username}`,{ConsistentRead: true, AttributesToGet : ['fcmToken']}) //from original function "get" changed to "getAsync"
.then(function(user){
return user.attrs.fcmToken;
})
.catch(function(err){
console.log(err)
});
fcmTokens = userTopic; // i was able to get the list of record im expecting.
Reference Library : https://github.com/servel333/vogels-promisified
I need some advice on how to structure this function as at the moment it is not happening in the correct order due to node being asynchronous.
This is the flow I want to achieve; I don't need help with the code itself but with the order to achieve the end results and any suggestions on how to make it efficient
Node routes a GET request to my controller.
Controller reads a .csv file on local system and opens a read stream using fs module
Then use csv-parse module to convert that to an array line by line (many 100,000's of lines)
Start a try/catch block
With the current row from the csv, take a value and try to find it in a MongoDB
If found, take the ID and store the line from the CSV and this id as a foreign ID in a separate database
If not found, create an entry into the DB and take the new ID and then do 6.
Print out to terminal the row number being worked on (ideally at some point I would like to be able to send this value to the page and have it update like a progress bar as the rows are completed)
Here is a small part of the code structure that I am currently using;
const fs = require('fs');
const parse = require('csv-parse');
function addDataOne(req, id) {
const modelOneInstance = new InstanceOne({ ...code });
const resultOne = modelOneInstance.save();
return resultOne;
}
function addDataTwo(req, id) {
const modelTwoInstance = new InstanceTwo({ ...code });
const resultTwo = modelTwoInstance.save();
return resultTwo;
}
exports.add_data = (req, res) => {
const fileSys = 'public/data/';
const parsedData = [];
let i = 0;
fs.createReadStream(`${fileSys}${req.query.file}`)
.pipe(parse({}))
.on('data', (dataRow) => {
let RowObj = {
one: dataRow[0],
two: dataRow[1],
three: dataRow[2],
etc,
etc
};
try {
ModelOne.find(
{ propertyone: RowObj.one, propertytwo: RowObj.two },
'_id, foreign_id'
).exec((err, searchProp) => {
if (err) {
console.log(err);
} else {
if (searchProp.length > 1) {
console.log('too many returned from find function');
}
if (searchProp.length === 1) {
addDataOne(RowObj, searchProp[0]).then((result) => {
searchProp[0].foreign_id.push(result._id);
searchProp[0].save();
});
}
if (searchProp.length === 0) {
let resultAddProp = null;
addDataTwo(RowObj).then((result) => {
resultAddProp = result;
addDataOne(req, resultAddProp._id).then((result) => {
resultAddProp.foreign_id.push(result._id);
resultAddProp.save();
});
});
}
}
});
} catch (error) {
console.log(error);
}
i++;
let iString = i.toString();
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(iString);
})
.on('end', () => {
res.send('added');
});
};
I have tried to make the functions use async/await but it seems to conflict with the fs.openReadStream or csv parse functionality, probably due to my inexperience and lack of correct use of code...
I appreciate that this is a long question about the fundamentals of the code but just some tips/advice/pointers on how to get this going would be appreciated. I had it working when the data was sent one at a time via a post request from postman but can't implement the next stage which is to read from the csv file which contains many records
First of all you can make the following checks into one query:
if (searchProp.length === 1) {
if (searchProp.length === 0) {
Use upsert option in mongodb findOneAndUpdate query to update or upsert.
Secondly don't do this in main thread. Use a queue mechanism it will be much more efficient.
Queue which I personally use is Bull Queue.
https://github.com/OptimalBits/bull#basic-usage
This also provides the functionality you need of showing progress.
Also regarding using Async Await with ReadStream, a lot of example can be found on net such as : https://humanwhocodes.com/snippets/2019/05/nodejs-read-stream-promise/
I have two problems with this code.
1) Only the last element of beerObjects is saved to the database.
2) There are n duplicates of the last element (n = beerObjects.length) saved to the database.
function addBeersToDatabase(beerObjects) {
for (i = 0; i < beerObjects.length; i++) {
console.log(beerObjects[i].beerId);
var currentBeer = beerObjects[i];
// check if beer is already in database
Beer.findOne({'beerId': currentBeer.beerId}, function(err, beer){
if (err) {
handleError(err);
}
if (beer) {
// beer is already in database
}
else {
// add new beer to database
console.log(currentBeer.beerId);
var newBeer = new Beer();
newBeer.beerId = currentBeer.beerId;
newBeer.name = currentBeer.name;
newBeer.description = currentBeer.description;
newBeer.abv = currentBeer.abv;
newBeer.image = currentBeer.image;
newBeer.save(function(err) {
if (err) {
throw err;
}
});
}
});
}
}
I want to loop through each beer and save its info to the database. I used findOne to prevent duplicates but this is not working. The first console.log() statement prints each beer id but the seconds console.log() statement prints just the last beer id multiple times.
The issue here is that in the findOne callback - your beerId will always be set to the last beer in beerObjects, because the loop finishes before you get to your first callback - welcome to asynchronous javascript.
One remedy for this is to wrap your findOne code in an IFFE (Immediately Invoked Function Expression). This code will complete before moving on to the next beer from beerObject.
Here is some more info on IFFE
Stack Overflow on IFFE
I took a quick pass at the code, I believe this should work, but you may have to make some adjustments with the internal code...
for(var i = 0; i < beerObjects.length; i++) {
console.log(beerObjects[i].beerId);
//var currentBeer = beerObjects[i]; dont need this now
(function (currentBeer) {
Beer.findOne({ beerId: currentBeer},
function(err, beer) {
if(!err && !beer) {
var newBeer = new Beer();
newBeer.beerId = currentBeer.beerId;
newBeer.name = currentBeer.name;
newBeer.description = currentBeer.description;
newBeer.abv = currentBeer.abv;
newBeer.image = currentBeer.image;
newBeer.save(function(err) {
// log your error here...
});
} else if(!err) {
console.log("Beer is in the system");
} else {
console.log("ERROR: " + err);
}
}
);
})(beerObjects[i].beerId);
}
I'm having an issue when trying to query based on the result of another query on mongodb.
I'm trying to make an initial query and then do another query for each one of the result of the first query. The reason I'm doing it like this is because I have two different collections and I need to join some data from one collection with the data of the other collection. In a SQL world I could easily do this with a JOIN, but as I'm using mongodb in this one I can't really use JOINs, so I guessed doing a for loop inside the first query's callback function would be the way to go.
Here's the code I'm using...
var resultSet = [];
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Sorry, wrong id.");
return e;
}
for(var m=0; m<docs.length; m++){
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
db.get('buscollection').find({id_bus: parseInt(docs[m].id_bus)}, function(e, busDocs){
auxRes.s_origin_description = busDocs[0].s_origin_description;
auxRes.s_destination_description = busDocs[0].id_destination_description;
resultSet.push(auxRes);
});
res.send(JSON.stringify(resultSet));
}
});
I need to res.send the resultSet array after all the values have been added.
I've tried some other ways of doing this, but the thing is that when the res.send line is reached the second query hasn't finished at all. I also tried doing that inside the inner query's callback, but I need to check if it's the last in the for loop, and checking the value o m won't do it as it always is equivalent to docs.length.
As far as I know there's no such thing as a synchronous query in mongodb, but maybe I'm wrong.
What's the right way of doing this?
EDIT
I found a way around it, but I'm sure there's got to be a better way. Here's how I'm doing it...
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Ha ocurrido un error, no existe esa parada");
return e;
}
var busIDs = [];
for(var m=0; m<docs.length; m++){
busIDs.push(parseInt(docs[m].id_bus));
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
resultSet.push(auxRes);
}
db.get('buscollection').find({id_bus: {$in: busIDs}}, function(e, busDocs){
for(var n = 0; n<busDocs.length; n++){
for(var k=0; k<resultSet.length; k++){
if(resultSet[k].id_bus == busDocs[n].id_bus){
resultSet[k].s_origin_description = busDocs[n].s_origin_description;
resultSet[k].s_destination_description = busDocs[n].id_destination_description;
}
}
}
res.send(JSON.stringify(resultSet));
});
});
Node.js behavior is asynchronous , programmer has to code taking consideration of this behavior. Use callbacks or promises or a flow control library . In your your program , you
have put mongo query inside loop , which is a bad approach of querying . Instead if querying multiple times , use $in operator . It will optimize your code performance and
solves your response sending problem also.
var resultSet = [];
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Sorry, wrong id.");
return e;
}
var bus_ids = [];
for(var m=0; m<docs.length; m++){
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
bus_ids.push(parseInt(docs[m].id_bus)); // collect all ids
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
resultSet.push(auxRes);
}
// Query at one time for all document
db.get('buscollection').find({id_bus: {$in : bus_ids}}).toArray( function(e, busDocs){
// Now find and merge in one go
busDocs.forEach(function(eachBusDoc){
for(var i=0,len = resultSet.length;i< len;i++){
if(resultSet[i].id_bus == busDocs.id_bus ){
resultSet[i].s_origin_description = eachBusDoc.s_origin_description;
resultSet[i].s_destination_description = eachBusDoc.id_destination_description;
}
}
});
res.send(JSON.stringify(resultSet));
});
});
Your updated solution in your question is generally fine, as using $in is an excellent way of fetching a set of results (you'll want to make sure that you've indexed the id_bus property).
Here are a few tweaks (with a bit of cleanup and optimization):
db.get('busstopcollection')
.find({id_bus_stop: parseInt(req.body.busstopid)}).toArray(function(e, docs){
var auxById = {}; // store a dictionary of all the results for later
if(e || docs === null || docs.length === 0) {
console.log("Ha ocurrido un error, no existe esa parada");
return e;
}
var busIDs = [];
docs.forEach(function(doc) {
busIDs.push(parseInt(doc.id_bus));
// consider just using the doc directly rather than copying each property
// especially if you're not manipulating any of the data as it passes
var auxRes = {
id_bus : doc.id_bus,
id_bus_stop : doc.id_bus_stop,
coord_x : doc.coord_x,
coord_y : doc.coord_y,
id_bus_variation : doc.id_bus_variation
};
// you could just use each doc directly ...
// var auxRes = doc; ??
// ** importantly, store off the id_bus for each one so you can
// ** avoid a costly loop trying to match an id below.
auxById[doc.id_bus] = auxRes;
resultSet.push(auxRes);
});
// might want to consider using a cursor ... here's an example
db.get('buscollection')
.find({id_bus: {$in: busIDs}}).each(function(e, busDoc){
// the last item in the cursor will be null
if (busDoc === null) {
res.send(JSON.stringify(resultSet));
return;
}
var res = auxById[busDoc.id_bus];
if (res) { // did we find it in our dictionary of results?
// yes, we did == copy the necessary field data
res.s_origin_description = busDoc.s_origin_description;
res.s_destination_description = busDoc.id_destination_description;
}
});
});
I am using node.js with mongoose. The problem i am facing is i am getting newModifier1 printed but outside that function the value is null.
Here is my code:
// Find userSchema
newModifier1 = "";
exports.findModifier = function(modifierName){
modifierModel.find({'name' : modifierName},function(err,result){
if(err){
console.log("Error : "+err);
throw err;
}
else{
newModifier1 = result;
// console.log("Modifier is searched successfully : "+newModifier1);
}
console.log("Modifier is searched successfully1 : "+newModifier1);
});
// newModifier1=temp;
return newModifier1; // it takes newModifier1 = "" value here
}
Any ideas what the problem could be?
This is what is happening:
// this is "global" an would be weirdly overwritten
// if function is called multiple times before finishing
newModifier1 = "";
exports.findModifier = function(modifierName){
// TIMESTAMP: 0
modifierModel.find({'name' : modifierName},function(err,result){
// TIMESTAMP: 2
if(err){
console.log("Error : "+err);
throw err;
}
else{
newModifier1 = result;
// console.log("Modifier is searched successfully : "+newModifier1);
}
console.log("Modifier is searched successfully1 : "+newModifier1);
});
// TIMESTAMP: 1
return newModifier1; // it takes newModifier1 = "" value here
}
I added some notes, when what is happening. As you can see and because of the async nature of node.js you return the value before you get a result back from the database.
You need familiarize yourself with the async flow and callback function.
Pass a callback function to findModifier and wait for the database to return a result.
modifierModel.find runs asynchronously and probably findModifier method is returning before the callback of find method executes. Although you see it being printed out what is returned from the method is en empty string anyway. You can use a library like async.