I'm trying to add in Agenda into my node application so I can run some background tasks on a daily basis, e.g. Deactivating users who have not logged in for 60 days.
I've tried following the example on the GitHub associated with the module, but seem to be running into a problem with my app hanging whenever I try to load the site, and ultimately getting an "Error 504: Gateway Server Timeout". I'm also seeing undefined in the console.
I know Agenda is up and working correctly, as I have a simple job at the moment that just does a console.log every minute.
In my app.js I require my worker.js file:
var agenda = require('./worker.js');
My worker.js is just a simple 1 line:
require('./lib/agenda.js');
agenda.js:
var Agenda = require('agenda');
var connectionString = "mongodb://" + process.env.MONGODB_USER + ":" +
process.env.MONGODB_PASSWORD + "#" +
process.env.DATABASE_SERVICE_NAME + ':' +
process.env.MONGODB_PORT + '/' +
process.env.MONGODB_DATABASE;
var agenda = new Agenda({db: {address: connectionString}});
var jobTypes = process.env.JOB_TYPES ? process.env.JOB_TYPES.split(',') : [];
jobTypes.forEach(function(type){
require('./jobs/' + type)(agenda);
});
if (jobTypes.length) {
agenda.on('ready', function() {
agenda.every('* * * * *', 'test job') //Run job at 0030 every day
agenda.start();
})
}
module.exports = agenda
and the test job is defined in a job file like so:
agenda.define('test job', function(job, done) {
console.log ('Agenda job executed');
done();
});
I feel like I am missing something really obvious!
It turns out that I needed to add an agenda.processEvery in the start command section of my agenda.js:
var Agenda = require('agenda');
var connectionString = "mongodb://" + process.env.MONGODB_USER + ":" +
process.env.MONGODB_PASSWORD + "#" +
process.env.DATABASE_SERVICE_NAME + ':' +
process.env.MONGODB_PORT + '/' +
process.env.MONGODB_DATABASE;
var agenda = new Agenda({db: {address: connectionString}});
var jobTypes = process.env.JOB_TYPES ? process.env.JOB_TYPES.split(',') : [];
jobTypes.forEach(function(type){
require('./jobs/' + type)(agenda);
});
if (jobTypes.length) {
agenda.on('ready', function() {
agenda.every('* * * * *', 'test job') //Run job at 0030 every day
agenda.processEvery('one minute'); //<====== This is the new line
agenda.start();
})
}
module.exports = agenda
Related
The problem that I am running into is that I am trying to create an express route that I can run a seed script or cron job against which when hit will then run a GET request, with Axios against an external API endpoint on a scheduled basis, get the appropriate data, and store it in a MongoDB database.
I was able to successfully run this in my backend app.js file, but the problem is that I don't want to run this query every single time the backend app starts or restarts as it will input duplicate data into the database and will not scale, which is why I've separated it out into Models and Routes and am trying to create an endpoint that I can hit with a cron job or seed script.
I have created a Model that will create the appropriate Schema and Model in MongoDB. The model creates fine, which I can see when I restart my backend server.
I have created a Route called /schedule that when hit should run the appropriate code to:
Use the Mongo Schema and Model
Run axios.get against the API endpoint
Use an onSuccess function to query the JSON response data, pull out the appropriate fields, and then send that data up to my MongoDB collection.
Here is my schedule.Route.js file:
const express = require("express");
const router = express.Router();
const axios = require('axios');
const Schedule = require("../models/scheduleModel");
const { response } = require('express')
var date_ob = new Date();
var day = ("0" + date_ob.getDate()).slice(-2);
var month = ("0" + (date_ob.getMonth() + 1)).slice(-2);
var year = date_ob.getFullYear();
var date = year + "/" + month + "/" + day;
// Route
router.get("/schedule", (response) => {
axios.get('https://api.sportradar.us/ncaamb/trial/v7/en/games/' + date + '/schedule.json?api_key=' + apiKey)
.then(function (response) {
onSuccess(response)
})
.catch(function (error) {
console.log(error);
});
});
function onSuccess(response) {
var gameData = response;
var gameArray = gameData.data.games
var arrayLength = Object.keys(gameArray).length
console.log(arrayLength)
for(var i = 0; i < arrayLength; i++) {
var homeTeam = gameData.data.games[i].home.name;
var homeAlias = gameData.data.games[i].home.alias;
var homeId = gameData.data.games[i].home.id;
var awayTeam = gameData.data.games[i].away.name;
var awayAlias = gameData.data.games[i].away.alias;
var awayId = gameData.data.games[i].away.id;
console.log( homeTeam + " " + homeAlias + " " + homeId + " " + awayTeam + " " + awayAlias + " " + awayId);
assignScheduleValue(homeTeam, homeAlias, homeId, awayTeam, awayAlias, awayId)
}
}
// Store values in MongoDB
function assignScheduleValue(homeTeam, homeAlias, homeId, awayTeam, awayAlias, awayId) {
var upSchedule = new Schedule()
upSchedule.homeAlias = homeAlias;
upSchedule.homeTeam = homeTeam;
upSchedule.awayTeam = awayTeam;
upSchedule.homeId = homeId;
upSchedule.awayId = awayId;
upSchedule.awayAlias = awayAlias;
upSchedule.save();
}
module.exports = router;
From my app.js file I'm just running:
app.use("/schedule", require("./routes/scheduleRoute"));
Again the goal with this route is so that I can hit it with a CURL or some other command and just run the axios.get request to pull that data into MongoDB daily.
This is the code that I deployed on Heroku. It's function is to update a certain key that i'm displaying every ten seconds.
const express = require('express');
const timestamp = require('time-stamp');
var app = express();
var port = 3000;
var server = app.listen(process.env.PORT || port, '0.0.0.0', function() {
console.log(`Listening at port: ${port}`);
});
app.get("/", encr)
code = 5
function encr(req, res){
console.log(`Code accessed, code = ${code}`)
res.send({code})
}
setInterval(async function() {
code = Math.floor((Math.random() * 100) + 1);
console.log(`Key updated. Time: ${timestamp.utc('DD/mm:ss:ms')}`)
}, 10 * 1000);
This setInterval function needs to run every 10 seconds, but ends up running every 10.01 seconds, which after a bit begins to add up.
Logs:
2021-02-08T20:15:45.414054+00:00 app[web.1]: Key updated. Time: 08/15:45:413
2021-02-08T20:15:55.423763+00:00 app[web.1]: Key updated. Time: 08/15:55:423
[...]
2021-02-08T20:18:25.543898+00:00 app[web.1]: Key updated. Time: 08/18:25:543
2021-02-08T20:18:35.551720+00:00 app[web.1]: Key updated. Time: 08/18:35:551
2021-02-08T20:18:45.559856+00:00 app[web.1]: Key updated. Time: 08/18:45:559
How can I fix this so it's exactly 10 seconds?
Timeouts/intervals aren't entirely accurate at the millisecond timescale. Instead of trying to reassign code every precise interval, consider instead saving the timestamp whenever code gets updated, and when it gets accessed again, check whether the timestamp has expired - if so, reassign it:
const makeCode = () => {
code = Math.floor((Math.random() * 100) + 1);
lastAssignTimestamp = performance.now();
};
let lastAssignTimestamp;
let code;
makeCode();
function encr(req, res){
if (performance.now() - lastAssignTimestamp > 10_000) {
// then the code has expired...
makeCode();
}
console.log(`Code accessed, code = ${code}`)
res.send({code})
}
I've edited my code to also print the last update's timestamp, that way I can just check directly in my other code without time keeping shenanigans
const express = require('express');
var app = express();
var port = 3000;
var server = app.listen(process.env.PORT || port, '0.0.0.0', function() {
console.log(`Listening at port: ${port}`);
});
app.get("/", encr)
code = 5
last_time = 0
const makeCode = () => {
code = Math.floor((Math.random() * 100) + 1);
lastAssignTimestamp = +new Date()
console.log(`New Code: ${code}`)
};
let lastAssignTimestamp;
makeCode();
setInterval(function(){
let x = +new Date()
if (x - lastAssignTimestamp > 1000 * 60){
// then the code has expired... every minute
makeCode();
}
last_time = lastAssignTimestamp
console.log(`Time left till next update: ${Math.round(((1000 * 60) - (x - lastAssignTimestamp))/1000)}`)
}, 1000);
function encr(req, res){
res.send(
{code, last_time}
)
}
I have a node.js script that creates a websocket connection to a crypto trading site (bitmex). It streams price data.
For some reason after an hour or two the stream goes bad and the prices (if streamed at all) are inacurate.
For now I restart it manually every hour but I need that to be done automatically. How can I do that?
Here is the code of the script:
var WebSocket = require('ws');
var ws = new WebSocket("wss://www.bitmex.com/realtime");
var couchbase = require('couchbase')
var cluster = new couchbase.Cluster('couchbase://localhost/');
cluster.authenticate('xxxxxx', 'xxxxxxx');
var bucket = cluster.openBucket('test');
var N1qlQuery = couchbase.N1qlQuery;
let num_msg = 0;
ws.onopen = function(){
ws.send(JSON.stringify({"op": "subscribe", "args": [
"trade:XBTUSD",
"trade:LTCZ18"]
}))
};
ws.onmessage = function(msg){
var response = JSON.parse(msg.data);
num_msg++
if(num_msg > 50) {
var coin = response['data'][0]['symbol'];
var price = response['data'][0]['price'];
//console.log(coin + ":" + price + "\n");
bucket.manager().createPrimaryIndex(
function(){
bucket.upsert( coin,
{
'price': price
},
function (err, result){
});
});
}
};
EDIT: I missed to mention that currently I use Windows 7 system (eventhough I do need to move to Ubuntu or similar).
EDIT 2: Final version of my code :)
const cron = require("node-cron");
var WebSocket = require('ws');
var couchbase = require('couchbase');
var dateTime = require('node-datetime');
let now = new Date();
minutes = now.getMinutes() + 1;
if(minutes + 30 > 59) {
minutes1 = minutes - 30;
} else {
minutes1 = minutes - 30;
}
if(minutes > minutes1) {
s_cron = minutes1 + "," + minutes + " * * * *";
} else {
s_cron = minutes + "," + minutes1 + " * * * *";
}
cron.schedule(s_cron, () => {
console.log("---------------------");
console.log("Running Cron Job");
var dt = dateTime.create();
var formatted = dt.format('Y-m-d H:M:S');
console.log(formatted);
// create bitmex ws
var ws = new WebSocket("wss://www.bitmex.com/realtime");
// connect to couchbase
var cluster = new couchbase.Cluster('couchbase://localhost/');
cluster.authenticate('xxxxxxxx', 'xxxxxxxxxx');
var bucket = cluster.openBucket('test');
var N1qlQuery = couchbase.N1qlQuery;
let num_msg = 0;
ws.onopen = function(){
ws.send(JSON.stringify({"op": "subscribe", "args": [
"trade:XBTUSD",
"trade:LTCZ18"]
}))
};
ws.onmessage = function(msg){
var response = JSON.parse(msg.data);
num_msg++
if(num_msg > 50) {
var coin = response['data'][0]['symbol'];
var price = response['data'][0]['price'];
//console.log(coin + ":" + price + "\n");
bucket.manager().createPrimaryIndex(
function(){
bucket.upsert( coin,
{
'price': price
},
function (err, result){
//bucket.disconnect()
});
});
}
};
});
Try 'node-cron': more at https://www.npmjs.com/package/node-cron Hope that works.
Consider using cron to restart every hour. Your crontab entry would look like:
0 * * * * <command to restart your app>
If you can't or don't want to use your system crontab or equivalent (not sure what it would be on Windows), you can use pm2.
pm2: https://www.npmjs.com/package/pm2
For how to make pm2 restart every hour, see https://github.com/Unitech/pm2/issues/1076 or https://stackoverflow.com/a/38062307/436641.
Another option would be node-cron: https://www.npmjs.com/package/node-cron
On init of my application i start function with settings.
const schedule = require('node-schedule');
const monk = require('monk');
const db = monk(config.mdbConnect);
var TGusers = db.get('TG-users');
...
...
TGusers.find({'cron.status': true}, {chatid:1, cron: 1})
.then((users) => {
console.log('twise');
for(var i=0;i<users.length;i++){
cronbox[users[i]] = {};
cronbox[users[i]].reminder = schedule.scheduleJob('*/1 * * * *', function(chatid){
console.log('chatid = '+ chatid);
tg.api.sendMessage(chatid, '🐳 Nuclear launch detected! / ' +chatid,
{'parse_mode': 'Markdown'}
);
}.bind(null, users[i].chatid));
}
}).catch((err) => {
if(err) saveerr(err);
}).then(() => db.close())
And this Monk find finction fired twice. I got two "twice" in my console, and .. looks like two cron tasks... What I did wrong?
I am trying to use mocha as a test driver for my API application.
I have a json of user names to song names, that I am trying to feed to an API endpoint.
json:
{
"description": "hold the lists that each user heard before",
"userIds":{
"a": ["m2","m6"],
"b": ["m4","m9"],
"c": ["m8","m7"],
"d": ["m2","m6","m7"],
"e": ["m11"]
}
}
Let's say the User and Song schema is just _id with name. What I want to do is after parsing the json file, convert "a" to user_id, by doing a mongoose lookup of "a" by name, let's say id=0. Convert each of the "m2" into song_id by name let's say id=1, and then call
the request, http://localhost:3001/listen/0/1
Here is what I have so far for my mocha test:
test.js:
it('adds the songs the users listen to', function(done){
var parsedListenJSON = require('../listen.json')["userIds"];
//console.log(parsedListenJSON);
for (var userName in parsedListenJSON) {
if (parsedListenJSON.hasOwnProperty(userName)) {
var songs = parsedListenJSON[userName];
var currentUser;
//console.log(userName + " -> " + songs);
var userQuery = User.findOne({'name': userName})
userQuery.then(function(user){
//console.log("user_id: " + user["_id"]);
currentUser = user;
});
for (var i=0; i<songs.length; i++)
{
//console.log(songs[i]);
var songQuery = Song.findOne({'name': songs[i]})
songQuery.then(function(song){
console.log("user_id: " + currentUser["_id"]);
console.log("song_id: " + song["_id"]);
// need to ensure we have both the user_id and song_id
api_request = "http://localhost:3001/listen/" + currentUser["_id"] + "/" + song["_id"];
console.log(api_request);
// listen/user_id/song_id
//.post('http://localhost:3001/listen/0/1')
request
.post(api_request)
.end(function(err, res){
expect(res.status).to.equal(200);
});
});
}
}
}
done();
})
Questions:
1) I need to make sure I have both the user_id and song_id before I make the API cal. Right now, each of the User.findOne and Song.findOne query has their own then clauses. I am passing the user_id through the currentUser variable into the then block of the Song query. But since the query is asynchronous, I don't think this is proper. How can I structure this code so that I proceed with the API call when both the then block have executed.
2) When I run the code as is, only the first user gets executed, and not the rest,
ie. the print out is:
user_id: 0
song_id: 1
http://localhost:3001/listen/0/1
user_id: 0
song_id: 5
http://localhost:3001/listen/0/5
3) The API endpoint works with postman, and in a simpler Mocha test below. But it doesn't seem to work in my original code.
var request = require('superagent');
var expect = require('expect.js');
var User = require('../models/user');
var Song = require('../models/song');
var mongoose = require('mongoose');
mongoose.connect('localhost/TestSongRecommender');
...
it('adds the songs', function(){
request
.post('http://localhost:3001/listen/0/2')
.end(function(res){
expect(res.status).to.equal(200);
});
});
Update:
The async.forEach approach works. Here's my final snippet:
updated test.js
var request = require('superagent');
var expect = require('expect.js');
var async = require('async');
var User = require('../models/user');
var Song = require('../models/song');
var mongoose = require('mongoose');
mongoose.connect('localhost/Test');
describe('song recommendations', function(){
it('adds the songs the users listen to', function(done){
var parsedListenJSON = require('../listen.json')["userIds"];
//console.log(parsedListenJSON);
async.forEach(Object.keys(parsedListenJSON), function forAllUsers(userName, callback) {
var songs = parsedListenJSON[userName];
//var currentUser;
//console.log(userName + " -> " + songs);
var userQuery = User.findOne({'name': userName})
userQuery.then(function(user){
//console.log("user_id: " + user["_id"]);
//console.log(songs);
//currentUser = user;
async.forEach(songs, function runSongQuery(songName, smallback) {
//console.log(songName);
var songQuery = Song.findOne({'name': songName})
songQuery.then(function(song){
//console.log("user_id: " + user["_id"]);
//console.log("song_id: " + song["_id"]);
// need to ensure we have both the user_id and song_id
api_request = "http://localhost:3001/listen/" + user["_id"] + "/" + song["_id"];
console.log(api_request);
// listen/user_id/song_id
//.post('http://localhost:3001/listen/0/1')
request
.post(api_request)
.end(function(err, res){
expect(res.status).to.equal(200);
smallback()
});
});
}, function allSongs(err) {
callback();
})
});
}, function allUserNames(err) {
done()
})
})
});
1&2) You're going to want to use the async package from NPM. You can get it with the command npm install async --save in your main folder and var async = require('async') in your code.
You'll have to replace each for loop with async.forEach. async.forEach takes an array and two functions as arguments. It calls the first function on each item in the array and the second function once all callbacks have returned.
The way you're currently doing it, with for loops isn't going to work. By the time asynchronous things return, your loop has iterated past them (non-blocking IO, remember) and your variables are no longer set correctly.
You also need things set up to call done only once all your code has run.
It will end up looking something like this if written properly:
it('adds the songs the users listen to', function(done){
var parsedListenJSON = require('../listen.json')["userIds"];
//console.log(parsedListenJSON);
async.forEach(parsedListenJSON, function forAllUsers(userName, callback) {
var songs = parsedListenJSON[userName];
//console.log(userName + " -> " + songs);
var userQuery = User.findOne({'name': userName})
userQuery.then(function(user){
//console.log("user_id: " + user["_id"]);
var currentUser = user;
async.forEach(songs, function runSongQuery(songName, smallback) {
var songQuery = Song.findOne({'name': songName})
songQuery.then(function(song){
console.log("user_id: " + currentUser["_id"]);
console.log("song_id: " + song["_id"]);
// need to ensure we have both the user_id and song_id
api_request = "http://localhost:3001/listen/" + currentUser["_id"] + "/" + song["_id"];
console.log(api_request);
// listen/user_id/song_id
//.post('http://localhost:3001/listen/0/1')
request
.post(api_request)
.end(function(res){
expect(res.status).to.equal(200);
smallback()
});
});
}, function allRan(err) {
callback();
})
});
}, function allUserNames(err) {
done()
})
})
3) You're going to want to use a testing framework for testing your API. I recommend supertest. Once you have supertest, require your app and supertest:
var request = require('supertest');
var app = require('./testApp.js');
Use it in tests like:
request(app)
.post(api_request)
.end(function(res){
expect(res.status).to.equal(200);
smallback()
});