I am currently trying to stream two tracks at the same time on the same connection, the problem I have is that a new connection is opened for the second stream in which Twitter rejects with error code 7, which is Twitter throwing off the oldest connection to make way for the newest, is there anything I can do programatically to prevent this?
This is the code I am using
var request = oa.get("https://stream.twitter.com/1.1/statuses/filter.json?track=tweet1", access_token, access_token_secret );
request.addListener('response', function (response) {
response.setEncoding('utf8');
response.addListener('data', function (chunk) {
var theTweets = JSON.parse(chunk);
console.log(theTweets);
MongoClient.connect("mongodb://localhost:27017/db", function(error, database) {
var collection = database.collection('coll');
collection.insert(theTweets, function(err, result) {});
});
});
response.addListener('end', function () {
console.log('--- END ---');
});
});
var requestTweet2 = oa.get("https://stream.twitter.com/1.1/statuses/filter.json?track=tweet2", access_token, access_token_secret );
requestTweet2.addListener('response', function (response) {
response.setEncoding('utf8');
response.addListener('data', function (chunk) {
var theTweets = JSON.parse(chunk);
MongoClient.connect("mongodb://localhost:27017/db", function(error, database) {
var collection = database.collection('coll');
collection.insert(theTweets, function(err, result) {});
});
});
response.addListener('end', function () {
console.log('--- END ---');
});
});
requestTweet2.end();
request.end();
The Twitter public stream will only accept one connection per IP address. You can find this documented here.
Each account may create only one standing connection to the public
endpoints, and connecting to a public stream more than once with the
same account credentials will cause the oldest connection to be
disconnected.
As an alternative you might use the user stream, which limits the incoming tweets to the account of the authenticated user.
Related
I'm writing a REST API in Sails.js and alongside the regular HTTP routes, I need the application to listen for notifications on a socket from Salesforce.
I have a controller with some logic but I don't know how to get it to subscribe to the socket on startup, so right now nothing is reaching it.
Controller:
pushTopicHandler: function(req, res) {
if (!req.isSocket) {
return res.badRequest();
}
var nforce = require('nforce');
var org = nforce.createConnection({
clientId: sails.config.client_id,
clientSecret: sails.config.client_secret,
redirectUri: sails.config.callback_url + '/oauth/_callback',
mode: 'multi',
environment: 'sandbox'
});
org.authenticate({ username: sails.config.sfUsername, password: sails.config.sfPassword }, function(err, oauth) {
if(err) return res.serverError(err);
var str = org.stream({ topic: sails.config.push_topic, oauth: oauth });
str.on('connect', function(){
console.log('Connected to pushtopic: ' + sails.config.push_topic);
});
str.on('error', function(error) {
console.log('Error received from pushtopic: ' + error);
});
str.on('data', function(data) {
console.log('Received the following from pushtopic ---');
console.log(data);
});
});
}
Sails has a bootstrap.js file which lets you write anything you want to run before the server lifts.
I was able to subscribe to the push topic in a small function before the cb() in that file and it works, the server starts the REST API normally and it's still listening for events.
Am trying to design a REST api which will throw an aggregated response from multiple apis.
Following is the NodeJS code am trying to execute -
Pseudo Code start
//endpoint to be called from a browser / REST client
router.get('/api/v1/getItems', (req, response, next) => {
var result = {} // hold the aggregated response from multiple apis
//internally fire another endpoint & add the response over to the var result
http.get(endpoint 1, function(resp){
add response to result})
http.get(endpoint 2, function(resp){
add response to result
})
return response.json(result);
}
Pseudo Code end
// endpoint to be called from the browser or REST Client.
router.get('/api/v1/getItems', (req, response, next) => {
var results = {};
// Nested Endpoint 1
var optionsgetmsg = {
host : 'host.domain.com', // tthe domain name
port : 9043,
path : '/services/itemdata', // the rest of the url
method : 'GET' // do GET
};
//child endpoint
var reqGet = http.request(optionsgetmsg, function(res) {
res.on('data', function(d) {
console.log("d "+ d); // child response
results.itemdata = d;
return response.send(results);
//process.stdout.write(d);
});
res.on('end', function(d){
})
});
reqGet.end();
reqGet.on('error', function(e) {
console.error(e);
});
});
The result in the above case should be the output 'd'. The output 'd' is the response from the child endpoint.
Actual result am getting is an empty object. {}
If you are sending JSON, you must set the headers correctly and the response:
//child endpoint
var reqGet = http.request(optionsgetmsg, function(res) {
res.on('data', function(d) {
res.setHeader('Content-Type', 'application/json');
var results = d;
response.send(JSON.stringify(results));
});
It is unclear as to what exactly you are asking for.
I am trying to process Wikipedia articles, and want to receive a list of all Wikipedia articles. In order to do this I am frequently sending http requests to the Wikipedia API, which allows you to receive 500 titles at time and also returns an apcontinue string, which, when used in the following request, returns title starting from that string.
In order to do this, I am using the agentkeepalive module:
var http = require('http');
var Agent = require('agentkeepalive');
var keepaliveAgent = new Agent({
keepAlive: true,
maxSockets: 5,
timeout: 5000,
keepAliveTimeout: 3000
});
To send an http request to Wikipedia, I use the following code:
function wikipediaApiCall(params, callback) {
var options = {
host: 'en.wikipedia.org',
path: '/w/api.php?' + createParamString(params),
method: 'GET',
agent: keepaliveAgent
};
var callbackFunc = function(response) {
var err;
var str = '';
if (('' + response.statusCode).match(/^5\d\d$/)) {
err = new Error('Server error');
}
//another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
response.on('error', function (e) {
err = new Error('Request error');
});
response.on('timeout', function () {
err = new Error('Timeout');
response.abort();
callback(err);
});
response.on('end', function () {
var obj = JSON.parse(str);
if (obj.warnings) {
err = new Error('Request error');
}
callback(err, obj);
});
}
var req = http.request(options, callbackFunc);
req.setTimeout(5000);
req.on('error', function(err) {
callback(err, null);
return;
});
req.on('timeout', function () {
err = new Error('Timeout');
response.abort();
callback(err);
});
req.on('finish', function(){
console.log('ended');
});
req.end();
}
However, after sending between 16 and 20 request, I am not getting any response, but my request also does not time out.
Any ideas why this is happening?
Update
The request I send to Wikipedia contains the following parameters:
var params = {
list: 'allpages',
aplimit: limit,
apfrom: from,
continue: cont,
// apfilterredir: 'nonredirects'
};
Interestingly, after leaving out the nonredirects setting, I was able to send and receive up to 330 requests, but no more than that.
Update 2
I was able to register a finished event. It appears to be fired for the request that is failing as well. I modified the code accordingly.
Perhaps you need a bot flag to have higher API limits. Maybe there are too many requests in parallel; WMF recommendation is to make requests serially in case of such big tasks. Also, you should use the maxlag parameter with low values, per WMF API Etiquette.
I'm trying to create a basic app in node.js that a) tracks a keyword in twitter and temporarily stores messages relating to that keyword, b) after enough messages have been accumulated, return it to the user. I'm using the ntwitter library.
I've a basic long polling system implemented on my client and server side, but I'm having some trouble on verification. The way I set it up currently, it verifies the user each time /api/streamfeed is called, so potentially every 30sec (since I have a 30s timeout schedule) before checking the stream. I'm thinking this will get me into trouble since I believe verification is rate-limited? Is there a way to check whether I'm verified without having to ping Twitter's API (perhaps store a boolean after the first attempt)?
Client side:
//upon receiving a response, poll again
function getStreamFeed() {
console.log('calling getStreamFeed');
$http.get('/api/streamfeed').success(function(data) {
console.log(data)
getStreamFeed();
});
};
setTimeout(getStreamFeed, 1000);
Server side:
app.get('/api/streamfeed', function(req, res) {
/*
...
polling code
...
*/
twit.verifyCredentials(function(err, data) {
if (err) res.send(404);
twit.stream('statuses/filter', {
track: 'justin bieber'
}, function(stream) {
stream.on('data', function(data) {
console.log(data.text)
messages.push(data.text);
});
})
});
});
I'd send the credentials back and resend them again... this could be a bool, or actual credentials to use. these aren't your private keys or anything, only the user's.
could also be sent in headers and cookies and properly hashed etc.
this just simply shows a pattern that should work.
client side:
function getStreamFeed(credentials) {
//upon receiving a response, poll again
console.log('calling getStreamFeed');
var url = '/api/streamfeed';
if (credentials) {
url += '&credentials=' + credentials;
}
$http
.get(url)
.success(function(data) {
console.log(data)
getStreamFeed(true);
});
};
setTimeout(getStreamFeed, 1000);
Server side:
app.get('/api/streamfeed', function(req, res) {
function twitStream () {
twit.stream('statuses/filter', {track: 'justin bieber'}, function(stream) {
stream.on('data', function(data) {
console.log(data.text)
messages.push(data.text);
});
}
}
var credentials = req.query.credentials;
if (credentials) {
twitStream()
}
twit.verifyCredentials(function(err, data) {
if (err) res.send(404);
twitStream()
});
});
I have a socket.io connection running with
Client
var socket = io.connect('http://localhost:8080');
socket.on('news', function (data) {
console.log(data);
socket.emit('my other event', { my: 'data' });
});
And server
var app = require('http').createServer(handler)
, io = require('socket.io').listen(app)
, fs = require('fs')
var url = 'pathtojson';
app.listen(8080);
function handler (req, res) {
fs.readFile(__dirname + '/index.html', function (err, data) {
if (err) {
res.writeHead(500);
return res.end('Error loading index.html');
}
res.writeHead(200);
res.end(data);
});
}
io.sockets.on('connection', function (socket) {
socket.emit('news', url);
socket.on('my other event', function (data) {
console.log(data, 'server');
});
});
This is just an example from socket.io. I want to send json data to the client whenever it is updated.
But where do I start ?
You need to have an event fire whenever the data you're interested in updating has changed, and then you need to have the client listen for that event and respond as desired.
You don't really give a context other than "send json data to the client whenever it is updated", so assuming that you are in whatever process is updating your JSON on the server:
if (req.url === '/endpoint') {
yourJSON.foo = 'bar';
// doing whatever you're interested in to the JSON
socket.emit('JSON changed', yourJSON);
//an event is defined by the first argument,
//a value is passed with it as the second
}
NB: Getting more fancy/thoughtful with this means changing your JSON in such a manner that the socket only emits in response to the data change (event, callback, etc). Explicating this pattern is perhaps outside the scope of the question.
Then on the client, you want to define a function to handle those changes:
socket.on('JSON changed', updateFunction);
//where updateFunction is a function you define
//that is expecting arguments that match the output
//from the connected socket.emit event
function updateFunction(newJSON) {
//do whatever with new JSON data
}
This is assuming there is some external endpoint being accessed to update JSON; having it come from a client over socket.io would simply involve defining another event, but this time having it emit from the client, and be listened to by the server.