Retrieve the last 3200 tweets of a specific user in node.js - node.js

Question

I am new to javascript and node.js and this is my first post, so please bear with me.
I am using ntwitter to get all previous tweets of a specific user.
My problem is that if the user has more than 200 tweets, I need to create a loop and I am not sure if I do it right.
This is the async function that gets the 200 latest tweets:
exports.getUserTimeline = function(user, callback) {
twit.getUserTimeline({ screen_name: user, count: 200 }, function(err, data) {
if (err) {
return callback(err);
}
callback(err, data);
});
}
I found a solution to do this using a recursive function, but it's quite ugly.. How can I improve it ?
exports.getUserHistory = function(user, callback) {
recursiveSearch(user, callback);
function recursiveSearch(user, callback, lastId, data) {
var data = data || []
, args = {screen_name: user, count: 200};
if(typeof lastId != "undefined") args.max_id = lastId;
twit.getUserTimeline(args, function(err, subdata) {
if (err) {
console.log('Twitter search failed!');
return callback(err);
}
if (data.length !== 0) subdata.shift();
data = data.concat(subdata);
var lastId = parseInt(data[data.length-1].id_str);
if (subdata.length !== 0) {
recursiveSearch(user, callback, lastId, data);
} else {
callback(err, data);
}
});
}
}
Thank's a lot!
Update: This is the improved (refactored) function suggested by hunterloftis with two modifications:
property max_id should not be specified on the first iteration
the case where the user exists but no tweets have been posted must be handled
code:
function getUserHistory(user, done) {
var data = [];
search();
function search(lastId) {
var args = {
screen_name: user,
count: 200,
include_rts: 1
};
if(lastId) args.max_id = lastId;
twit.getUserTimeline(args, onTimeline);
function onTimeline(err, chunk) {
if (err) {
console.log('Twitter search failed!');
return done(err);
}
if (!chunk.length) {
console.log('User has not tweeted yet');
return done(err);
}
//Get rid of the first element of each iteration (not the first time)
if (data.length) chunk.shift();
data = data.concat(chunk);
var thisId = parseInt(data[data.length - 1].id_str);
if (chunk.length) return search(thisId);
console.log(data.length + ' tweets imported');
return done(undefined, data);
}
}
}
When retrieving tweets I noticed that my tweet count wasn't always the same as the 'statuses_count' property of the user. It took me some time to figure out that this difference corresponds to the number of deleted tweets :)

Answer 1

Does your recursive function work? Doesn't look too bad to me. I might refactor it just a little into something more like this:
function getUserHistory(user, done) {
var data = [];
search();
function search(lastId) {
var args = {
screen_name: user,
count: 200,
max_id: lastId
};
twit.getUserTimeline(args, onTimeline);
function onTimeline(err, chunk) {
if (err) {
console.log('Twitter search failed!');
return done(err);
}
if (data.length) chunk.shift(); // What is this for?
data = data.concat(chunk);
var thisId = parseInt(data[data.length - 1].id_str);
if (chunk.length) return search(thisId);
return done(undefined, data);
}
}
}