Q promises and mongo db q.all - node.js

Question

I'm making calls to a mongodb database - pulling data out... reading it, and then making further requests based on that data. Once all the data has been received, I wish to process it.
I've been using Q.promises library but don't know what I'm doing. I thought that q.all would only trigger once everything has completed? However, my processPlaylist function runs twice. I've commented the code below:
Thanks,
Rob
var PlaylistCollection = require('./models/playlist');
var AssetCollection = require('./models/asset');
var screenID = '############';
var playerData = [];
// array continaing playlistys which have been synced
var alreadySynced = [];
// Process our playlist once downloaded
var processPlaylist = function (playerData) {
console.log('----Processing Playerlist-----')
console.log(playerData);
// DO STUFF
}
// Get playlist by id. Return playlist Data
var getSubLists = function (id) {
return PlaylistCollection.findById(id);
}
// Get sub-playlist function
function getSubListRecursive(id) {
return getSubLists(id).then(function (playlist) {
// store all our returned playlist data into a playlist array
playerData.push(playlist)
// an Array to keep tabs on what we've already pulled down
alreadySynced.push(playlist.id)
// get all our playlist.resources, and only return those which are unique
var playlistResources = _.uniq(playlist.resources, 'rid');
// console.log('Playlist Resources: ', playlistResources)
// console.log(alreadySynced);
var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
// remove playlists which have already been synced. We don't want to pull them down twice
sublists = _.difference(sublists, alreadySynced);
// console.log('sublists: ', sublists)
// Get the next playlist and so on...
var dbops = sublists.map(function (sublist) {
// console.log(sublist)
return getSubListRecursive(sublist)
});
q.all(dbops).then(function () {
console.log('All Done - so process the playlist')
return processPlaylist(playerData);
});
})
}
// Trigger the whole process..... grab our first playlist / ScreenID
getSubListRecursive(screenID);
and I get the following output:
----Processing Playerlist-----
[ { _id: 554d1df16ce4c438f8e2225b,
title: 'list 1',
__v: 29,
daily: true,
endTime: '',
startTime: '',
resources:
[ { rid: '55650cebef204ab70302a4d9',
title: 'list 4',
type: 'playlist' },
{ rid: '554d1df16ce4c438f8e2225b',
title: 'list 1',
type: 'playlist' } ] },
{ _id: 55650cebef204ab70302a4d9,
title: 'list 4',
__v: 1,
daily: false,
endTime: '',
startTime: '',
resources:
[ { rid: '55650647ef204ab70302a4d8',
title: 'list 3',
type: 'playlist' } ] } ]
All Done - so process the playlist
----Processing Playerlist-----
[ { _id: 554d1df16ce4c438f8e2225b,
title: 'list 1',
__v: 29,
daily: true,
endTime: '',
startTime: '',
resources:
[ { rid: '55650cebef204ab70302a4d9',
title: 'list 4',
type: 'playlist' },
{ rid: '554d1df16ce4c438f8e2225b',
title: 'list 1',
type: 'playlist' } ] },
{ _id: 55650cebef204ab70302a4d9,
title: 'list 4',
__v: 1,
daily: false,
endTime: '',
startTime: '',
resources:
[ { rid: '55650647ef204ab70302a4d8',
title: 'list 3',
type: 'playlist' } ] },
{ _id: 55650647ef204ab70302a4d8,
title: 'list 3',
__v: 5,
daily: false,
endTime: '',
startTime: '',
resources:
[ { rid: '55650637ef204ab70302a4d7',
title: 'list 2',
type: 'playlist' },
{ rid: '554d1df16ce4c438f8e2225b',
title: 'list 1',
type: 'playlist' },
{ rid: '55650cebef204ab70302a4d9',
title: 'list 4',
type: 'playlist' } ] } ]
EDIT
There were a number of things wrong with what I wrote. I discussed it with a buddy of mine - who pointed out that getSubListRecursive is being invoked recursively several times so the q.all statement is being executed several times...
So I refactored...
// Get sub-playlist function
function getSubListRecursive(id) {
console.log(id)
return getSubLists(id).then(function (playlist) {
if (playlist) {
// store all our returned playlist data into a playlist array
playerData.push(playlist)
// an Array to keep tabs on what we've already pulled down
alreadySynced.push(playlist.id)
// get all our playlist.resources, and only return those which are unique
var playlistResources = _.uniq(playlist.resources, 'rid');
// console.log('Playlist Resources: ', playlistResources)
// console.log(alreadySynced);
var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
// remove playlists which have already been synced. We don't want to pull them down twice
sublists = _.difference(sublists, alreadySynced);
// console.log('sublists: ', sublists)
return sublists.map(function (sublist) {
// console.log(sublist)
if (sublists.length > 0) {
return getSubListRecursive(sublist)
} else {
return processPlaylist(playerData);
}
});
} else {
return processPlaylist(playerData);
}
});
}
this works. I'm basically using promises to control the flow here - which probably isn't the best way of doing it? I no longer use an all statement, and ultimately end up with array populated with all the playlist data - which I can manipulate in my processPlaylist function.
However, I've not marked the question as solved, as I'd really like to know how I can do this with Q.all (properly use promises)
Thanks,
Rob

Answer 1

I think you were just confused about when the entire process was finished. You need to wait until the entire recursive promise chain has resolved. I think you could use the original code with a slight change to where processPlaylist() is called:
var PlaylistCollection = require('./models/playlist');
var AssetCollection = require('./models/asset');
var screenID = '############';
var playerData = [];
// array continaing playlistys which have been synced
var alreadySynced = [];
// Process our playlist once downloaded
var processPlaylist = function (playerData) {
console.log('----Processing Playerlist-----')
console.log(playerData);
// DO STUFF
}
// Get playlist by id. Return playlist Data
var getSubLists = function (id) {
return PlaylistCollection.findById(id);
}
// Get sub-playlist function
function getSubListRecursive(id) {
return getSubLists(id).then(function (playlist) {
// store all our returned playlist data into a playlist array
playerData.push(playlist)
// an Array to keep tabs on what we've already pulled down
alreadySynced.push(playlist.id)
// get all our playlist.resources, and only return those which are unique
var playlistResources = _.uniq(playlist.resources, 'rid');
// console.log('Playlist Resources: ', playlistResources)
// console.log(alreadySynced);
var sublists = _.pluck(_.filter(playlistResources, { 'type': 'playlist' }), 'rid');
// remove playlists which have already been synced. We don't want to pull them down twice
sublists = _.difference(sublists, alreadySynced);
// console.log('sublists: ', sublists)
// Get the next playlist and so on...
var dbops = sublists.map(function (sublist) {
// console.log(sublist)
return getSubListRecursive(sublist)
});
return q.all(dbops);
});
}
// Trigger the whole process..... grab our first playlist / ScreenID
getSubListRecursive(screenID).then(function() {
console.log('All Done - so process the playlist')
return processPlaylist(playerData);
});