Aggregate out from Multiple Collections - node.js

Question

How I can output result of MongoDB aggregation into collection without replacing the collection from another aggregation output?
I need to get data only with $out: 'tempCollection', because I have 500mln documents, and getting pipeline stage limit
var q = [
{$match: query},
{$group: {_id: '$hash'}},
{$out: 'tempCollection'}
];
async.parallel([
function(callback) {
firstCollection.aggregate(q, callback);
},
function(callback) {
secondCollection.aggregate(q, callback);
},
...
], function() {
// I want to get all from tempCollection (with pagination) here
});

Answer 1

The bottom line here is that the $out option only ever "replaces" output on the target collection. So to do anything else you must work through a client connection rather than just outputting to the server.
Your best option here with mongoose is to step straight into the underlying driver and get access to the node stream interface as supported by the driver.
Trival example, but it shows the basic way to structure:
var async = require('async'),
mongoose = require('mongoose'),
Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/aggtest');
var testSchema = new Schema({},{ "_id": false, strict: false });
var ModelA = mongoose.model( 'ModelA', testSchema ),
ModelB = mongoose.model( 'ModelB', testSchema ),
ModelC = mongoose.model( 'ModelC', testSchema );
function processCursor(cursor,target,callback) {
cursor.on("end",callback);
cursor.on("error",callback);
cursor.on("data",function(data) {
cursor.pause();
target.update(
{ "_id": data._id },
{ "$setOnInsert": { "_id": data._id } },
{ "upsert": true },
function(err) {
if (err) callback(err);
cursor.resume();
}
);
});
}
async.series(
[
// Clean data
function(callback) {
async.each([ModelA,ModelB,ModelC],function(model,callback) {
model.remove({},callback);
},callback);
},
// Sample data
function(callback) {
async.each([ModelA,ModelB],function(model,callback) {
async.each([1,2,3],function(id,callback) {
model.create({ "_id": id },callback);
},callback);
},callback);
},
// Run merge
function(callback) {
async.parallel(
[
function(callback) {
var cursor = ModelA.collection.aggregate(
[
{ "$group": { "_id": "$_id" } }
],
{ "batchSize": 25 }
);
processCursor(cursor,ModelC,callback)
},
function(callback) {
var cursor = ModelB.collection.aggregate(
[
{ "$group": { "_id": "$_id" } }
],
{ "batchSize": 25 }
);
processCursor(cursor,ModelC,callback)
}
],
callback
);
},
// Get merged
function(callback) {
ModelC.find({},function(err,results) {
console.log(results);
callback(err);
});
}
],
function(err) {
if (err) throw err;
mongoose.disconnect();
}
);
Oustide of that, then you are going to need to $out to "separate" collections, and then merge them in with a similar .update() process, but to keep it "server side" then you need to use .eval().
It's not nice, but that is the only way to keep operations on the server. You can also modify this with "Bulk" operations ( again through the same native .collection interface ) for a bit more throughput. But the options come down to "read through the client" or "eval".