I need to fetch 1000 records from mongodb , how can i do this in node js?
i have tried using pagination by sending the limit to 1000 even thought it takes 25 seconds to return the response, so is there any other way to minimize the response time ?
exports.getAll = function(filters, sort, skip, limit, callback) {
let query = {
removed: false
};
let filter = {};
let options = {
sort: []
};
req.skip = 0;
options.limit = 10000
options.sort = [
['_id', 'desc']
];
collections.findItems(query, filter, options, (err, result) => {
if (err) {
logger.error('db error', err);
return callback('Something went wrong');
}
callback(null,result);
});
};
exports.getAll = function getAll(req, res, next) {
service.getAll(req.body.filters, req.body.sort, req.body.skip, req.body.limit, (err, data) => {
if (err) {
return next({
status: 400,
error: err
});
}
res.json(data);
});
};
I guess you are using mongoose find in findItems method. find() returns mongoose documents with quite lot of stuff in it.
What you can do is use find().lean().
With lean(), simple plain javascript object of data from mongodb will be returned. It will optimize the response time a lot.
But I also suggest you to read this for more query optimizations at mongodb level: https://docs.mongodb.com/manual/core/query-optimization/
Are you using any ORM ? Like sequelize ?
What's your structure of document?
Check your mongdb logs:
tail -f /var/log/mongodb/mongod.log
If your document is straight forward then you should use indexing.
If you have index then try to rebuild them.
Sorting works efficiently when you have applied indexing. Avoid sorting on server if you can,Sort data at client side.
Because database imposes a 32MB memory limit on sorting operations
Related
I am new to mongoDb, as I am trying to query from different collection and in order to do that, when I am fetching data from category collection I mean when I am running select * from collection it is throwing error, MongoError: pool destroyed.
As per my understanding it is because of some find({}) is creating a pool and that is being destroyed.
The code which I am using inside model is below,
const MongoClient = require('mongodb').MongoClient;
const dbConfig = require('../configurations/database.config.js');
export const getAllCategoriesApi = (req, res, next) => {
return new Promise((resolve, reject ) => {
let finalCategory = []
const client = new MongoClient(dbConfig.url, { useNewUrlParser: true });
client.connect(err => {
const collection = client.db(dbConfig.db).collection("categories");
debugger
if (err) throw err;
let query = { CAT_PARENT: { $eq: '0' } };
collection.find(query).toArray(function(err, data) {
if(err) return next(err);
finalCategory.push(data);
resolve(finalCategory);
// db.close();
});
client.close();
});
});
}
When my finding here is when I am using
let query = { CAT_PARENT: { $eq: '0' } };
collection.find(query).toArray(function(err, data) {})
When I am using find(query) it is returning data but with {} or $gte/gt it is throwing Pool error.
The code which I have written in controller is below,
import { getAllCategoriesListApi } from '../models/fetchAllCategory';
const redis = require("redis");
const client = redis.createClient(process.env.REDIS_PORT);
export const getAllCategoriesListData = (req, res, next, query) => {
// Try fetching the result from Redis first in case we have it cached
return client.get(`allstorescategory:${query}`, (err, result) => {
// If that key exist in Redis store
if (false) {
res.send(result)
} else {
// Key does not exist in Redis store
getAllCategoriesListApi(req, res, next).then( function ( data ) {
const responseJSON = data;
// Save the Wikipedia API response in Redis store
client.setex(`allstorescategory:${query}`, 3600, JSON.stringify({ source: 'Redis Cache', responseJSON }));
res.send(responseJSON)
}).catch(function (err) {
console.log(err)
})
}
});
}
Can any one tell me what mistake I am doing here. How I can fix pool issue.
Thanking you in advance.
I assume that toArray is asynchronous (i.e. it invokes the callback passed in as results become available, i.e. read from the network).
If this is true the client.close(); call is going to get executed prior to results having been read, hence likely yielding your error.
The close call needs to be done after you have finished iterating the results.
Separately from this, you should probably not be creating the client instance in the request handler like this. Client instances are expensive to create (they must talk to all of the servers in the deployment before they can actually perform queries) and generally should be created per running process rather than per request.
I am new to couchdb and trying to save some docs at the same time indexing them.
Currently what I do is the following:
app.post("/api/result-store/v1/results/:searchID", (req, res) => {
const dbName = "test_" + req.params.searchID;
database.checkAndCreateDatabase(dbName).then(
db => {
const docs = req.body.objects;
db.bulk({ docs }).then(
body => {
res.send({
writes: body.reduce((total, currentValue) => {
return total + (currentValue.ok ? 1 : 0);
}, 0)
});
},
err => {
res.send(err);
}
);
},
err => {
console.log(err);
}
);
});
So all I do above is saving but no indexing. Now if I query the database like:
{{url}}/api/result-store/v1/results/jmeter_test_db_size_90_k?q=*:*&limit=200&counts=["qid_name", "datasource"]
Then the indexing will start. But this is too late since indexing takes time and the customer needs to wait and wait until the indexing is done to get the result.
I am thinking to start indexing the data as soon as I insert the docs so saving and indexing at the same time or consequently. Is it possible at all? Any insight is appreciated
Since you're using nano, you can create the index by invoking db.createIndex just after creating the database (creates an index on database fields, as specified in CouchDB doc).
I need to fetch two different MongoDB collections (db.stats and db.tables ) for the same request req.
Now, in the code below, I am nesting the queries within the callback function.
router.post('/', (req, res) => {
let season = String(req.body.year);
let resultData, resultTable;
db.stats.findOne({Year: season}, function (err, data) {
if (data) {
resultData = getResult(data);
db.tables.findOne({Year: season}, function (err, data) {
if (data) {
resultTable = getTable(data);
res.render('index.html', {
data:{
result : resultData,
message: "Working"}
});
} else {
console.log("Error in Tables");
}
});
} else {
console.log("Error in Stats");
}
});
});
This code works, but there a few things that don't seem right. So my question is:
How do I avoid this nested structure? Because it not only looks ugly but also, while I am processing these requests the client side is unresponsive and that is bad.
What you have right now is known as the callback hell in JavaScript. This is where Promises comes in handy.
Here's what you can do:
router.post('/', (req, res) => {
let season = String(req.body.year);
var queries = [
db.stats.findOne({ Year: season }),
db.tables.findOne({ Year: season })
];
Promise.all(queries)
.then(results => {
if (!results[0]) {
console.log("Error in Stats");
return; // bad response. a better way is to return status 500 here
} else if (!results[1]) {
console.log("Error in Tables");
return; // bad response. a better way is to return status 500 here
}
let resultData = getResult(results[0]);
let resultTable = getTable(results[1]);
res.render('index.html', { data: {
result : resultData,
message: "Working"
} });
})
.catch(err => {
console.log("Error in getting queries", err);
// bad response. a better way is to return status 500 here
});
});
It looks like you are using Mongoose as your ODM to access your mongo database. When you don't pass in a function as the second parameter, the value returned by the function call (e.g. db.stats.findOne({ Year: season })) will be a Promise. We will put all of these unresolved Promises in an array and call Promise.all to resolve them. By using Promise.all, you are waiting until all of your database queries get executed before moving on to render your index.html view. In this case, the results of your database function calls will be stored in the results array in the order of your queries array.
Also, I would recommend doing something like res.status(500).send("A descriptive error message here") whenever there is an error on the server side in addition to the console.log calls.
The above will solve your nested structure problem, but latter problem will still be there (i.e. client side is unresponsive when processing these requests). In order to solve this, you need to first identify your bottleneck. What function calls are taking up most of the time? Since you are using findOne, I do not think that will be the bottleneck unless the connection between your server and the database has latency issues.
I am going to assume that the POST request is not done through AJAX since you have res.render in it, so this problem shouldn't be caused by any client-sided code. I suspect that either one of getResult or getTable (or both) is taking up quite a significant amount of time, considering the fact that it causes the client side to be unresponsive. What's the size of the data when you query your database? If the size of it is so huge that it takes a significant amount of time to process, I would recommend changing the way how the request is made. You can use AJAX on the front-end to make a POST request to the back-end, which will then return the response as a JSON object. That way, the page on the browser would not need to reload, and you'll get a better user experience.
mongodb driver return a promise if you dont send a callback so you can use async await
router.post('/', async(req, res) => {
let season = String(req.body.year);
let resultData, resultTable;
try {
const [data1,data2] = await Promise.all([
db.stats.findOne({Year: season}),
db.tables.findOne({Year: season})
]);
if (data1 && data2) {
resultData = getResult(data1);
resultTable = getTable(data2);
return res.render('index.html', {
data: {
result: resultData,
message: "Working"
}
});
}
res.send('error');
console.log("Error");
} catch (err) {
res.send('error');
console.log("Error");
}
});
I have a huge collection of documents in my DB and I'm wondering how can I run through all the documents and update them, each document with a different value.
The answer depends on the driver you're using. All MongoDB drivers I know have cursor.forEach() implemented one way or another.
Here are some examples:
node-mongodb-native
collection.find(query).forEach(function(doc) {
// handle
}, function(err) {
// done or error
});
mongojs
db.collection.find(query).forEach(function(err, doc) {
// handle
});
monk
collection.find(query, { stream: true })
.each(function(doc){
// handle doc
})
.error(function(err){
// handle error
})
.success(function(){
// final callback
});
mongoose
collection.find(query).stream()
.on('data', function(doc){
// handle doc
})
.on('error', function(err){
// handle error
})
.on('end', function(){
// final callback
});
Updating documents inside of .forEach callback
The only problem with updating documents inside of .forEach callback is that you have no idea when all documents are updated.
To solve this problem you should use some asynchronous control flow solution. Here are some options:
async
promises (when.js, bluebird)
Here is an example of using async, using its queue feature:
var q = async.queue(function (doc, callback) {
// code for your update
collection.update({
_id: doc._id
}, {
$set: {hi: 'there'}
}, {
w: 1
}, callback);
}, Infinity);
var cursor = collection.find(query);
cursor.each(function(err, doc) {
if (err) throw err;
if (doc) q.push(doc); // dispatching doc to async.queue
});
q.drain = function() {
if (cursor.isClosed()) {
console.log('all items have been processed');
db.close();
}
}
Using the mongodb driver, and modern NodeJS with async/await, a good solution is to use next():
const collection = db.collection('things')
const cursor = collection.find({
bla: 42 // find all things where bla is 42
});
let document;
while ((document = await cursor.next())) {
await collection.findOneAndUpdate({
_id: document._id
}, {
$set: {
blu: 43
}
});
}
This results in only one document at a time being required in memory, as opposed to e.g. the accepted answer, where many documents get sucked into memory, before processing of the documents starts. In cases of "huge collections" (as per the question) this may be important.
If documents are large, this can be improved further by using a projection, so that only those fields of documents that are required are fetched from the database.
var MongoClient = require('mongodb').MongoClient,
assert = require('assert');
MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {
assert.equal(err, null);
console.log("Successfully connected to MongoDB.");
var query = {
"category_code": "biotech"
};
db.collection('companies').find(query).toArray(function(err, docs) {
assert.equal(err, null);
assert.notEqual(docs.length, 0);
docs.forEach(function(doc) {
console.log(doc.name + " is a " + doc.category_code + " company.");
});
db.close();
});
});
Notice that the call .toArray is making the application to fetch the entire dataset.
var MongoClient = require('mongodb').MongoClient,
assert = require('assert');
MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {
assert.equal(err, null);
console.log("Successfully connected to MongoDB.");
var query = {
"category_code": "biotech"
};
var cursor = db.collection('companies').find(query);
function(doc) {
cursor.forEach(
console.log(doc.name + " is a " + doc.category_code + " company.");
},
function(err) {
assert.equal(err, null);
return db.close();
}
);
});
Notice that the cursor returned by the find() is assigned to var cursor. With this approach, instead of fetching all data in memory and consuming data at once, we're streaming the data to our application. find() can create a cursor immediately because it doesn't actually make a request to the database until we try to use some of the documents it will provide. The point of cursor is to describe our query. The 2nd parameter to cursor.forEach shows what to do when the driver gets exhausted or an error occurs.
In the initial version of the above code, it was toArray() which forced the database call. It meant we needed ALL the documents and wanted them to be in an array.
Also, MongoDB returns data in batch format. The image below shows, requests from cursors (from application) to MongoDB
forEach is better than toArray because we can process documents as they come in until we reach the end. Contrast it with toArray - where we wait for ALL the documents to be retrieved and the entire array is built. This means we're not getting any advantage from the fact that the driver and the database system are working together to batch results to your application. Batching is meant to provide efficiency in terms of memory overhead and the execution time. Take advantage of it, if you can in your application.
None of the previous answers mentions batching the updates. That makes them extremely slow 🐌 - tens or hundreds of times slower than a solution using bulkWrite.
Let's say you want to double the value of a field in each document. Here's how to do that fast 💨 and with fixed memory consumption:
// Double the value of the 'foo' field in all documents
let bulkWrites = [];
const bulkDocumentsSize = 100; // how many documents to write at once
let i = 0;
db.collection.find({ ... }).forEach(doc => {
i++;
// Update the document...
doc.foo = doc.foo * 2;
// Add the update to an array of bulk operations to execute later
bulkWrites.push({
replaceOne: {
filter: { _id: doc._id },
replacement: doc,
},
});
// Update the documents and log progress every `bulkDocumentsSize` documents
if (i % bulkDocumentsSize === 0) {
db.collection.bulkWrite(bulkWrites);
bulkWrites = [];
print(`Updated ${i} documents`);
}
});
// Flush the last <100 bulk writes
db.collection.bulkWrite(bulkWrites);
And here is an example of using a Mongoose cursor async with promises:
new Promise(function (resolve, reject) {
collection.find(query).cursor()
.on('data', function(doc) {
// ...
})
.on('error', reject)
.on('end', resolve);
})
.then(function () {
// ...
});
Reference:
Mongoose cursors
Streams and promises
Leonid's answer is great, but I want to reinforce the importance of using async/promises and to give a different solution with a promises example.
The simplest solution to this problem is to loop forEach document and call an update. Usually, you don't need close the db connection after each request, but if you do need to close the connection, be careful. You must just close it if you are sure that all updates have finished executing.
A common mistake here is to call db.close() after all updates are dispatched without knowing if they have completed. If you do that, you'll get errors.
Wrong implementation:
collection.find(query).each(function(err, doc) {
if (err) throw err;
if (doc) {
collection.update(query, update, function(err, updated) {
// handle
});
}
else {
db.close(); // if there is any pending update, it will throw an error there
}
});
However, as db.close() is also an async operation (its signature have a callback option) you may be lucky and this code can finish without errors. It may work only when you need to update just a few docs in a small collection (so, don't try).
Correct solution:
As a solution with async was already proposed by Leonid, below follows a solution using Q promises.
var Q = require('q');
var client = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
client.connect(url, function(err, db) {
if (err) throw err;
var promises = [];
var query = {}; // select all docs
var collection = db.collection('demo');
var cursor = collection.find(query);
// read all docs
cursor.each(function(err, doc) {
if (err) throw err;
if (doc) {
// create a promise to update the doc
var query = doc;
var update = { $set: {hi: 'there'} };
var promise =
Q.npost(collection, 'update', [query, update])
.then(function(updated){
console.log('Updated: ' + updated);
});
promises.push(promise);
} else {
// close the connection after executing all promises
Q.all(promises)
.then(function() {
if (cursor.isClosed()) {
console.log('all items have been processed');
db.close();
}
})
.fail(console.error);
}
});
});
The node-mongodb-native now supports a endCallback parameter to cursor.forEach as for one to handle the event AFTER the whole iteration, refer to the official document for details http://mongodb.github.io/node-mongodb-native/2.2/api/Cursor.html#forEach.
Also note that .each is deprecated in the nodejs native driver now.
You can now use (in an async function, of course):
for await (let doc of collection.find(query)) {
await updateDoc(doc);
}
// all done
which nicely serializes all updates.
let's assume that we have the below MongoDB data in place.
Database name: users
Collection name: jobs
===========================
Documents
{ "_id" : ObjectId("1"), "job" : "Security", "name" : "Jack", "age" : 35 }
{ "_id" : ObjectId("2"), "job" : "Development", "name" : "Tito" }
{ "_id" : ObjectId("3"), "job" : "Design", "name" : "Ben", "age" : 45}
{ "_id" : ObjectId("4"), "job" : "Programming", "name" : "John", "age" : 25 }
{ "_id" : ObjectId("5"), "job" : "IT", "name" : "ricko", "age" : 45 }
==========================
This code:
var MongoClient = require('mongodb').MongoClient;
var dbURL = 'mongodb://localhost/users';
MongoClient.connect(dbURL, (err, db) => {
if (err) {
throw err;
} else {
console.log('Connection successful');
var dataBase = db.db();
// loop forEach
dataBase.collection('jobs').find().forEach(function(myDoc){
console.log('There is a job called :'+ myDoc.job +'in Database')})
});
I looked for a solution with good performance and I end up creating a mix of what I found which I think works good:
/**
* This method will read the documents from the cursor in batches and invoke the callback
* for each batch in parallel.
* IT IS VERY RECOMMENDED TO CREATE THE CURSOR TO AN OPTION OF BATCH SIZE THAT WILL MATCH
* THE VALUE OF batchSize. This way the performance benefits are maxed out since
* the mongo instance will send into our process memory the same number of documents
* that we handle in concurrent each time, so no memory space is wasted
* and also the memory usage is limited.
*
* Example of usage:
* const cursor = await collection.aggregate([
{...}, ...],
{
cursor: {batchSize: BATCH_SIZE} // Limiting memory use
});
DbUtil.concurrentCursorBatchProcessing(cursor, BATCH_SIZE, async (doc) => ...)
* #param cursor - A cursor to batch process on.
* We can get this from our collection.js API by either using aggregateCursor/findCursor
* #param batchSize - The batch size, should match the batchSize of the cursor option.
* #param callback - Callback that should be async, will be called in parallel for each batch.
* #return {Promise<void>}
*/
static async concurrentCursorBatchProcessing(cursor, batchSize, callback) {
let doc;
const docsBatch = [];
while ((doc = await cursor.next())) {
docsBatch.push(doc);
if (docsBatch.length >= batchSize) {
await PromiseUtils.concurrentPromiseAll(docsBatch, async (currDoc) => {
return callback(currDoc);
});
// Emptying the batch array
docsBatch.splice(0, docsBatch.length);
}
}
// Checking if there is a last batch remaining since it was small than batchSize
if (docsBatch.length > 0) {
await PromiseUtils.concurrentPromiseAll(docsBatch, async (currDoc) => {
return callback(currDoc);
});
}
}
An example of usage for reading many big documents and updating them:
const cursor = await collection.aggregate([
{
...
}
], {
cursor: {batchSize: BATCH_SIZE}, // Limiting memory use
allowDiskUse: true
});
const bulkUpdates = [];
await DbUtil.concurrentCursorBatchProcessing(cursor, BATCH_SIZE, async (doc: any) => {
const update: any = {
updateOne: {
filter: {
...
},
update: {
...
}
}
};
bulkUpdates.push(update);
// Updating if we read too many docs to clear space in memory
await this.bulkWriteIfNeeded(bulkUpdates, collection);
});
// Making sure we updated everything
await this.bulkWriteIfNeeded(bulkUpdates, collection, true);
...
private async bulkWriteParametersIfNeeded(
bulkUpdates: any[], collection: any,
forceUpdate = false, flushBatchSize) {
if (bulkUpdates.length >= flushBatchSize || forceUpdate) {
// concurrentPromiseChunked is a method that loops over an array in a concurrent way using lodash.chunk and Promise.map
await PromiseUtils.concurrentPromiseChunked(bulkUpsertParameters, (upsertChunk: any) => {
return techniquesParametersCollection.bulkWrite(upsertChunk);
});
// Emptying the array
bulkUpsertParameters.splice(0, bulkUpsertParameters.length);
}
}
Does Mongoose v3.6+ support batch inserts now? I've searched for a few minutes but anything matching this query is a couple of years old and the answer was an unequivocal no.
Edit:
For future reference, the answer is to use Model.create(). create() accepts an array as its first argument, so you can pass your documents to be inserted as an array.
See Model.create() documentation
Model.create() vs Model.collection.insert(): a faster approach
Model.create() is a bad way to do inserts if you are dealing with a very large bulk. It will be very slow. In that case you should use Model.collection.insert, which performs much better. Depending on the size of the bulk, Model.create() will even crash! Tried with a million documents, no luck. Using Model.collection.insert it took just a few seconds.
Model.collection.insert(docs, options, callback)
docs is the array of documents to be inserted;
options is an optional configuration object - see the docs
callback(err, docs) will be called after all documents get saved or an error occurs. On success, docs is the array of persisted documents.
As Mongoose's author points out here, this method will bypass any validation procedures and access the Mongo driver directly. It's a trade-off you have to make since you're handling a large amount of data, otherwise you wouldn't be able to insert it to your database at all (remember we're talking hundreds of thousands of documents here).
A simple example
var Potato = mongoose.model('Potato', PotatoSchema);
var potatoBag = [/* a humongous amount of potato objects */];
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Update 2019-06-22: although insert() can still be used just fine, it's been deprecated in favor of insertMany(). The parameters are exactly the same, so you can just use it as a drop-in replacement and everything should work just fine (well, the return value is a bit different, but you're probably not using it anyway).
Reference
Mongo documentation
Aaron Heckman on Google Groups discussing bulk inserts
Mongoose 4.4.0 now supports bulk insert
Mongoose 4.4.0 introduces --true-- bulk insert with the model method .insertMany(). It is way faster than looping on .create() or providing it with an array.
Usage:
var rawDocuments = [/* ... */];
Book.insertMany(rawDocuments)
.then(function(mongooseDocuments) {
/* ... */
})
.catch(function(err) {
/* Error handling */
});
Or
Book.insertMany(rawDocuments, function (err, mongooseDocuments) { /* Your callback function... */ });
You can track it on:
https://github.com/Automattic/mongoose/issues/723
https://github.com/Automattic/mongoose/blob/1887e72694829b62f4e3547283783cebbe66b46b/lib/model.js#L1774
Indeed, you can use the "create" method of Mongoose, it can contain an array of documents, see this example:
Candy.create({ candy: 'jelly bean' }, { candy: 'snickers' }, function (err, jellybean, snickers) {
});
The callback function contains the inserted documents.
You do not always know how many items has to be inserted (fixed argument length like above) so you can loop through them:
var insertedDocs = [];
for (var i=1; i<arguments.length; ++i) {
insertedDocs.push(arguments[i]);
}
Update: A better solution
A better solution would to use Candy.collection.insert() instead of Candy.create() - used in the example above - because it's faster (create() is calling Model.save() on each item so it's slower).
See the Mongo documentation for more information:
http://docs.mongodb.org/manual/reference/method/db.collection.insert/
(thanks to arcseldon for pointing this out)
Here are both way of saving data with insertMany and save
1) Mongoose save array of documents with insertMany in bulk
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const data = [/* array of object which data need to save in db */];
Potato.insertMany(data)
.then((result) => {
console.log("result ", result);
res.status(200).json({'success': 'new documents added!', 'data': result});
})
.catch(err => {
console.error("error ", err);
res.status(400).json({err});
});
})
2) Mongoose save array of documents with .save()
These documents will save parallel.
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const saveData = []
const data = [/* array of object which data need to save in db */];
data.map((i) => {
console.log(i)
var potato = new Potato(data[i])
potato.save()
.then((result) => {
console.log(result)
saveData.push(result)
if (saveData.length === data.length) {
res.status(200).json({'success': 'new documents added!', 'data': saveData});
}
})
.catch((err) => {
console.error(err)
res.status(500).json({err});
})
})
})
You can perform bulk insert using mongoose, as the highest score answer.
But the example cannot work, it should be:
/* a humongous amount of potatos */
var potatoBag = [{name:'potato1'}, {name:'potato2'}];
var Potato = mongoose.model('Potato', PotatoSchema);
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Don't use a schema instance for the bulk insert, you should use a plain map object.
It seems that using mongoose there is a limit of more than 1000 documents, when using
Potato.collection.insert(potatoBag, onInsert);
You can use:
var bulk = Model.collection.initializeOrderedBulkOp();
async.each(users, function (user, callback) {
bulk.insert(hash);
}, function (err) {
var bulkStart = Date.now();
bulk.execute(function(err, res){
if (err) console.log (" gameResult.js > err " , err);
console.log (" gameResult.js > BULK TIME " , Date.now() - bulkStart );
console.log (" gameResult.js > BULK INSERT " , res.nInserted)
});
});
But this is almost twice as fast when testing with 10000 documents:
function fastInsert(arrOfResults) {
var startTime = Date.now();
var count = 0;
var c = Math.round( arrOfResults.length / 990);
var fakeArr = [];
fakeArr.length = c;
var docsSaved = 0
async.each(fakeArr, function (item, callback) {
var sliced = arrOfResults.slice(count, count+999);
sliced.length)
count = count +999;
if(sliced.length != 0 ){
GameResultModel.collection.insert(sliced, function (err, docs) {
docsSaved += docs.ops.length
callback();
});
}else {
callback()
}
}, function (err) {
console.log (" gameResult.js > BULK INSERT AMOUNT: ", arrOfResults.length, "docsSaved " , docsSaved, " DIFF TIME:",Date.now() - startTime);
});
}
You can perform bulk insert using mongoDB shell using inserting the values in an array.
db.collection.insert([{values},{values},{values},{values}]);
Sharing working and relevant code from our project:
//documentsArray is the list of sampleCollection objects
sampleCollection.insertMany(documentsArray)
.then((res) => {
console.log("insert sampleCollection result ", res);
})
.catch(err => {
console.log("bulk insert sampleCollection error ", err);
});