I'm a bit puzzled by the situation I have now.
I've a simple SQL statement I execute from NodeJs on a SQLite database. The SQL statement returns values with a lot of decimals; although my data only contain two decimals.
When I run the exact same query in DB Browser for SQLite, I have a correct result.
My NodeJs code
app.get('/payerComparison/', (req, res) => {
// Returns labels and values within response
var response = {};
let db = new sqlite3.Database('./Spending.db', sqlite3.OPEN_READONLY, (err) => {
if(err){console.log(err.message); return}
});
response['labels'] = [];
response['data'] = [];
db.each("SELECT payer, sum(amount) AS sum FROM tickets GROUP BY payer", (err, row) => {
if(err){console.log(err.message); return}
response['labels'].push(row.payer);
response['data'].push(row.sum);
});
db.close((err) => {
if(err){console.log(err.message); return}
// Send data
console.log(response);
res.send(JSON.stringify(response));
});
})
What I have in the command line
{
labels: [ 'Aurélien', 'Commun', 'GFIS', 'Pauline' ],
data: [ 124128.26, 136426.43000000008, 5512.180000000001, 39666.93 ]
}
The result in DB Browser
I hope you can help me clarify this mystery!
Thank you
Round the values up to 2 decimals :).
SELECT payer, round(sum(amount),2) AS sum FROM tickets GROUP BY payer
To sort the documents inside Ads Collection I am using the below query which takes parameters from the URL and its working perfectly.
router.get("/", auth, async (req, res) => {
let query;
let queryStr = JSON.stringify(req.query);
queryStr = queryStr.replace(
/\b(gt|gte|lt|lte|in)\b/g,
(match) => `$${match}`
);
console.log(queryStr);
query = Ads.find(JSON.parse(queryStr));
const ads = await query;
res.status(200).json({ data: ads });
});
I am using the text operator in the Ads Collection for searching with the below route .
router.get("/find/:query", (req, res) => {
let query = req.params.query;
Ads.find(
{
$text: { $search: query },
},
function (err, result) {
if (err) throw err;
if (result) {
res.json(result);
} else {
res.send(
JSON.stringify({
error: "Error",
})
);
}
}
);
});
Both the routes are working perfectly but How can I merge the above two in one?
For e.g, I want to do a text search on the first route after getting a response, and similarly for the second route, after getting a response I want to apply the query parameters and get a response .
How can I merge the above two to get the desired output?
From what I can infer, are you looking for a pipeline where the parallel running of the two is possible:
Please have a look at MongoDB Aggregate which works as a pipeline. Here you can have two pipelines for the same input and have different output, or output of one can be transferred to next level to process.
Mongodb Aggregate - Facet Command Link
[1]:https://docs.mongodb.com/manual/reference/operator/aggregation/facet/#pipe._S_facet
Mongodb Aggregate links
[2]: https://docs.mongodb.com/manual/reference/operator/query/
I need to fetch 1000 records from mongodb , how can i do this in node js?
i have tried using pagination by sending the limit to 1000 even thought it takes 25 seconds to return the response, so is there any other way to minimize the response time ?
exports.getAll = function(filters, sort, skip, limit, callback) {
let query = {
removed: false
};
let filter = {};
let options = {
sort: []
};
req.skip = 0;
options.limit = 10000
options.sort = [
['_id', 'desc']
];
collections.findItems(query, filter, options, (err, result) => {
if (err) {
logger.error('db error', err);
return callback('Something went wrong');
}
callback(null,result);
});
};
exports.getAll = function getAll(req, res, next) {
service.getAll(req.body.filters, req.body.sort, req.body.skip, req.body.limit, (err, data) => {
if (err) {
return next({
status: 400,
error: err
});
}
res.json(data);
});
};
I guess you are using mongoose find in findItems method. find() returns mongoose documents with quite lot of stuff in it.
What you can do is use find().lean().
With lean(), simple plain javascript object of data from mongodb will be returned. It will optimize the response time a lot.
But I also suggest you to read this for more query optimizations at mongodb level: https://docs.mongodb.com/manual/core/query-optimization/
Are you using any ORM ? Like sequelize ?
What's your structure of document?
Check your mongdb logs:
tail -f /var/log/mongodb/mongod.log
If your document is straight forward then you should use indexing.
If you have index then try to rebuild them.
Sorting works efficiently when you have applied indexing. Avoid sorting on server if you can,Sort data at client side.
Because database imposes a 32MB memory limit on sorting operations
I'm trying to achieve a very simple thing:
START TRANSACTION;
DELETE FROM table WHERE id = 1;
ROLLBACK;
Running this on the postgres database works perfectly. With massive.js it doesn't:
this.db.run(
"START TRANSACTION",
[]
);
setTimeout(() => {
this.db.run(
"DELETE FROM table WHERE id = $1"
[1]
);
}, 2000);
setTimeout(() => {
this.db.run(
"ROLLBACK;"
[]
);
}, 4000);
It doesn't rollback the changes, just deletes from the database. COMMIT doesn't work as well. What's wrong?
Is there some way to dump queries order?
Massive uses pg-promise underneath, which in turn supports transactions:
db.instance.tx(t => {
// BEGIN has been executed
return t.none('DELETE FROM table WHERE id = $1', [123])
.then(() => {
// Records have been deleted within the transaction
throw new Error('Random error to fail the transaction');
// = the same as returning Promise.reject(new Error('Random...'))
});
})
.catch(error => {
// ROLLBACK has been executed, no records end up deleted.
console.log(error);
});
See also:
Accessing the Driver
Monitoring Queries
Is there some way to dump queries order?
Monitoring Queries shows you how to do it, or you can add event query handler within db.driverConfig object when initializing massive.
Does Mongoose v3.6+ support batch inserts now? I've searched for a few minutes but anything matching this query is a couple of years old and the answer was an unequivocal no.
Edit:
For future reference, the answer is to use Model.create(). create() accepts an array as its first argument, so you can pass your documents to be inserted as an array.
See Model.create() documentation
Model.create() vs Model.collection.insert(): a faster approach
Model.create() is a bad way to do inserts if you are dealing with a very large bulk. It will be very slow. In that case you should use Model.collection.insert, which performs much better. Depending on the size of the bulk, Model.create() will even crash! Tried with a million documents, no luck. Using Model.collection.insert it took just a few seconds.
Model.collection.insert(docs, options, callback)
docs is the array of documents to be inserted;
options is an optional configuration object - see the docs
callback(err, docs) will be called after all documents get saved or an error occurs. On success, docs is the array of persisted documents.
As Mongoose's author points out here, this method will bypass any validation procedures and access the Mongo driver directly. It's a trade-off you have to make since you're handling a large amount of data, otherwise you wouldn't be able to insert it to your database at all (remember we're talking hundreds of thousands of documents here).
A simple example
var Potato = mongoose.model('Potato', PotatoSchema);
var potatoBag = [/* a humongous amount of potato objects */];
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Update 2019-06-22: although insert() can still be used just fine, it's been deprecated in favor of insertMany(). The parameters are exactly the same, so you can just use it as a drop-in replacement and everything should work just fine (well, the return value is a bit different, but you're probably not using it anyway).
Reference
Mongo documentation
Aaron Heckman on Google Groups discussing bulk inserts
Mongoose 4.4.0 now supports bulk insert
Mongoose 4.4.0 introduces --true-- bulk insert with the model method .insertMany(). It is way faster than looping on .create() or providing it with an array.
Usage:
var rawDocuments = [/* ... */];
Book.insertMany(rawDocuments)
.then(function(mongooseDocuments) {
/* ... */
})
.catch(function(err) {
/* Error handling */
});
Or
Book.insertMany(rawDocuments, function (err, mongooseDocuments) { /* Your callback function... */ });
You can track it on:
https://github.com/Automattic/mongoose/issues/723
https://github.com/Automattic/mongoose/blob/1887e72694829b62f4e3547283783cebbe66b46b/lib/model.js#L1774
Indeed, you can use the "create" method of Mongoose, it can contain an array of documents, see this example:
Candy.create({ candy: 'jelly bean' }, { candy: 'snickers' }, function (err, jellybean, snickers) {
});
The callback function contains the inserted documents.
You do not always know how many items has to be inserted (fixed argument length like above) so you can loop through them:
var insertedDocs = [];
for (var i=1; i<arguments.length; ++i) {
insertedDocs.push(arguments[i]);
}
Update: A better solution
A better solution would to use Candy.collection.insert() instead of Candy.create() - used in the example above - because it's faster (create() is calling Model.save() on each item so it's slower).
See the Mongo documentation for more information:
http://docs.mongodb.org/manual/reference/method/db.collection.insert/
(thanks to arcseldon for pointing this out)
Here are both way of saving data with insertMany and save
1) Mongoose save array of documents with insertMany in bulk
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const data = [/* array of object which data need to save in db */];
Potato.insertMany(data)
.then((result) => {
console.log("result ", result);
res.status(200).json({'success': 'new documents added!', 'data': result});
})
.catch(err => {
console.error("error ", err);
res.status(400).json({err});
});
})
2) Mongoose save array of documents with .save()
These documents will save parallel.
/* write mongoose schema model and export this */
var Potato = mongoose.model('Potato', PotatoSchema);
/* write this api in routes directory */
router.post('/addDocuments', function (req, res) {
const saveData = []
const data = [/* array of object which data need to save in db */];
data.map((i) => {
console.log(i)
var potato = new Potato(data[i])
potato.save()
.then((result) => {
console.log(result)
saveData.push(result)
if (saveData.length === data.length) {
res.status(200).json({'success': 'new documents added!', 'data': saveData});
}
})
.catch((err) => {
console.error(err)
res.status(500).json({err});
})
})
})
You can perform bulk insert using mongoose, as the highest score answer.
But the example cannot work, it should be:
/* a humongous amount of potatos */
var potatoBag = [{name:'potato1'}, {name:'potato2'}];
var Potato = mongoose.model('Potato', PotatoSchema);
Potato.collection.insert(potatoBag, onInsert);
function onInsert(err, docs) {
if (err) {
// TODO: handle error
} else {
console.info('%d potatoes were successfully stored.', docs.length);
}
}
Don't use a schema instance for the bulk insert, you should use a plain map object.
It seems that using mongoose there is a limit of more than 1000 documents, when using
Potato.collection.insert(potatoBag, onInsert);
You can use:
var bulk = Model.collection.initializeOrderedBulkOp();
async.each(users, function (user, callback) {
bulk.insert(hash);
}, function (err) {
var bulkStart = Date.now();
bulk.execute(function(err, res){
if (err) console.log (" gameResult.js > err " , err);
console.log (" gameResult.js > BULK TIME " , Date.now() - bulkStart );
console.log (" gameResult.js > BULK INSERT " , res.nInserted)
});
});
But this is almost twice as fast when testing with 10000 documents:
function fastInsert(arrOfResults) {
var startTime = Date.now();
var count = 0;
var c = Math.round( arrOfResults.length / 990);
var fakeArr = [];
fakeArr.length = c;
var docsSaved = 0
async.each(fakeArr, function (item, callback) {
var sliced = arrOfResults.slice(count, count+999);
sliced.length)
count = count +999;
if(sliced.length != 0 ){
GameResultModel.collection.insert(sliced, function (err, docs) {
docsSaved += docs.ops.length
callback();
});
}else {
callback()
}
}, function (err) {
console.log (" gameResult.js > BULK INSERT AMOUNT: ", arrOfResults.length, "docsSaved " , docsSaved, " DIFF TIME:",Date.now() - startTime);
});
}
You can perform bulk insert using mongoDB shell using inserting the values in an array.
db.collection.insert([{values},{values},{values},{values}]);
Sharing working and relevant code from our project:
//documentsArray is the list of sampleCollection objects
sampleCollection.insertMany(documentsArray)
.then((res) => {
console.log("insert sampleCollection result ", res);
})
.catch(err => {
console.log("bulk insert sampleCollection error ", err);
});