Getting the latest documents from a Cloudant/CouchDB database from NodeJS - node.js

I would like to take the last n documents from my Cloudant database using a Node query. So far I have narrowed it down to the find() function, but the documentation only really explains how to retrieve all documents containing an absolute value, for example:
db.find({selector:{name:'Alice'}}, function(er, result) {
...
});
(taken from https://www.npmjs.com/package/cloudant#cloudant-query)
What I'm looking for is the equivalent of this SQL:
SELECT * FROM db WHERE name = "Alice" LIMIT 10
The code I have so far is this:
var cloudant = require('cloudant');
cloudant({account: username, password: password}, function (err, conn) {
if (err) {
callback("Could not initialize connection to Cloudant: " + err);
} else {
var db = conn.db.use('mydb');
db.find(???, function(err, data) {
if (err) {
callback("No data found: " + err);
} else {
...
}
});
}
});
If I need to make design documents, I'd do so in the Cloudant online interface, so don't worry too much about making an executable answer for that if it's necessary.

It's important to note that Cloudant Query requires you to define your index before performing the query e.g. to index the 'name' field from your documents:
db.index( {name:'nameindex', type:'json', index:{fields:['name']}}
We can now query the data using the find function as you indicated:
var query = { selector: { name: 'Alice' }};
db.find(query, function(err, data) {
});
The interesting thing about your question is the phrase 'the last n documents'. We can retrieve 'n' documents by adding a 'limit' to our query.
var query = { selector: { name: 'Alice' }, limit: 10};
db.find(query, function(err, data) {
});
but this doesn't necessarily indicate the last ten documents; it just limits the result set to ten.
If you want your query results to appear in time order, then you'll need something in your documents that indicates the time e.g.
a time string : { "name": "Alice", "datetime": "2015-11-26 10:22:24 Z" }
a timestamp : { "name": "Alice", "ts": "123456678" }
When your document contains a field which represents time, then your index can be created incorporate this into the index e.g.
db.index( {name:'nameindex', type:'json', index:{fields:['name','time']}}
and documents can be queried to appear in *reverse order * (to get the latest first):
var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"]};
db.find(query, function(err, data) {
});
You may also want to look at type:"text" indexes too. See https://docs.cloudant.com/cloudant_query.html

var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"]};
should be
var query = { selector: { name: 'Alice' }, sort: [ { name: "desc"}, { ts: "desc"}];

Related

Cannot read property 'columnName' of undefined in sails Js Mongo DB

I am using sails JS with Mongo DB.
My model is:
module.exports = {
attributes: {
title:{type:"string",required:true},
content:{type:"string",required:true},
date:{type:"string",required:true},
filename:{type:"string",required:true},
},
};
My Controller is:
fetchposts:function(req,res){
console.log("in fetch posts")
mysort={$id:-1}
Cyberblog.find().sort(mysort).limit(5).exec(function(err, result) {
if (err || !result) {
message="no records fetched";
console.log(message);
res.redirect('/newpost');
}
else{
console.log(result)
}
I am facing an error saying that
"Warning: The sort clause in the provided criteria is specified as a dictionary (plain JS object),
meaning that it is presumably using Mongo-Esque semantics (something like { fullName: -1, rank: 1 }).
But as of Sails v1/Waterline 0.13, this is no longer the recommended usage. Instead, please use either
a string like 'fullName DESC', or an array-like [ { fullName: 'DESC' } ].
(Since I get what you mean, tolerating & remapping this usage for now...)
and I am unable to fetch any records. It is showing no records fetched.
So I have one warning on Sort and no records coming from DB. Please help me resolve the issue.
Sort clause allow send string:
var users = await User.find({ name: 'Jake'})
.sort('age ASC');
return res.json(users);
Or an array:
var users = await User.find({ name: 'Finn'})
.sort([
{ age: 'ASC' },
{ createdAt: 'ASC' },
]);
return res.json(users);
Check this out in the documentation:
https://sailsjs.com/documentation/reference/waterline-orm/queries/sort

MongoDB - find one and add a new property

Background: Im developing an app that shows analytics for inventory management.
It gets an office EXCEL file uploaded, and as the file uploads the app convert it to an array of JSONs. Then, it comapers each json object with the objects in the DB, change its quantity according to the XLS file, and add a timestamp to the stamps array which contain the changes in qunatity.
For example:
{"_id":"5c3f531baf4fe3182cf4f1f2",
"sku":123456,
"product_name":"Example",
"product_cost":10,
"product_price":60,
"product_quantity":100,
"Warehouse":4,
"stamps":[]
}
after the XLS upload, lets say we sold 10 units, it should look like that:
{"_id":"5c3f531baf4fe3182cf4f1f2",
"sku":123456,
"product_name":"Example",
"product_cost":10,
"product_price":60,
"product_quantity":90,
"Warehouse":4,
"stamps":[{"1548147562": -10}]
}
Right now i cant find the right commands for mongoDB to do it, Im developing in Node.js and Angular, Would love to read some ideas.
for (let i = 0; i < products.length; i++) {
ProductsDatabase.findOneAndUpdate(
{"_id": products[i]['id']},
//CHANGE QUANTITY AND ADD A STAMP
...
}
You would need two operations here. The first will be to get an array of documents from the db that match the ones in the JSON array. From the list you compare the 'product_quantity' keys and if there is a change, create a new array of objects with the product id and change in quantity.
The second operation will be an update which uses this new array with the change in quantity for each matching product.
Armed with this new array of updated product properties, it would be ideal to use a bulk update for this as looping through the list and sending
each update request to the server can be computationally costly.
Consider using the bulkWrite method which is on the model. This accepts an array of write operations and executes each of them of which a typical update operation
for your use case would have the following structure
{ updateOne :
{
"filter" : <document>,
"update" : <document>,
"upsert" : <boolean>,
"collation": <document>,
"arrayFilters": [ <filterdocument1>, ... ]
}
}
So your operations would follow this pattern:
(async () => {
let bulkOperations = []
const ids = products.map(({ id }) => id)
const matchedProducts = await ProductDatabase.find({
'_id': { '$in': ids }
}).lean().exec()
for(let product in products) {
const [matchedProduct, ...rest] = matchedProducts.filter(p => p._id === product.id)
const { _id, product_quantity } = matchedProduct
const changeInQuantity = product.product_quantity - product_quantity
if (changeInQuantity !== 0) {
const stamps = { [(new Date()).getTime()] : changeInQuantity }
bulkOperations.push({
'updateOne': {
'filter': { _id },
'update': {
'$inc': { 'product_quantity': changeInQuantity },
'$push': { stamps }
}
}
})
}
}
const bulkResult = await ProductDatabase.bulkWrite(bulkOperations)
console.log(bulkResult)
})()
You can use mongoose's findOneAndUpdate to update the existing value of a document.
"use strict";
const ids = products.map(x => x._id);
let operations = products.map(xlProductData => {
return ProductsDatabase.find({
_id: {
$in: ids
}
}).then(products => {
return products.map(productData => {
return ProductsDatabase.findOneAndUpdate({
_id: xlProductData.id // or product._id
}, {
sku: xlProductData.sku,
product_name: xlProductData.product_name,
product_cost: xlProductData.product_cost,
product_price: xlProductData.product_price,
Warehouse: xlProductData.Warehouse,
product_quantity: productData.product_quantity - xlProductData.product_quantity,
$push: {
stamps: {
[new Date().getTime()]: -1 * xlProductData.product_quantity
}
},
updated_at: new Date()
}, {
upsert: false,
returnNewDocument: true
});
});
});
});
Promise.all(operations).then(() => {
console.log('All good');
}).catch(err => {
console.log('err ', err);
});

MongoDB Query Optimize Search Text

I have an application developed in NodeJS, which works as a REST API and consumes data from MongoDB
In MongoDB I have a collection called 'ftp' with more than 10 million documents with the following structure
{
"_id" : ObjectId("59e7c66911506bd1725cf145"),
"ip" : "72.32.177.76",
"timestamp" : "2017-10-16T02:30:26-04:00",
"data" : {
"banner" : "220-FileZilla Server version 0.9.41 beta\r\n"
}
}
The "data.banner" field is a hased index
From NoodeJs I make an aggregate query that filters a string of text using a regular expression, groups and counts the results.
function getData(timeInit, table, textSearch, res) {
MongoClient.connect(url, function (err, db) {
if (err) throw err;
db.collection(table).aggregate([
{
$match: { 'data.banner': $regex:textSearch}
}, {
$group: {
_id: '$data.banner',
num: { $sum: 1 },
}
},
{
$sort: {
num: -1
}
},{
$limit:5
}
], {
allowDiskUse: true
}
).toArray(function (err, result) {
if (err) throw err;
var timeFinal = new Date();
var data = {
result: result,
timeLapse: (timeFinal - timeInit) / 1000,
numResult: result.length
};
res.send(data);
db.close();
});
});
};
The query with regular expression takes about 8 seconds to return results, an excessive time in my opinion, since the regular expressions are not optimal.
My question is how should I make the filter to search for documents that contain text in an optimal way reducing the response time.
If someone knows how to optimize this type of query I would appreciate it a lot.

Is it possible to get count of number of docs returned from find() query in mongoose

I am trying to get count of data fetched from the database using find() query in mongoose. Now can anyone tell me can i do something like below or do i have to write other function to do that
merchantmodel.find({merchant_id: merchant_id, rating: {'$ne': -1 }, review: {'$ne': "" }}, {'review':1, '_id':0}, {sort: {time_at: -1}}, function(err, docs) {
if (err) {
} else {
if (docs) {
console.log(docs[1].review);
console.log(docs.size()); // Here by writing something is it possible to get count or not
res.json({success: 1, message : "Successfully Fetched the Reviews"});
}
}
});
Convert returned value to array and then use length property
var query = { merchant_id : merchant_id, rating : { '$ne': -1 }, review: { '$ne': "" }};
var projection = { 'review':1, '_id':0 };
var options = { sort: { time_at: -1 } };
merchantmodel.find(query, projection, options).toArray(function(err, docs) {
if (err) {
throw(err);
}
console.log(docs[1].review);
console.log(docs.length);
res.json({success: 1, message : "Successfully Fetched the Reviews"});
});
You can simply do this:
console.log(docs.length);
The docs variable returned by the find() method is an array so docs.length would do the job.
The mongodb native way to do this would be:
db.collection.find( { a: 5, b: 5 } ).count()

Order and limit results in a query with a callback

Using Mongoose, I'd like to make a query with MongoDB and order and limit the results I get. I am doing this with Node.js so I am using callbacks.
So far, I have managed to order my results like this:
myModel.find({ $query: {}, $orderby: { created_at : -1 }}, function (err, items) {
callback( null, items )
});
How can I limit the results I get selecting and index and the number of items I want to get?
Using mongodb native:
http://mongodb.github.io/node-mongodb-native/api-generated/collection.html#find
myModel.find(filter)
.limit(pageSize)
.skip(skip)
.sort(sort)
.toArray(callback);
You can also specify the items in your query:
myModel.find(filter, {sort: {created_at: -1}, limit: 10}, function(err, items){
});
There is no $orderby in node mongodb native, so I'm not sure what library or other tool you're using.
...
Now that you've clarified Mongoose (which in general I recommend against):
myModel.find(filter).limit(10).exec(function(err, items){
//process
});
To sort documents, we can apply sort on a cursor object. To enforce order of sort, instead of passing an object, we need to pass an array to the sort method.
var MongoClient = require('mongodb').MongoClient,
commandLineArgs = require('command-line-args'),
assert = require('assert');
var options = commandLineOptions();
MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {
assert.equal(err, null);
console.log("Successfully connected to MongoDB.");
var query = queryDocument(options);
var projection = {
"_id": 0,
"name": 1,
"founded_year": 1,
"number_of_employees": 1
};
var cursor = db.collection('companies').find(query);
cursor.project(projection);
cursor.limit(options.limit);
cursor.skip(options.skip);
cursor.sort([
["founded_year", 1],
["number_of_employees", -1]
]);
var numMatches = 0;
cursor.forEach(
function(doc) {
numMatches = numMatches + 1;
console.log(doc.name + "\n\tfounded " + doc.founded_year +
"\n\t" + doc.number_of_employees + " employees");
},
function(err) {
assert.equal(err, null);
console.log("Our query was:" + JSON.stringify(query));
console.log("Documents displayed: " + numMatches);
return db.close();
}
);
});
function queryDocument(options) {
console.log(options);
var query = {
"founded_year": {
"$gte": options.firstYear,
"$lte": options.lastYear
}
};
if ("employees" in options) {
query.number_of_employees = {
"$gte": options.employees
};
}
return query;
}
function commandLineOptions() {
var cli = commandLineArgs([{
name: "firstYear",
alias: "f",
type: Number
}, {
name: "lastYear",
alias: "l",
type: Number
}, {
name: "employees",
alias: "e",
type: Number
}, {
name: "skip",
type: Number,
defaultValue: 0
}, {
name: "limit",
type: Number,
defaultValue: 20000
}]);
var options = cli.parse()
if (!(("firstYear" in options) && ("lastYear" in options))) {
console.log(cli.getUsage({
title: "Usage",
description: "The first two options below are required. The rest are optional."
}));
process.exit();
}
return options;
}
One thing to notice is the order in which MongoDB applies skip, limit and sort
sort
skip
limit
There's also a possibility that we can sort data on the MongoDB side as well, provided that we've setup the indexing.
Notice that MongoDB driver will send a query when we call a cursor method passing a callback function to process query results.

Resources