How to count and do jointure like stuff on couchDB - couchdb

I'm currently trying to get the number of events for one organizer.
This is what my organizer document looks like:
{
"doc_type": "User",
"email": "xxx#gmail.com",
"blebleble: "blebleble",
}
This is what my event document looks like:
{
"doc_type": "Event",
"email": "xxx#gmail.com",
"blablabla: "blablabla",
}
I still couldn't figure out how to do some kind of jointure between both docs and do a count on the number of event that shares the same. I think I can work around the email that both docs shares but I don't know how I can do that. I'm still having trouble with CouchDB. Doesn't seems like a hard thing to do in SQL, but can't find out for nosql.
Thanks you in advance.

"jointure" is not not a term I've encountered in my field so I am left to guess what is meant is join.
Joins are possible with CouchDB views, but what I read from the requirement in the OP is to get counts of events by email. See CouchDB's Joins With Views documentation. For that, I don't see documents with an ancestral relation rather a one-to-many relation, i.e. user ==> events.
Consider this design document:
{
"_id": "_design/SO-68999682",
"views": {
"user_events": {
"map": `function (doc) {
if(doc.doc_type === 'Event') {
emit(doc.email);
}
}`,
"reduce": '_count'
}
}
The view's map function simply adds doc.email to the 'user_events' index when appropriate. Of particular interest the reduce function specifies the built-in reduce function _count.
Given such a view index one may apply the /db/_design/design-doc/_view/view-name endpoint to, for example,
View all events
{
reduce: false,
include_docs: true
}
Get a count of all events
{
reduce: true
}
Get a count of events for every email (summary)
{
reduce: true,
group_level: 1
}
Get a count of events for a specific email
{
reduce: true,
group_level: 1,
key: email
}
Get all events for a specific email
{
reduce: false,
include_docs: true,
key: email
}
The _count reduce built-in provides high performance. The snippet below demonstrates the above using the very handy and compatible PouchDB.
async function showAllEventDocs() {
let result = await db.query('SO-68999682/user_events', {
reduce: false,
include_docs: true
});
//show
gel('user_events_view').innerText = result.rows.map(row => [row.doc.email, row.doc.date].join('\t\t')).join('\n');
}
async function showEventCountTotal() {
let result = await db.query('SO-68999682/user_events', {
reduce: true
});
gel('event_count_total').innerText = result.rows[0].value;
}
async function showEventCountSummary() {
let result = await db.query('SO-68999682/user_events', {
reduce: true,
group_level: 1
});
//show key/value (email, count)
gel('event_count_summary').innerText = result.rows.map(row => [row.key, row.value].join('\t\t')).join('\n');
}
async function showUserEventCount(email, displayElement) {
let result = await db.query('SO-68999682/user_events', {
reduce: true,
group_level: 1,
key: email
});
//show value (count)
gel(displayElement).innerText = result.rows[0].value;
}
async function showUserEvents(email, displayElement) {
let result = await db.query('SO-68999682/user_events', {
reduce: false,
include_docs: true,
key: email
});
//show
gel(displayElement).innerText = result.rows.map(row => [row.doc.email, row.doc.date].join('\t\t')).join('\n');
}
function getDocsToInstall(count) {
const docs = [{
"doc_type": "User",
"email": "Jerry#gmail.com"
},
{
"doc_type": "User",
"email": "Bobby#gmail.com"
},
{
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Bobby#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Bobby#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
},
];
// design document
const ddoc = {
"_id": "_design/SO-68999682",
"views": {
"user_events": {
"map": `function (doc) {
if(doc.doc_type === 'Event') {
emit(doc.email);
}
}`,
"reduce": '_count'
}
}
};
docs.push(ddoc);
return docs;
}
const db = new PouchDB('SO-68999682', {
adapter: 'memory'
});
// install docs and show view in various forms.
(async() => {
await db.bulkDocs(getDocsToInstall(20));
await showAllEventDocs();
await showEventCountTotal();
await showEventCountSummary();
await showUserEventCount('Jerry#gmail.com', 'jerry_event_count');
await showUserEventCount('Bobby#gmail.com', 'bobby_event_count');
await showUserEvents('Jerry#gmail.com', 'jerry_events');
await showUserEvents('Bobby#gmail.com', 'bobby_events');
})();
const gel = id => document.getElementById(id);
function getDocDate() {
const today = new Date();
const day = Math.random() * 100 % today.getDay() + 1; // keep it basic
return new Date(today.getFullYear(), today.getMonth(), day)
}
.bold {
font-weight: bold
}
.plain {
font-weight: normal
}
<script src="https://cdn.jsdelivr.net/npm/pouchdb#7.1.1/dist/pouchdb.min.js"></script>
<script src="https://github.com/pouchdb/pouchdb/releases/download/7.1.1/pouchdb.memory.min.js"></script>
<pre>All user_events (entire view)</pre>
<pre id='user_events_view'></pre>
<hr/>
<pre>Total number of events: <span id='event_count_total'></span> events</pre>
<hr/>
<pre>Event count summary (user, count)</pre>
<pre id='event_count_summary'></pre>
<hr/>
<pre>Event count by email (specific to user)</pre>
<pre>Bobby#gmail.com has <span id='bobby_event_count'></span> events</pre>
<pre>Jerry#gmail.com has <span id='jerry_event_count'></span> events</pre>
<hr/>
<pre>Events by email</pre>
<pre class="bold">Bobby#gmail.com <pre class="plain" id='bobby_events'></pre></pre>
<pre class="bold">Jerry#gmail.com <pre class="plain" id='jerry_events'></pre></pre>
<hr/>
Notice the demo snippet's documents have a date field. If such a field existed in the OPs Event documents, then changing the emit to
emit(doc.email + '/' + doc.date);
would allow all the aforementioned queries plus the option to query by a date or date range, an exercise which I'll leave readers to explore.

Related

CouchDB count all documents and return a number?

We have a college project in CouchDB and I'm using node, I want to create a view that returns a number of all my documents by email.
I cannot find anything that works and I'm not sure what I'm missing, I tried a lot of different reduce functions and emit methods.
Thanks for any answers.
The documents have 2 fields, name and email
Do not use the db endpoint because the response field doc_count includes design documents along with other documents that may not have an email field.
A straight forward way to do this is with a view. The code snippet demonstrates the difference between db info doc_count and a view's total_rows using PouchDB. I'd guess there's probably more interesting uses for the index.
The design doc is trivial
{
_id: '_design/my_index',
views: {
email: {
map: function(doc) {
if (doc.email) emit(doc.email);
}.toString()
}
}
}
And the view query is very efficient and simple.
db.query('my_index/email', {
include_docs: false,
limit: 0
})
const gel = id => document.getElementById(id);
let db;
function setJsonToText(elId, json) {
gel(elId).innerText = JSON.stringify(json, undefined, 3);
}
async function view() {
// display db info
setJsonToText('info', await db.info());
// display total number or rows in the email index
const result = await db.query('my_index/email', {
include_docs: false,
limit: 0
});
setJsonToText('view', result);
}
// canned test documents
function getDocsToInstall() {
return [{
email: 'jerry#garcia.com',
},
{
email: 'bob#weir.com',
},
{
email: 'phil#lesh.com'
},
{
email: 'wavy#gravy.com'
},
{
email: 'samson#delilah.com'
},
{
email: 'cosmic#charlie.com'
},
// design doc
{
_id: '_design/my_index',
views: {
email: {
map: function(doc) {
if (doc.email) emit(doc.email);
}.toString()
}
}
}
]
}
// init example db instance
async function initDb() {
db = new PouchDB('test', {
adapter: 'memory'
});
await db.bulkDocs(getDocsToInstall());
};
(async() => {
await initDb();
await view();
})();
<script src="https://github.com/pouchdb/pouchdb/releases/download/7.1.1/pouchdb-7.1.1.min.js"></script>
<script src="https://github.com/pouchdb/pouchdb/releases/download/7.1.1/pouchdb.memory.min.js"></script>
<pre>Info</pre>
<pre id='info'></pre>
<div style='margin-top:2em'></div>
<pre>email view</pre>
<pre id='view'>
</pre>
You can use GET /{db}, which returns information about the specified database. This is a JSON object that contains the property doc_count.
doc_count (number) – A count of the documents in the specified database.
With Angular for example, this could be done with the following method:
async countDocuments(database: string): Promise<number> {
return this.http.get<any>(this.url('GET', database), this.httpOptions).toPromise()
.then(info => info['doc_count']);
}
Assumption:
Assuming that following documents are present in the Customers database:
[
{
"_id": "93512c6c8585ab360dc7f535ff00bdfa",
"_rev": "1-299289ee89275a8618cd9470733035f4",
"name": "Tom",
"email": "tom#domain.com"
},
{
"_id": "93512c6c8585ab360dc7f535ff00c930",
"_rev": "1-a676883d6f1b5bce3b0a9ece92da6964",
"name": "Tom Doe",
"email": "tom#domain.com"
},
{
"_id": "93512c6c8585ab360dc7f535ff00edc0",
"_rev": "1-09b5bf64cfe66af7e1134448e1a328c3",
"name": "John",
"email": "john#domain.com"
},
{
"_id": "93512c6c8585ab360dc7f535ff010988",
"_rev": "1-88e347af11cfd1e40e63920fa5806fd2",
"name": "Alan",
"email": "alan#domain.com"
}
]
If I understand your query correctly, then based on above data, You need below given result set.
{
"tom#domain.com": 2,
"alan#domain.com": 1,
"john#domain.com": 1
}
Solution:
In order to achieve above, Consider following design document containing a View which has Map and Reduce functions.
{
"_id": "_design/Customers",
"views": {
"by-email": {
"map": "function (doc) {
if(doc.email){
emit(doc.email, doc._id);
}
}",
"reduce": "_count"
}
},
"language": "javascript"
}
The above view function emits value of the key email of the document if the key exists in the document.
The reduce function _count is a built in reducer (provided by CouchDB) that does the counting logic.
Executing View Query:
In order to query this view, you need to: select the view function, mark reduce to be executed (as it is optional to run reduce) and set 1 as group level.
Here is how you can do it through the UI:
Result:
Here is the result given by above query:
[![result of map reduce query
Hope this helped.
For more details about other reduce functions and group level, please refer CouchDB documentation.
Cheers.

Firebase database collection returns empty array when trying to get all documents

I'm trying to get all documents from my database collection "posts" but I'm getting an empty array instead.
The strange thing is that I'm able to get all documents from another collection called "users" that has the same structure and using the exact same code.
I've spent days looking for an answer but I haven't been able to find the solution.
This is the request:
const db = admin.firestore();
exports.getAllPosts = (req, res) => {
db.collection('posts')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let posts = [];
snapshot.forEach((doc) => {
posts.push({
id: doc.id,
body: doc.data().body,
author: doc.data().author,
createdAt: doc.data().timestamp,
voteScore: doc.data().voteScore
});
});
return res.json(posts);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And this is the response:
[]
This is what my current collection looks like:
Posts collection screenshot
This the response that I get when I return "snapshot":
{
"_query": {
"_firestore": {
"_settings": {
"projectId": "readable-bf7a6",
"firebaseVersion": "9.6.0",
"libName": "gccl",
"libVersion": "4.10.0 fire/9.6.0"
},
"_settingsFrozen": true,
"_serializer": {
"allowUndefined": false
},
"_projectId": "readable-bf7a6",
"registeredListenersCount": 0,
"bulkWritersCount": 0,
"_backoffSettings": {
"initialDelayMs": 100,
"maxDelayMs": 60000,
"backoffFactor": 1.3
},
"_clientPool": {
"concurrentOperationLimit": 100,
"maxIdleClients": 1,
"activeClients": {},
"failedClients": {},
"terminated": false,
"terminateDeferred": {
"promise": {}
}
}
},
"_queryOptions": {
"parentPath": {
"segments": []
},
"collectionId": "posts",
"converter": {},
"allDescendants": false,
"fieldFilters": [],
"fieldOrders": [
{
"field": {
"segments": [
"createdAt"
]
},
"direction": "DESCENDING"
}
],
"kindless": false
},
"_serializer": {
"allowUndefined": false
},
"_allowUndefined": false
},
"_readTime": {
"_seconds": 1622395245,
"_nanoseconds": 513743000
},
"_size": 0,
"_materializedDocs": null,
"_materializedChanges": null
}
Notice how the request for the collection "users" works successfully:
const db = admin.firestore();
exports.getAllUsers = (req, res) => {
db.collection('users')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let users = [];
snapshot.forEach((doc) => {
let users = [];
snapshot.forEach((doc) => {
users.push({
id: doc.data().userId,
email: doc.data().email,
handle: doc.data().handle
});
});
return res.json(users);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And the response:
[
{
"id": "EPoHBxhQFUXbcL3TCVx1LdUG2nO2",
"email": "ruben#gmail.com"
},
{
"id": "RqEa3dEq8TSDcZYeolXafju67rB2",
"email": "user10#gmail.com"
},
{
"id": "dxveb4n2iMQej5Q14uprsKRxFp23",
"email": "user4#gmail.com",
"handle": "user4"
},
{
"id": "YQPzBPcsqlVZk9iJEuZTHKUNuVG2",
"email": "user2#gmail.com",
"handle": "user2"
},
{
"id": "CZ05BJxi3TUOpIrmBaz539OWlbC3",
"email": "user#gmail.com",
"handle": "user"
},
{
"id": "t0t83BVwt4gVgJkDv7HL1r1MaKr1",
"email": "userJose2#gmail.com",
"handle": "Jose"
}
]
This is what the users collection looks like in Firebase:
Users collection screenshot
Why is one collection failing when the other works fine and I'm using the same code? What am I missing here?
Thanks in advance and I hope I've made it as clear as possible. Please let me know if you need me to provide anything else.
Very simple my friend, your posts documents can't be ordered like this:
.orderBy('createdAt', 'desc')
Because the post documents does not have the createdAt property, but they have a timestamp property, you should use that property to order your posts like this:
.orderBy('timestamp', 'desc')
I hope that helps 👍
I am not answering directly to #Ruben Garcia Bri, but for future firebase developers who may run into the problem of getting empty documents, I also ran into the same problem, but I solved it by adding a field to the particular document I am trying to retrieve.
Sometimes the cause is because the documents you are trying to get have no field in them.
I mean that a document must have a field before the server can recognize it as an existing document.
So if you run into this problem, consider adding a field to that document before you can successfully retrieve it.

Get delta of nested array element in a document using change stream (Node.js)

I have a document that has an array field called telephone which can have multiple extension array objects that can have multiple objects in it. So its an array inside an array. I am listening to the db using changeStream. If I change telephone[0].extension[0].valueBoolean = true where telephone[0].extension[0].url == "https//google.com",
I get the whole telephone array back in change.updateDescription.updatedFields NOT just telephone[0].extension[0]
updatedFields
{
"telephone": [{
"use": "offline",
"extension": [
{
"url": "https//gmail.com",
"valueDateTime": "2021-01-12T06:31:48.000Z"
}, {
"url": "https//yahoo.com",
"valueDateTime": "1700-01-01T00:00:00.000Z"
}, {
"url": "https//google.com",
"TimeLastModified": "2021-02-23T11:06:06.000Z",
"valueBoolean": false
}],
"value": "+123456789",
"system": "phone"
}, {
"use": "internet",
"extension": [
{
"url": "https//gmail.com",
"valueDateTime": "2021-01-12T06:31:48.000Z"
}, {
"url": "https//yahoo.com",
"valueDateTime": "1700-01-01T00:00:00.000Z"
}, {
"url": "https//google.com",
"TimeLastModified": "2021-02-23T11:06:06.000Z",
"valueBoolean": false
}],
"value": "+123456799",
"system": "phone"
}]
}
Here's what i have so far
MongoClient.connect(CONNECTION_STRING, {
useUnifiedTopology: true,
})
.then((client) => {
console.log("Connected successfully to server");
dbConnected = true;
}
// specify db and collections
const db = client.db(DB_NAME);
const myCollection = db.collection(COLLECTION_NAME);
const options = { fullDocument: "updateLookup" };
const changeStream = myCollection.watch(options);
// start listening to changes
changeStream.on("change", async (change) => {
// console.log("CHANGE!");
// console.log(JSON.stringify(change));
// check operationType
try {
if (
change.operationType == "insert" ||
change.operationType == "update" ||
change.operationType == "replace"
) {
const updatedFields = change.updateDescription.updatedFields
console.log("updatedFields", JSON.stringify(change.updateDescription.updatedFields));
}
} catch (e) {
console.log(e);
}
});
})
.catch((e) => {
console.log(`Error: ${e}`);
});
How do I see what exact element in a nested array changed with changeStream ?
Unfortunately it seems that this is currently not supported - there's an open Jira-ticket that is related to your problem, see https://jira.mongodb.org/browse/SERVER-41559 for further details.

Node mongo count query items

I have a simple GET search. Problem is that when I use some criteria like status or name or email it shows correct results BUT pagination behave like there were no criteria.
For example, there is one user in DB with name John so I search for John and it gives me exactly him as a result. But pagination shows me 10pages and I don't know how to fix it. When I parse count it gives me all results not only one,
Here is my controller
/**
* GET /admin/users/:page
* Find users
*/
exports.findUsers = (req, res, next) => {
const perPage = 13
const page = Number(req.params.page) || 1
var query = {};
const possibleQueryProps = [{
"key": "id",
"queryKey": "_id"
},
{
"key": "email",
"queryKey": "email"
},
{
"key": "firstname",
"queryKey": "profile.firstname"
},
{
"key": "lastname",
"queryKey": "profile.lastname"
},
{
"key": "location",
"queryKey": "profile.location"
},
{
"key": "status",
"queryKey": "profile.status"
}
];
possibleQueryProps.forEach(prop => {
if (req.query[prop.key]) {
query[prop.queryKey] = req.query[prop.key];
}
});
console.log(query)
User
.find(query)
.skip((perPage * page) - perPage)
.limit(perPage)
.exec(function (err, users) {
User.countDocuments().exec(function (err, count) {
if (err) return next(err)
res.render('admin/users', {
layout: "admin",
users: users,
query: req.query,
active: {
users: true
},
current: page,
pages: Math.ceil(count / perPage),
helpers,
pagesArr: Array.from(Array(((Math.ceil(count / perPage)))).keys()).map(i => 1 + i),
pagination: core.pagination(page, (Math.ceil(count / perPage))),
})
})
})
};
I will be thankful for any advice.
But pagination shows me 10pages and I don't know how to fix it. When I parse count it gives me all results not only one,
You need to count only documents that match your search criteria, so rather than User.countDocuments() you'll want something like User.countDocuments(query).

Find when the keys are unkown in mongodb

How can I getthe data that has email as abc#gmail.com in mongoDB?I don't know the Key Name and I want to iterate through all the data.
I have data like this:
{
"_id":"5c0a1589a5a41b2ae707317b",
"test1":{
"email":"abc#gmail.com",
"phoneNo":"123456897",
"endpointId":"test1"
}
}
{
"_id":"5c0a1989a5a41b2ae807317b",
"test2":{
"email":"abc#gmail.com",
"phoneNo":"123456897",
"endpointId":"test2"
}
}
{
"_id":"5c0a1989a5a41b2ae807317b",
"test2":{
"email":"pqr#gmail.com",
"phoneNo":"123456897",
"endpointId":"test3"
}
}
But the object key is not known at the time of searching. I want to iterate through all the data and get matched data that has specific email.
If I know the key name like test1,test2 etc then I can use find({test1:{...}}) but Here I don't know the key value.
So, how can I do that?
You can use below aggregation using $objectToArray in mongodb 3.4 and above
db.collection.aggregate([
{ "$addFields": {
"field": { "$objectToArray": "$$ROOT" }
}},
{ "$match": { "field.v.email": "abc#gmail.com" }},
{ "$project": { "field": 0 }}
])
I am assuming you get the objects in array type.
I made a method named findObject. This method will take the object array and the desired email.
Finally, return the first object, that matched with the email.
const data = [{
"_id":"5c0a1589a5a41b2ae707317b",
"test1":{
"email": "abc#gmail.com",
"phoneNo": "123456897",
"endpointId":"test1"
}
},
{
"_id":"5c0a1989a5a41b2ae807317b",
"test2":{
"email": "abc#gmail.com",
"phoneNo": "123456897",
"endpointId":"test2"
}
},
{
"_id":"5c0a1989a5a41b2ae807317b",
"test2":{
"email": "pqr#gmail.com",
"phoneNo": "123456897",
"endpointId": "test3"
}
}];
const findObject = (data, email) => {
for (let index=0; index<data.length; index++) {
const currentData = data[index];
for (let property in currentData) {
if (property != '_id' && currentData[property].email == email) {
return currentData;
}
}
}
return null;
}
let desiredObject;
const desiredEmail = 'abc#gmail.com';
desiredObject = findObject(data, desiredEmail);
console.log(desiredObject);
And the output will be
{ _id: '5c0a1589a5a41b2ae707317b',
test1:
{ email: 'abc#gmail.com',
phoneNo: '123456897',
endpointId: 'test1' } }
I think you can't do query on totally unknown field! if you could change your schema see here for more info, also you could write script to migrate to a new DB with new schema:
// new doc instance
{
"_id":"5c0a1589a5a41b2ae707317b",
"obj": {
"name": "test1"
"email":"abc#gmail.com"
"phoneNo":"123456897",
"endpointId":"test1"
}
},
{
"_id":"5c0a1989a5a41b2ae807317b",
"obj": {
"name": "test2"
"email":"abc#gmail.com"
"phoneNo":"123456897",
"endpointId":"test2"
}
},
{
"_id":"5c0a1989a5a41b2ae807317b",
"obj": {
"name": "test3"
"email":"pqr#gmail.com"
"phoneNo":"123456897",
"endpointId":"test3"
}
}
otherwise, check this may works correctly. if all of them is not effective so make a query to get all of your data as an Array and use filter method on it:
Model.find({}, (err, docs) => {
const result = docs.filter((doc) => {
for (key in doc) {
if (doc[key].email === 'abc#gmail.com')
return doc;
}
});
console.log(result);
});

Resources