MongoDB can't get all the rows - node.js

I've got a project using the MEAN stack and inside the Mongo DB collection I've got a collection called 'services' with the following rows (a small subset):
{ "_id" : ObjectId("55af611de2be6d817d000001"), "client" : "55a8e5fa586f94752a000002", "collaborators" : null, "date" : ISODate("2015-07-22T09:23:41.917Z"), "latestUpdateDate" : ISODate("2015-07-22T09:23:41.917Z"), "backofficeRequested" : false, "description" : "Uma descrição qq", "wasSeen" : 0, "currentStatusId" : "0", "lastStatusId" : 4, "__v" : 2, "addresses" : [ ], "dates" : [ ], "estimateDuration" : 61, "estimatePrice" : "55", "notes" : [ ], "serviceDate" : ISODate("2015-07-23T10:07:00Z"), "totalPrice" : "600", "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55afa8662c1e98c80d2fd2ff"), "client" : "55a8e5fa586f94752a000002", "estimatePrice" : "12", "estimateDuration" : NumberLong(720), "description" : "12", "serviceDate" : ISODate("1212-12-12T12:48:45Z"), "lastStatusId" : null, "currentStatusId" : "0", "collaborators" : [ { "id" : "55acb6022c1e98d05f2fd2ff" } ], "dates" : [ ], "totalPrice" : "", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55afa9f62c1e98370e2fd2ff"), "client" : "55ae68d82c1e98ac782fd303", "estimatePrice" : "12", "estimateDuration" : NumberLong(720), "description" : "12", "serviceDate" : ISODate("1212-12-12T12:48:45Z"), "lastStatusId" : null, "currentStatusId" : "1", "collaborators" : [ { "id" : "55acb6022c1e98d05f2fd2ff" } ], "dates" : [ ], "totalPrice" : "500", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55afaa7e2c1e98bc0d2fd2ff"), "client" : "55a8e5fa586f94752a000002", "estimatePrice" : "12", "estimateDuration" : NumberLong(720), "description" : "12", "serviceDate" : ISODate("1212-12-12T12:48:45Z"), "lastStatusId" : null, "currentStatusId" : "0", "collaborators" : [ { "id" : "55acb6022c1e98d05f2fd2ff" } ], "dates" : [ ], "totalPrice" : "", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55afaf582c1e98d80d2fd303"), "client" : "55ae68d82c1e98ac782fd303", "estimatePrice" : "1", "estimateDuration" : NumberLong(60), "description" : "1", "serviceDate" : ISODate("1111-11-11T11:47:45Z"), "lastStatusId" : null, "currentStatusId" : "4", "collaborators" : [ { "id" : "55ae40772c1e98007a2fd2ff" } ], "dates" : [ ], "totalPrice" : "", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55afb8472c1e98370e2fd300"), "client" : "55a8e5fa586f94752a000002", "estimatePrice" : "1", "estimateDuration" : NumberLong(60), "description" : "1", "serviceDate" : ISODate("1111-11-11T11:47:45Z"), "lastStatusId" : null, "currentStatusId" : NumberLong(2), "collaborators" : [ { "id" : "55ae40772c1e98007a2fd2ff" } ], "dates" : [ ], "totalPrice" : "", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
{ "_id" : ObjectId("55aff2dc2c1e98ac112fd2ff"), "client" : "55a8e5fa586f94752a000002", "estimatePrice" : "1", "estimateDuration" : NumberLong(720), "description" : "1", "serviceDate" : ISODate("1989-12-12T12:12:00Z"), "lastStatusId" : null, "currentStatusId" : NumberLong(2), "collaborators" : [ { "id" : "55ae40772c1e98007a2fd2ff" } ], "dates" : [ ], "totalPrice" : "", "backofficeRequested" : NumberLong(0), "serviceInvoiced" : "No", "evaluation" : "", "positiveFeatures" : "", "negativeFeatures" : "" }
I want to get all the rows belonging to the client with the ID "55a8e5fa586f94752a000002".
It's pretty easy to do in the mongo Shell:
db.services.find({"client":"55a8e5fa586f94752a000002"})
But I am having a hard time inside Node.js because it either returns no services at all or it ends up returning just one service.
Here's what I'm using right now:
(...)
console.log(checkDatas[0]._id);
Service.find().where('client', checkDatas[0]._id)
.exec(function(err, services)
{
console.log("output: " + services);
(...)
And finally, here's the output:
**55a8e5fa586f94752a000002**
**output**: { _id: 55aff719af19b58913000001,
client: 55a8e5fa586f94752a000002,
__v: 1,
collaborators: [ { id: '55acb6022c1e98d05f2fd2ff' } ],
notes: [],
dates: [],
addresses: [],
date: Wed Jul 22 2015 22:03:37 GMT+0200 (CEST),
latestUpdateDate: Wed Jul 22 2015 22:03:37 GMT+0200 (CEST),
backofficeRequested: false,
totalPrice: null,
estimatePrice: null,
estimateDuration: null,
description: '',
serviceDate: null,
wasSeen: 0,
currentStatusId: 3,
lastStatusId: 1 }
It's as if I were using findOne but I'm not.
On top of that, yesterday I had a .sort() method also in that query and things worked fine after commenting it but today the problem's back.
Do MongoDB / Node have some sort of caching system I'm not aware of?
Thanks
EDIT:
Running
Service.find({'client': checkDatas[0]._id})
yields the same result

It looks like you're mixing ObjectId's and strings, which are two different things.
Your Mongoose query is looking for an ObjectId:
Service.find().where('client', checkDatas[0]._id)
Quick tip: you can distinguish both types in the output:
// ObjectId, because it doesn't have quotes around it
client: 55a8e5fa586f94752a000002
// String
collaborators: [ { id: '55acb6022c1e98d05f2fd2ff' } ]
Your MongoDB shell query is looking for a string:
db.services.find({"client":"55a8e5fa586f94752a000002"})
The subset you're posting shows only strings, but I think that your database may actually contain both types. You can check and see if this works better:
Service.find().or([
{ client : checkDatas[0]._id },
{ client : String(checkDatas[0]._id) }
])...
Obviously this isn't ideal, you should consider normalizing your database if both types are mixed.
EDIT: the or query probably won't work when using Mongoose, since it will cast both clauses to the type defined in the schema. You can still check from the Mongo shell:
db.services.find({ $or : [
{ client : "55a8e5fa586f94752a000002" },
{ client : ObjectId("55a8e5fa586f94752a000002") }
]})

Related

ElasticSearch can't get multiple suggestor values from the same document

Can you help me please?
I have a problem with Completion Suggester in ElasticSearch
Example: I have this mapping :
PUT music
{
"mappings": {
"properties": {
"suggest": {
"type": "completion"
},
"title": {
"type": "keyword"
}
}
}
}
and index multiple suggestions for a document as follows:
PUT music/_doc/1?refresh
{
"suggest": [
{
"input": "Nirva test",
"weight": 10
},
{
"input": "Nirva hola",
"weight": 3
}
]
}
Querying: you can do this request on kibana
POST music/_search?pretty
{
"suggest": {
"song-suggest": {
"prefix": "nirv",
"completion": {
"field": "suggest"
}
}
}
}
and the result I retrieve only the first value but not both.
I did the test on kibana dev tool too and this is the result
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 0,
"relation" : "eq"
},
"max_score" : null,
"hits" : [ ]
},
"suggest" : {
"song-suggest" : [
{
"text" : "nir",
"offset" : 0,
"length" : 3,
"options" : [
{
"text" : "Nirvana test",
"_index" : "music",
"_type" : "_doc",
"_id" : "1",
"_score" : 10.0,
"_source" : {
"suggest" : [
{
"input" : "Nirvana test",
"weight" : 10
},
{
"input" : "Nirvana best",
"weight" : 3
}
]
}
}
]
}
]
}
}
expected result :
"suggest" : {
"song-suggest" : [
{
"text" : "nirvana",
"offset" : 0,
"length" : 7,
"options" : [
{
"text" : "Nirvana test",
"_index" : "music",
"_type" : "_doc",
"_id" : "1",
"_score" : 10.0,
"_source" : {
"suggest" : [
{
"input" : "Nirvana test",
"weight" : 10
},
{
"input" : "Nirvana best",
"weight" : 3
}
]
}
}
]
},
{
"text" : "nirvana b",
"offset" : 0,
"length" : 9,
"options" : [
{
"text" : "Nirvana best",
"_index" : "music",
"_type" : "_doc",
"_id" : "1",
"_score" : 3.0,
"_source" : {
"suggest" : [
{
"input" : "Nirvana test",
"weight" : 10
},
{
"input" : "Nirvana best",
"weight" : 3
}
]
}
}
]
}
]
}
This is the default behavior of current implementations. You can check #31738. Below is one of the comment for an explanation why it is returning only one document/suggestion.
The completion suggester is document-based by design so we cannot
return one entry per matching suggestion. It is documented that it
returns documents not suggestions and a single input can be indexed in
multiple suggestions (if you have synonyms in your analyzer for
instance) so it is not trivial to differentiate a match from its
variations. Also the completion suggester does not visit all
suggestions to select the top N, it has a special structure (a
weighted FST) that can visit suggestions in the order of their scores
and early terminates the query once enough documents have been found.

"How to get 'parent to child' relation with mongodb aggregation"

I have one collection called "location". in this collection all child and parent collection are stores. now I want to create a query who returns me parent to child spaces separated string.
Collection
businessId: { type: mongoose.Schema.Types.ObjectId, ref: 'admin' },
parentId: { type: mongoose.Schema.Types.ObjectId, ref: 'location' },
name: { type: String },
image: { type: String },
imageManipulation: { type: String },
locationColor: [{ range: { type: String }, color: { type: String } }],
area: {},
settings: {},
status: { type: String, enum: [0, 1], default: 1 },
isChild: { type: String, enum: [0, 1] },
parentPosition: { type: String }
In the above collection, you can see parentId field. if the location is a child then it have parentId. if the location is a parent then parentId will null. parent location can N level child location.
Collection Data
[{
"_id" : ObjectId("5cee1002a01ad50f5c222982"),
"status" : "1",
"name" : "Ground Floor",
"settings" : {
"zoom" : "0",
"positionX" : "0",
"positionY" : "0",
"width" : "498",
"height" : "498"
},
"image" : "1559105538977.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:52:18.999Z"),
"createdAt" : ISODate("2019-05-29T04:52:18.999Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee103ca01ad50f5c222983"),
"status" : "1",
"name" : "Kitchen",
"settings" : {
"zoom" : "0",
"positionX" : "0",
"positionY" : "0",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":3,\"points\":[{\"x\":20,\"y\":178},{\"x\":19,\"y\":75},{\"x\":56,\"y\":71},{\"x\":57,\"y\":52},{\"x\":80,\"y\":18},{\"x\":138,\"y\":17},{\"x\":165,\"y\":52},{\"x\":165,\"y\":94},{\"x\":174,\"y\":96},{\"x\":173,\"y\":179}],\"fill\":\"rgba(178,40,40,0.58)\"}",
"parentId" : ObjectId("5cee1002a01ad50f5c222982"),
"image" : "1559105596975.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:53:16.990Z"),
"createdAt" : ISODate("2019-05-29T04:53:16.990Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee1078a01ad50f5c222984"),
"status" : "1",
"name" : "Cbot",
"settings" : {
"zoom" : "0",
"positionX" : "0",
"positionY" : "0",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":3,\"points\":[{\"x\":20,\"y\":311},{\"x\":17,\"y\":59},{\"x\":84,\"y\":58},{\"x\":88,\"y\":312}],\"fill\":\"rgba(20,205,123,0.67)\"}",
"parentId" : ObjectId("5cee103ca01ad50f5c222983"),
"image" : "1559105656049.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:54:16.070Z"),
"createdAt" : ISODate("2019-05-29T04:54:16.070Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee10c1a01ad50f5c222985"),
"status" : "1",
"name" : "Drower 1",
"settings" : {
"zoom" : "5",
"positionX" : "470",
"positionY" : "70",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":3,\"points\":[{\"x\":21,\"y\":102},{\"x\":81,\"y\":104},{\"x\":79,\"y\":43},{\"x\":21,\"y\":43}],\"fill\":\"rgba(16,77,193,0.5)\"}",
"parentId" : ObjectId("5cee1078a01ad50f5c222984"),
"image" : "1559105729881.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:55:29.901Z"),
"createdAt" : ISODate("2019-05-29T04:55:29.901Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee110ea01ad50f5c222986"),
"status" : "1",
"name" : "Drawer 2",
"settings" : {
"zoom" : "5",
"positionX" : "484",
"positionY" : "103",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":1,\"coordinates\":{\"x\":23,\"y\":125,\"width\":58,\"height\":56},\"points\":[{\"x\":23,\"y\":125},{\"x\":81,\"y\":181}],\"fill\":\"rgba(117,37,109,0.74)\"}",
"parentId" : ObjectId("5cee1078a01ad50f5c222984"),
"image" : "1559105806551.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:56:46.574Z"),
"createdAt" : ISODate("2019-05-29T04:56:46.574Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee1148a01ad50f5c222987"),
"status" : "1",
"name" : "Drawer 3",
"settings" : {
"zoom" : "5",
"positionX" : "477",
"positionY" : "94",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":3,\"points\":[{\"x\":22,\"y\":205},{\"x\":20,\"y\":290},{\"x\":84,\"y\":288},{\"x\":85,\"y\":205}],\"fill\":\"rgba(164,108,54,0.57)\"}",
"parentId" : ObjectId("5cee1078a01ad50f5c222984"),
"image" : "1559105864947.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:57:44.972Z"),
"createdAt" : ISODate("2019-05-29T04:57:44.972Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee5e683b9f67a9f501f818"),
"status" : "1",
"name" : "Washroom",
"settings" : {
"zoom" : "5",
"positionX" : "477",
"positionY" : "94",
"width" : "498",
"height" : "498"
},
"area" : "{\"type\":3,\"points\":[{\"x\":22,\"y\":205},{\"x\":20,\"y\":290},{\"x\":84,\"y\":288},{\"x\":85,\"y\":205}],\"fill\":\"rgba(164,108,54,0.57)\"}",
"parentId" : ObjectId("5cee1002a01ad50f5c222982"),
"image" : "1559105864947.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:57:44.972Z"),
"createdAt" : ISODate("2019-05-29T04:57:44.972Z"),
"__v" : 0
},
{
"_id" : ObjectId("5cee5f593b9f67a9f501fa01"),
"status" : "1",
"name" : "Third Floor",
"settings" : {
"zoom" : "0",
"positionX" : "0",
"positionY" : "0",
"width" : "498",
"height" : "498"
},
"image" : "1559105538977123.jpg",
"businessId" : ObjectId("5cbd61dc3b56b902284ea388"),
"locationColor" : [],
"updatedAt" : ISODate("2019-05-29T04:52:18.999Z"),
"createdAt" : ISODate("2019-05-29T04:52:18.999Z"),
"__v" : 0
}]
Expected result in JSON
[
{
"_id": "5cee1002a01ad50f5c222982",
"name": "Ground Floor"
},
{
"_id": "5cee103ca01ad50f5c222983",
"name": " Kitchen"
},
{
"_id": "5cee1078a01ad50f5c222984",
"name": " Cbot"
},
{
"_id": "5cee110ea01ad50f5c222986",
"name": " Drawer 2"
},
{
"_id": "5cee1148a01ad50f5c222987",
"name": " Drawer 3"
},
{
"_id": "5cee10c1a01ad50f5c222985",
"name": " Drower 1"
},
{
"_id": "5cee5e683b9f67a9f501f818",
"name": " Washroom"
},
{
"_id": "5cee5f593b9f67a9f501fa01",
"name": "Third Floor"
}
]
I do not think you should let mongodb take care of name formatting. So my solution is about finding how many spaces a certain name needs before, so that js can deal with formatting. This is the query:
db.collection.aggregate([
{
$graphLookup: {
from: "collection",
startWith: "$parentId",
connectFromField: "parentId",
connectToField: "_id",
as: "hierarchy"
}
},
{
$project: {
"_id": 1,
"name": 1,
"hierarchy_size": { $size: "$hierarchy" }
}
}
]);
With the $graphLookup, the db is building an in memory graph of edges between connectFromField and connectToField. From the graph you only need the depth of your hierarchy, so I computed hierarchy_size. This is the output:
/* 1 */
{
"_id" : ObjectId("5cee1002a01ad50f5c222982"),
"name" : "Ground Floor",
"hierarchy_size" : 0
}
/* 2 */
{
"_id" : ObjectId("5cee103ca01ad50f5c222983"),
"name" : "Kitchen",
"hierarchy_size" : 1
}
/* 3 */
{
"_id" : ObjectId("5cee1078a01ad50f5c222984"),
"name" : "Cbot",
"hierarchy_size" : 2
}
/* 4 */
{
"_id" : ObjectId("5cee10c1a01ad50f5c222985"),
"name" : "Drower 1",
"hierarchy_size" : 3
}
/* 5 */
{
"_id" : ObjectId("5cee110ea01ad50f5c222986"),
"name" : "Drawer 2",
"hierarchy_size" : 3
}
/* 6 */
{
"_id" : ObjectId("5cee1148a01ad50f5c222987"),
"name" : "Drawer 3",
"hierarchy_size" : 3
}
/* 7 */
{
"_id" : ObjectId("5cee5e683b9f67a9f501f818"),
"name" : "Washroom",
"hierarchy_size" : 1
}
/* 8 */
{
"_id" : ObjectId("5cee5f593b9f67a9f501fa01"),
"name" : "Third Floor",
"hierarchy_size" : 0
}
The only problem here might be query performances, but that depends on how much data you need to process. Also consider the memory limit.

Mongodb Count multiple subobjects and group by

Hello This query is running fine for me I want to add other arrays as well like contact and I want separate count for them. If I add contact it add the result in count P.S : I used array name with $sum it start showing me zero count
return Company.aggregate(
{"$unwind":"$agreement"},
{"$unwind":"$contact"},
{"$unwind":"$companySite"},
{ $group: { _id: {
"Organization": "$orgId",
"Company": "$name"
}, "count": { $sum: 1 } } },{
"$project": {
"_id": 0,
"Company": "$_id.Company",
"Organization":"$_id.Organization",
"Count": "$count"
}
})
expected output
"Company": "Multi-Metal Manufacturing",
"Organization": "1",
"AgreementCount": 1,
"ContactCount" : 4
"ABC Count": 5
Sample Document is :
{
"_id" : ObjectId("57c6a97a90c933a2d54117dc"),
"ITBCompanyId" : 1034,
"updatedAt" : ISODate("2016-08-31T12:47:03.679Z"),
"createdAt" : ISODate("2016-08-31T09:55:06.217Z"),
"identifier" : "INL10",
"name" : "Inline Data Systems",
"addressLine1" : "7 Park Place",
"addressLine2" : "Ste D",
"city" : "Swansea",
"orgId" : "1",
"deletedAt" : null,
"_info" : {
"lastUpdated" : ISODate("2016-02-10T21:22:15.000Z"),
"updatedBy" : "Clarissa"
},
"whois" : [],
"configuration" : [],
"contact" : [
{
"email" : "justin#inlinedatasystems.com",
"mobileGuid" : "d8852942-8f5a-406f-a646-a5f8697f7885",
"presence" : null,
"gender" : null,
"title" : null,
"country" : null,
"zip" : null,
"state" : null,
"city" : null,
"addressLine2" : null,
"addressLine1" : null,
"lastName" : "Wilkerson",
"firstName" : "Justin",
"id" : 191,
"_id" : ObjectId("57c6abdfb966968130b1671c"),
"communicationItems" : [],
"customFields" : null,
"company" : {
"name" : "Inline Data Systems",
"id" : "19331"
},
"relationship" : {
"name" : null,
"id" : 0
}
}
],
"agreement" : [
{
"periodType" : null,
"billAmount" : "0",
"billTermsId" : 12,
"billOneTimeFlag" : false,
"billCycleId" : "2",
"expiredDays" : "0"
},
{
"id" : "40",
"name" : "Managed Hosted Server Agreement",
"billAmount" : "0",
"periodType" : null,
"_id" : ObjectId("57c6ab96b966968130b16616"),
"_info" : {
"lastUpdated" : ISODate("2014-10-03T14:25:42.000Z"),
"updatedBy" : "ali "
},
"contact" : {
"id" : "191",
"name" : "Justin Wilkerson"
},
"agreementType" : {
"id" : "33",
"name" : "HDCloud - Hosted Server"
}
},
{
"id" : "41",
"name" : "Managed Backup Agreement",
"parentAgreementId" : "42",
"customerPO" : ""
}
],
"companySite" : [
{
"id" : 1037,
"name" : "Main",
"city" : "Swansea",
"state" : "IL",
"zip" : "62226",
}
]

AQL Query Really Slow (~20 seconds)

The following query is taking around 20 seconds to execute:
FOR p IN PATHS(locations, connections, "outbound", { maxLength: 1 }) FILTER p.source._key == "26094" RETURN p.vertices[*].name
I believe this is a simple query (and the database is not that big) and it should execute fairly quick... I must be doing something wrong... Here is the query result:
==> [object ArangoQueryCursor - count: 286, hasMore: false]
The locations (vertices) collection has 23753 documents, and the connections (edges) collection has 123414 documents.
I tried to filter by _id as well but the performance is somewhat the same.
Is there anything I could do to get a better performance?
Here is the query's .explain() report:
{
"plan" : {
"nodes" : [
{
"type" : "SingletonNode",
"dependencies" : [ ],
"id" : 1,
"estimatedCost" : 1,
"estimatedNrItems" : 1
},
{
"type" : "CalculationNode",
"dependencies" : [
1
],
"id" : 2,
"estimatedCost" : 2,
"estimatedNrItems" : 1,
"expression" : {
"type" : "function call",
"name" : "PATHS",
"subNodes" : [
{
"type" : "array",
"subNodes" : [
{
"type" : "collection",
"name" : "locations"
},
{
"type" : "collection",
"name" : "connections"
},
{
"type" : "value",
"value" : "outbound"
},
{
"type" : "object",
"subNodes" : [
{
"type" : "object element",
"name" : "maxLength",
"subNodes" : [
{
"type" : "value",
"value" : 1
}
]
}
]
}
]
}
]
},
"outVariable" : {
"id" : 2,
"name" : "2"
},
"canThrow" : true
},
{
"type" : "EnumerateListNode",
"dependencies" : [
2
],
"id" : 3,
"estimatedCost" : 102,
"estimatedNrItems" : 100,
"inVariable" : {
"id" : 2,
"name" : "2"
},
"outVariable" : {
"id" : 0,
"name" : "p"
}
},
{
"type" : "CalculationNode",
"dependencies" : [
3
],
"id" : 4,
"estimatedCost" : 202,
"estimatedNrItems" : 100,
"expression" : {
"type" : "compare ==",
"subNodes" : [
{
"type" : "attribute access",
"name" : "_key",
"subNodes" : [
{
"type" : "attribute access",
"name" : "source",
"subNodes" : [
{
"type" : "reference",
"name" : "p",
"id" : 0
}
]
}
]
},
{
"type" : "value",
"value" : "26094"
}
]
},
"outVariable" : {
"id" : 3,
"name" : "3"
},
"canThrow" : false
},
{
"type" : "FilterNode",
"dependencies" : [
4
],
"id" : 5,
"estimatedCost" : 302,
"estimatedNrItems" : 100,
"inVariable" : {
"id" : 3,
"name" : "3"
}
},
{
"type" : "CalculationNode",
"dependencies" : [
5
],
"id" : 6,
"estimatedCost" : 402,
"estimatedNrItems" : 100,
"expression" : {
"type" : "expand",
"subNodes" : [
{
"type" : "iterator",
"subNodes" : [
{
"type" : "variable",
"name" : "1_",
"id" : 1
},
{
"type" : "attribute access",
"name" : "vertices",
"subNodes" : [
{
"type" : "reference",
"name" : "p",
"id" : 0
}
]
}
]
},
{
"type" : "attribute access",
"name" : "name",
"subNodes" : [
{
"type" : "reference",
"name" : "1_",
"id" : 1
}
]
}
]
},
"outVariable" : {
"id" : 4,
"name" : "4"
},
"canThrow" : false
},
{
"type" : "ReturnNode",
"dependencies" : [
6
],
"id" : 7,
"estimatedCost" : 502,
"estimatedNrItems" : 100,
"inVariable" : {
"id" : 4,
"name" : "4"
}
}
],
"rules" : [
"move-calculations-up",
"move-filters-up",
"move-calculations-up-2",
"move-filters-up-2"
],
"collections" : [
{
"name" : "connections",
"type" : "read"
},
{
"name" : "locations",
"type" : "read"
}
],
"variables" : [
{
"id" : 0,
"name" : "p"
},
{
"id" : 1,
"name" : "1_"
},
{
"id" : 2,
"name" : "2"
},
{
"id" : 3,
"name" : "3"
},
{
"id" : 4,
"name" : "4"
}
],
"estimatedCost" : 502,
"estimatedNrItems" : 100
},
"warnings" : [ ],
"stats" : {
"rulesExecuted" : 21,
"rulesSkipped" : 0,
"plansCreated" : 1
}
}
PATHS() will build all paths of the graph and then post-filter the results using the FILTER on the _key attribute. This may create a huge result set first (for all paths) before filtering out all non-matches.
If all that's required is to find connected vertices on depth 1, I think it will be more efficient to do something like this:
querying using TRAVERSAL:
This is more efficient because it will build all paths in the graph but only those starting at the specified start vertex:
FOR p IN TRAVERSAL(locations, connections, "1", "outbound", { minDepth: 1, maxDepth: 1, paths: true })
RETURN p.path.vertices[*].name
querying direct neighbors using NEIGHBORS:
This may be slightly more efficient even because it will construct a smaller intermediate result.
Additionally, it won't return the start vertex (26094) but all vertices directly connected to it:
FOR p IN NEIGHBORS(locations, connections, "26094", "outbound")
RETURN p.vertex.name
querying the edges directly (not using graph functions)
Finally you can query the edge collection directly.
Again, this won't return the start vertex (26094) but all vertices directly connected to it:
FOR edge IN connections
FILTER edge._from == "locations/26094"
FOR vertex IN locations
FILTER vertex._id == edge._to
RETURN vertex.name

Query Rule for non-indexed attribute FILTER

I observere an enormous runtime difference between those two AQL statements an a DB set with about 20 Mio records:
FOR e IN EAll
FILTER e.lastname == "Kmp" // <-- skip-index
FILTER e.lastpaff != "" // <-- no index
RETURN e
// runs in less than a second
AND
FOR e IN EAll
FILTER e.lastpaff != "" // <-- no index
FILTER e.lastname == "Kmp" // <-- skip-index
RETURN e
// needs about a minute to execute.
In addition to be (or not) indexed, the selectivity of those statements is highly different: the indexedAttribute is highly selective where-as the nonIndexedAttribute only filters 50%.
Is it possible that there is not yet an optimization rule for that? I currently am using ArangoDB 2.4.0.
DETAILS:
There is a SKIP-Index on the indexed Attribute (which seems to be used in the execuation plan 1).
Here are the execuation plan, in which only the order of the filters are changed:
FAST QUERY:
arangosh [Uni]> stmt.explain()
{
"plan" : {
"nodes" : [
{
"type" : "SingletonNode",
"dependencies" : [ ],
"id" : 1,
"estimatedCost" : 1,
"estimatedNrItems" : 1
},
{
"type" : "IndexRangeNode",
"dependencies" : [
1
],
"id" : 8,
"estimatedCost" : 170463.32,
"estimatedNrItems" : 170462,
"database" : "Uni",
"collection" : "EAll",
"outVariable" : {
"id" : 0,
"name" : "i"
},
"ranges" : [
[
{
"variable" : "i",
"attr" : "lastname",
"lowConst" : {
"bound" : "Kmp",
"include" : true,
"isConstant" : true
},
"highConst" : {
"bound" : "Kmp",
"include" : true,
"isConstant" : true
},
"lows" : [ ],
"highs" : [ ],
"valid" : true,
"equality" : true
}
]
],
"index" : {
"type" : "skiplist",
"id" : "13295598550318",
"unique" : false,
"fields" : [
"lastname"
]
},
"reverse" : false
},
{
"type" : "CalculationNode",
"dependencies" : [
8
],
"id" : 5,
"estimatedCost" : 340925.32,
"estimatedNrItems" : 170462,
"expression" : {
"type" : "compare !=",
"subNodes" : [
{
"type" : "attribute access",
"name" : "lastpaff",
"subNodes" : [
{
"type" : "reference",
"name" : "i",
"id" : 0
}
]
},
{
"type" : "value",
"value" : ""
}
]
},
"outVariable" : {
"id" : 2,
"name" : "2"
},
"canThrow" : false
},
{
"type" : "FilterNode",
"dependencies" : [
5
],
"id" : 6,
"estimatedCost" : 511387.32,
"estimatedNrItems" : 170462,
"inVariable" : {
"id" : 2,
"name" : "2"
}
},
{
"type" : "ReturnNode",
"dependencies" : [
6
],
"id" : 7,
"estimatedCost" : 681849.3200000001,
"estimatedNrItems" : 170462,
"inVariable" : {
"id" : 0,
"name" : "i"
}
}
],
"rules" : [
"move-calculations-up",
"move-filters-up",
"move-calculations-up-2",
"move-filters-up-2",
"use-index-range",
"remove-filter-covered-by-index"
],
"collections" : [
{
"name" : "EAll",
"type" : "read"
}
],
"variables" : [
{
"id" : 0,
"name" : "i"
},
{
"id" : 1,
"name" : "1"
},
{
"id" : 2,
"name" : "2"
}
],
"estimatedCost" : 681849.3200000001,
"estimatedNrItems" : 170462
},
"warnings" : [ ],
"stats" : {
"rulesExecuted" : 19,
"rulesSkipped" : 0,
"plansCreated" : 1
}
}
SLOW Query:
arangosh [Uni]> stmt.explain()
{
"plan" : {
"nodes" : [
{
"type" : "SingletonNode",
"dependencies" : [ ],
"id" : 1,
"estimatedCost" : 1,
"estimatedNrItems" : 1
},
{
"type" : "EnumerateCollectionNode",
"dependencies" : [
1
],
"id" : 2,
"estimatedCost" : 17046233,
"estimatedNrItems" : 17046232,
"database" : "Uni",
"collection" : "EAll",
"outVariable" : {
"id" : 0,
"name" : "i"
},
"random" : false
},
{
"type" : "CalculationNode",
"dependencies" : [
2
],
"id" : 3,
"estimatedCost" : 34092465,
"estimatedNrItems" : 17046232,
"expression" : {
"type" : "compare !=",
"subNodes" : [
{
"type" : "attribute access",
"name" : "lastpaff",
"subNodes" : [
{
"type" : "reference",
"name" : "i",
"id" : 0
}
]
},
{
"type" : "value",
"value" : ""
}
]
},
"outVariable" : {
"id" : 1,
"name" : "1"
},
"canThrow" : false
},
{
"type" : "FilterNode",
"dependencies" : [
3
],
"id" : 4,
"estimatedCost" : 51138697,
"estimatedNrItems" : 17046232,
"inVariable" : {
"id" : 1,
"name" : "1"
}
},
{
"type" : "CalculationNode",
"dependencies" : [
4
],
"id" : 5,
"estimatedCost" : 68184929,
"estimatedNrItems" : 17046232,
"expression" : {
"type" : "compare ==",
"subNodes" : [
{
"type" : "attribute access",
"name" : "lastname",
"subNodes" : [
{
"type" : "reference",
"name" : "i",
"id" : 0
}
]
},
{
"type" : "value",
"value" : "Kmp"
}
]
},
"outVariable" : {
"id" : 2,
"name" : "2"
},
"canThrow" : false
},
{
"type" : "FilterNode",
"dependencies" : [
5
],
"id" : 6,
"estimatedCost" : 85231161,
"estimatedNrItems" : 17046232,
"inVariable" : {
"id" : 2,
"name" : "2"
}
},
{
"type" : "ReturnNode",
"dependencies" : [
6
],
"id" : 7,
"estimatedCost" : 102277393,
"estimatedNrItems" : 17046232,
"inVariable" : {
"id" : 0,
"name" : "i"
}
}
],
"rules" : [
"move-calculations-up",
"move-filters-up",
"move-calculations-up-2",
"move-filters-up-2"
],
"collections" : [
{
"name" : "EAll",
"type" : "read"
}
],
"variables" : [
{
"id" : 0,
"name" : "i"
},
{
"id" : 1,
"name" : "1"
},
{
"id" : 2,
"name" : "2"
}
],
"estimatedCost" : 102277393,
"estimatedNrItems" : 17046232
},
"warnings" : [ ],
"stats" : {
"rulesExecuted" : 19,
"rulesSkipped" : 0,
"plansCreated" : 1
}
}
Indeed, conditions like the following disabled the usage of indexes even though an index could be used:
FILTER doc.indexedAttribute != ... FILTER doc.indexedAttribute == ...
Interestingly an index is used when the two conditions are put into the same FILTER condition and combined with &&:
FILTER doc.indexedAttribute != ... && doc.indexedAttribute == ...
Though these two statements are equivalent, they trigger a slightly different code path. The former will be AND-combining two existing FILTER ranges, the latter one will produce a range from a single FILTER. The case of AND-combination for the FILTER ranges was overly defensive and rejected both sides even if only a single side (in this case the one with the non-equality operator) could not be used for an index scan.
This has been fixed in 2.4, and the fix will be contained in 2.4.2. A workaround for now is to combine the two FILTER statements in a single one.

Resources