Elasticsearch js query with must and must_not - node.js

I am trying to get a query to run with both must and must_not, but have not had any luck with the syntax I am attempting. I see a lot of people on StackOverflow using quotes on both sides like they would be in a Curl call, but this is straight out of a node application.
I will show the query that does work, and I am simply trying to add what I do not want to be included in the outcome. In either case, because this is just trash data that is on a local dev environment, the outcome should match.
First the working query:
client.search({
index: config.ES_INDEX,
type: "issue",
body: {
query: {
match: {
issue_state: 'Closed'
}
},
size: 1000
}
}).then(function(resp){
console.log(util.inspect(resp, {showHidden: false, depth: null}));
}).catch(function(err){
console.log('Failed to search. ' + err.message);
});
Output:
{ took: 5,
timed_out: false,
_shards: { total: 5, successful: 5, failed: 0 },
hits:
{ total: 1,
max_score: 1,
hits:
[ { _index: 'noc_tool',
_type: 'issue',
_id: 'Sy2IQFMLe',
_score: 1,
_source:
{ job_name: 'Job Name 1',
is_maintenance: 'no',
servicenow_id: 'lkjjklh',
type: 'Chase',
start_time: '1970-01-01T23:15:00.000Z',
maint_reminder: null,
update_duration: '4 Hours',
location: 'Test Group',
issue_state: 'Closed',
notes: [ { created_on: 1484063571941, body: 'lkjlkjhlkj' } ],
emailService: { lastEmailAt: 1484237594114 },
created_on: 1484063571941,
updated_on: 1484240538801,
reason: 'because I want to' } } ] } }
Now, the failed query:
client.search({
index: config.ES_INDEX,
type: "issue",
body: {
query: {
bool: {
must: [
{
term: {
issue_state: 'Closed'
}
}
],
must_not: [
{
term: {
is_maintenance: 'yes'
}
}
]
}
},
size: 1000
}
}).then(function(resp){
console.log(util.inspect(resp, {showHidden: false, depth: null}));
}).catch(function(err){
console.log('Failed to search. ' + err.message);
});
Output:
{ took: 6,
timed_out: false,
_shards: { total: 5, successful: 5, failed: 0 },
hits: { total: 0, max_score: null, hits: [] } }
Any help here would be much appreciated.

I ended up using a little "reverse logic" but here is what is working..
return new Promise(function(resolve, reject) {
client.search({
index: config.ES_INDEX,
type: "issue",
body: {
query: {
bool: {
must:[
{
match: {
issue_state: 'Closed'
}
},
{
match: {
is_maintenance: 'no'
}
}
]
}
},
size: 1000
}
}).then(function (resp) {
resolve (resp.hits.hits);
}).catch(function (err) {
reject('Failed to search. ' + err.message);
});

Related

How to request data from mongodb (mongoose) from subdocument within the interval "from" and "to"?

I need to return data that matches the range, and that's it. When requested, I will be returned an entire document that meets the conditions.
Do not know how to properly compose a query so that the database returns a ready data rage, without the need to filter after response on the server side.
Document structure:
{
symbol: "test",
data: [{
timestamp: 1,
value: 10,
},
{
timestamp: 2,
value: 20,
},
{
timestamp: 3,
value: 30,
},
{
timestamp: 4,
value: 40,
},
]
}
My request:
function request(symbol, from, to) {
return model.findOne({
symbol,
data: {
$elemMatch: {
timestamp: {
$gte: from,
$lt: to
}
}
}
}).then(res => res.data)
}
request('test', 2, 3)
Got response as full document structure. Using res.data.filter bad approach, because it has a big performance impact. Only the required data needs to be returned.
Example response as needed:
{
symbol: "test",
data: [
{
timestamp: 2,
value: 20,
},
{
timestamp: 3,
value: 30,
},
]
}
// or better
[
{
timestamp: 2,
value: 20,
},
{
timestamp: 3,
value: 30,
},
]
Thanks for the advice. Tried different variants from stackoverflow, nothing helped.
don't need the elementMatch function , you can query from subdocument as :
function request(symbol, from, to) {
return model.findOne({
symbol : symbol,
'data.timestamp' : { $gte: from, $lt: to }
}
}).then(res => res.data)
}

How to calculate based on the result of an aggregation?

For example, I can get the result of the average counts of request to a specific url base on parent id with the code below:
client.search({
index: 'console-*',
body: {
query: {
bool: {
query_string: {
query: 'meta.http.url:"https://www.google.com"'
}
}
},
aggs: {
parent_id: {
terms: {
field: 'parent_id'
}
}
}
},
size: 0
}).then(res => {
console.log(res.hits.total/res.aggregations.total.value)
})
Now let's say I have a number of urls like:
https://www.google.com
https://www.bing.com
https://www.apple.com
And I can use:
client.search({
index: 'console-*',
body: {
sort: {
'#timestamp': {
order: 'asc'
}
},
query: {
bool: {
must: [{
range: {
'#timestamp': {
gte: 'now-5m',
lte: 'now'
}
}
}, {
query_string: { query: '_exists_:meta.http.url' }
}]
}
},
aggs: {
urls: {
terms: {
field: 'meta.http.url'
},
aggs: {
total: {
cardinality: {
field: 'parent_id'
}
}
}
}
}
},
size: 0
}).then(res => {
console.log(JSON.stringify(res))
console.log(res.aggregations.urls.buckets.map(o => {
const res = {};
res[o.key] = o.doc_count / o.total.value;
return res;
}))
})
Is it possible to get the result without doing any additional calculation in Node.js?
Yes, you can leverage pipeline aggregations, and more specifically, the bucket_script one.
The aggs section would look like this instead and in each bucket you'll get the result of the document count divided by the total value stored in the compute section:
aggs: {
urls: {
terms: {
field: 'meta.http.url'
},
aggs: {
total: {
cardinality: {
field: 'parent_id'
}
},
compute: {
bucket_script: {
buckets_path: {
count: "_count",
total: "total"
},
script: "params.count / params.total"
}
}
}
}
}

How to insert multiple JSON document in elastic search

Input Data
[{
"_index": "abc",
"_type": "_doc",
"_id": "QAE",
"_score": 6.514091,
"_source": {
"category": "fruits",
"action": "eating",
"metainfo": {
"hash": "nzUZ1ONm0e167p"
},
"createddate": "2019-10-03T12:37:45.297Z"
}},
{
"_index": "abc",
"_type": "_doc",
"_id": "PQR",
"_score": 6.514091,
"_source": {
"category": "Vegetables",
"action": "eating",
"metainfo": {
"hash": "nzUZ1ONm0e167p"
},
"createddate": "2019-10-03T12:37:45.297Z"
}
}-----------------
----------------]
I have around 30,000 records as input data. How to insert this data in a single query. I tried by
var elasticsearch = require('elasticsearch');
var client = new elasticsearch.Client({
host: '********',
log: 'trace'
});
client.index({
index: "abc",
body: ****input data*****
}).then((res) => {
console.log(res);
}, (err) => {
console.log("err", err);
});
In this code, send input data in the body. but it returns an error. Please suggest to me.
This seems like what are you looking for:
'use strict'
require('array.prototype.flatmap').shim()
const { Client } = require('#elastic/elasticsearch')
const client = new Client({
node: 'http://localhost:9200'
})
async function run () {
await client.indices.create({
index: 'tweets',
body: {
mappings: {
properties: {
id: { type: 'integer' },
text: { type: 'text' },
user: { type: 'keyword' },
time: { type: 'date' }
}
}
}
}, { ignore: [400] })
const dataset = [{
id: 1,
text: 'If I fall, don\'t bring me back.',
user: 'jon',
date: new Date()
}, {
id: 2,
text: 'Winter is coming',
user: 'ned',
date: new Date()
}, {
id: 3,
text: 'A Lannister always pays his debts.',
user: 'tyrion',
date: new Date()
}, {
id: 4,
text: 'I am the blood of the dragon.',
user: 'daenerys',
date: new Date()
}, {
id: 5, // change this value to a string to see the bulk response with errors
text: 'A girl is Arya Stark of Winterfell. And I\'m going home.',
user: 'arya',
date: new Date()
}]
// The major part is below:
const body = dataset.flatMap(doc => [{ index: { _index: 'tweets' } }, doc])
const { body: bulkResponse } = await client.bulk({ refresh: true, body })
//
if (bulkResponse.errors) {
const erroredDocuments = []
// The items array has the same order of the dataset we just indexed.
// The presence of the `error` key indicates that the operation
// that we did for the document has failed.
bulkResponse.items.forEach((action, i) => {
const operation = Object.keys(action)[0]
if (action[operation].error) {
erroredDocuments.push({
// If the status is 429 it means that you can retry the document,
// otherwise it's very likely a mapping error, and you should
// fix the document before to try it again.
status: action[operation].status,
error: action[operation].error,
operation: body[i * 2],
document: body[i * 2 + 1]
})
}
})
console.log(erroredDocuments)
}
const { body: count } = await client.count({ index: 'tweets' })
console.log(count)
}
run().catch(console.log)
Reference link: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/bulk_examples.html

Get RANDOM ids with es query with node.js

could anyone help me? Im new in elasticSearch and node.js.
REQUEST
app.get('/', function (req, res, next){
try {
client.search({
index: 'dbcatalogo',
type: 'cars',
size: 10,
body: {
query: {
function_score:{
functions:[{
random_score:{
seed: 1
}
}]
},
match_all: {}
},
sort: {
'AnoModelo': 'desc'
}
}
}).then(function (json) {
res.json(json.hits);
},
function (err) {res.json(error.HandleError(err));});
}
catch (err) { res.json(error.HandleError(err)); }});
RESPONSE
[parse_exception] failed to parse search source. expected field name but got [START_OBJECT]
You are almost there the match_all needs to be within the query object of function_score as shows in function score below .
Example:
client.search({
index: 'dbcatalogo',
type: 'cars',
size: 10,
body: {
query: {
function_score:{
functions:[{
random_score:{
seed: 1
}
}],
query : {
match_all: {}
}
}
},
sort: {
'AnoModelo': 'desc'
}
}
}).then(function (json) {
res.json(json.hits);
}
Also if you are sorting on a field other than score the random_score does not make much sense.

Mongoose: select method does not return the include field

I have one document like that
var ConfigSchema = new Schema({
basicConfig: {
levelId: {
type: Number,
required: true
},
hostId: {
type: Number,
required: true
},
Name: {
type: String,
trim: true
},
Settings: {
Type1: {
// some types here...
}
Type2: {
// some types here...
}
}
},
enrolls: {
name: {type: String},
list: [String]
},
awards: {
enable: {type: Boolean},
Primary: {
amount: {type: Number},
type: {type: String}
}
}
Now I want to find configs with hostId matches 60, and selecting basicConfig field.
Config.findOne({ 'basicConfig.hostId': 60 })
.select('basicConfig').exec(function(err, configs) {
if (err) {
console.error(err);
return ;
}
console.log(configs);
});
However, all fields of this document will be returned. It seems that the select does NOT work? Why?
Output:
{ _id: 555c4144c0bff1541d0e4059,
enrolls: {},
awards: { primary: { PrimarySettings: {}, primaryAck: {} } },
basicConfig:
{ levelId: 24,
hostId: 60,
poolName: 'LC' } }
Also, those following codes have been test, it does not work.
BonusConfig.findOne({ 'basicConfig.hostId': 60 }, 'basicConfig', function(err, configs) {
if (err) {
console.error(err);
return ;
}
console.log(configs);
});
But, without the basicConfig field with select with the following codes, it work well.
BonusConfig.findOne({ 'basicConfig.hostId': 60 })
.select('-basicConfig').exec(function(err, configs) {
if (err) {
console.error(err);
return ;
}
console.log(configs);
});
What's wrong with my codes?
Mongoose version: 3.8.24
Mongodb version: 2.6.7
Edit 1
Here is the query log in mongoose debug mode.
Mongoose: configs.findOne({ 'basicConfig.hostId': 60 }) { fields: { basicConfig: 1 } }
Edit 2
After further investigation.
The output result of Config.findOne({ 'basicConfig.hostId': 60 }).select('basicConfig'):
{ _id: 555c4144c0bff1541d0e4059,
enrolls: {},
awards: { primary: { PrimarySettings: {}, primaryAck: {} } },
basicConfig:
{ levelId: 24,
hostId: 60,
poolName: 'LC' } }
Other fields are empty value except basicConfig. However, I want the result is
{ _id: 555c4144c0bff1541d0e4059,
basicConfig:
{ levelId: 24,
hostId: 60,
poolName: 'LC' } }
Your projection of fields that need to be returned is missing the additional parameter. Try this:
BonusConfig.findOne({ 'basicConfig.hostId': 60 }, {'basicConfig':1}, function(err, configs) {
if (err) {
console.error(err);
return ;
}
console.log(configs);
});
Quoting this
This behavior is by design, but admittedly it's not very well-designed. Mongoose is over-eager when it comes to creating sub-docs when loading from the database. Planning on changing that in v5.

Resources