I have a document that has an array field called telephone which can have multiple extension array objects that can have multiple objects in it. So its an array inside an array. I am listening to the db using changeStream. If I change telephone[0].extension[0].valueBoolean = true where telephone[0].extension[0].url == "https//google.com",
I get the whole telephone array back in change.updateDescription.updatedFields NOT just telephone[0].extension[0]
updatedFields
{
"telephone": [{
"use": "offline",
"extension": [
{
"url": "https//gmail.com",
"valueDateTime": "2021-01-12T06:31:48.000Z"
}, {
"url": "https//yahoo.com",
"valueDateTime": "1700-01-01T00:00:00.000Z"
}, {
"url": "https//google.com",
"TimeLastModified": "2021-02-23T11:06:06.000Z",
"valueBoolean": false
}],
"value": "+123456789",
"system": "phone"
}, {
"use": "internet",
"extension": [
{
"url": "https//gmail.com",
"valueDateTime": "2021-01-12T06:31:48.000Z"
}, {
"url": "https//yahoo.com",
"valueDateTime": "1700-01-01T00:00:00.000Z"
}, {
"url": "https//google.com",
"TimeLastModified": "2021-02-23T11:06:06.000Z",
"valueBoolean": false
}],
"value": "+123456799",
"system": "phone"
}]
}
Here's what i have so far
MongoClient.connect(CONNECTION_STRING, {
useUnifiedTopology: true,
})
.then((client) => {
console.log("Connected successfully to server");
dbConnected = true;
}
// specify db and collections
const db = client.db(DB_NAME);
const myCollection = db.collection(COLLECTION_NAME);
const options = { fullDocument: "updateLookup" };
const changeStream = myCollection.watch(options);
// start listening to changes
changeStream.on("change", async (change) => {
// console.log("CHANGE!");
// console.log(JSON.stringify(change));
// check operationType
try {
if (
change.operationType == "insert" ||
change.operationType == "update" ||
change.operationType == "replace"
) {
const updatedFields = change.updateDescription.updatedFields
console.log("updatedFields", JSON.stringify(change.updateDescription.updatedFields));
}
} catch (e) {
console.log(e);
}
});
})
.catch((e) => {
console.log(`Error: ${e}`);
});
How do I see what exact element in a nested array changed with changeStream ?
Unfortunately it seems that this is currently not supported - there's an open Jira-ticket that is related to your problem, see https://jira.mongodb.org/browse/SERVER-41559 for further details.
Related
I am facing an issue with sorting values in an elastic search query. I am doing a simple search with sort but getting the following error. The query works without a sort parameter.
Elastic search client version: Version 7.6.1(Using this version because I am using opensearch)
search_phase_execution_exception: [illegal_argument_exception] Reason:
Text fields are not optimised for operations that require per-document
field data like aggregations and sorting, so these operations are
disabled by default. Please use a keyword field instead.
Alternatively, set fielddata=true on [subtype] in order to load field
data by uninverting the inverted index. Note that this can use
significant memory.
Code Sample:
const {Client} = require('#elastic/elasticsearch') // Version 7.6.1
var connectionString = 'https://admin:admin#localhost:9200'
const client = new Client({
node: connectionString,
ssl: {
rejectUnauthorized: false
}
})
client.info()
.then(async response => {
console.log('success', response.statusCode)
var query = {
"query": {
"match": {
"revhostname": {
"query": "ten.tsacmoc.ac.1dsh.631-651-14-37-c",
},
},
},
"sort": [
{
"revhostname": {"order": "asc"},
"subtype": {"order": "asc"},
"value": {"order": "asc"},
}
],
};
var response = await client.search({
index: 'r7',
body: query,
});
console.log("Search results:", JSON.stringify(response));
})
.catch(error => {
console.error('error', JSON.stringify(error))
})
Mapping:
{
"properties": {
"revhostname": {
"type" : "keyword"
},
"value": {
"type" : "keyword"
},
"subtype": {
"type" : "keyword"
},
"timestamp": {
"type" : "long"
},
"ip": {
"type" : "ip"
}
}
}
I tried adding fielddata=true in mapping but the issue was not solved. Your help is much appreciated.
Thank you.
As you mentioned mapping in a comment, your revhostname field is defined as text and keyword both type of field and Elasticsearch dont allow sorting on text type of field.
If your mapping is still same as you mentioned in comment then you need to use the field name like revhostname.keyword which will resolved issue.
const {Client} = require('#elastic/elasticsearch') // Version 7.6.1
var connectionString = 'https://admin:admin#localhost:9200'
const client = new Client({
node: connectionString,
ssl: {
rejectUnauthorized: false
}
})
client.info()
.then(async response => {
console.log('success', response.statusCode)
var query = {
"query": {
"match": {
"revhostname": {
"query": "ten.tsacmoc.ac.1dsh.631-651-14-37-c",
},
},
},
"sort": [
{
"revhostname.keyword": {"order": "asc"},
"subtype.keyword": {"order": "asc"},
"value.keyword": {"order": "asc"},
}
],
};
var response = await client.search({
index: 'r7',
body: query,
});
console.log("Search results:", JSON.stringify(response));
})
.catch(error => {
console.error('error', JSON.stringify(error))
})
I'm trying to get all documents from my database collection "posts" but I'm getting an empty array instead.
The strange thing is that I'm able to get all documents from another collection called "users" that has the same structure and using the exact same code.
I've spent days looking for an answer but I haven't been able to find the solution.
This is the request:
const db = admin.firestore();
exports.getAllPosts = (req, res) => {
db.collection('posts')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let posts = [];
snapshot.forEach((doc) => {
posts.push({
id: doc.id,
body: doc.data().body,
author: doc.data().author,
createdAt: doc.data().timestamp,
voteScore: doc.data().voteScore
});
});
return res.json(posts);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And this is the response:
[]
This is what my current collection looks like:
Posts collection screenshot
This the response that I get when I return "snapshot":
{
"_query": {
"_firestore": {
"_settings": {
"projectId": "readable-bf7a6",
"firebaseVersion": "9.6.0",
"libName": "gccl",
"libVersion": "4.10.0 fire/9.6.0"
},
"_settingsFrozen": true,
"_serializer": {
"allowUndefined": false
},
"_projectId": "readable-bf7a6",
"registeredListenersCount": 0,
"bulkWritersCount": 0,
"_backoffSettings": {
"initialDelayMs": 100,
"maxDelayMs": 60000,
"backoffFactor": 1.3
},
"_clientPool": {
"concurrentOperationLimit": 100,
"maxIdleClients": 1,
"activeClients": {},
"failedClients": {},
"terminated": false,
"terminateDeferred": {
"promise": {}
}
}
},
"_queryOptions": {
"parentPath": {
"segments": []
},
"collectionId": "posts",
"converter": {},
"allDescendants": false,
"fieldFilters": [],
"fieldOrders": [
{
"field": {
"segments": [
"createdAt"
]
},
"direction": "DESCENDING"
}
],
"kindless": false
},
"_serializer": {
"allowUndefined": false
},
"_allowUndefined": false
},
"_readTime": {
"_seconds": 1622395245,
"_nanoseconds": 513743000
},
"_size": 0,
"_materializedDocs": null,
"_materializedChanges": null
}
Notice how the request for the collection "users" works successfully:
const db = admin.firestore();
exports.getAllUsers = (req, res) => {
db.collection('users')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let users = [];
snapshot.forEach((doc) => {
let users = [];
snapshot.forEach((doc) => {
users.push({
id: doc.data().userId,
email: doc.data().email,
handle: doc.data().handle
});
});
return res.json(users);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And the response:
[
{
"id": "EPoHBxhQFUXbcL3TCVx1LdUG2nO2",
"email": "ruben#gmail.com"
},
{
"id": "RqEa3dEq8TSDcZYeolXafju67rB2",
"email": "user10#gmail.com"
},
{
"id": "dxveb4n2iMQej5Q14uprsKRxFp23",
"email": "user4#gmail.com",
"handle": "user4"
},
{
"id": "YQPzBPcsqlVZk9iJEuZTHKUNuVG2",
"email": "user2#gmail.com",
"handle": "user2"
},
{
"id": "CZ05BJxi3TUOpIrmBaz539OWlbC3",
"email": "user#gmail.com",
"handle": "user"
},
{
"id": "t0t83BVwt4gVgJkDv7HL1r1MaKr1",
"email": "userJose2#gmail.com",
"handle": "Jose"
}
]
This is what the users collection looks like in Firebase:
Users collection screenshot
Why is one collection failing when the other works fine and I'm using the same code? What am I missing here?
Thanks in advance and I hope I've made it as clear as possible. Please let me know if you need me to provide anything else.
Very simple my friend, your posts documents can't be ordered like this:
.orderBy('createdAt', 'desc')
Because the post documents does not have the createdAt property, but they have a timestamp property, you should use that property to order your posts like this:
.orderBy('timestamp', 'desc')
I hope that helps 👍
I am not answering directly to #Ruben Garcia Bri, but for future firebase developers who may run into the problem of getting empty documents, I also ran into the same problem, but I solved it by adding a field to the particular document I am trying to retrieve.
Sometimes the cause is because the documents you are trying to get have no field in them.
I mean that a document must have a field before the server can recognize it as an existing document.
So if you run into this problem, consider adding a field to that document before you can successfully retrieve it.
Problem
Hi dev,
I have the problem that when I try to make a get request to the series by id it shows me null.
I have noticed from the Atlas Mongos platform that I created the collection but it does not show me the data, only the structure of the scheme shows me
Function.js
const fs = require('fs');
const fetch = require('node-fetch');
const BASE_URL = " http://localhost:8081/api/v1/"
async function getSeries() {
return new Promise((resolve , reject) =>{
setTimeout(() => {
const res = require('./simple_database/series/1.json' , 'utf8');
resolve(res)
}, 1000);
})
}
module.exports = {
getSeries
}
Router
The route allseries allows me to access all the content. What I want to do is pass that content to the SeriesModel, maybe it is there where I have the problem that the data is not being inserted correctly.
In the route series/:id is where the null value is returning to me
const express = require('express');
const router = express.Router();
const f = require('./function');
const SeriesModel = require('./models/series');
router.get('/allseries', (req, res) => {
f.getSeries().then((series) =>{
res.status(200).json({
series
})
}).then((doc) =>{
SeriesModel.insertMany(doc , function(err , docs){
if(err){
console.error(err)
}else{
console.log(docs);
console.info('%d serie were successfully stored.', docs.length);
}
})
})
});
router.get('/series/:id' , (req , res , next) =>{
const id = req.params.id;
SeriesModel.findById(id)
.exec()
.then((doc) =>{
console.log("From database " , doc);
res.status(200).json(doc)
}).catch((err) =>{
console.error(err);
res.status(500).json({error: err})
})
})
module.exports = router;
Model/series.js
const mongoose = require('mongoose');
const serieSchema = mongoose.Schema({
"_id": {
"$oid": {
"type": "ObjectId"
}
},
"series_id": {
"type": "String"
},
"aggregateRating": {
"reviewCount": {
"type": "Number"
},
"ratingCount": {
"type": "Number"
},
"#type": {
"type": "String"
},
"ratingValue": {
"type": "Number"
}
},
"episodes": {
"1x": {
"07 Ghost": {
"type": [
"Mixed"
]
}
}
},
"metadata": {
"description": {
"type": "String"
},
"url": {
"type": "String"
},
"image": {
"type": "String"
},
"type": {
"type": "String"
},
"id": {
"type": "String"
},
"name": {
"type": "String"
}
},
"1x": {
"07 Ghost": {
"type": [
"Mixed"
]
}
}
});
module.exports = mongoose.model("cr_series" , serieSchema);
It is because findById takes it's parameter in form of object like this
SeriesModel.findById({_id:id})
You need to tell your query to which json object you want to match your incoming object.
const results = await visionClient.labelDetection(imageUri).safeSearchDetection(imageUri);
i am trying to get an image response with cloud vision.
Below is an example of code for an HTTPS Cloud Function that will do the OCR (i.e. text detection) of an image stored in Firebase Storage. You would, for example, call it from your app after you have uploaded an image to Firebase Storage (in the gs://myproject.com/imgtoocr/ bucket), by passing the image name in the body of the HTTP Request.
....
const vision = require('#google-cloud/vision');
const client = new vision.ImageAnnotatorClient();
exports.imageOCR = functions.https.onRequest((req, res) => {
cors(req, res, () => {
const imageFilename = req.body.imageFilename;
let result;
return client
.documentTextDetection(
'gs://myproject.com/imgtoocr/' + imageFilename
)
.then(results => {
const blocks = results[0].fullTextAnnotation.pages[0].blocks;
blocks.forEach(blockDetail => {
blockDetail.paragraphs.forEach(paragraph => {
//Here you build the result you want to send back
});
});
return {
result: result
};
})
.then(ocrResult => {
return res.status(200).json(ocrResult);
})
.catch(err => {
console.error('ERROR:', err);
res.status(400).send(err);
});
});
});
You will find more info and examples (in particular for Label Detection) in the following documentation for node.js:
https://cloud.google.com/vision/docs/ocr-tutorial
https://cloud.google.com/vision/docs/detecting-labels
https://cloud.google.com/nodejs/docs/reference/vision/0.19.x/
Solved it this way for version 0.21.0
import * as vision from '#google-cloud/vision';
const visionClient = new vision.ImageAnnotatorClient();
const request = {
"image": {
"source": {
"imageUri": imageUri
}
},
"features": [
{
"type": "FACE_DETECTION"
},
{
"type": "LABEL_DETECTION"
},
{
"type": "SAFE_SEARCH_DETECTION"
},
{
"type": "WEB_DETECTION"
},
{
"type": "CROP_HINTS"
},
{
"type": "IMAGE_PROPERTIES"
},
{
"type": "DOCUMENT_TEXT_DETECTION"
},
{
"type": "TEXT_DETECTION"
},
{
"type": "LOGO_DETECTION"
},
{
"type": "LANDMARK_DETECTION"
},
{
"type": "TYPE_UNSPECIFIED"
},
// Other detection types here...
]
};
return await visionClient.annotateImage(request).then((res) => {
console.log(JSON.stringify(res));
});
How to update the multiple documents in MongoDB and set the value of the element in an increasing order?
I have got the document as follows
{
"_id" : ObjectId("5b162a31dfaf342dc44c920d")
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920f")
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920c")
}
How can I update the whole documents with a single query so that I can have a new element called "order" in every single field in an increasing order as below
{
"_id" : ObjectId("5b162a31dfaf342dc44c920d"),
"order": 1
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920f"),
"order": 2
}
{
"_id" : ObjectId("5b162a31dfaf342dc44c920c"),
"order": 3
}
Currently I am using the following way to solve the problem
for(let i = 0; i <= req.body.id.length;i++) {
const queryOpts = {
_id: ObjectId(req.body.id[i])
};
const updateOpts = {
$set: {
'order': i + 1
}
};
const dataRes = await req.db.collection('GalleryImage').updateOne(queryOpts, updateOpts);
if(i === req.body.id.length-1) {
return commonHelper.sendResponseMessage(res, dataRes, {
_id: req.body.id
}, moduleConfig.message.updateGalleryOrder);
}
If there any better way than this so that it would not be the expensive operation if there are large number of documents ?
Use bulkWrite() with Array.map() to construct the statement:
try {
let response = await req.db.collection('GalleryImage').bulkWrite(
req.body.id.map((_id,order) =>
({ updateOne: {
filter: { _id: ObjectId(_id) },
update: {
$set: { order: order+1 }
}
}})
)
);
} catch(e) {
// deal with any errors
}
Array.map() has the "index" of the array element being processed within it's second function argument. So simply use that to get the order and set that on all statements.
Rather than writing/responding with the database n times, this only needs happen "once".
There is no other way to get a "sequence" other than introducing it yourself, but at least we can do it with "one" write this way instead of several. Note also to "trap your possible errors" when using async/await syntax.
Example listing
const { MongoClient, ObjectID: ObjectId } = require('mongodb');
const uri = 'mongodb://localhost:27017';
const data = [
"5b162a31dfaf342dc44c920d",
"5b162a31dfaf342dc44c920f",
"5b162a31dfaf342dc44c920c"
];
const log = data => console.log(JSON.stringify(data, undefined, 2));
(async function() {
try {
const client = await MongoClient.connect(uri);
let db = client.db('test');
// Set up
await db.collection('gallery').removeMany({});
await db.collection('gallery').insertMany(
data.map(_id => ({ _id: ObjectId(_id) }))
);
// Update with indexes
let response = await db.collection('gallery').bulkWrite(
data.map((_id,idx) =>
({
updateOne: {
filter: { _id: ObjectId(_id) },
update: { $set: { order: idx+1 } }
}
})
)
);
log({ response });
let items = await db.collection('gallery').find().toArray();
log({ items });
client.close();
} catch(e) {
console.error(e)
} finally {
process.exit()
}
})()
And the output
{
"response": {
"ok": 1,
"writeErrors": [],
"writeConcernErrors": [],
"insertedIds": [],
"nInserted": 0,
"nUpserted": 0,
"nMatched": 3,
"nModified": 3,
"nRemoved": 0,
"upserted": [],
"lastOp": {
"ts": "6563535160225038345",
"t": 18
}
}
}
{
"items": [
{
"_id": "5b162a31dfaf342dc44c920d",
"order": 1
},
{
"_id": "5b162a31dfaf342dc44c920f",
"order": 2
},
{
"_id": "5b162a31dfaf342dc44c920c",
"order": 3
}
]
}
Clearly shows nMatched: 3 and nModified: 3 just as is expected.