I Am running a lambda function (NodeJS) to upload some documents to AWS Cloud Search. I keep getting the following error.
{
"errorMessage": "{ [\"The value of tags cannot be a JSON array or object\"] }",
"errorType": "DocumentServiceException",
"stackTrace": [
"Object.extractError (/var/task/node_modules/aws-sdk/lib/protocol/json.js:48:27)",
"Request.extractError (/var/task/node_modules/aws-sdk/lib/protocol/rest_json.js:37:8)",
"Request.callListeners (/var/task/node_modules/aws-sdk/lib/sequential_executor.js:105:20)",
"Request.emit (/var/task/node_modules/aws-sdk/lib/sequential_executor.js:77:10)",
"Request.emit (/var/task/node_modules/aws-sdk/lib/request.js:678:14)",
"Request.transition (/var/task/node_modules/aws-sdk/lib/request.js:22:10)",
"AcceptorStateMachine.runTo (/var/task/node_modules/aws-sdk/lib/state_machine.js:14:12)",
"/var/task/node_modules/aws-sdk/lib/state_machine.js:26:10",
"Request.<anonymous> (/var/task/node_modules/aws-sdk/lib/request.js:38:9)",
"Request.<anonymous> (/var/task/node_modules/aws-sdk/lib/request.js:680:12)"
]
}
I have followed the document format of
var item = {
type: 'add',
id: key,
fields: {
userid: value.userId,
storyid: value.storyId,
description: value.description,
title: value.title,
type: 'xyz'
}
}
This is the code I am using to upload the data
exports.handle = function(e, ctx, cb) {
ctx.callbackWaitsForEmptyEventLoop = false;
var documentsBatch = e.data;
var params = {
contentType: 'application/json',
documents: JSON.stringify(documentsBatch)
};
var req = cloudsearchdomain.uploadDocuments(params, function(err, data) {
if (err){
// an error occurred
cb(err, null);
}else{
// successful response
}
});
req.send();
}
My stringified data when logged looks something similar to
[
{
"type": "add",
"id": "FpgAxxxxKrM4utxosPy23--KhO6FgvxK",
"fields": {
"userid": "FpgARscKlxaxutxosPy23",
"storyid": "-KhxbPpRP7REEK",
"description": "xyz 🔥 🔥",
"title": "umm",
"type": "story"
}
},
{
"type": "add",
"id": "FccccxosPy23--KiYbrrPjtJVk2bghO-W",
"fields": {
"userid": "FpgARfPy23",
"storyid": "-KiYbrfggO-W",
"description": "noo",
"title": "lalaa out",
"type": "story"
}
}
]
Can someone point me in the right direction?
The problem was with another JSON object which had an additional JSON attribute other than fields. Once I was able to find and remove it everything worked. There should be a lint-er for the same, or the SDK should throw a better exception.
Related
I want to remove items (an object) from an array on a document in elasticsearch, however whenever I try and run my update script using painless, I receive an Array Index Out of Bounds exception.
I'm using the javascript elasticsearch npm package to search elasticsearch for the relevant documents which then returns me data like:
"_index": "centres",
"_type": "doc",
"_id": "51bc77d1-b514-4f4e-85fa-412def6829f5",
"_score": 1,
"_source": {
"id": "cbaa7daa-f1a2-4ac3-8d7c-fc981245d21c",
"name": "Five House",
"openDays": [
{
"title": "new open Day",
"endDate": "2022-03-22T00:00:00.000Z",
"id": "82be934b-eeb1-419c-96ed-a58808b30df7"
},
{
"title": "last open Day",
"endDate": "2020-12-24T00:00:00.000Z",
"id": "8cc339b9-d2f8-4252-b68a-ed0a49cbfabd"
}
]
}
I then want to go through and remove certain items from the openDays array. I've created an array of the items I want to remove, so for the above example:
[
{
id: '51bc77d1-b514-4f4e-85fa-412def6829f5',
indexes: [
{
"title": "last open Day",
"endDate": "2020-12-24T00:00:00.000Z",
"id": "8cc339b9-d2f8-4252-b68a-ed0a49cbfabd"
}
]
}
]
I'm then trying to run an update via the elasticsearch node client like this:
for (const centre of updates) {
if (centre.indexes.length) {
await Promise.all(centre.indexes.map(async (theIndex) => {
const updated = await client.update({
index: 'centres',
type: 'doc',
id: centre.id,
body: {
script: {
lang: 'painless',
source: "ctx._source.openDays.remove(ctx._source.openDays.indexOf('openDayID'))",
params: {
"openDayID": theIndex.id
}
}
}
}).catch((err) => {throw err;});
}))
.catch((err) => {throw err;});
await client.indices.refresh({ index: 'centres' }).catch((err) => { throw err;});
}
}
When I run this though, it returns a 400 with an "array_index_out_of_bounds_exception" error:
-> POST http://localhost:9200/centres/doc/51bc77d1-b514-4f4e-85fa-412def6829f5/_update
{
"script": {
"lang": "painless",
"source": "ctx._source.openDays.remove(ctx._source.openDays.indexOf(\u0027openDayID\u0027))",
"params": {
"openDayID": "8cc339b9-d2f8-4252-b68a-ed0a49cbfabd"
}
}
}
<- 400
{
"error": {
"root_cause": [
{
"type": "remote_transport_exception",
"reason": "[oSsa7mn][172.17.0.2:9300][indices:data/write/update[s]]"
}
],
"type": "illegal_argument_exception",
"reason": "failed to execute script",
"caused_by": {
"type": "script_exception",
"reason": "runtime error",
"script_stack": [],
"script": "ctx._source.openDays.remove(ctx._source.openDays.indexOf(\u0027openDayID\u0027))",
"lang": "painless",
"caused_by": {
"type": "array_index_out_of_bounds_exception",
"reason": null
}
}
},
"status": 400
}
I'm not quite sure where I'm going wrong with this. Am I using the indexOf painless script correctly? Does indexOf allow for the searching of properties on objects in arrays?
I stumbled across this question and answer: Elasticsearch: Get object index with Painless script
The body of the update script needs changing like so:
Promise.all(...
const inline = `
def openDayID = '${theIndex.id}';
def openDays = ctx._source.openDays;
def openDayIndex = -1;
for (int i = 0; i < openDays.length; i++)
{
if (openDays[i].id == openDayID)
{
openDayIndex = i;
}
}
if (openDayIndex != -1) {
ctx._source.openDays.remove(openDayIndex);
}
`;
const updated = await client.update({
index: 'centres',
type: 'doc',
id: centre.id,
body: {
script: {
lang: 'painless',
inline: inline,
},
}
}).catch((err) => {throw err;});
await client.indices.refresh({ index: 'centres' }).catch((err) => { throw err;});
})).catch(... //end of Promise.all
I am not au fait with painless scripting, so there are most likely better ways of writing this e.g. breaking once the index of the ID is found.
I have also had to move the refresh statement into the Promise.all since if you're trying to remove more than one item from the array of objects, you'll be changing the document and changing the index. There is probably a better way of dealing with this too.
'openDayID' should be params.openDayID
And use removeIf:
"ctx._source.openDays.removeIf(el -> (el.id == params.openDayID))"
I want to make pagination of "car_types" using aws dynamo db and node js. I don't want to use js, Can we make it using dynamo db ? I want total items, total page, page size, current page and data in response.
{
"uid": "222-3333",
"car_types": [
{
"description": "fsdf",
"title": "sdfsd"
},
{
"description": "fdfdfdf",
"title": "dfdfd"
},
{
"description": "dasda",
"title": "asdas"
},
{
"description": "dasd",
"title": "asdas"
},
{
"description": "dasdasd",
"title": "asdas"
}
]
}
Aws Dynamo DB and Node js Code, Which I used to get result.
export function get_car_types_list(){
var params = {
TableName : "cms_cars",
KeyConditionExpression: "#uid = :uid",
ExpressionAttributeNames:{
"#uid": "uid"
},
ExpressionAttributeValues: {
":uid": "222-3333"
}
};
return docClient.query(params).promise()
.then(function(data) {
console.log(data);
return data;
}).catch( (err) => {
console.log('got Error', err);
});
}
I want Result using dynamo db query:
{
"totalItem":5,
"totalPage":1,
"pageSize":"1",
"currentPage":"1",
"car_types": [
{
"description": "fsdf",
"title": "sdfsd"
},
{
"description": "fdfdfdf",
"title": "dfdfd"
},
{
"description": "dasda",
"title": "asdas"
},
{
"description": "dasd",
"title": "asdas"
},
{
"description": "dasdasd",
"title": "asdas"
}
]
}
DynamoDb will return 1 mb data when scan/query is executed, also LastEvaluatedKey is added to result if there are any remaining data. If you pass ExclusiveStartKey: LastEvaluatedKey you can scan/query with pagination. I added some tweaks to your approach, it may help you.
Edit: You can limit the result by passing Limit: Number to your params. This will allow you to limit the returning item count and you can get more with LastEvaluatedKey.
export function get_car_types_list(LastEvaluatedKey){
var params = {
TableName : "cms_cars",
KeyConditionExpression: "#uid = :uid",
ExpressionAttributeNames:{
"#uid": "uid"
},
ExpressionAttributeValues: {
":uid": "222-3333"
},
Limit: 5,
};
if (LastEvaluatedKey) {
params.ExclusiveStartKey = LastEvaluatedKey;
}
return docClient.query(params).promise()
.then(function(data) {
console.log(data);
return data;
}).catch( (err) => {
console.log('got Error', err);
});
}
I followed this link to get the mapping of the Elastic Search Index and would like to insert data(not include all fields) into it as below, but failed.
Mapping of index:
{
"mymapping": {
"mappings": {
"_meta": {
"beat": "apm",
"version": "7.5.0"
},
"dynamic_templates": [
{
"labels": {
"path_match": "labels.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"properties": {
"#timestamp": {
"type": "date"
},
"people": {
"dynamic": "false",
"properties": {
"id": {
"type": "keyword",
"ignore_above": 1024
},
"name": {
"type": "keyword",
"ignore_above": 1024
}
}
}
}
}
}
}
I prepared client.js without problem, here is InsertData.js:
const esClient = require('./client');
const insertDoc = async function(indexName, _id, mappingType, data){
return await esClient.index({
index: indexName,
type: mappingType,
id: _id,
body: data
});
}
module.exports = insertDoc;
async function test(){
const data = {
beat: 'apm',
version: '7.5.0'
}
try {
const resp = await insertDoc('mymapping', 2, '_meta', data);
console.log(resp);
} catch (e) {
console.log(e);
}
}
test();
When I tried to insert data, there was exception.
Error output:
message:
'[illegal_argument_exception] Rejecting mapping update to [mymapping] as the final mapping would have more than 1 type: [_doc, _meta]',
path: '/mymapping/_doc/2',
query: { type: '_meta' },
body:
{ error:
{ root_cause: [Array],
type: 'illegal_argument_exception',
reason:
'Rejecting mapping update to [mymapping] as the final mapping would have more than 1 type: [_doc, _meta]' },
status: 400 },
statusCode: 400,
response:
'{
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "Rejecting mapping update to [mymapping] as the final mapping would have more than 1 type: [_doc, _meta]"
}
],
"type": "illegal_argument_exception",
"reason": "Rejecting mapping update to [mymapping] as the final mapping would have more than 1 type: [_doc, _meta]"
},
"status": 400
}'
How can insert data properly?
Starting elasticsearch 6+ multiple types have been deprecated. You are trying to explicitly put the type while creating the mapping which is meta and the one which is getting inserted be default is _doc.
See this for more details : Removal of types
Hi I am new to nodejs and mongodb, I have json file with below structure,
I have defined one shipment schema with the "comments" section as nested schema
{
"buyerId": "B58",
"sellerId": "SL8",
"comments": {
"title": "title5",
"body": "body5",
"date": "12-07-2017"
}
}
I have defined one function like below
exports.post = function(req, res) {
const comments = []
var s = new shipment();
s.sellerId = req.body.sellerId;
s.buyerId = req.body.buyerId;
s.poId = req.body.poId;
s.comments.push({
title: req.body.comments.title,
body: req.body.comments.body,
date: req.body.comments.date
});
s.save(function(err) {
if (err) {
res.send(err);
}
console.log("added");
res.send({
message: 'shipment Created !'
})
})
}
The above 'post' function will work properly when I have only one "comments" section, I mean the data gets properly
inserted into the mongodb as shown below
{
"_id": ObjectId("59689bc59058dbc812000002"),
"buyerId": "B58",
"sellerId": "SL8",
"comments": [{
"title": "title5",
"body": "body5",
"date": ISODate("2017-12-06T18:30:00Z"),
"_id": ObjectId("59689bc59058dbc812000003")
}],
"__v": 0
}
but when I have multiple "comments" section as shown below,
{
"buyerId": "B58",
"sellerId": "SL8",
"comments": [{
"title": "title5",
"body": "body5",
"date": "12-07-2017"
},
{
"title": "title8",
"body": "body7",
"date": "12-07-2017"
}
]
}
then no comments section gets inserted into the mongodb as shown below.
{
"_id": ObjectId("5968c04d4c02336800000002"),
"buyerId": "B57",
"sellerId": "SL7",
"comments": [{
"_id": ObjectId("5968c04d4c02336800000003")
}],
"__v": 0
}
what changes should I do in the function to get all the comments section being inserted into the mongodb properly ?
Instead of assigning value of every property, make instance and pass body directly into it.
const s = new shipment(req.body)
And then when you send data to the request, send in the following format
{ "buyerId": "B58", "sellerId": "SL8", "comments": [{ "title": "title5", "body": "body5", "date": "12-07-2017" }, { "title": "title8", "body": "body7", "date": "12-07-2017" } ] }
I tried like below and it worked.
for( var i = 0; i < req.body.comments.length; i++){
s.comments.push(
{ title: req.body.comments[i].title,
body : req.body.comments[i].body,
date : req.body.comments[i].date });
}
originally comments is an array in the second example comments is an array.
your function
s.comments.push({
title: req.body.comments.title,
body: req.body.comments.body,
date: req.body.comments.date
})
will only work if comments is an object. Put that in a for loop to make it work with arrays like so
for( var i = 0; i < req.body.comments.length; i++){
s.comments.push({
title: req.body.comments[i].title,
body: req.body.comments[i].body,
date: req.body.comments[i].date
})
}
I'm trying to set up a small api from AWS Lambda to DynamoDB and I am having trouble figuring out if and how I can write an array of objects into a key.
I have an object like
{
"teamName": "Team Awesome",
"members": [
{
"email": "person-1#example.com",
"name": "Bob"
},
{
"email": "person-2#example.com",
"name": "Alice"
}
]
}
The members array is giving me issues, in the docs it looks like it can be done considering the list types, but there is just no example how HOW to do it, and I am running out of ways to try it.
So is it possible to write something in this format at all and how do you in that case do it?
Example code - what do I put at ???
var AWS = require('aws-sdk');
var dynamodb = new AWS.DynamoDB();
exports.handler = function(event, context) {
var tableName = "GDCCompetition";
var datetime = new Date().getTime().toString();
DynamoDB.putItem({
"TableName": tableName,
"Item": {
"datetime": {
"N": datetime
},
"teamName": {
"S": event.teamName
},
"members": ???
}
});
}
The documentation is not really obvious, but there is a thing called DocClient, you can pass a usual JS object to it and it will do all the parsing and transformation into AWS object (with all the types). You can use it like this:
var AWS = require("aws-sdk");
var DynamoDB = new AWS.DynamoDB.DocumentClient();
var params = {
TableName: "MyTable",
Item: {
"teamName": "Team Awesome",
"members": [
{
"email": "person-1#example.com",
"name": "Bob"
},
{
"email": "person-2#example.com",
"name": "Alice"
}
]
}
};
DynamoDB.put(params, function (err) {
if (err) {
return throw err;
}
//this is put
});
You could convert the object to DynamoDb record first
const AWS = require('aws-sdk');
var tableName = "GDCCompetition";
var datetime = new Date().getTime().toString();
const members = [
{
"email": "person-1#example.com",
"name": "Bob"
},
{
"email": "person-2#example.com",
"name": "Alice"
}
];
const marshalled = AWS.DynamoDB.Converter.marshall({ members });
const params = {
"TableName": tableName,
"Item": {
"datetime": {
"N": datetime
},
"teamName": {
"S": event.teamName
},
"members": marshalled.members,
},
}
AWS.DynamoDB.putItem(params, function (err) {
if (err) {
return throw err;
}
});