I can't modify the filtering parameter with
String.replace()
I can get the filtering keys from the URL as an object but its badly fromatted from me.
Filtering: {{URL}}/api/v1/bootcamps?averageCost[lt]=10000
Current format: { averageCost: { lt: '10000' } }
Right fromat: { averageCost: { $lt: '10000' } }
So I tried to convert it as a String and replace that value. But that value can be: lt, lte, gt, gte, in but it has some problem because the line after the .replace() method doesnt executed and of course the catch block cathes the error...
My code snippet:
try {
console.log(req.query);
const queryStr = JSON.stringify(req.query);
console.log(queryStr); //thats the last thing I get
queryStr = queryStr.replace(
/\b(gt|gte|lt|lte|in)\b/g,
match => `$${match}`
);
console.log(queryStr); // I dont get this
const bootcamps = await Bootcamp.find();
res.status(200).json({
succes: true,
count: bootcamps.length,
data: bootcamps
});
} catch (err) {
return res.status(404).json({ succes: false });
}
To replace it correctly you should use something like this:
let queryStr = '{ "averageCost": { "lt": "10000" }, "test": { "gt": "12345"} }';
const regex = /\b(gt|gte|lt|lte|in)\b/g;
queryStr = queryStr.replace(regex, '$$' + "$1"); // <-- here is the correct replace
This will replace queryStr with:
{ "averageCost": { "$lt": "10000" }, "test": { "$gt": "12345"} }
JSFiddle https://jsfiddle.net/c52z8ewr/
If you need the object back just do JSON.parse(queryStr)
You can also Try This
const queryCpy = { ...this.query };
// console.log(queryCpy, 'before filter');
console.log(queryCpy, 'after filter');
let queryString = JSON.stringify(queryCpy);
queryString = queryString.replace(
/\b(gt|gte|lt|lte)\b/g,
(rep) => `$${rep}`,
);
console.log(queryString, 'after filter');
If want an object in return do:
const bootcamps = await Bootcamp.find(JSON.parse(queryStr));
Related
I've used this approach, it should give me the Expected Output which I've mentioned below.
But, due to asynchronous execution, it is giving the Actual Output.
So, Please help me to solve the same.
See, I'm calling one async function, inside i'm running three nested map loops. And in the third loop I'm connecting the database and fetching values. Based on the value, I'm generating the values of the Object which is empty at the first. And once the function called, it is triggering the then method.That's where the problem arises, I need that then function to be executed after the called function executed completely. This is the complete problem statement
Given the following code:
let messageObject = {};
async function storeManager() {
// storing the defects or modifying
console.log('hii from storeManager()');
await Promise.all(
Object.keys(filledDefects).map((defectName) => {
Object.keys(filledDefects[defectName]).map((subDefectName) => {
Object.keys(filledDefects[defectName][subDefectName]).map(
async (zone) => {
const result = await dbConnectedPool.query(
`SELECT * FROM defect_table WHERE body_number=${enteredBodyNumber} AND category='${selectedCategory}' AND subcategory='${selectedSubCategory}' AND defect='${defectName}' AND subdefect='${subDefectName}' AND zone = ${zone.replace(
'_',
''
)}`
);
if (result.rows.length == 0) {
// block to save defects record for the first time
console.log(
`INSERT INTO defect_table (body_number,mode,category,subcategory,defect,subdefect,zone,defectCount,date,time,username) VALUES (${enteredBodyNumber},'${mode}','${selectedCategory}','${selectedSubCategory}','${defectName}','${subDefectName}',${zone.replace(
'_',
''
)},${
filledDefects[defectName][subDefectName][zone]
},'${date}','${time}','${username}');`
);
await dbConnectedPool.query(
`INSERT INTO defect_table (body_number,mode,category,subcategory,defect,subdefect,zone,defectCount,date,time,username) VALUES (${enteredBodyNumber},'${mode}','${selectedCategory}','${selectedSubCategory}','${defectName}','${subDefectName}',${zone.replace(
'_',
''
)},${
filledDefects[defectName][subDefectName][zone]
},'${date}','${time}','${username}');`
);
mod.set(
messageObject,
`Newly Saved Zone.${zone}.${defectName}.${subDefectName}`,
filledDefects[defectName][subDefectName][zone]
);
console.log('inside: ', messageObject);
} else {
// block to modify existing defect records
console.log(
`UPDATE defect_table SET defectCount=${
filledDefects[defectName][subDefectName][zone]
},date='${date}',time='${time}',username='${username}' WHERE body_number=${enteredBodyNumber} AND category='${selectedCategory}' AND subcategory='${selectedSubCategory}' AND defect='${defectName}' AND subdefect='${subDefectName}' AND zone=${zone.replace(
'_',
''
)}`
);
await dbConnectedPool.query(
`UPDATE defect_table SET defectCount=${
filledDefects[defectName][subDefectName][zone]
},date='${date}',time='${time}',username='${username}' WHERE body_number=${enteredBodyNumber} AND category='${selectedCategory}' AND subcategory='${selectedSubCategory}' AND defect='${defectName}' AND subdefect='${subDefectName}' AND zone=${zone.replace(
'_',
''
)}`
);
mod.set(
messageObject,
`Overwritten Zone.${zone}.${defectName}.${subDefectName}`,
filledDefects[defectName][subDefectName][zone]
);
console.log('inside: ', messageObject);
}
// checking whether already record exists with same aspects
}
);
});
})
);
console.log('bye from storeManager()');
}
storeManager().then(() => {
console.log('message outside:', messageObject);
});
Expected Output:
hii from storeManager()
bye from storeManager()
UPDATE defect_table SET defectCount=12,date='2022-10-12',time='12:52:33',username='Vasanth'
WHERE body_number=1234 AND category='LH SHELL BODY MAIN-LINE' AND subcategory='FENDER - LH
SBML' AND defect='Surface' AND subdefect='Dent' AND zone=210
inside: { 'Overwritten Zone': { _210: { Surface: [Object] } } }
UPDATE defect_table SET defectCount=12,date='2022-10-12',time='12:52:33',username='Vasanth'
WHERE body_number=1234 AND category='LH SHELL BODY MAIN-LINE' AND subcategory='FENDER - LH
SBML' AND defect='Surface' AND subdefect='Dent' AND zone=215
inside: {
'Overwritten Zone': { _210: { Surface: [Object] }, _215: { Surface: [Object] } }
}
message outside: {
'Overwritten Zone': { _210: { Surface: [Object] }, _215: { Surface: [Object] } }
}
Actuall Output:
hii from storeManager()
bye from storeManager()
message outside: {}
UPDATE defect_table SET defectCount=12,date='2022-10-12',time='12:52:33',username='Vasanth'
WHERE body_number=1234 AND category='LH SHELL BODY MAIN-LINE' AND subcategory='FENDER - LH
SBML' AND defect='Surface' AND subdefect='Dent' AND zone=210
inside: { 'Overwritten Zone': { _210: { Surface: [Object] } } }
UPDATE defect_table SET defectCount=12,date='2022-10-12',time='12:52:33',username='Vasanth'
WHERE body_number=1234 AND category='LH SHELL BODY MAIN-LINE' AND subcategory='FENDER - LH
SBML' AND defect='Surface' AND subdefect='Dent' AND zone=215
inside: {
'Overwritten Zone': { _210: { Surface: [Object] }, _215: { Surface: [Object] } }
}
You'll need to use Promise.all everywhere you are producing an array of promises, not just on the outermost call. And you'll need to make the map callbacks actually return those promises!
await Promise.all(Object.entries(filledDefects).map(async ([defectName, defect]) => {
await Promise.all(Object.entries(defect).map(async ([subDefectName, subDefect]) => {
await Promis.all(Object.entries(subDefect).map(async ([zoneName, zone]) => {
await …;
}));
}));
}));
Alternatively you can also write this without some of the async/await:
await Promise.all(Object.entries(filledDefects).map(([defectName, defect]) =>
Promise.all(Object.entries(defect).map(async ([subDefectName, subDefect]) =>
Promis.all(Object.entries(subDefect).map(async ([zoneName, zone]) => {
await …;
}));
));
));
Promise.all expects an array of promises as argument, but you pass it an array of undefined values instead. That means Promise.all will return a promise that is resolved immediately.
Two causes for this problem:
the outer map callbacks don't have a return statement (so they map each defectName and each subDefectName to undefined)
If you would return the result of the inner .map calls, then each defectName maps to an array (of arrays ...), which still isn't what you need. You don't want a nested array, but a flat array, so use return Object.keys().flatMap instead of Object.keys().map
i have one problem in result of my code, i tring consult one script ORACLE with nodejs using TS but i don't know why this error apear in my console i tring many ways to fix this error and i can't fix them, i hope your can help me whit this, bellow follow my code and screenshot of my error.
Controller
async bipagem(req: Request, res: Response) {
try {
let credentials = super.openToken(req);
let { p_fil_filial, p_set_cdgo, p_mini_fab, p_codigo_barra } = req.query;
let info = await this.rep.bipagem(
p_fil_filial as string,
p_set_cdgo as string,
p_mini_fab as string,
p_codigo_barra as string,
credentials as string
);
res.json(info);
} catch (error) {
catchErr(res, error);
}
}
Repository
public async bipagem(
p_fil_filial: string,
p_set_cdgo: string,
p_mini_fab: string,
p_codigo_barra: string,
userPool: string
) {
let conn;
try {
conn = await connection(userPool);
const resultado = await conn.execute(
`DECLARE
result SYS_REFCURSOR;
BEGIN
-- Call the function
:result := brio.pck_fab0024.bipagem(p_fil_filial => :p_fil_filial,
p_set_cdgo => :p_set_cdgo,
p_mini_fab => :p_mini_fab,
p_codigo_barra => :p_codigo_barra,
p_msg => :p_msg);
END;`,
{
p_fil_filial,
p_set_cdgo,
p_mini_fab,
p_codigo_barra,
p_msg: { type: oracledb.STRING, dir: oracledb.BIND_OUT },
}
);
return resultado;
} catch (erro) {
console.log(erro);
} finally {
if (conn) conn.close();
}
}
Screenshot error
ORA-01008 means "not all variables bound". It looks like you have 6 variables in your PL/SQL block, but only 5 variables that are assigned to those. :result is not bound.
const body = {
query: {
geo_shape: {
geometry: {
relation: 'within',
shape: {
type: 'polygon',
coordinates: [$polygon],
},
},
},
},
pit: {
id: "t_yxAwEPZXNyaS1wYzYtMjAxN3IxFjZxU2RBTzNyUXhTUV9XbzhHSk9IZ3cAFjhlclRmRGFLUU5TVHZKNXZReUc3SWcAAAAAAAALmpMWQkNwYmVSeGVRaHU2aDFZZExFRjZXZwEWNnFTZEFPM3JReFNRX1dvOEdKT0hndwAA",
keep_alive: "1m",
},
};
Query fails with search_phase_execution_exception at onBody
Without pit query works fine but it's needed to retrieve more than 10000 hits
Well, using PIT in NodeJS ElasticSearch's client is not clear, or at least is not well documented. You can create a PIT using the client like:
const pitRes = await elastic.openPointInTime({
index: index,
keep_alive: "1m"
});
pit_id = pitRes.body.id;
But there is no way to use that pit_id in the search method, and it's not documented properly :S
BUT, you can use the scroll API as follows:
const scrollSearch = await elastic.helpers.scrollSearch({
index: index,
body: {
"size": 10000,
"query": {
"query_string": {
"fields": [ "vm_ref", "org", "vm" ],
"query": organization + moreQuery
},
"sort": [
{ "utc_date": "desc" }
]
}
}});
And then read the results as follows:
let res = [];
try {
for await (const result of scrollSearch) {
res.push(...result.body.hits.hits);
}
} catch (e) {
console.log(e);
}
I know that's not the exact answer to your question, but I hope it helps ;)
The usage of point-in-time for pagination of search results is now documented in ElasticSearch. You can find more or less detailed explanations here: Paginate search results
I prepared an example that may give an idea about how to implement the workflow, described in the documentation:
async function searchWithPointInTime(cluster, index, chunkSize, keepAlive) {
if (!chunkSize) {
chunkSize = 5000;
}
if (!keepAlive) {
keepAlive = "1m";
}
const client = new Client({ node: cluster });
let pointInTimeId = null;
let searchAfter = null;
try {
// Open point in time
pointInTimeId = (await client.openPointInTime({ index, keep_alive: keepAlive })).body.id;
// Query next chunk of data
while (true) {
const size = remained === null ? chunkSize : Math.min(remained, chunkSize);
const response = await client.search({
// Pay attention: no index here (because it will come from the point-in-time)
body: {
size: chunkSize,
track_total_hits: false, // This will make query faster
query: {
// (1) TODO: put any filter you need here (instead of match_all)
match_all: {},
},
pit: {
id: pointInTimeId,
keep_alive: keepAlive,
},
// Sorting should be by _shard_doc or at least include _shard_doc
sort: [{ _shard_doc: "desc" }],
// The next parameter is very important - it tells Elastic to bring us next portion
...(searchAfter !== null && { search_after: [searchAfter] }),
},
});
const { hits } = response.body.hits;
if (!hits || !hits.length) {
break; // No more data
}
for (hit of hits) {
// (2) TODO: Do whatever you need with results
}
// Check if we done reading the data
if (hits.length < size) {
break; // We finished reading all data
}
// Get next value for the 'search after' position
// by extracting the _shard_doc from the sort key of the last hit
searchAfter = hits[hits.length - 1].sort[0];
}
} catch (ex) {
console.error(ex);
} finally {
// Close point in time
if (pointInTime) {
await client.closePointInTime({ body: { id: pointInTime } });
}
}
}
I'm absolutely brand new to DynamoDb and I'm trying to simply write an object from a NodeJS Lambda. Based on what I've read and researched I should probably be using DocumentClient from the aws-sdk. I also found the following question here regarding issues with DocumentClient, but it doesn't seem to address my specific issue....which I can't really find/pinpoint unfortunately. I've set up a debugger to help with SAM local development, but it appears to be only providing some of the errors.
The code's implementation is shown here.
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": {"S": randstring.generate(9)},
"School":{"S": team_name},
"Seed": {"S": seed},
"ESPN_Id": {"S": espn_id}
}
}
console.log(JSON.stringify(params))
dynamodb.put(params, (error,data) => {
if (error) {
console.log("Error ", error)
} else {
console.log("Success! ", data)
}
})
Basically I'm scrubbing a website utilizing cheerio library and cherry picking values from the DOM and saving them into the json object shown below.
{
"TableName": "March-Madness-Teams",
"Item": {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
When I attempt to push this json object to Dynamo, I get errors says
Error MultipleValidationErrors: There were 2 validation errors:
* MissingRequiredParameter: Missing required key 'TableName' in params
* MissingRequiredParameter: Missing required key 'Item' in params
The above error is all good in well....I assume it didn't like the fact that I had wrapped those to keys in strings, so I removed the quotes and sent the following
{
TableName: "March-Madness-Teams",
Item: {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
However, when I do that...I kind of get nothing.
Here is a larger code snippet.
return new Promise((resolve,reject) => {
axios.get('http://www.espn.com/mens-college-basketball/bracketology')
.then(html => {
const dynamodb = new aws.DynamoDB.DocumentClient()
let $ = cheerio.load(html.data)
$('.region').each(async function(index, element){
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(async function(index2, element2){
var seed = $(element2).siblings('span.rank').text()
if (seed.length > 2){
seed = $(element2).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element2).attr('href').split('/').slice(-2)[0]
var team_name = $(element2).text()
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School":team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
}
console.log(JSON.stringify(params))
// dynamodb.put(params)
// .then(function(data) {
// console.log(`Success`, data)
// })
})
})
})
})
Can you try without the type?
Instead of
"School":{"S": team_name},
for example, use
"School": team_name,
From your code, I can see the mis promise on the dynamodb request. Try to change your lines :
dynamodb.put(params).then(function(data) {
console.log(`Success`, data)
})
to be :
dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
you can combine with await too :
await dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
exports.lambdaHandler = async (event, context) => {
const html = await axios.get('http://www.espn.com/mens-college-basketball/bracketology')
let $ = cheerio.load(html.data)
const schools = buildCompleteSchoolObject(html, $)
try {
await writeSchoolsToDynamo(schools)
return { statusCode: 200 }
} catch (error) {
return { statusCode: 400, message: error.message }
}
}
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(async school => {
await dynamodb.put(school).promise()
})
await Promise.all(promises)
}
const buildCompleteSchoolObject = (html, $) => {
const schools = []
$('.region').each(loopThroughSubRegions(schools, $))
return schools
}
const loopThroughSubRegions = (schools, $) => {
return (index, element) => {
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(populateSchoolObjects(schools, $))
}
}
const populateSchoolObjects = (schools, $) => {
return (index, element) => {
var seed = $(element).siblings('span.rank').text()
if (seed.length > 2) {
seed = $(element).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element).attr('href').split('/').slice(-2)[0]
var team_name = $(element).text()
schools.push({
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School": team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
})
}
}
I know this is drastically different from what I started with but I did some more digging and kind of kind of worked to this...I'm not sure if this is the best way, but I seemed to get it to work...Let me know if something should change!
Oh I understand what you want.
Maybe you can see the code above works, but there is one concept you have to improve here about async - await and promise especially on lambda function.
I have some notes here from your code above, maybe can be your consideration to improve your lambda :
Using await for every promise in lambda is not the best approach because we know the lambda time limitation. But sometimes we can do that for other case.
Maybe you can change the dynamodb.put method to be dynamodb.batchWriteItem :
The BatchWriteItem operation puts or deletes multiple items in one or more tables.
Or If you have to use dynamodb.put instead, try to get improve the code to be like so :
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(school => {
dynamodb.put(school).promise()
})
return Promise.all(promises)
}
Still very much a beginner, Just editing to make a little more sense
Here is an example payload.
{
"Status":
{
"result":[
{
"LastUpdate":"2017-09-07 06:47:09",
"Type":"2",
"Value":"' s the inside temperature",
"idx":"4"
}
],
"status":"OK",
"title":"GetUserVariable"
},
"Devices":
{
"28":
{
"rid":"28",
"regex":"(the )?(AC|(Air( )?(Con)?(ditioner)?))( Power)?( )?$/i"
},
"71":
{
"rid":"71",
"regex":"(the )?inside temp/i"
}
}
}
I want to filter the "Devices" array down to entires that match Status.result[0].Value.
I have the follow code working, but it is in working in reverse, it returns the matching string, not the filtered array, just not sure how to reverse it.
var devices = msg.payload.Devices;
var request = [ msg.payload.Status.result[0].Value ];
var matches = request.filter(function (text) {
return devices.some(function (regex) {
var realregex = new RegExp(regex, "i");
return realregex.test(text);
});
});
msg = { topic:"Inputs", payload: { devices:devices, request:request } };
msg2 = { topic:"Output", payload: { matches:matches } };
return [ [ msg, msg2 ] ];
Thanks,
Wob