NodeJS organize loop with repeated records - node.js

I have a table where I store the sales of certain products. I need to fetch the records by client id.
I did like this:
public async getData(clientID: any): Promise<any> {
try {
return await client
.scan({
TableName: "dbSales",
FilterExpression: "contains(clientID, :clientID)",
ExpressionAttributeValues: {
":clientID": clientID,
},
})
.promise()
.then(async (response) => {
let data = [];
for (let i = 0; i < response.Count; i++) {
if(data.filter(product => product.productId == response.Items[i]['productId']) != undefined){
const resultData = await this.getProduct(response.Items[i]['productId']);
data.push(resultData);
}
}
return {
status: 200,
data: data,
};
})
.catch((error: AxiosError) => {
throw error;
});
} catch (e) {
throw new HttpError(500, e.message);
}
}
There it will get all the records with that customer id, and then through a condition it will get the product name of each record it found, through another function (getProduct), with the repetition loop.
The problem is that I'm getting a lot of repeated results, i.e. for the same product id and customer.
I need that when there is more than one record in dbSales with the same customer id and with the same product id, it returns me only one result for those records.
That is, one row for each record that contains the same customer ID and product ID.
This is generating a long delay in the search for data, as I only need which products a particular customer has purchased, without the information being repeated when he has more purchases of the same product.

Problem solved! Solution:
The solution presented by our friend #jarmod solved my problem, in a simple way.
let result = response.Items.filter((e, i) => {
return response.Items.findIndex((x) => {
return x.productId == e.productId}) == i;
});
for (let i = 0; i < result.length; i++) {
if(data.filter(product => product.productId == result[i]['productId']) != undefined){
const resultData = await this.getProduct(result[i]['productId']);
data.push(resultData);
}
}
Proposed solution: How to remove duplicates objects in array based on 2 properties?

Related

Updating fields in array -NodeJs

I have this code that loop through all users in DB then look for specific events based on the id value and if there is a match it should update the field caption with a new given data,
for the code :
1- search all potential user = OK
2 - search and find events based on id = OK
3- update the field caption = NOK
this is my code hope I mentioned everything
router.post('/delete/:id',async (req, res) => {
const eventId = req.params.id // this is the ID of the event
User.find({}).lean(true).exec((err, users) => {
let getTEvent = [];
for (let i = 0; i < users.length; i++) {
if (users[i].events && users[i].events.length) {
for (let j = 0; j < users[i].events.length; j++) {
if (users[i].events[j]._id === eventId) {
console.log('you event is :'+ eventId) // this statement to verify if really got the correct ID or not
users[i].events[j].caption ="deleted" // from here the problem
users[i].save(err => {
if (err) throw err;
console.log("status changed saved");
// Redirect back after the job is done.
});
}
}
}
}
});
})
the error that I get is that users[i].save is not a function I don't know with what should I replace it,
As per comments #whoami
router.post('/delete/:id', (req, res) =>
User.findOneAndUpdate({
"events._id": req.params.id
},
{ $set: { "events.caption": "yesssss" }
}, {upsert: true}, (err, user) => {
if (err) {
res.send('error updating ');
} else {
console.log(user);
console.log(req.params.id)
}
}));
Below the mongoDb and event datastructure
Hope I clarified everything ,
Best Regards,
Actual Issue :
.save() works on mongoose document but not on javaScript object. In your code you've already converted mongoose documents returned from .find() call to .Js objects using : .lean(true).
.lean(true) is used to convert mongoose docs to .Js objects to work manipulate fields inside docs in code.
Fixing code :
const mongoose = require('mongoose');
router.post("/delete/:id", async (req, res) => {
const eventId = req.params.id; // this is the ID of the event
await User.find({})
.lean(true)
.exec((err, users) => {
let getTEvent = [];
for (let i = 0; i < users.length; i++) {
if (users[i].events && users[i].events.length) {
for (let j = 0; j < users[i].events.length; j++) {
if (users[i].events[j]._id === eventId) {
console.log("you event is :" + eventId);
users[i].events[j].caption = "deleted";
users[i]._id = mongoose.Types.ObjectId(users[i]._id); // Converting `_id` string to `ObjectId()` to match with type of `_id` in DB
let user = new User(users[i]); // Create a mongoose doc out of model
user.save((err) => { // As we're passing `_id` in doc to `.save()` it will be an update call rather-than insert
if (err) throw err;
console.log("status changed saved");
// Redirect back after the job is done.
});
}
}
}
}
});
});
As I've mentioned this can be done with out this extra process of reading docs/iteration/update call. Using .updateOne() or .updateMany() along with $ positional operator things can be done in one DB call :
const mongoose = require('mongoose');
router.post("/delete/:id", async (req, res) => {
const eventId = req.params.id; // this is the ID of the event
/** We're using `.updateMany` with filter on `events._id` -
* So that all user docs which has `'events._id': eventId` will be updated,
* If you've a particular user needs to be updated used `.updateOne()` with a filter to find that user document - filter kind of `userName`
*
* Both `.updateMany()` or `.update()` will return write result but not the docs,
* if you need docs in response use `.findOneAndUpdate` or `.findAndModify()`
*/
/** `$` helps to update particular object's caption field in `events` array (Object where `{ 'events._id': eventId }` ) */
await User.updateMany({ 'events._id': eventId },{$set : {'events.$.caption': 'deleted'}}).exec((err)=>{
if (err) throw err;
console.log("status changed saved");
// Redirect back after the job is done.
})
});
Ref : update-documents

BatchWrite in AWS dynamo db skipping some items

I am trying to write Items to AWS dynamo db using node SDK. The problem I am facing is that when I write batch items to AWS in parallel using threads, some of the items are not written to database. The number of items are written are random. For instance, If I run my code 3 times, at one time it would be 150, next it would 200 and third time it could be 135. In addition, when I write the items sequentially without threads, even then some of the items are not written.However, in this case the items are less missing. For instance if the total number of items is 300 then the items written are 298. I investigated the problem to see if there any unprocessed items but the batchWrite method returns nothing. It means that all the items are being processed correctly. Please note that I have OnDemand provision for my respective database so I do not expect any throttling issues. So here is my code.
exports.run = async function() {
**This is the function which runs first !!!!!**
const data = await getArrayOfObjects();
console.log("TOTAL PRICE CHANGES")
console.log(data.length)
const batchesOfData = makeBatches(data)
const threads = new Set();
console.log("**********")
console.log(batchesOfData.length)
console.log("**********")
for(let i = 0; i < batchesOfData.length; i++) {
console.log("BATCH!!!!!")
console.log(i)
console.log(batchesOfData[i].length)
// Sequential Approach
const response = await compensationHelper.createItems(batchesOfData[i])
console.log("RESPONSE")
console.log(response)
Parallel approach
// const workerResult = await runService(batchesOfData[i])
// console.log("WORKER RESUULT!!!!")
// console.log(workerResult);
}
}
exports.updateItemsInBatch = async function(data, tableName) {
console.log("WRITING DATA")
console.log(data.length)
const batchItems = {
RequestItems: {},
};
batchItems.RequestItems[tableName] = data;
try {
const result = await documentClient.batchWrite(batchItems).promise();
console.log("UNPROCESSED ITEMS")
console.log(result)
if (result instanceof Error) {
console.log(`[Error]: ${JSON.stringify(Error)}`);
throw new Error(result);
}
return Promise.resolve(true);
} catch (err) {
console.error(`[Error]: ${JSON.stringify(err.message)}`);
return Promise.reject(new Error(err));
}
};
exports.convertToAWSCompatibleFormat = function(data) {
const awsCompatibleData = [];
data.forEach(record => awsCompatibleData.push({ PutRequest: { Item: record } }));
return awsCompatibleData;
};
const createItems = async function(itemList) {
try {
const objectsList = [];
for (let index = 0; index < itemList.length; index++) {
try {
const itemListObj = itemList[index];
const ObjToBeInserted = {
// some data assignments here
};
objectsList.push(ObjToBeInserted);
if (
objectsList.length >= AWS_BATCH_SIZE ||
index === itemList.length - 1
) {
const awsCompatiableFormat = convertToAWSCompatibleFormat(
objectsList
);
await updateItemsInBatch(
awsCompatiableFormat,
process.env.myTableName
);
}
} catch (error) {
console.log(`[Error]: ${JSON.stringify(error)}`);
}
}
return Promise.resolve(true);
} catch (err) {
return Promise.reject(new Error(err));
}
};
const makeBatches = products => {
const productBatches = [];
let countr = -1;
for (let index = 0; index < products.length; index++) {
if (index % AWS_BATCH_SIZE === 0) {
countr++;
productBatches[countr] = [];
if (countr === MAX_BATCHES) {
break;
}
}
try {
productBatches[countr].push(products[index]);
} catch (error) {
continue;
}
}
return productBatches;
};
async function runService(workerData) {
return new Promise((resolve, reject) => {
const worker = new Worker(path.join(__dirname, './worker.js'), { workerData });
worker.on('message', resolve);
worker.on('error', reject);
worker.on('exit', (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with exit code ${code}`));
})
})
}
// My worker file
'use strict';
const { workerData, parentPort } = require('worker_threads')
const creatItems = require('myscripts')
// You can do any heavy stuff here, in a synchronous way
// without blocking the "main thread"
console.log("I AM A NEW THREAD")
createItems(workerData)
// console.log('Going to write tons of content on file '+workerData);
parentPort.postMessage({ fileName: workerData, status: 'Done' })
From boto3 documentation:
If one or more of the following is true, DynamoDB rejects the entire batch write operation:
One or more tables specified in the BatchWriteItem request does not exist.
Primary key attributes specified on an item in the request do not match those in the corresponding table's primary key schema.
You try to perform multiple operations on the same item in the same BatchWriteItem request. For example, you cannot put and delete the same item in the same BatchWriteItem request.
Your request contains at least two items with identical hash and range keys (which essentially is two put operations).
There are more than 25 requests in the batch.
Any individual item in a batch exceeds 400 KB.
The total request size exceeds 16 MB.
To me, it looks some of this is true. At my job, we also had a problem that one batch contained 2 identical primary and secondary keys in the batch so the whole batch was discarded. I know it's not node.js, but we used this to overcome that problem.
It is batch_writer(overwrite_by_pkeys) and it is used to overwrite the last occurance of the same primary and last key in the batch. If only a small portion of your data is duplicate data and you do not need to save it, you can use this. BUT if you need to save all your data, I do not advise you to use this functionality.
I don't see where you are checking the response for UnprocessedItems. Batch operations will often return a list of items it didn't process. As is documented, BatchWriteItem "can write up to 16 MB of data, which can comprise as many as 25 put or delete requests."
I had duplicate keys issue which means that primary and the sort key had duplicate values in the batch, however, in my case this error was not returned from the AWS BatchWrite method if my timestamp was in fraction of seconds 2020-02-09T08:02:36.71, which was a bit surprising. I resolved the issue by making my createdAt(sort key) to be more granular like this => 2020-02-09T08:02:36.7187 Thus making it non-repetitive.

How to add row in google sheet synchronously?

I'm using google-spreadsheet library to access and insert data to Google Sheets. I was able to add multiple rows but each row is being added in no particular order, I guess due to the addRow() async behavior.
Example data is:
let myData = [{id: "1", fname: "John"}, {id: "2", fname: "Matt"}, {id: "3", fname: "Paul"}]
for (let i = 0; i < myData.length; i++) {
let row = myData[i];
// Async call to insert row.
doc.addRow(sheetIndex, row, (err) => {
if (err) throw err;
});
}
For-loop iterates and passes each row object to addRow() but it doesn't care what the order is when each row gets inserted to Googlesheets.
I've tried doing an async-await approach to wait for doc.addRow() before it iterates to next row object but I have no luck. Is there a way to achieve the insertion in the same order I pass it? Thanks in advance! :)
You couldn't await the function probably because it does't return Promise. So, lets promisify that function first.
function asyncAddRow(sheetIndex, row) {
return new Promise((res, rej) => {
doc.addRow(sheetIndex, row, (err) => {
if (err) rej(err);
else res(true);
});
})
}
Now, instead of calling doc.addRow you call our asyncAddRow. Since it returns a promise so you can await it now. Like this:
for (let i = 0; i < myData.length; i++) {
let row = myData[i];
await asyncAddRow(sheetIndex, row);
}
You should use the method append()
let values = [
[
// Cell values ...
],
// Additional rows ...
];
let resource = {
values,
};
this.sheetsService.spreadsheets.values.append({
spreadsheetId,
range,
valueInputOption,
resource,
}, (err, result) => {
if (err) {
// Handle error.
console.log(err);
} else {
console.log(`${result.updates.updatedCells} cells appended.`);
}
});
More info there

Node Postgres COPY FROM failing silently

I am trying to use PostgreSQL's COPY FROM API to stream potentially-thousands of records into a database as they are dynamically generated in node.js code. To do so, I wrote this generic wrapper function:
function streamRows(client, { table, columns, data }) {
return new Promise((resolve, reject) => {
const sqlStream = client.query(
copyFrom(`COPY ${ table } (${ columns.join(', ') }) FROM STDIN`));
const rowStream = new Readable();
rowStream.pipe(sqlStream)
.on('finish', resolve)
.on('error', reject);
for (const row of data) {
rowStream.push(`${ row.join('\t') }\n`);
}
rowStream.push('\\.\n');
rowStream.push(null);
});
}
The database table I'm writing into looks like this:
CREATE TABLE devices (
id SERIAL PRIMARY KEY,
group_id INTEGER REFERENCES groups(id),
serial_number CHAR(12) NOT NULL,
status INTEGER NOT NULL
);
And I am calling it as follows:
function *genRows(id, devices) {
let count = 0;
for (const serial of devices) {
yield [ id, serial, UNSTARTED ];
count++;
if (count % 10 === 0) log.info(`Streamed ${ count } rows...`);
}
log.info(`Streamed ${ count } rows.`);
}
await streamRows(client, {
table: 'devices',
columns: [ 'group_id', 'serial_number', 'status' ],
data: genRows(id, devices),
});
The log statements in my generator function that's producing the per-row data all run as expected, and the output indicates that it is in fact always running the generator to completion, and streaming all the data rows I want. No errors are ever thrown. But if I wait for it to complete, the table sometimes ends up with 0 rows added to it--i.e., it looks like I sent all that data to Postgres, but none of it was actually inserted. What am I doing wrong?
I do not know exactly what parts of this made the difference and what is purely stylistic, but after playing around with a bunch of different examples from across the web, I managed to cobble together this function which works:
function streamRows(client, { table, columns, data }) {
return new Promise((resolve, reject) => {
const iterator = data[Symbol.iterator]();
const rs = new Readable();
const ws = client.query(copyFrom(`COPY ${ table } (${ columns.join(', ') }) FROM STDIN`));
rs._read = function() {
const { value, done } = iterator.next();
rs.push(done ? null : `${ value.join('\t') }\n`);
};
rs.on('error', reject);
ws.on('error', reject);
ws.on('end', resolve);
rs.pipe(ws);
});
}

Returning a recursive function with a promise

I am trying to have a function that will page through an endpoint to get all of the contacts. Right now my promise is returning just the number 2 which I do not understand. I want it to return all of the contacts. Here is the code I have at the moment. I am hoping someone can help me understand how to return the array of contacts properly.
function getContacts(vid,key){
return axios.get('https://api.hubapi.com/contacts/v1/lists/all/contacts/all?hapikey=' + key + '&vidOffset=' + vid)
.then(response =>{
//console.log(response.data['has-more'])
//console.log(response.data['vid-offset'])
if (response.data['has-more']){
contacts.push(getContacts(response.data['vid-offset'],key))
if(vid === 0){
return contacts.push(response.data.contacts)
}else{
return response.data.contacts
}
}else{
//console.log(contacts)
return response.data.contacts
}
})
}
I would make the getContacts function return a promise that resolves to a list of all contacts. Within that function you can chain the individual promises that load a page of your data:
function getContacts(key){
const url = 'https://api.hubapi.com/contacts/v1/lists/all/contacts/all'
let contacts = []; // this array will contain all contacts
const getContactsPage = offset => axios.get(
url + '?hapikey=' + key + '&vidOffset=' + offset
).then(response => {
// add the contacts of this response to the array
contacts = contacts.concat(response.data.contacts);
if (response.data['has-more']) {
return getContactsPage(response.data['vid-offset']);
} else {
// this was the last page, return the collected contacts
return contacts;
}
});
// start by loading the first page
return getContactsPage(0);
}
Now you can use the function like this:
getContacts(myKey).then(contacts => {
// do something with the contacts...
console.log(contacts);
})
Here is the result I came up with.
function getContacts(vid,key){
var contacts = []
return new Promise(function(resolve,reject){
toCall(0)
//need this extra fn due to recursion
function toCall(vid){
axios.get('https://api.hubapi.com/contacts/v1/lists/all/contacts/all?hapikey=########-####-####-####-############&vidOffset='+vid)
.then(response =>{
contacts = contacts.concat(response.data.contacts)
if (response.data['has-more']){
toCall(response.data['vid-offset'])
}else{
resolve(contacts)
}
})
}
})
}

Resources