How to create object in async and await function in Node js? - node.js

Hi i am creating a shopify api node js script which runs with async and await.Basically i am using this to fetch data form paginated page and i am getting the data properly in console log for each page. the issue is i am unable to make a array in which i get all the data in the end of function.
Here is the code
const fetch = require("node-fetch");
const priceRule = "641166639179"
const totalresult = []
async function findCodeId(price_rule, url=null){
//get(priceRuleId, id)
let urlneww = url ? url : `https://xxxxxxxxxxxx.myshopify.com/admin/api/2020-01/price_rules/${price_rule}/discount_codes.json`;
await fetch(urlneww, {
method: 'GET',
headers: {
"Content-Type": "application/json",
"Authorization": "Basic xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx="
}
})
//.then(response => response.json())
.then(result => {
let rrr = result.json() ;
rrr.then((data)=>{
totalresult.push(data)
console.log(data)
}).catch(error => error);
if(result.headers.get('link').includes("next")){
let str = result.headers.get('link');
let arrStr = str.split('<').pop().split('>')[0]; // returns 'two'
//console.log(arrStr);
findCodeId(priceRule,arrStr);
//console.log(totalresult)
}else{
}
//return totalresult
})
.catch(error => console.log('error', error));
}
findCodeId(priceRule)
i am trying to push the data in totalresult constant but it is not working. Could you please suggest how can i do this so that on each result it pushes the data in the totalresult and at end of function i got all result data collected in totalresult.

You are mixing promise/async-await style which is making it complex. Also you are not awaiting while recursively calling function. Try this
const fetch = require("node-fetch");
const priceRule = "641166639179";
const totalresult = [];
async function findCodeId(price_rule, url = null) {
try {
// get(priceRuleId, id)
const urlneww = url
? url
: `https://xxxxxxxxxxxx.myshopify.com/admin/api/2020-01/price_rules/${price_rule}/discount_codes.json`;
const result = await fetch(urlneww, {
"method": "GET",
"headers": {
"Content-Type": "application/json",
"Authorization": "Basic xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx="
}
});
const data = await result.json();
totalresult.push(data);
console.log(data);
if (result.headers.get("link").includes("next")) {
const str = result.headers.get("link");
const arrStr = str
.split("<")
.pop()
.split(">")[0]; // returns 'two'
// console.log(arrStr);
await findCodeId(priceRule, arrStr);
// console.log(totalresult)
} else {
console.log(totalresult);
}
} catch (Err) {
console.error(Err);
}
}
findCodeId(priceRule);

Related

res.json returns undefined from express server after retrieving data from firebase realtime database

I currently have a realtime database set up that saves a list of a specific user's list of saved jobs for a job search application with react on the front end. However, when I try to return that data through res.json, console.logging it prints out undefined. Would appreciate any help or insight on why it is behaving this way and/or possible fixes.
Thanks in advance.
The data looks something like this:
{
"users" : {
"<userId>" : {
"savedJobs" : [ {
"company" : "PepsiCo",
"id" : 7693578,
"location" : [ {
"name" : "Valhalla, NY"
} ]
} ]
}
}
}
It is initialized like so:
var admin = require("firebase-admin");
let db = admin.database();
var userRef = db.ref("users");
Old Backend logic:
app.get("/fetchSavedJobs/:userId", (req, res) => {
console.log("inside fetch saved jobs");
console.log("fetch saved jobs user id " + req.params.userId);
const userId = req.params.userId;
let list = [];
userRef.orderByChild(userId).limitToLast(1).once('value').then((querySnapshot) => {
if(!querySnapshot.numChildren) {
throw new Error("user not in database, no saved jobs");
}
let dataSnapshot;
querySnapshot.forEach((snap) => {
dataSnapshot = snap;
})
if (!dataSnapshot.exists()) { // value may be null, meaning idToFind doesn't exist
throw new Error(`Entry ${userId} not found.`);
}
const jobsList = dataSnapshot.val().savedJobs;
jobsList.forEach((x) => list.push({company: x.company, id: x.id}));
return res.json(list);
})
I've re-edited my back-end to to the following:
console.log(userId, typeof userId)
prints "userId_value, string"
app.get("/fetchSavedJobs/:userId", (req, res) => {
const userId = req.params.userId;
var userRef = db.ref("users/" + userId); //initializing userRef
let list = [];
userRef.once("value").then((snapshot) => {
console.log("snapshot val", snapshot.val());
res.json(snapshot.val().savedJobs);
})
Front end:
export const fetchSavedJobs = (userUid) => {
return async (dispatch) => {
const fetchData = async () => {
console.log("fetch data is called");
const response = await fetch("/fetchSavedJobs/" + userUid, {
method: "GET",
headers: {
'Accept': 'application/json',
"Content-Type": "application/json",
},
});
if (!response.ok) {
throw new Error("Fetching saved cart data failed");
}
};
try {
const savedJobsData = await fetchData();
console.log("came back from server with saved jobs");
console.log("retrieved Saved Jobs data" + savedJobsData); //this prints out undefined
} catch (error) {}
};
};

Fetch in vue 3 stops backend and does nothing after working fine one time

So, my problem is when I try to login in my vue app, the backend automatically stops when I try to fetch from it an array of objects.
To be more specific.
This is my fetch "attempt" to retrieve the objects from the database.
let url = utils.url;
let requestParam = utils.globalRequestParameters;
requestParam.method = "GET";
requestParam.body = null;
if (cars.value.length == 0) {
fetch(url + "cars", requestParam).then((res) =>
res.json().then(async (res) => {
store.dispatch("Car/fetchCars", res);
fetch(url + "users", requestParam).then((users) =>
users.json().then((users) => {
for (let car of res) {
let userCar = Object.values(users).find(
(a) => a.id == car.userId
);
car.userName = userCar.lastName + " " + userCar.firstName;
}
})
);
})
);
}
And login in view Login.vue
let requestParameters = utils.globalRequestParameters;
requestParameters.method = "POST";
requestParameters.body = JSON.stringify(data);
fetch(utils.url + "login", requestParameters).then((res) => {
res.json().then((res) => {
this.mesaj = res.message;
console.log("token:" + res.token);
if (res.token) {
localStorage.setItem("token", res.token);
console.log("token:" + res.token);
console.log("id:" + res.id);
let id = res.id;
requestParameters.method = "GET";
requestParameters.body = null;
this.$store.dispatch("login", true);
fetch(utils.url + "users/" + id, requestParameters).then(
(res) => {
res.json().then((res) => {
console.log("Jos de tot");
this.$store.dispatch("User/setUser", res);
console.log(this.$store.state.User.user);
this.$router.push("/");
});
}
);
}
});
});
}
},
Note.
cars is a computed value that return store.state.cars
and utils is
let url = "http://127.0.0.1:3000/";
let globalRequestParameters = {
method: "GET",
mode: "cors",
cache: "no-cache",
credentials: "same-origin",
headers: {
"Content-Type": "application/json",
},
redirect: "follow",
referrerPolicy: "no-referrer",
};
module.exports.globalRequestParameters = globalRequestParameters;
module.exports.url = url;
Here at the first fetch the backend stops and also the fetch it is not done.
And the backend route is
router.get('/cars', async (req, res) => {
res.json(await functions.getAllCars(req,res));
})
getAllCars = async (req, res) => {
const snapshot = await db.collection("Cars").get();
let cars = [];
snapshot.forEach((doc) => {
let car = {
id: doc.id,
userId: doc.data().userId,
manufacturer: doc.data().manufacturer,
model: doc.data().model,
color: doc.data().color,
plate: doc.data().plate,
price: doc.data().price,
description: doc.data().description
};
cars.push(car);
});
res.status(200).send(cars);
return
};
router.get("/users/:id", async (req, res) => {
res.json(await functions.getUserById(req.params.id, res));
});
getUserById = (id, res) => {
db
.collection("Users")
.doc(id)
.get()
.then((response) => {
let user = {};
user.id = response.id;
user.firstName = response.data().firstName;
user.lastName = response.data().lastName;
user.gender = response.data().gender;
user.jobTitle = response.data().jobTitle;
user.phone = response.data().phone;
user.email = response.data().email;
user.isAdmin = response.data().isAdmin;
res.status(200).send(user);
return
})
.catch((err) => {
res.status(404).send({ message: "User not found" });
return
});
};
The user is retrieved correctly, I see it in console through a console log, but the messages that I get in the terminal and console are:
*As a final note. I use vue 3, node.js version 16.13.0 and Firestore as Database. And yesterday everything was working perfectly fine on my other computer but I had to go somewhere and use my laptop. Maybe there is something about my laptop. All I did was just to install the modules for the front and back
I think this has nothing to do with Vue - it is simply the problem of your Express backend code
ERR_HTTP_HEADERS_SENT: Cannot set headers after they are sent to the client
As described here:
That particular error occurs whenever you try to send more than one response to the same request and is usually caused by improper asynchronous code.
getAllCars
getAllCars is async function with await inside - as soon as this await is hit (together with db.collection("Cars").get() call), function returns Promise which is awaited at res.json(await functions.getAllCars(req,res));
When the DB call finishes, the rest of the method is executed including res.status(200).send(cars) - this will send the cars array to the client and returns undefined (this is what simple return does) and res.json(undefined) is executed causing the error above (you are trying to send second response)
getUserById
You say that this handler works fine but I really doubt it - from what I see, this should NOT work either
You are calling it with res.json(await functions.getUserById(req.params.id, res));. To await actually doing something, the awaited function must return a Promise (either implicitly by using await inside or explicitly) or general "thenable" object. The getUserById function returns nothing (return statements inside then() or catch() does not count! ...those are different functions)
This problem can be fixed by doing return db.collection("Users").doc(id).get().then() but then you will get same error as in getAllCars case
Correct pattern
Do not use res.status(200).send() and res.json() together
For the sake of sanity (at least until you really know what you are doing) do not mix promises with async/await
async functions should return the data (do not use return without "argument")
Following code shows both Promise based and async/await style (it "pseudo code" in the sense I did not tested it but hopefully you get the idea)
router.get('/cars', async (req, res) => {
try {
const response = await functions.getAllCars()
res.status(200).json(response);
} catch() {
res.sendStatus(500)
}
})
getAllCars = async () => {
const snapshot = await db.collection("Cars").get();
let cars = [];
snapshot.forEach((doc) => {
let car = {
id: doc.id,
userId: doc.data().userId,
manufacturer: doc.data().manufacturer,
model: doc.data().model,
color: doc.data().color,
plate: doc.data().plate,
price: doc.data().price,
description: doc.data().description
};
cars.push(car);
});
// res.status(200).send(cars); //* should be handled by caller
return cars //* return the data
};
router.get("/users/:id", async (req, res) => {
functions.getUserById(req.params.id)
.then((response) => {
if(response === null)
res.status(404).json({ message: "User not found" });
else
res.status(200).json(response);
})
.catch(er) {
res.status(500).send(er.message)
}
});
getUserById = (id) => {
return db //* return the promise
.collection("Users")
.doc(id)
.get()
.then((response) => {
let user = {};
user.id = response.id;
user.firstName = response.data().firstName;
user.lastName = response.data().lastName;
user.gender = response.data().gender;
user.jobTitle = response.data().jobTitle;
user.phone = response.data().phone;
user.email = response.data().email;
user.isAdmin = response.data().isAdmin;
// res.status(200).send(user); //* should be handled by caller
return user //* return the data
})
.catch((err) => {
return null
});
};

Why the nodejs heap out of memory for creating Excel file with big data?

I am creating an excel file at nodejs end and returning base64 data to reactJS to download the file. At nodejs end, I am using promise all and fetch data from a server in chunks and append data into Excel as
worksheet.addRows(data);
For data around 20-30k, it is working fine but for data like 100k, it shows me an error heap out of memory at nodejs end.
I have increase memory allocate to nodejs also but same error
node --max_old_space_size=5000 app.js
What I am doing wrong any suggestions?
Nodejs
const axios = require('axios');
var excel = require("exceljs");
const workbook = new excel.Workbook();
const worksheet = workbook.addWorksheet("My Sheet");
worksheet.columns = [
{ header: "TicketId", key: "ticketId" },
{ header: "Email", key: 'user_email' },
{ header: "User", key : 'user_name' },
{ header: "Subject", key: "subject" },
...//many more headers
];
exports.getTicketData = async (req, res, next) => {
res.connection.setTimeout(0);
const { body } = req;
const token = body.token;
const organization_id = body.organization_id;
const server = body.server;
const sideFilter = body.sideFilter;
let baseurl = 'url for server end to fetch data';
if (baseurl) {
let data = new Array();
let limit = 3000;
const promises = [];
try {
let count = await getCount(token,limit, organization_id, baseurl, sideFilter);
for(var i = 1;i<=count;i++) {
promises.push(getData(i,limit,organization_id,token, baseurl, sideFilter));
}
await Promise.all(promises).then((results) => {
}).catch((e) => {
throw e;
});
var base64File = await writeExcelAndUpload(workbook);
return res.status(200).json({ file:base64File });
} catch (err) {
return res.status(400).json({ type:'error', msg:'File not generated please contact support staff' });
}
} else {
return res.status(400).json({ type:'error', msg:'please define server name' });
}
};
let getData = (page,limit, organization_id,token, baseurl, sideFilter) =>{
return new Promise((resolve, reject) => {
axios.post(baseurl+`/v2/get-export`, {
page:page,
organization_id:organization_id,
per_page:limit,
filter: "",
sorted:"",
...sideFilter
},{ headers: {"Authorization" : `Bearer ${token}`} }).then(function (response) {
let dataTemp = response.data.data.data.map((t,i)=>{
return {
...t,
name:t.name,
...//many more columns like 70
}
});
worksheet.addRows(dataTemp);
resolve(true);
}).catch(function (error) {
reject(error);
});
});
}
let getCount = (token,limit, organization_id, baseurl, sideFilter) => {
// run an api and return count against limit
}
let writeExcelAndUpload = async (workbook) => {
const fileBuffer = await workbook.xlsx.writeBuffer();
let base64File = Buffer.from(fileBuffer).toString('base64');
base64File = 'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,'+base64File;
return base64File;
}
Client side reactjs
exportLink = () => {
postData ={
...
};
return axios.post(`${baseurl}/api/ticketing/get-ticket`, postData).then(function (response) {
const downloadLink = document.createElement("a");
const fileName = "export.xlsx";
downloadLink.href = response.data.file;
downloadLink.download = fileName;
downloadLink.click();
}).catch(function(error){
throw error;
});
}
Well, it is kinda expected that you may get a heap out of memory when working with such an amount of entries like 100k.
I could suggest you start using pagination, and instead of fetching e.g. 100k of entries at once fetch 1k of entries do what you need with them, then fetch the next 1k of entries repeat until you processed all entries.

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Using async/await recursively with node request package

I'm making an http request using the response node library and I am trying to call it recursively (If the user made a commit on one day, check the previous day. If not, count up all the days to get the streak).
The problem is that the line
const githubResponse = await request(options);
Spits out the error
Unexpected token o in JSON at position 1
await request(options) doesn't seem to return the JSON GitHub API response I am expecting, but instead githubResponse seems to be an object that I can't use. I'm guessing I'm using async/await improperly, but I'm not sure how to fix it.
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await request(options)
// I get an error on the next line
if (JSON.parse(githubResponse).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
UPDATE: It seems like this is not a promise, so I need to format this differently (as a callback). When I try this:
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
request(options, async function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
if (JSON.parse(body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
});
}
The line
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
becomes the problem as streakCounter is undefined, making the log NaN.
As said in the comments request uses callbacks instead of returning a promise and you dont really need to promisify it by yourself since there's already a pacakge for that called request-promise.
Using it in your code should directly work out of the box with async/await
I used promisify example from here to convert it to this and it worked!
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await promisify(request)(options);
if (JSON.parse(githubResponse.body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
function promisify(fn) {
return function (...args) {
return new Promise((resolve, reject) => {
fn(...args, (err, result) => {
if (err) return reject(err);
resolve(result);
});
});
};
};
If you are upgraded to Node 8 LTS, then native util.promisfy can be used as below:
const { promisify } = require('util')
const request = promisify(require('request'))
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview',
'json':true
}
};
try{
const githubResponse = await request(options);
if (githubResponse.body.total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
catch(err){
console.error(err)
return 0;
}
}
Using json:true in the options, will reduce another step to parse as the response will be in JSON format.

Resources