ASYNC/AWAIT for Chaining multiple http.requests - node.js

How would I go about chaining an http.request within the response of another http.request and then push them into an array before going to the front end?
router.get("/:team", (req, res) => {
let teamParams = teams[req.params.team];
twitter.get("search/tweets", teamParams, (err, data, resp) => {
let tweetArr = [];
let text = data.statuses;
text.map((dat) => {
let im = dat.entities.urls[0].url
dat.links = im;
tweetArr.push(dat);
});
res.json({ message: "Success", tweets: tweetArr });
});
});
Currently I get my data object loop through it and add a url as a property. Now I want to chain another http request to make an API call to another API and get a response, before I use res.json.
I've tried a workaround with promises but I can never return the full object with the response from the second api call.
This is what I have so far, I have managed to get to a point where my object contains the requests from the second link. How can I return all the tweets into an array I can finally resolve?
require("dotenv").config();
const Twitter = require("twitter");
const API_IMAGE_PREV = "http://api.linkpreview.net/";
const request = require("request");
const key = process.env.IM_PREV_KEY;
let twitter = new Twitter({
consumer_key: process.env.TWITTER_CONSUMER_KEY,
consumer_secret: process.env.TWITTER_CONSUMER_SECRET,
bearer_token: process.env.TWITTER_BEARER_TOKEN
});
let teamParams = {
q: "from:ManUtdMEN MUFC",
count: 2,
result_type: "recent"
};
var third = function thirdUrl(next) {
var promise = new Promise(function(resolve, reject) {
next.forEach(x => {
let ln = x.links;
const options = {
url: "http://api.linkpreview.net/?key=" + key + "&q=" + ln,
method: "get"
};
request.get(options, (err, req, res) => {
if (err) {
console.log(err);
} else {
x.desc = res;
}
});
});
});
};
var second = function secondUrl(previous) {
var promise = new Promise(function(resolve, reject) {
let p = previous.statuses;
let links = [];
p.forEach(t => {
let l = t.entities.urls[0].url;
t.links = l;
});
resolve(p);
});
return promise;
};
twitter
.get("search/tweets", teamParams)
.then(second)
.then(third)
.catch(function(error) {
throw error;
});

What module are you using for the http requests? Here's an example with axios.
const axios = require('axios');
router.get("/:team", (req, res) => {
let teamParams = teams[req.params.team];
twitter.get("search/tweets", teamParams, async (err, data, resp) => {
let tweetArr = [];
let text = data.statuses;
text.map((dat) => {
let im = dat.entities.urls[0].url
dat.links = im;
tweetArr.push(dat);
});
let res = await Promise.all(tweetArr.map(dat => axios.get(dat.links));
res.json({ message: "Success", tweets: res });
});
})

Related

Why the nodejs heap out of memory for creating Excel file with big data?

I am creating an excel file at nodejs end and returning base64 data to reactJS to download the file. At nodejs end, I am using promise all and fetch data from a server in chunks and append data into Excel as
worksheet.addRows(data);
For data around 20-30k, it is working fine but for data like 100k, it shows me an error heap out of memory at nodejs end.
I have increase memory allocate to nodejs also but same error
node --max_old_space_size=5000 app.js
What I am doing wrong any suggestions?
Nodejs
const axios = require('axios');
var excel = require("exceljs");
const workbook = new excel.Workbook();
const worksheet = workbook.addWorksheet("My Sheet");
worksheet.columns = [
{ header: "TicketId", key: "ticketId" },
{ header: "Email", key: 'user_email' },
{ header: "User", key : 'user_name' },
{ header: "Subject", key: "subject" },
...//many more headers
];
exports.getTicketData = async (req, res, next) => {
res.connection.setTimeout(0);
const { body } = req;
const token = body.token;
const organization_id = body.organization_id;
const server = body.server;
const sideFilter = body.sideFilter;
let baseurl = 'url for server end to fetch data';
if (baseurl) {
let data = new Array();
let limit = 3000;
const promises = [];
try {
let count = await getCount(token,limit, organization_id, baseurl, sideFilter);
for(var i = 1;i<=count;i++) {
promises.push(getData(i,limit,organization_id,token, baseurl, sideFilter));
}
await Promise.all(promises).then((results) => {
}).catch((e) => {
throw e;
});
var base64File = await writeExcelAndUpload(workbook);
return res.status(200).json({ file:base64File });
} catch (err) {
return res.status(400).json({ type:'error', msg:'File not generated please contact support staff' });
}
} else {
return res.status(400).json({ type:'error', msg:'please define server name' });
}
};
let getData = (page,limit, organization_id,token, baseurl, sideFilter) =>{
return new Promise((resolve, reject) => {
axios.post(baseurl+`/v2/get-export`, {
page:page,
organization_id:organization_id,
per_page:limit,
filter: "",
sorted:"",
...sideFilter
},{ headers: {"Authorization" : `Bearer ${token}`} }).then(function (response) {
let dataTemp = response.data.data.data.map((t,i)=>{
return {
...t,
name:t.name,
...//many more columns like 70
}
});
worksheet.addRows(dataTemp);
resolve(true);
}).catch(function (error) {
reject(error);
});
});
}
let getCount = (token,limit, organization_id, baseurl, sideFilter) => {
// run an api and return count against limit
}
let writeExcelAndUpload = async (workbook) => {
const fileBuffer = await workbook.xlsx.writeBuffer();
let base64File = Buffer.from(fileBuffer).toString('base64');
base64File = 'data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,'+base64File;
return base64File;
}
Client side reactjs
exportLink = () => {
postData ={
...
};
return axios.post(`${baseurl}/api/ticketing/get-ticket`, postData).then(function (response) {
const downloadLink = document.createElement("a");
const fileName = "export.xlsx";
downloadLink.href = response.data.file;
downloadLink.download = fileName;
downloadLink.click();
}).catch(function(error){
throw error;
});
}
Well, it is kinda expected that you may get a heap out of memory when working with such an amount of entries like 100k.
I could suggest you start using pagination, and instead of fetching e.g. 100k of entries at once fetch 1k of entries do what you need with them, then fetch the next 1k of entries repeat until you processed all entries.

context.hasRole is not a function in Nodejs sinon testing issue

I had made function to fetch cloudwatch details from AWS.I trying to create a testcase in node.js and using sinon but i am getting a context.hasRole is not defined because i am checking this in my function file which is cloudwatch.js.
Can you please help me to fake this test
"-----cloudwatch.spec.js-------"
describe('cloudwatch', () => {
let sandbox = null;
beforeEach(() => {
sandbox = sinon.createSandbox(AWS.config);
})
afterEach(() => {
sandbox.restore()
})
it('Should return queryid', async () => {
let queryId = {
queryId: "12ab3456-12ab-123a-789e-1234567890ab"
};
const body = {
endTime: 34568765,
queryString: 'filter #message like /Audit/',
startTime: 34565678,
limit: 100,
logGroupName: '/aws/lambda/dev-api--service-sandbox-api'
};
let params = {}
let queryid = {
queryId: 6786971301298309123
};
await cloudwatch.startQuery(context, params, body, callback);
sinon.match(queryId)
})
})
"------Cloudwatch.js----"
let cloudwatch = module.exports = {};
const AWS = require('aws-sdk');
const nconf = require('nconf');
const {
HttpResult,
HttpUnauthorizedError,
AmazonCloudWatchLogsClient
} = require('api-lib');
AWS.config = nconf.get('amazonCloudWatchLogsClient');
cloudwatch.startQuery = async function(context, params, body,
callback) {
body.startTime = new Date(body.startTime).valueOf();
body.endTime = new Date(body.endTime).valueOf();
if (!context.hasRole("read:cloudwatch"))
return callback(new HttpUnauthorizedError("context missing role
read:cloudwatch"));
const amazonCloudWatchLogsClient = new
AmazonCloudWatchLogsClient(AWS.config);
let result = await amazonCloudWatchLogsClient.startQuery(body,
function(err) {
console.log("Error", err);
});
callback(null, new HttpResult(result));
};
cloudwatch.getQueryResults = async function(context, params,
requestBody, callback) {
console.log(requestBody)
let test = requestBody.queryId;
test = test.toString();
requestBody.queryId = test;
if (!context.hasRole("read:cloudwatch"))
return callback(new HttpUnauthorizedError("context missing role
read:cloudwatch"));
const amazonCloudWatchLogsClient = new
AmazonCloudWatchLogsClient(AWS.config);
let result2 = await
amazonCloudWatchLogsClient.getQueryResults(requestBody.queryId,
function(err) {
console.log("Error", err);
});
callback(null, new HttpResult(result2));
};
I am using eslint and except from chai for comparing the output to the sample output.

How to read body of response from fetch call

should probably mention I am a noob at fetch api, still learning. So sorry if this seems like a basic question. Anyways. To sum up my code, I have a fetch that runs once a button is clicked, it takes the content of an input element, and posts that to my server. Once my server validates it, it sends a response as res.send("success"). I want to be able to console.log that, or run an if else statement. Here is the code:
Front end
<script>
document.getElementById('checktradeurl').addEventListener('click', setTradeURL);
function setTradeURL(){
var tradeofferURL = document.getElementById('tradeurl').value
fetch('/api/tradeurl/', {
method: 'POST',
body: JSON.stringify({url:tradeofferURL})
})
.then(response=>response.json()
)
.then((res)=>{
console.log(res)
})
.catch((err) => {
console.log(err)
})
};
</script>
Backend
router.post('/api/tradeurl/', async (req, res) => {
console.log(req.body.url)
let link = req.body.url;
if (URL.parse(link, true).query.partner && URL.parse(link, true).query.token) {
var sid = new SteamID.fromIndividualAccountID(URL.parse(link, true).query.partner);
var sid64 = sid.getSteamID64();
var token = URL.parse(link, true).query.token;
console.log(sid64);
console.log(token)
var key = process.env.apiKey
let url = `https://api.steampowered.com/IEconService/GetTradeHoldDurations/v1/?key=${key}&steamid_target=${sid64}&trade_offer_access_token=${token}`
console.log(url)
fetch(url).then((data) => {
return responsecode = data.headers.get('x-eresult')
}).then(code => {
if (code == 1) {
res.send("SUCCESS")
}
})
}
// var key = process.env.apiKey
// res.sendStatus(200);
});

May Promise.all().finally() return unresolved data?

I found quite unusual behavior of Promise.all().finally() - look like it returning data before map() was applied to this.
1. Data received from database.
2. Making call to Google Maps API inside map(), applied to the data retrieved from database and adding to the object property "Distance" with result from Google API call.
3. Return data in Promise.all() - data received without new property.
I can't get how this even possible?
public static get = async (req: Request, res: Response) => {
const latitude = req.query.lat;
const longitude = req.query.long;
const pool = await new sql.ConnectionPool(CommonConstants.connectionString).connect();
const request = pool.request();
const result = await request.execute('SuppliersSP');
sql.close();
const rows = result.recordset.map(async (supplier) => {
const data = { origin: [latitude, longitude], destination: [supplier.Latitude, supplier.Longitude] };
const distance = await GetDistance(data) || 0;
Object.defineProperty(supplier, 'Distance', {
enumerable: true,
configurable: true,
writable: true,
value: distance
});
return supplier;
})
Promise.all(rows).finally(() => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.status(200).json(rows);
});
}
Here is GetDistance function:
import { Constants } from "./constants";
const https = require('https');
export function GetDistance(coords) {
const { origin, destination } = coords;
return new Promise((resolve, reject) => {
https.get(`${Constants.GoogleMapsUrl}?origins=${origin[0]},${origin[1]}
&destinations=${destination[0]},${destination[1]}
&key=${Constants.GoogleMapsApiKey}`, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
const distance = JSON.parse(data);
resolve(distance.rows[0].elements[0].distance.value);
});
}).on("error", (err) => {
reject("Error: " + err.message);
});
});
}
It was solved by a change .finally() to .then().

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Resources