Using async/await recursively with node request package - node.js

I'm making an http request using the response node library and I am trying to call it recursively (If the user made a commit on one day, check the previous day. If not, count up all the days to get the streak).
The problem is that the line
const githubResponse = await request(options);
Spits out the error
Unexpected token o in JSON at position 1
await request(options) doesn't seem to return the JSON GitHub API response I am expecting, but instead githubResponse seems to be an object that I can't use. I'm guessing I'm using async/await improperly, but I'm not sure how to fix it.
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await request(options)
// I get an error on the next line
if (JSON.parse(githubResponse).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
UPDATE: It seems like this is not a promise, so I need to format this differently (as a callback). When I try this:
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
request(options, async function (error, response, body) {
console.log('error:', error); // Print the error if one occurred
if (JSON.parse(body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
});
}
The line
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
becomes the problem as streakCounter is undefined, making the log NaN.

As said in the comments request uses callbacks instead of returning a promise and you dont really need to promisify it by yourself since there's already a pacakge for that called request-promise.
Using it in your code should directly work out of the box with async/await

I used promisify example from here to convert it to this and it worked!
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview'
}
};
const githubResponse = await promisify(request)(options);
if (JSON.parse(githubResponse.body).total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
function promisify(fn) {
return function (...args) {
return new Promise((resolve, reject) => {
fn(...args, (err, result) => {
if (err) return reject(err);
resolve(result);
});
});
};
};

If you are upgraded to Node 8 LTS, then native util.promisfy can be used as below:
const { promisify } = require('util')
const request = promisify(require('request'))
async function checkUserCommitForDate(user, date) {
const options = {
url: `https://api.github.com/search/commits?q=author:${user}+author-date:${date}`,
headers: {
'User-Agent': 'request',
'Accept': 'application/vnd.github.cloak-preview',
'json':true
}
};
try{
const githubResponse = await request(options);
if (githubResponse.body.total_count > 0) {
const previousDaysDate = moment(date).subtract(1, 'day').format('YYYY-MM-DD');
let streakCounter = await checkUserCommitForDate(user, previousDaysDate);
streakCounter++;
console.log('streakCounter', streakCounter);
return streakCounter;
} else {
return 0;
}
}
catch(err){
console.error(err)
return 0;
}
}
Using json:true in the options, will reduce another step to parse as the response will be in JSON format.

Related

Nodejs await does not waits for the function to return data. The function needs to wait for the response called by POST call using https

const https = require('https');
async function A() {
let postData = JSON.stringify({
"f1": "D1",
"f2": "D2",
"encrypted": true
});
let options = {
hostname: "dummyurl1122.com",
path: "/api/wow",
method: "POST",
timestamp: new Date(),
headers: {
'Content-Type': 'application/json',
}
}
let chunks = [];
let reqGetToken = await https.request(options, (resGetToken) => {
if (resGetToken.statusCode != 200) {
console.log("***** Error = " + resGetToken.statusMessage);
return;
}
resGetToken.on("data", (d) => {
chunks.push(d);
});
resGetToken.on("end", async () => {
let output = JSON.parse(await (Buffer.concat(chunks)).toString());
return output.token; //****** I want this in output
});
});
reqGetToken.on("error", (e) => {
console.error(e);
});
reqGetToken.write(postData);
reqGetToken.end();
}
let output = await A(); // I want the output but it does not waits for the https response
console.log(output + 2);
When I call the function A it does not waits for the response and goes to next line.
How can I make it wait for the output and then go to next line.
The function A calls a external POST api and I need its response in output and then execute further lines.
Since https.request does not return a promise, you can't await it.
See this answer for how to use await with https.

Promise.all returning undefined in Node JS

I have a code to fetch directory names from first API. For every directory, need to get the file name from a second API. I am using something like this in my Node JS code -
async function main_function(req, res) {
const response = await fetch(...)
.then((response) => {
if (response.ok) {
return response.text();
} else {
return "";
}
})
.then((data) => {
dirs = ...some logic to extract number of directories...
const tempPromises = [];
for (i = 0; i < dirs.length; i++) {
tempPromises.push(getFilename(i));
}
console.log(tempPromises); // Prints [ Promise { <pending> } ]
Promise.all(tempPromises).then((result_new) => {
console.log(result_new); // This prints "undefined"
res.send({ status: "ok" });
});
});
}
async function getFilename(inp_a) {
const response = await fetch(...)
.then((response) => {
if (response.ok) {
return response.text();
} else {
return "";
}
})
.then((data) => {
return new Promise((resolve) => {
resolve("Temp Name");
});
});
}
What am I missing here?
Your getFilename() doesn't seem to be returning anything i.e it's returning undefined. Try returning response at the end of the function,
async function getFilename(inp_a) {
const response = ...
return response;
}
Thanks to Mat J for the comment. I was able to simplify my code and also learn when no to use chaining.
Also thanks to Shadab's answer which helped me know that async function always returns a promise and it was that default promise being returned and not the actual string. Wasn't aware of that. (I am pretty new to JS)
Here's my final code/logic which works -
async function main_function(req,res){
try{
const response = await fetch(...)
const resp = await response.text();
dirs = ...some logic to extract number of directories...
const tempPromises = [];
for (i = 0; i < dirs.length; i++) {
tempPromises.push(getFilename(i));
}
Promise.all(tempPromises).then((result_new) => {
console.log(result_new);
res.send({ status: "ok" });
});
}
catch(err){
console.log(err)
res.send({"status" : "error"})
}
}
async function getFilename(inp_a) {
const response = await fetch(...)
respText = await response.text();
return("Temp Name"); //
}

How to set timeout with abort controller and retry in ES7 async await function

I am trying fetch request, if request takes time longer than specified timeout then I am aborting it using abort controller or if there is error in response like socket hangup then try to fetch it again recursively. I want to repeat it i.e 5 times and in last if it fails then return just error. But in my code after 1st try it fails instantly and shows fetch request aborted.
Here is my code.
Thank you in advance.
const getData = async () => {
let url = `https://www.example.com`
let options = {
method: 'POST',
agent: proxyagent,
signal: signal,
body: JSON.stringify({
username: 'abc'
})
}
const fetchData = async (retries) => {
try {
let timer = setTimeout(() => controller.abort(), 1000)
let response = await fetch(url, options)
clearTimeout(timer)
if (response.ok) return response.json()
} catch (e) {
console.log(e)
if (retries === 1 || retries < 1) {
return {
message: 'Something went wrong'
}
}
return await fetchData(retries - 1)
}
}
return await fetchData(5)
}
You cannot reuse an aborted signal. You will have to create a new one for each fetch
const getData = async () => {
let url = `https://www.example.com`;
let controller;
let options = {
method: 'POST',
agent: proxyagent,
body: JSON.stringify({
username: 'abc'
})
}
const fetchData = async (retries) => {
controller = new AbortController();
const optionsWithSignal = {
...options,
signal: controller.signal
};
try {
let timer = setTimeout(() => controller.abort(), 1000)
let response = await fetch(url, optionsWithSignal)
clearTimeout(timer);
if (response.ok) return response.json()
} catch (e) {
console.log(e)
if (retries === 1 || retries < 1) {
return {
message: 'Something went wrong'
}
}
return await fetchData(retries - 1)
}
}
return await fetchData(5)
}

Call DynamoDb scan recursively when Promisified

I need to get some data from DynamoDb, using the scan() method. I have implemented some basic pagination by calling my function recursively n number of times to get the correct page.
Currently, I call my function and inside the scan() callback, if the data can be send back, I use the handler callback to return the data.
CURRENT CODE
const AWS = require('aws-sdk')
const docClient = new AWS.DynamoDB.DocumentClient()
const TABLE_NAME = process.env.TABLE_NAME
const DEFAULT_PAGE_SIZE = 500
const DEFAULT_PAGE_NUMBER = 1
const self = {
handler: (event, context, callback) => {
const {pageNumber, pageSize} = event.queryStringParameters ? event.queryStringParameters : {pageNumber: DEFAULT_PAGE_NUMBER, pageSize: DEFAULT_PAGE_SIZE}
const params = {
TableName: ORGANISATION_TYPES_TABLE_NAME,
Limit: pageSize ? pageSize : DEFAULT_PAGE_SIZE
}
return self.scan(params, pageNumber, 1, callback)
},
scan: (params, pageNumber, pageCount, callback) => {
docClient.scan(params, (err, data) => {
if (err) {
callback(null, {
statusCode: 500,
body: JSON.stringify(err)
})
};
if (data.LastEvaluatedKey && pageCount < pageNumber) {
pageCount += 1
params.ExclusiveStartKey = data.LastEvaluatedKey
self.scan(params, pageNumber, pageCount, callback)
} else {
callback(null, {
statusCode: 200,
body: JSON.stringify(data)
})
}
})
}
}
module.exports = self
The above code does work, allowing me to specify a pageSize and pageNumber query parameter.
However, I want to Promisify self.scan.
I tried the following, but it results in the response being undefined
DESIRED CODE
const AWS = require('aws-sdk')
const docClient = new AWS.DynamoDB.DocumentClient()
const ORGANISATION_TYPES_TABLE_NAME = process.env.ORGANISATION_TYPES_TABLE_NAME
const DEFAULT_PAGE_SIZE = 500
const DEFAULT_PAGE_NUMBER = 1
const self = {
handler: (event, context, callback) => {
const {pageNumber, pageSize} = event.queryStringParameters ? event.queryStringParameters : {pageNumber: DEFAULT_PAGE_NUMBER, pageSize: DEFAULT_PAGE_SIZE}
const params = {
TableName: ORGANISATION_TYPES_TABLE_NAME,
Limit: pageSize ? pageSize : DEFAULT_PAGE_SIZE
}
return self.scan(params, pageNumber, 1).then((response) => {
callback(null, {
statusCode: 200,
body: JSON.stringify(response)
})
}).catch((err) => {
callback(null, {
statusCode: 500,
body: JSON.stringify(err)
})
})
},
scan: (params, pageNumber, pageCount) => {
return new Promise((resolve, reject) => {
docClient.scan(params, (err, data) => {
if (err) {
reject(err)
};
if (data.LastEvaluatedKey && pageCount < pageNumber) {
pageCount += 1
params.ExclusiveStartKey = data.LastEvaluatedKey
self.scan(params, pageNumber, pageCount, callback)
} else {
resolve(data)
}
})
})
}
}
module.exports = self
I also tried just doing return Promise.resolve(data) inside the docClient.scan() callback, but that doesn't work either. It's as if promises cannot be resolved inside a callback?
I have recently helped someone with this problem, there's actually quite an elegant solution that we hit upon that uses the hasNextPage property on the response you get from the SDK. The key is to have your recursive function pass an array that holds your results through the recursive calls and just concat until you run out of pages and then just return the array.
const scan = async params => {
function scanRec(promise, xs) {
return promise
.then(async result => {
const response = result.$response;
const items = xs.concat(result.Items);
response.hasNextPage() ? scanRec(response.nextPage().promise(), items) : items
})
}
return scanRec(docClient.query(params).promise(), []);
}
You'd then use the function in the normal way:
const params = { /** params **/ };
scan(params).then(x => {
// ...
})

How can I get the completed value of an async function when calling a function from index.js?

I'm new to node and I'm trying to create a function to make API requests with pagination. The function successfully gives me the desired output, but I'm confused as to how I can use the .then() function in index.js as it's async. If I use await in my index.js then it throws an error. I hoping to get some guidance as to how I can fix this, and how I can understand async/await better.
//hs-api.js
const request = require('request-promise');
const settings = require('./settings');
var all = []
let getReq = async (url) => {
var options = {
'method': 'GET',
'url': url,
'headers': {
'Content-Type': 'application/json'
}
}
let results = await request(options, function async (error, response) {
if (error) {
reject(error)
} else {
res = JSON.parse(response.body)
}
})
all = all.concat(res.results)
if(res.hasOwnProperty("paging")) {
await getReq(`${res.paging.next.link}&apikey=${settings.api_key}`)
} else {
console.log(all)
return all
}
}
Here is where I call the function
//index.js
let apiResponse = api.getReq(`https://apiexample.com/?apikey=${settings.api_key}`)
console.log(apiResponse)
As of today (Node.js 14), you need to wrap it in a function.
Something like this
(async () => {
let apiResponse = await api.getReq(`https://apiexample.com/?apikey=${settings.api_key}`)
console.log(apiResponse)
})()
There is a proposal for ES for top-level await which will make it possible to run
let apiResponse = await api.getReq(`https://apiexample.com/?apikey=${settings.api_key}`)
console.log(apiResponse)
without wrapping it in an async function.
Node.js (since version 10) has an experimental support for this feature. You need to run it with --experimental-repl-await flag
You need to wrap it in async function in index.js.
// index.js
async someFn() {
let apiResponse = await api.getReq(`https://apiexample.com/?apikey=${settings.api_key}`)
console.log(apiResponse)
}
// call
someFn();
or use '.then'
api.getReq(`https://apiexample.com/?apikey=${settings.api_key}`)
.then(apiResponse => {
console.log(apiResponse);
})
.catch(console.log);
UPD:
//hs-api.js
const results = await request(options);
// don`t sure, that it is correct error indicator for your library
if(result.error) {
throw new Error(// describe error here);
}
const res = JSON.parse(response.body);
all = all.concat(res.results);
if(res.hasOwnProperty("paging")) {
return await getReq(`${res.paging.next.link}&apikey=${settings.api_key}`)
} else {
console.log(all);
return all;
}

Resources