I am trying to run the following function which performs request.get to an array of URL's, and then write to an array, the whole operation needs to be synchronous. But my code is not synchronous, and prints different output each time:
var arrayPart = [];
fileDecode : async function(fileName,filePath){
for (a=0; a< arr.length; a++){
var partID = JSON.parse(arr[a].id)
var uri = listID[remainder]+'/download/'+'?id='+partID
await request.get(uri, this.onRequestDone);
}
onRequestDone: async function(err, resp, body){
await new Promise(function (resolve, reject) {
if(err){
reject(err)
}else{
const buf = Buffer.from(body)
console.log("buf", buf)
arrayPart.push(buf);
fs.writeFileSync('message.txt', arrayPart)
resolve(body)
}
});
}
}
my onRequestDone function does not behave correctly and prints buff differently.
The fact that request.get accepts second callback argument means that it it's callback-based and doesn't support promises. It doesn't return a promise to await. It generally doesn't make sense to provide async function as a callback in places where returned promise is ignored.
request-promise package can be used instead:
const request = require('request-promise');
...
const res = await request.get(uri);
const buf = Buffer.from(res.body);
...
Related
I build an array of promises. Some promises require making http.get() before resolving. Others resolve without this. I push all promises to an array and then iterate. The promises including http.get() are still pending.
I've tried doing promise.all. I've tried replacing the http.get() with resolve(200). This always executes the promise.
const https = require('https');
const http = require('http');
let promises = [];
exports.RegisterHTTPDependency = function(url, name, severity) {
let promise = [];
let pr = GeneratePromise(url);
promise.push(pr, name, severity)
promises.push(promise)
}
exports.Check = function() {
let results = {};
for (let i = 0; i < promises.length; i++) {
const check = promises[i];
console.log('promise: ', check[0]) // this returns "Promise { <pending> }" or 20,
// depending on if the resolve(20) is included in GeneratePromise
}
}
let GeneratePromise = function(url){
return new Promise(function(resolve, reject) {
// resolve(20) // if I include this code, it returns 20
http.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
resolve(resp.statusCode); // this promise does come back <pending>
});
}).on("error", (err) => {
reject(err);
});
});
}
The value 20 is a return value. I want the resp.statusCode to be a return value as well, instead of pending.
Since from GeneratePromise(), it is indeed returning a 200. The reason why you get a pending is where this function is called did not wait for the promise to resolve when you call the GeneratePromise() function.
In short, a promise can have 3 stages (pending, fulfilled, rejected). When you output the promise, it is still pending(not yet resolved or rejected)
What you can do is to use the async, await keywords to instruct the program to block the execution until the promise resolve, turning the promise from pending stage to fulfiled.
exports.RegisterHTTPDependency = async function(url, name, severity) {
let promise = [];
let pr = await GeneratePromise(url);
promise.push(pr, name, severity)
promises.push(promise)
}
I think if you have an array of promises the correct way to "wait" for them to resolve would be with Promise.all().
Promise.all(promisesArray)
.then(result => {
// will execute when every promise has been fulfilled
// result is an array of the values returned by your promises.
})
https://developer.mozilla.org/es/docs/Web/JavaScript/Referencia/Objetos_globales/Promise/all
I'm working with we sockets and I have a lot of requests, so I figured it'd be best to pull the data out of the asynchronus funtion and put it in a variable outside of it. My code looks like this but I just get request [Pending]:
var variable;
var Request = require("request");
function getAsyncValue() {
return new Promise((resolve, reject) => {
Request.get(url, (error, response, body) => {
if(error) {
return reject(err);
}
var object = JSON.parse(body);
var value = object.data.available_balance;
resolve(value);
});
});
}
async function asyncWrapper() {
variable = await getAsyncValue();
return(variable);
}
printVariable = asyncWrapper();
console.log(printVariable);
Any idea on how I can achieve my goal?
Your asyncWrapper is a promise. You have to use await.
To use an await, you need an async function. So you can use a IIFE(Immediately Invoked Functions Expressions) async function.
async function(){
printVariable = await asyncWrapper();
console.log(printVariable);
}();
This was what I was looking for:
var request = require('sync-request');
var returnCode;
var getUrl = "url";
returnCode = httpGet(getUrl);
var object = JSON.parse(returnCode);
var balance = objekt.data.available_balance;
console.log(balance);
function httpGet(url){
var response = request(
'GET',
url
);
return response.body;
}
put your console.log inside the asyncWrapper() as following:
async function asyncWrapper() {
variable = await getAsyncValue();
console.log(printVariable);
return(variable);
}
printVariable = asyncWrapper();
the reason for doing this is because when u had console.log() after the asyncWrapper(), it will be called immediately and not in an sync manner as u would need.
or you can do the following:
Just wrap the asyncWrapper function call inside an another async method and await for the result of asynWrapper() in that. Look below:
var function1 = async function () {
printVariable = await asyncWrapper();
console.log(printVariable);
}
I have written the following code to retrieve song lyrics from the apiseeds lyric api.
const apiseeds = require("apiseeds-lyrics");
const apiseedskey = "MY_API_KEY";
async function getLyrics(artistName, songName)
{
return await apiseeds.getLyric(apiseedskey, artistname, songName,
(response) => {
return response;
});
}
var artist = "Darius Rucker";
var title = "Wagon Wheel";
var lyrics = await getLyrics(artist, title)
console.log(lyrics);
I should also mention that the second block of code there is enclosed within an eventEmitter.on event with an asynchronous callback function.
Whenever the code runs, I get undefined in the console.
async and await can only be used to treat asynchronous functions that returns Promises, not callbacks. You should be able to transform your call to use Promises, or use another library.
The main reason we use await is to wait for the promise to resolve before continuing the code execution:
const result = await codeThatReturnsPromise()
console.log(result)
We could transform your code to this:
// async here means it returns a promise
async function getLyrics(artistName, songName)
{
return new Promise((resolve, reject) => {
apiseeds.getLyric(apiseedskey, artistname, songName, (response) => resolve(response))
})
}
var artist = "Darius Rucker";
var title = "Wagon Wheel";
var lyrics = await getLyrics(artist, title)
console.log(lyrics);
How come this async/await doesn't work?
I've spent all day trying different combinations, watching videos and reading about async/await to find why this doesn't work before posting this here.
I'm trying to make a second nodejs app that will run on a different port, and my main app will call this so it scrap some data and save it to the db for cache.
What it's suppose to do:
Take a keyword and send it to a method called scrapSearch, this method create a complete URI link and send it to the method that actually get the webpage and returns it up to the first caller.
What is happening:
The console.log below the initial call is triggered before the results are returned.
Console output
Requesting : https://www.google.ca/?q=mykeyword
TypeError: Cannot read property 'substr' of undefined
at /DarkHawk/srv/NodesProjects/_scraper/node_scrapper.js:34:18
at <anonymous>
app.js:
'use strict';
var koa = require('koa');
var fs = require('fs');
var app = new koa();
var Router = require('koa-router');
var router = new Router();
app
.use(router.routes())
.use(router.allowedMethods());
app.listen(3002, 'localhost');
router.get('/scraptest', async function(ctx, next) {
var sfn = require('./scrap-functions.js');
var scrapFunctions = new sfn();
var html = await scrapFunctions.scrapSearch("mykeyword");
console.log(html.substr(0, 20));
//Normally here I'll be calling my other method to extract content
let json_extracted = scrapFunctions.exGg('mykeywords', html);
//Save to db
});
scrap-functions.js:
'use strict';
var request = require('request');
var cheerio = require('cheerio');
function Scraper() {
this.html = ''; //I tried saving html in here but the main script seems to have issues
retrieving that
this.kw = {};
this.tr = {};
}
// Search G0000000gle
Scraper.prototype.scrapSearch = async function(keyword) {
let url = "https://www.google.ca/?q="+keyword";
let html = await this.urlRequest(url);
return html;
};
// Get a url'S content
Scraper.prototype.urlRequest = async function(url) {
console.log("Requesting : "+url);
await request(url, await function(error, response, html) {
if(error) console.error(error);
return response;
});
};
module.exports = Scraper;
I tried a lot of things but I finally gave up - I tried putting await/async before each methods - didn't work either.
Why that isn't working?
Edit: wrong function name based on the fact that I created 2 different projects for testing and I mixed the file while copy/pasting.
You are not returning anything from urlRequest. Because it is an async function, it will still create a promise, but it will resolve with undefined. Therefore your html is undefined as seen in the error.
The problematic part is the request function which is a callback style function, but you're treating it as a promise. Using await on any value that is not a promise, won't do anything (technically it creates a promise that resolves directly with the value, but the resulting value remains the same). Both awaits within the urlRequest are unnecessary.
request(url, function(error, response, html) {
if(error) console.error(error);
// This return is for the callback function, not the outer function
return response;
});
You cannot return a value from within the callback. As it's asynchronous, your function will already have finished by the time the callback is called. With the callback style you would do the work inside the callback.
But you can turn it into a promise. You have to create a new promise and return it from urlRequest. Inside the promise you do the asynchronous work (request) and either resolve with the value (the response) or reject with the error.
Scraper.prototype.urlRequest = function(url) {
console.log("Requesting : "+url);
return new Promise((resolve, reject) => {
request(url, (err, response) => {
if (err) {
return reject(err);
}
resolve(response);
});
});
};
When an error occurred you want to return from the callback, so the rest (successful part) is not executed. I also removed the async keyword, because it's manually creating a promise.
If you're using Node 8, you can promisify the request function with the built-in util.promisify.
const util = require('util');
const request = require('request');
const requestPromise = util.promisify(request);
Scraper.prototype.urlRequest = function(url) {
console.log("Requesting : " + url);
return requestPromise(url);
};
Both versions will resolve with the response and to get the HTML you need to use response.body.
Scraper.prototype.scrapSearch = async function(keyword) {
let url = "https://www.google.ca/?q=" + keyword;
let response = await this.urlRequest(url);
return response.body;
};
You still need to handle errors from the promise, either with .catch() on the promise, or using try/catch when you await it.
It is absolutely essential to understand promises when using async/await, because it's syntactic sugar on top of promises, to make it look more like synchronous code.
See also:
Understand promises before you start using async/await
Async functions - making promises friendly
Exploring ES6 - Promises for asynchronous programming
I have a problem with a promise using node.js. My code is below:
var p1 = new Promise(function(resolve, reject) {
// my function here
});
p1.then(function(result){
// my result
});
This code works but to get values from p1 I must use the .then method and my result values can be accessed just on p1.then. How do I access p1 values without .then?
Below are my expected results:
var p1 = new Promise(function(resolve, reject) {
// my function here
});
var abc = NextFunction(p1);
The p1 values will be used afterwards in code outside of the p1 variable.
p1 is a Promise, you have to wait for it to evaluate and use the values as are required by Promise.
You can read here: http://www.html5rocks.com/en/tutorials/es6/promises/
Although the result is available only inside the resolved function, you can extend it using a simple variable
var res;
p1.then(function(result){
res = result; // Now you can use res everywhere
});
But be mindful to use res only after the promise resolved, if you depend on that value, call the next function from inside the .then like this:
var res;
p1.then(function(result){
var abc = NextFunction(result);
});
You can use await after the promise is resolved or rejected.
function resolveAfter2Seconds(x) {
return new Promise(resolve => {
setTimeout(() => {
resolve(x);
}, 2000);
});
}
async function f1() {
var x = await resolveAfter2Seconds(10);
console.log(x); // 10
}
f1();
Be aware await expression must be inside async function though.
You can do this, using the deasync module
var node = require("deasync");
// Wait for a promise without using the await
function wait(promise) {
var done = 0;
var result = null;
promise.then(
// on value
function (value) {
done = 1;
result = value;
return (value);
},
// on exception
function (reason) {
done = 1;
throw reason;
}
);
while (!done)
node.runLoopOnce();
return (result);
}
function test() {
var task = new Promise((resolve, reject)=>{
setTimeout(resolve, 2000, 'Hello');
//resolve('immediately');
});
console.log("wait ...");
var result = wait(task);
console.log("wait ...done", result);
}
In nodejs 14.8.0+, you are able to use top level awaits.
So your script will now look like:
var p1 = await new Promise(function(resolve, reject) {
// my function here
});
// p1 is now the result of the promise, not a promise
If you are doing this in the REPL, it is not enabled by default, so you have to run the REPL with the --experimental-repl-await flag. Using nvm, I was able to test even node version 12.20.1 and the repl flag works.