I am learning NodeJS (generally I write in PHP). Please help me figure out the Promises. Now the 'getDataFromDir' function returns 'undefined'. I read the documentation, but apparently I don't fully understand something. Thanks in advance!
const host = 'localhost';
const port = 3000;
const __dirname = process.cwd();
async function getDataFromDir(fileName){
fsp
.readdir(path.join(fileName))
.then(async (indir) => {
const list = []
for (const item of indir) {
const src = await fsp.stat(path.join(fileName, item))
list.push(item)
}
return list;
})
}
const server = http.createServer(async (req, res) => {
const result = await getDataFromDir(__dirname);
result
.then((list) => {
console.log(list);
});
});
It seems like your return statement is returning only for the callback function under your .then statement. You should be able to use the await statement with your initial request and achieve a similar result.
async function getDataFromDir(fileName){
let indir = await fsp.readdir(path.join(fileName));
const list = [];
for (const item of indir) {
const src = await fsp.stat(path.join(fileName, item));
list.push(item);
}
return list;
}
Related
The script by itself works great (entering the url manually, writing a json file using the fs module, node script_name.js) but within a Express get request it returns undefined.
So I've built a simple frontend to let the user enter the subreddit name to be scraped.
And here's where the problem is:
Express controller
const run = require("../run");
requestPosts: async (req, res) => {
try {
const { subreddit } = req.body;
const response = await run(subreddit);
//console.log(response);
res.json(response);
} catch (error) {
console.error(error);
}
},
Cheerio functions
const axios = require("axios");
const { load } = require("cheerio");
let posts = [];
async function getImage(postLink) {
const { data } = await axios(postLink);
const $ = load(data);
return $("a.post-link").attr("href");
}
async function run(url) {
try {
console.log(url);
const { data } = await axios(url);
const $ = load(data);
$(".thing.linkflair.link").map(async (i, e) => {
const title = $(e)
.find(".entry.unvoted .top-matter .title .title")
.text();
const user = $(e)
.find(".entry.unvoted .top-matter .tagline .author")
.text();
const profileLink = `https://old.reddit.com/user/${user}`;
const postLink = `https://old.reddit.com/${$(e).find("a").attr("href")}`;
// const thumbail = $(e).find("a img").attr("src");
const image = await getImage(postLink);
posts.push({
id: i + 1,
title,
postLink,
image,
user: { user, profileLink },
});
});
const nextPage = $(".next-button a").attr("href");
if (nextPage) {
await run(nextPage);
} else {
return posts;
}
} catch (error) {
console.error(error);
}
}
module.exports = run;
I've tried working with Promise((resolve, reject) => {}).
I think it's returning undefined because maybe the code its not synchronized.
(idk if it makes sense, i've just started programming)
.map() is not promise-aware and does not wait for your promises to finish. So, $(".thing.linkflair.link").map() finishes long before any of the asynchronous functions inside its callback do. Thus you try to return posts BEFORE it has been populated.
Passing an async callback to .map() will return an array of promises. You can use Promise.all() on those promises to know when they are done and once you're doing that, you may as well just return each post object rather that using a higher level scoped/shared object, thus making the code more self contained.
I would suggest this code:
async function run(url) {
try {
console.log(url);
const { data } = await axios(url);
const $ = load(data);
const posts = await Promise.all($(".thing.linkflair.link").map(async (i, e) => {
const title = $(e)
.find(".entry.unvoted .top-matter .title .title")
.text();
const user = $(e)
.find(".entry.unvoted .top-matter .tagline .author")
.text();
const profileLink = `https://old.reddit.com/user/${user}`;
const postLink = `https://old.reddit.com/${$(e).find("a").attr("href")}`;
// const thumbail = $(e).find("a img").attr("src");
const image = await getImage(postLink);
// return a post object
return {
id: i + 1,
title,
postLink,
image,
user: { user, profileLink },
};
}));
const nextPage = $(".next-button a").attr("href");
if (nextPage) {
const newPosts = await run(nextPage);
// add these posts to the ones we already have
posts.push(...newPosts);
}
return posts;
} catch (error) {
console.error(error);
}
}
I am struggling to understand callbacks, promises, and async/await.
What I want to do is read a .csv file inside my project folder that contains 150+ post ID's.
For each one of those ID's I want to make a https GET request to fetch a JSON response from my Wordpress website.
Then for each one of those posts that gets returned I want to insert them in my Firestore database.
I'm struggling with how to properly set up the callback functions.
Please help.
Thanks in advance.
const express = require('express');
const router = express.Router();
const https = require("https");
const Recipe = require("../includes/newrecipe");
var admin = require('firebase-admin');
var serviceAccount = require("../service_key.json");
const collectionKey = "recipes"; //name of the collection
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: "<MY_FIRESTORE_URL>"
});
const firestore = admin.firestore();
const fs = require('fs');
const parse = require('csv-parser');
function prepareCsvData() {
return new Promise((resolve, reject) => {
//establish empty csvData array and filename to be referenced
var csvData = [];
var filename = 'wprm_recipe_ids.csv';
//read the csv file and push the data object into the array
fs.createReadStream(filename)
.pipe(parse(['ID']))
.on('data', (data) => csvData.push(data))
.on('end', () => { resolve(csvData); });
});
}
function getRecipeFromBlog(recipeId) {
return new Promise((resolve, reject) => {
//make the get request to my website to get the recipe
https.get('<MY_WEBSITE_URL>' + recipeId, (response) => {
var body = "";
response.on('data', function (chunk) { body += chunk; });
response.on('end', () => {
var { recipe } = JSON.parse(body);
//build new recipe to be exported
var newRecipe = new Recipe(recipe);
resolve(newRecipe);
});
});
});
}
/* GET recipes. */
router.get('/', async (req, res, next) => {
//first prepare the csv data
//function returns a promise with the csv data
//that I can then use in the next step
const csvData = await prepareCsvData();
for (var i = 0; csvData.length < i; i++) {
getRecipeFromBlog(csvData[i].ID)
.then((newRecipe) => {
//when I have a recipe for a given recipe ID
//update database in firestore
firestore
.collection(collectionKey)
.doc(""+newRecipe.id)
.set(newRecipe)
.then(function() {
console.log('document written');
});
});
}
res.send('done');
});
You need to do something like below:
Play around this, You'll get it working hopefully!
Let me know if that worked!
router.get("/", async (req, res, next) => {
const csvData = await prepareCsvData();
const recipePromises = [];
// check if data is empty or not
if (!csvData.length) {
return res.send("Empty data");
}
csvData.forEach((element) => {
recipePromises.push(getRecipeFromBlog(element.id));
});
// await for all promises parallelly.
const result = await Promise.all(recipePromises);
// Get a new write batch
const batch = db.batch();
result.forEach((recipe) => {
const ref = db.collection("recipes").doc(`${recipe.id}`);
batch.set(ref, recipe);
});
// Commit the batch
await batch.commit();
res.send("done");
});
The OP code looks pretty close to working. Have the promise-returning functions been tested? Assuming they work, first decorate them as async...
async function prepareCsvData() {...}
async function getRecipeFromBlog(recipeId) {...}
Create another promise-returning function to insert many recipes into firebase...
async function writeRecipesToFB(recipes) {
const collectionRef = collection(collectionKey);
const promises = recipes.map(recipe => {
return collectionRef.doc(`${recipe.id}`).set(recipe);
});
return Promise.all(promises)
}
As another answer suggests, as an alternative, firebase's batch write is a good idea...
async function writeRecipesToFB(recipes) {
// as a set of promises
const collectionRef = collection(collectionKey);
const batch = db.batch();
recipes.forEach(recipe => {
const docRef = collectionRef.doc(`${recipe.id}`)
batch.set(docRef, recipe)
});
return batch.commit();
}
Now the express function is easy to write...
router.get('/', async (req, res, next) => {
const csvData = await prepareCsvData();
let promises = csvData.map(row => {
return getRecipeFromBlog(row.ID);
});
const recipes = await Promise.all(promises);
await writeRecipesToFB(recipes);
res.send('done');
});
I'm here doing a scraper using node.js request and request-promise, cheerio.
My code:
const request = require("request");
const cheerio = require("cheerio");
const rp = require("request-promise");
const url = "https://singapore.craigslist.org/d/automotive-services/search/aos"
const scrapeResults = [];
async function scrapeJobHeader() {
try {
const htmResult = await rp.get(url);
const $ = await cheerio.load(htmResult);
$(".result-info").each((index, element) => {
const resultTitle = $(element).children(".result-title");
title = resultTitle.text();
link = resultTitle.attr("href");
const datePosted = $(element).children("time").attr("datetime");
const scrapResult = {title, link, datePosted};
scrapeResults.push(scrapResult);
return scrapeResults;
});
} catch (err) {
console.error(err);
}
}
async function scrapeDescription(jobWithHeaders) {
return await Promise.all(
jobWithHeaders.map(async job => {
const htmResult = await rp.get(job.url);
const $ = await cheerio.load(htmResult);
$(".print-qrcode-container").remove();
job.description = $("#postingbody").text();
})
);
}
async function scrapeCraigslist() {
const jobWithHeaders = await scrapeJobHeader();
const jobsFullData = await scrapeDescription();
console.log(jobFullData);
}
scrapeCraigslist();
When I run the code I get error like:
C:\Users\Ahmed-PC\craigslist>node index.js
(node:19808) UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'map' of undefined
at scrapeDescription (C:\Users\Ahmed-PC\craigslist\index.js:42:24)
at scrapeCraigslist (C:\Users\Ahmed-PC\craigslist\index.js:62:32)
How I can fix this error and what wrong I'm doing here ?
You're doing this await scrapeDescription();, but you can't call that function without passing it an array.
When you do, then your argument jobWithheaders is undefined and you then try to do undefined.map() which gives you the error you see.
It looks like maybe you just need to change this:
async function scrapeCraigslist() {
const jobWithHeaders = await scrapeJobHeader();
const jobsFullData = await scrapeDescription();
console.log(jobFullData);
}
to this:
async function scrapeCraigslist() {
const jobWithHeaders = await scrapeJobHeader();
const jobsFullData = await scrapeDescription(jobWithHeaders); // <===
console.log(jobFullData);
}
Also, there's no reason to do:
return await Promise.all(...)
Change that to:
return Promise.all(...)
Either way, you're returning a promise that resolves to the same value. Basically, there's never any reason inside an async function to do return await somePromise. Just return the promise directly without the await. All the await does (if not optimized out by the interpreter) is wait for the promise to resolve, get value out of it, then take the promise that was already returned from the async function and make that value the resolved value of that promise. Which gives you the identical result as just returning the promise you already had without the await.
Change this:
const scrapeResults = [];
async function scrapeJobHeader() {
try {
const htmResult = await rp.get(url);
const $ = await cheerio.load(htmResult);
$(".result-info").each((index, element) => {
const resultTitle = $(element).children(".result-title");
title = resultTitle.text();
link = resultTitle.attr("href");
const datePosted = $(element).children("time").attr("datetime");
const scrapResult = {title, link, datePosted};
scrapeResults.push(scrapResult);
return scrapeResults;
});
} catch (err) {
console.error(err);
}
}
to this:
async function scrapeJobHeader() {
const scrapeResults = [];
const htmResult = await rp.get(url);
const $ = await cheerio.load(htmResult);
$(".result-info").each((index, element) => {
const resultTitle = $(element).children(".result-title");
const title = resultTitle.text();
const link = resultTitle.attr("href");
const datePosted = $(element).children("time").attr("datetime");
const scrapResult = {title, link, datePosted};
scrapeResults.push(scrapResult);
});
return scrapeResults;
}
And, then change this:
scrapeCraigslist();
to this:
scrapeCraigslist().then(results => {
// use the results in here only
console.log(results);
}).catch(err => {
console.log(err);
});
Then, change this:
async function scrapeDescription(jobWithHeaders) {
return await Promise.all(
jobWithHeaders.map(async job => {
const htmResult = await rp.get(job.url);
const $ = await cheerio.load(htmResult);
$(".print-qrcode-container").remove();
job.description = $("#postingbody").text();
})
);
}
to this:
function scrapeDescription(jobWithHeaders) {
return Promise.all(
jobWithHeaders.map(async job => {
const htmResult = await rp.get(job.url);
const $ = await cheerio.load(htmResult);
$(".print-qrcode-container").remove();
job.description = $("#postingbody").text();
return job;
});
);
}
I tried to get the document and the subcollection data at once in firestore.
And I used the async and await to deal with the forEach loop.
It still has some problem. The console.log 4 always executes first.
But what I expect the should be 1 -> 2 -> 3 -> 4.
Could anyone help me how to redesign my code?
let data = {};
toolboxesRef.get()
.then(async snapshot => {
let toolboxes = [];
// get toolbox
await snapshot.forEach(async doc => {
let toolbox = {};
await toolboxesRef.doc(doc.id).collection('tags').get()
.then(snapshot => {
let tags = []
snapshot.forEach(doc => {
tags.push(doc.id);
console.log(1)
})
toolbox.tags = tags;
toolbox.id = doc.id;
toolbox.data = doc.data();
console.log(2)
})
console.log(3)
toolboxes.push(toolbox)
})
console.log(4);
data.toolboxes = toolboxes
return data;
})
export const asyncForEach = async (dataSnapshot, callback) => {
const toWait = [];
dataSnapshot.forEach(childSnapshot => {
toWait.push(childFunction((childSnapshot)));
});
await Promise.all(toWait);
};
Hey i updated the code because it seems that Firebase integrate it's own foreach function. Then in order to resolve i decided to call every function and store the promise that it return into an array then i use Promise.all to resolve an array of async function
You are using async operations inside forEach which doesn't work as you expect thm. You need to either use for..of or Promise.all. Try this version
const snapshot = await toolboxesRef.get();
const toolboxes = [];
for(const doc of snapshot) {
const toolbox = {};
const snapshot1 = await toolboxesRef.doc(doc.id).collection("tags").get();
const tags = [];
snapshot1.forEach(doc => {
tags.push(doc.id);
console.log(1);
});
toolbox.tags = tags;
toolbox.id = doc.id;
toolbox.data = doc.data();
console.log(2);
console.log(3);
toolboxes.push(toolbox);
}
console.log(4);
data.toolboxes = toolboxes;
return data;
You might need to tweak few things here and there but you will get an idea.
I'm trying to get the mongo client in another file. The problem is, when I try getting the mongoClient variable, it returns undefined.
How can I wait till a the mongoClient variable is declared before trying to get it?
File 1
let mongoClient;
module.exports = async function() {
const mongooseOptions = {...};
mongoClient = await mongoose.connect(dbUrl, mongooseOptions);
};
exports.getMongoClient = () => mongoClient;
File 2
const { getMongoClient } = require('../../startups/db');
console.log(getMongoClient); // Returns undefined
You should use this logic
File 1
const axios = require("axios");
async function getData() {
return await axios.get("https://jsonplaceholder.typicode.com/todos");
}
module.exports = { getData };
File 2
const { getData } = require("./file1");
getData().then(data => console.log(data));
You should use global variable for access in in any file like as bellow.
File 1
global.mongoClient;
module.exports = async function() {
const mongooseOptions = {...};
global.mongoClient = await mongoose.connect(dbUrl, mongooseOptions);
};
File 2
console.log(global.getMongoClient);