SWAPI Pagination with Node.JS, Express and Axios - node.js

I wanted to fetch all people/films/etc. from SWAPI.
I tried a lot of things before finally get something usable. However, it only shows the first 10. people (in the people case)
const express = require("express");
const router = express.Router();
const axios = require("axios");
router.get("/people", (req, res) => {
axios
.get("https://swapi.dev/api/people/?format=json")
.then((data) => {
return res.send(data.data.results);
})
.catch((error) => {
console.log("error: ", error);
});
});
module.exports = router;
What I tried, thanks to a lot of searches from Stack Overflow, is this :
const express = require("express");
const router = express.Router();
const axios = require("axios");
router.get("/people", async (req, res) => {
let nextPage = `https://swapi.dev/api/people/`;
let people = [];
while (nextPage) {
res = await axios(nextPage)
const { next, results } = await res.data;
nextPage = next
people = [...people, ...results]
}
console.log(people.length) // 82
console.log(people) // shows what I wanted, all people !
return people;
});
module.exports = router;
When starting the server, the page doesn't finish loading (it's still loading at this moment), but the console.log managed to show exactly what I wanted.
So, how can I manage to show this in the page ?
My goal is to use that route for axios API calls from a React front-end (searching for a specific name)

Do not overwrite the res in your code and finish it with res.send:
let nextPage = `https://swapi.dev/api/people/`;
let people = [];
while (nextPage) {
let nextres = await axios(nextPage)
const { next, results } = await nextres.data;
nextPage = next
people = [...people, ...results]
}
console.log(people.length) // 82
console.log(people) // shows what I wanted, all people !
res.send(people);

Related

Nodejs/Express: Facing issues while working on routings

viewController.js
const Book = require('./../models/bookModel')
const APIFeatures = require('./../utils/apiFeatures.js')
exports.getOverview=async(req,res,next)=>{
const features = new APIFeatures(Book.find(),req.query).filter().paginations()
let getItAllProd = await features.getBook;
res.render('overview',{
data:getItAllProd,
title:"trending books"
})
next()
}
exports.getOneBook =async(req,res,next)=>{
const getThatBook = await Book.findOne({slug:req.params.id})
res.render('onebook',{
data:getThatBook
})
next()
}
exports.categories= async(req,res,next)=>{
const getCate = await Book.find({category:req.params.category});
res.render('overview',{
data:getCate,
name:req.params
})
next()
}
exports.paginationOfBook = async(req,res,next)=>{
let pages = req.params.page
pages = pages * 6 || 6;
const limit = 6;
let skip = pages-6;
const getlimitedbook = await Book.find({}).limit(limit).skip(skip);
res.render('overview',{
data:getlimitedbook,
})
next()
}
viewRoutes.js
const express = require('express');
const viewController = require('./../controllers/viewController')
const router = express.Router();
router.get('/overview',viewController.getOverview)
router.get('/overview/:category',viewController.categories)
router.get('/overview/:page',viewController.paginationOfBook)
router.get('/overview/:id',viewController.getOneBook)
module.exports = router;
using http://localhost:3000/overview I can get all the books and http://localhost:3000/overview/:category - using this I get the books by categories. but if I try to load the the for http://localhost:3000/overview/:page and http://localhost:3000/overview?:id - the data will not load up.. so below I did some changes in routing..
router.get('/overview',viewController.getOverview)
router.get('/overview/:id',viewController.getOneBook)
router.get('/overview/:category',viewController.categories)
router.get('/overview/:page',viewController.paginationOfBook)
Now I have access to the '/overview' and '/overview/:Id'. but not for others. the data is not loading for them. do you have any solution? so I can access all the routes.uuu
First of all please elaborate you're question more. But I think the problem is in the following line:
const getThatBook = await Book.findOne({slug:req.params.id})
edit this line to:
const getThatBook = await Book.findById(req.params.id)
and call this route like this http://localhost:3000/overview/34234232...
Or change your "id" to "slug" in the following line.
const getThatBook = await Book.findOne({slug:req.params.slug})
and call this route like this http://localhost:3000/overview/xyz_book_slug...
router.get('/overview/:slug',viewController.getOneBook)
and pass the book's slug property value when calling this route.

Express Rest Api: question about routing?

I have a Rest API where a user can create a list and then put list items in it (for a quiz). My schema structure is this:
const verbListSchema = {
title: String,
verbs: [{verb: String}]
};
Here are the url endpoints I have so far:
/lists/ (gets back all the lists)
/lists/verbs (gets all the verbs from all the lists)
My question is - I want to get, post, patch and delete a specific list using its id, like /lists?list_id=123/verbs or /lists/123/verbs and then one step further to get individual verbs I want to do something like /lists/123/verbs/124 or /lists?list_id=123/verbs?verb_id=124 the last doesn't work because it counts the last endpoint as a query param.
In terms of best practice what's the best way to do this. I could do something like this (I use express.js)?
app.[request-type]("/lists") {...}
app.[request-type]("/lists/:list_id") {...}
app.[request-type]("/lists/:list_id/verbs") {...}
app.[request-type]("/lists/:list_id/verbs/:verb_id") {...}
and then if I want to retrieve all the lists, not just a specific one I can check if the list_id is "all" like, /lists/all/verbs?
And here is my code so far:
const express = require("express");
const verbRouter = require("./verbRoutes");
const router = express.Router();
const VerbList = require("../../verb-list-db");
const isOriginal = async (req,res,next) => {
const listExists = await VerbList.find({title: req.body.listTitle})
if (listExists.length > 0 ) return res.status(400).json({message: "list already exists"});
next();
};
router.route("/")
.get(async (req,res,next) => {
try {
const listId = req.query.list_id;
if (listId) return res.json(await VerbList.find({_id: listId}));
const lists = await VerbList.find({});
res.json(lists);
} catch(err) {next(err)}
})
.post(isOriginal, async (req,res,next) => {
const newList = new VerbList({ // creates a new list
title: req.body.listTitle
})
newList.save()
.then(() => {return res.send("list successfully added!")})
.catch(err => next(err));
})
.patch(isOriginal, async (req,res,next) => {
try {
const listId = req.query.list_id;
if (!listId) throw new Error("you must have a list_id to patch!")
res.json(await VerbList.updateOne({_id: req.query.list_id}, {title: req.body.listTitle}))
} catch(err) {next(err)}
})
.delete(async (req,res,next) => {
try {
const listId = req.query.list_id;
if (!listId) throw new Error("you must have a list_id to delete!");
res.json(await VerbList.deleteOne({_id: req.query.list_id}))
} catch(err) {next(err)}
})
Any suggestions would be appreciated :)
You can try to modularize your express code by separating your /lists routes from your main server.js (or index.js) file.
index.js
const express = require('express');
const app = express();
//Now lets route all the API requests that start with '/list' to a file called lists.js
app.use('/lists', require('/path/to/lists.js')
app.listen(3000, () => console.log(`\nServer started on port 3000`))
lists.js
const express = require('express');
const router = express.Router();
// now you know all the requests in this file would be for /list so lets implement a router for fetching all the lists
router.get('/', async(req, res) => {
*** add all your logic for /lists endpoints here**
res.status(200).json(lists_json_response); //send a response back to your client
})
//you can create as many endpoints as you want in this file for endpoints that start with '/lists'
router.[request-method]("/lists/:list_id") {...} // endpoint for requesting a specific list
router.[request-method]("/:list_id/verbs") {...} //endpoint for requesting all the verbs for a specific list
router.[request-method]("/lists/all/verbs") {...} // all the verbs in all the lists
module.exports = router;
Also you cant put query parameters in the middle of an endpoint. if it is going to be a variable that you need and its not in the end of the URL, you have to pass it as a param, e.g.:
instead of doing /lists?list_id=123/verbs?verb_id=124, you can do something like /lists/123/verbs/124 to look for the verb with id 124 in a list with id 123.
so to listen to a request to this endpoint, you can design another endpoint in your lists.js file like this:
router[request-method].('/:list_id/verb/:verb_id', async(req, res)=> {
var list_id = req.params.list_id
var verb_id = req.params.verb_id
***
now use the list_id and verb_id to query the requested data and send a response back to the client
***
})

fs.writefile is making POST request loop infinitly within my express app

I have this current server code:
const express = require("express")
const fs = require("fs")
const router = express.Router()
const path = require("path")
const todos = JSON.parse(fs.readFileSync(path.join(__dirname, "../db", "todolist.json"), "utf8"))
router.get("/", async (req, res) => {
res.send(todos)
})
router.post("/new", async (req, res) => {
const { title, description } = req.body
const todoItem = {
id: "3",
title,
description
}
todos.todos.push(todoItem)
const data = JSON.stringify(todos, null, 2)
fs.writeFile(path.join(__dirname, "../db", "todolist.json"), data, () => {})
res.status(201).json(todoItem)
})
client:
console.log("Hello world!")
const somedata = {
title: "A new boy",
description: "Recieved from the client"
}
const main = async () => {
const response1 = await fetch("http://localhost:3000/todo", {
method: "GET",
})
const data1 = await response1.json()
const response2 = await fetch("http://localhost:3000/todo/new", {
method: "POST",
body: JSON.stringify(somedata),
headers: {
'Content-Type': 'application/json',
"Accept": "application/json"
}
})
const data2 = await response2.json()
return { data1, data2 }
}
main().then(data => console.log(data))
When I make a /POST request to create a new entity the browser just loops the request over and over until I manually have to quit the server. This does not happen if I use postman for some reason. Does anybody see any obvious error here with how the writeFile-method is used and why it continuously reloads the browser to keep pushing POST requests?
Thanks! :)
i had the same problem! And it took me about 1 hour to understand what my Problem is:
If you use "live server extension", the server will restart everytime, when you write, change or delete a file in the project folder!
So, if your node-app wirte a file, the live-server will restart and the app writes the file again! => loop
In my case, i write a pdf-file. All i had to do, is to tell the live server extension to ignore pdf files:
So i just add to "settings.json":
"liveServer.settings.ignoreFiles":["**/*.pdf"]
fs.writeFile is asynchronous function. So, to send a response after file written you must do it in the callback. And of course, don't forget about error checking. I.e.
router.post("/new", async (req, res) => {
const { title, description } = req.body
const todoItem = {
id: "3",
title,
description
}
todos.todos.push(todoItem)
const data = JSON.stringify(todos, null, 2)
fs.writeFile(path.join(__dirname, "../db", "todolist.json"), data, (err) => {
if(err) {
throw err;
}
res.status(201).json(todoItem)
})
})
Or you can use fs.writeFileSync as Muhammad mentioned earlier.
I think I found the problem. It seemed that the live server extension was messing things up when I had the client and server on separate ports, making the browser refresh for every request made somehow. I switched back to them sharing port, which then makes it work. I have to find a good way of separating them on a later basis without this bug happening, but that is for another time.
Thanks for your help :)
I share my working sample.body-parser dependency is need to get body in post request.Please don't change the order in server.js.Check and let me know.
and also check once whether your client code is in in loop.
My server.js
const express = require("express")
const fs = require("fs")
const router = express.Router()
const path = require("path")
const app = express();
const bodyParser = require("body-parser")
const todos = JSON.parse(fs.readFileSync(path.join(__dirname, "../db", "todolist.json"), "utf8"))
app.use(bodyParser.json());
app.use("/",router)
router.get("/todo", async (req, res) => {
res.send(todos)
})
router.post("/todo/new", async (req, res) => {
const { title, description } = req.body
const todoItem = {
id: "3",
title,
description
}
todos.todos.push(todoItem)
const data = JSON.stringify(todos, null, 2)
fs.writeFile(path.join(__dirname, "../db", "todolist.json"), data, () => {})
res.status(201).json(todoItem)
});
app.listen(3000, () => {
console.log(`Server running in Port`);
});
todolist.json
{
"todos": []
}
I think you should use fs.writeFileSync() or write some code in its callback

All my scraped text ends up in one big object instead of separate objects with Cheerio

I'm following a web scraping course that uses Cheerio. I practice on a different website then they use in the course and now I run into the problem that all my scraped text end up in one big object. But every title should end up in it's own object. Can someone see what I did wrong? I already bumbed my head 2 hours on this problem.
const request = require('request-promise');
const cheerio = require('cheerio');
const url = "https://huurgoed.nl/gehele-aanbod";
const scrapeResults = [];
async function scrapeHuurgoed() {
try {
const htmlResult = await request.get(url);
const $ = await cheerio.load(htmlResult);
$("div.aanbod").each((index, element) => {
const result = $(element).children(".item");
const title = result.find("h2").text().trim();
const characteristics = result.find("h4").text();
const scrapeResult = {title, characteristics};
scrapeResults.push(scrapeResult);
});
console.log(scrapeResults);
} catch(err) {
console.error(err);
}
}
scrapeHuurgoed();
This is the link to the repo: https://github.com/danielkroon/huurgoed-scraper/blob/master/index.js
Thanks!
That is because of the way you used selectors. I've modified your script to fetch the content as you expected. Currently the script is collecting titles and characteristics. Feel free to add the rest within your script.
This is how you can get the required output:
const request = require('request-promise');
const cheerio = require('cheerio');
const url = "https://huurgoed.nl/gehele-aanbod";
const scrapeResults = [];
async function scrapeHuurgoed() {
try {
const htmlResult = await request.get(url);
const $ = await cheerio.load(htmlResult);
$("div.item").each((index, element) => {
const title = $(element).find(".kenmerken > h2").text().trim();
const characteristics = $(element).find("h4").text().trim();
scrapeResults.push({title,characteristics});
});
console.log(scrapeResults);
} catch(err) {
console.error(err);
}
}
scrapeHuurgoed();

express router, req.body undefined

I have a post interface that does not submit data correctly.
test shows: req.body undefined
const express = require('express');
const router = express.Router();
const passport = require("passport");
const passportInfo = passport.authenticate('jwt',{ session: false });
const HomeSchema = require('../../../models/AnvizHome');
const homeBannerValidator = require('../../../validation/anviz/homeBanner');
router.post("/banner",passportInfo,(req,res) => {
const {msg,isValid} = homeBannerValidator(req.body);
if(!isValid){
return res.status(400).json(msg);
}
HomeSchema.findOne({handle:req.body.handle}).then(banner => {
console.log('current: ' + req.body);
const newBanner = {
bannerBg:req.body.bannerBg,
bannerName:req.body.bannerName,
bannerSubName:req.body.bannerSubName,
bannerFeather:req.body.bannerFeather,
bannerLink:req.body.bannerLink
};
banner.prodcutBanner = newBanner;
banner.then(home => res.json(home));
})
.catch((err) => res.json(err));
});
module.exports = router;
postman test:
In fact, the terminal can see the returned data.
[Object: null prototype] {
bannerBg: '5555555555555555555555555555555555',
bannerName: 'THE GLOBAL LEADING',
bannerSubName: 'PROVIDER OF INTELLIGENT SECURITY',
bannerLink: 'www.anviz.com',
handle: 'true' }
Seeking one or two!Thank you!
you forgot to import and use body parser
https://www.npmjs.com/package/body-parser
var bodyParser = require('body-parser');
var express = require('express');
express.use(bodyParser.json());
express.use(bodyParser.urlencoded({ extended: false }));
I think I know the reason, because this is based on the user's token to submit data, so must be in the post according to the user id to determine whether the data is successful?
Here is the code that has been successfully saved, but I don't know right and wrong:
router.post("/banner",passportInfo,(req,res) => {
const {msg,isValid} = homeBannerValidator(req.body);
if(!isValid){
return res.status(400).json(msg);
}
const profileFields = {};
profileFields.prodcutBanner = {};
if(req.body.handle) profileFields.handle = req.body.handle;
if(req.body.bannerBg) profileFields.prodcutBanner.bannerBg = req.body.bannerBg;
if(req.body.bannerName) profileFields.prodcutBanner.bannerName = req.body.bannerName;
if(req.body.bannerSubName) profileFields.prodcutBanner.bannerSubName = req.body.bannerSubName;
if(req.body.bannerFeather) profileFields.prodcutBanner.bannerFeather = req.body.bannerFeather;
if(req.body.bannerLink) profileFields.prodcutBanner.bannerLink = req.body.bannerLink;
HomeSchema.findOne({user: req.user.id}).then(profile => {
if(profile){
HomeSchema.findByIdAndUpdate({user: req.user.id},{$set:profileFields},{new:true}).then(profile => res.json(profile));
}else{
HomeSchema.findOne({handle:profileFields.handle}).then(profile => {
if(profile){
msg.handle = "The user's handle personal information already exists, please do not re-create it!";
res.status(400).json(msg);
}
new HomeSchema(profileFields).save().then(profile => res.json(profile));
})
}
})
.catch((err) => res.json(err));});
module.exports = router;

Resources