How to get json data from multiple url-pages Node.js? - node.js

Question

I am quite new to Node.js and haven't been working with json data before so really hope that you can help me.
I am trying to get all event information from Ticketmaster's API and add specific variables to mongoDB. However, the APIs' that I am currently using are limited to 200 events per page. It is therefore not possible for me to connect the event information with venue information since these are added seperately to mongoDB and are not exhaustive of all event and venue information (not able to connect on ids because of missing event and venue data).
My question is therefore in regards to how I can get all pages into my database at once?
The code that I have written so far looks something like below:
app.get('/tm', (req, res) => {
axios // getting venues
.get('https://app.ticketmaster.com/discovery/v2/venues.json?apikey=myApiKey&page=0&size=200&countryCode=DK')
.then(response => {
const venuesToBeInserted = response.data._embedded.venues.map(venue => { // preparing venues
return {
sourceID: venue.id,
venue: venue.name,
postalCode: venue.postalCode,
city: venue.city.name,
country: venue.country.name,
countryCode: venue.country.countryCode,
address: !!venue.address ? venue.address.line1 : null,
longitude: !!venue.location ? venue.location.longitude : null,
latitude: !!venue.location ? venue.location.latitude : null,
source: 'ticketmaster'
}
})
// Create all venues at once
Venue.create(venuesToBeInserted).then(venues => {
console.log("venues inserted")
axios // getting events and shows - note the page parameter in the api link
.get('https://app.ticketmaster.com/discovery/v2/events.json?apikey=myApiKey&countryCode=DK&size=200&page=0')
.then(response => {
const eventsToBeInserted = response.data._embedded.events.map(events => { // preparing events
const event = events._embedded.attractions[0]
return {
sourceID: event.id,
name: event.name,
slug: slugify(event.name).toLowerCase(),
tags: !!event.classifications ? [event.classifications[0].genre.name, event.classifications[0].subGenre.nam] : [], // duplicate genres occur
// possible tags from ticketmaster: type and subtype
}
})
// Create all events at once
Event.create(eventsToBeInserted).then(events => {
console.log("events inserted")
const showsToBeInserted = response.data._embedded.events.map(show => {
const event = events.find(event => event.sourceID == show._embedded.attractions[0].id);
const venue = venues.find(venue => venue.sourceID == show._embedded.venues[0].id);
if (!!event && !!venue) {
return {
event: event._id,
venue: venue._id,
timezone: show.dates.timezone,
dateStart: !!show.dates.start.dateTime ? show.dates.start.dateTime : show.dates.start.localDate,
tickets: !!show.priceRanges ? {
minPrice: show.priceRanges[0].min,
maxPrice: show.priceRanges[0].max,
currency: show.priceRanges[0].currency
}: {}
}
}
})
// Let's see what we have created in the database
Venue.find({}).select({
name: 1,
slug: -1
}).limit(10).populate('event').populate('venue').then(events => {
console.log(util.inspect(events));
}).catch(err => {
console.error(err);
});
}).catch( err => {
console.error(err)
})
}).catch( err => {
console.error(err)
})
}).catch(err => {
console.error(err)
});
}).catch(err => {
console.error(err)
})
})
EDIT
Using the approach that Jake suggested gave me an error (Error: Requested failed with status code 401). I have tried to search for it online but I cannot figure out why the error happens.. See picture below of part of the error message in my console.log.
error message

Answer 1

You can do this using promises, which you already are using, you just need to chain them together using recursion.
function getVenues(page, size, venues) {
page = page || 0;
size = size || 200;
venues = venues || [];
return axios
.get(`https://app.ticketmaster.com/discovery/v2/venues.json?apikey=myApiKey&page=${page}&size=${size}&countryCode=DK`)
.then(response => response.data._embedded.venues)
.then(rawVenues => {
rawVenues.forEach(venue => venues.push(venue));
if (rawVenues.length < size) {
// All done return the compiled list.
return venues;
}
// Recurse over the next set of venues by adding another promise to the chain.
return getVenues(page + 1, size, venues);
});
}
function getEvents(page, size, events) {
page = page || 0;
size = size || 200;
events = events || [];
return axios
.get(`https://app.ticketmaster.com/discovery/v2/events.json?apikey=myApiKey&countryCode=DK&size=${size}&page=${page}`)
.then(response => response.data._embedded.events)
.then(rawEvents => {
rawEvents.forEach(event => events.push(event));
if (rawEvents.length < size) {
// All done return the compiled list.
return events;
}
// Recurse over the next set of events by adding another promise to the chain.
return getEvents(page + 1, size, events);
});
}
app.get('/tm', (req, res) => {
getVenues().then(rawVenues => {
const venuesToBeInserted = rawVenues.map(venue => {
return {
sourceID: venue.id,
venue: venue.name,
postalCode: venue.postalCode,
city: venue.city.name,
country: venue.country.name,
countryCode: venue.country.countryCode,
address: !!venue.address ? venue.address.line1 : null,
longitude: !!venue.location ? venue.location.longitude : null,
latitude: !!venue.location ? venue.location.latitude : null,
source: 'ticketmaster'
};
});
// Return promise so errors bubble up the chain...
return Venue.create(venuesToBeInserted).then(venues => {
console.log("venues inserted");
// Return promise so errors bubble up the chain...
return getEvents().then(rawEvents => {
const eventsToBeInserted = rawEvents.map(rawEvent => {
const event = events._embedded.attractions[0];
return {
sourceID: event.id,
name: event.name,
slug: slugify(event.name).toLowerCase(),
tags: !!event.classifications ? [event.classifications[0].genre.name, event.classifications[0].subGenre.nam] : []
};
});
// Return promise so errors bubble up the chain...
return Event.create(eventsToBeInserted).then(events => {
console.log("events inserted");
const showsToBeInserted = rawEvents.map(show => {
const event = events.find(event => event.sourceID == show._embedded.attractions[0].id);
const venue = venues.find(venue => venue.sourceID == show._embedded.venues[0].id);
if (!!event && !!venue) {
return {
event: event._id,
venue: venue._id,
timezone: show.dates.timezone,
dateStart: !!show.dates.start.dateTime ? show.dates.start.dateTime : show.dates.start.localDate,
tickets: !!show.priceRanges ? {
minPrice: show.priceRanges[0].min,
maxPrice: show.priceRanges[0].max,
currency: show.priceRanges[0].currency
} : {}
}
}
});
// Do something with the found shows...
});
});
});
}).then(() => { // This then is fired after all of the promises above have resolved...
return Venue.find({}).select({
name: 1,
slug: -1
}).limit(10).populate('event').populate('venue').then(events => {
console.log(util.inspect(events));
res.send(events);
});
}).catch(err => { // Catches any error during execution.
console.error(err);
res.status(500).send(err);
});;
});