how to allow multiple async routes in express.js - node.js

I'm fairly new to Node and Express and am struggling with creating a route that takes a user uploaded file and processes it into another file. The problem is that the second time the user sends a request, I am having to wait for the first process to complete to allow the user to upload a new file.
The basic structure of route that I have is below
THE PROBLEM is that the function convertFile below is a time taking process and it keeps the server busy from accepting new requests. How do I make it so that once the project is saved in mongo db at newProject.save() - the process to convertFile runs in the background while the server accepts new requests from the same route?
I'm sending a response back the user after newProject.save() and was hoping that would allow the user to send another request. And although this sends another request, the server doesn't accept it since its busy with the previous process to convertFile
router.post('/', upload.fields([{ name: 'zip' }]), async (req, res, next) => {
let data = {
title: req.body.title,
description: req.body.description,
}
if (req.files && req.files.zip) {
const newProject = new MongoProject(data);
newProject.save()
.then(async (project) => {
res.send(project);
console.log("Project created");
const uploadedFilePath = path.normalize(req.files.zip[0].path);
// below method - "convertFile" is a time taking method
const extractZipinfo = await convertFile(uploadedFilePath , data.masterFile).then((zipInfo) => {
console.log({ zipInfo })
data.zipInfo = {
sizes: zipInfo.sizes
}
})
})
.catch(err => {
console.log("Error creating project");
return res.send(err);
})
}
})
Below is the simplified version of code in convertFile function (code modified for brevity):
I know that this can be improvised a lot, but i'm struggling with getting it to function as expected first (allowing multiple routes)
async function convertFile(inputFilePath, outputInfo) {
const outputFilePath = "output.abc";
const jsonFilePath = "output.json";
const doc = new Document(); // this is a class to store all data of the output file that we will write at the end
const _FileAPI = new fileAPI();
const outputFinalData = await _FileAPI.Init() // this is an async method
.then(() => {
const dataClass = initiateClass(); // this is a class to store data in JSON format
const paragraphs = _FileAPI.GetallParagraphs(inputFilePath);
for (let i = 0, len = paragraphs.size(); i < len; i++) {
for (let j = 0, lenj = paragraphs.size(); j < lenj; j++) {
const para = paragraphs.get(j);
// read each para and Capitalize each word
dataClass.paragraphs.push(para);
}
}
fs.writeFileSync(jsonFilePath, JSON.stringify(dataClass, null, 2), 'utf-8');
console.log("then")
}).then(() => {
const io = new NodeIO(); // this class helps in writing the file in the desired output format
const outData = io.write(outputFilePath, doc).then(() => {
outputInfo.sizes.push(fs.statSync(outputFilePath).size);
return outputInfo;
});
return outData;
});
return outputFinalData;
}

Related

BotFramework TypeError: Cannot perform 'get' on a proxy that has been revoked

I am trying to develop a MS Teams bot that sends content to students module(unit) wise. I have created 3 classes:
methods.js = Contains all the methods for sending texts, attachments etc.
teamBot.js = Captures a specific keyword from the users and based on that executes a function.
test.js = Connects the bot with Airtable and sends the content accordingly
I am facing Cannot perform 'get' on a proxy that has been revoked error. I figured it might be because of the context. I am passing context as a parameter, which I feel might not be the correct way, how can I achieve the result, and retain the context between files.
teamsBot.js
const test = require("./test");
class TeamsBot extends TeamsActivityHandler {
constructor() {
super();
// record the likeCount
this.likeCountObj = { likeCount: 0 };
this.onMessage(async (context, next) => {
console.log("Running with Message Activity.");
let txt = context.activity.text;
// const removedMentionText = TurnContext.removeRecipientMention(context.activity);
// if (removedMentionText) {
// // Remove the line break
// txt = removedMentionText.toLowerCase().replace(/\n|\r/g, "").trim();
// }
// Trigger command by IM text
switch (txt) {
case "Begin": {
await test.sendModuleContent(context)
}
// By calling next() you ensure that the next BotHandler is run.
await next();
});
// Listen to MembersAdded event, view https://learn.microsoft.com/en-us/microsoftteams/platform/resources/bot-v3/bots-notifications for more events
this.onMembersAdded(async (context, next) => {
const membersAdded = context.activity.membersAdded;
for (let cnt = 0; cnt < membersAdded.length; cnt++) {
if (membersAdded[cnt].id) {
const card = cardTools.AdaptiveCards.declareWithoutData(rawWelcomeCard).render();
await context.sendActivity({ attachments: [CardFactory.adaptiveCard(card)] });
break;
}
}
await next();
});
}
test.js
const ms = require('./methods')
async function sendModuleContent(context) {
data = module_text //fetched from Airtable
await ms.sendText(context, data)
}
methods.js
const {TeamsActivityHandler, ActivityHandler, MessageFactory } = require('botbuilder');
async function sendText(context, text){
console.log("Sending text")
await context.sendActivity(text);
}
Refer this: TypeError: Cannot perform 'get' on a proxy that has been revoked
make the following changes to test.js
const {
TurnContext
} = require("botbuilder");
var conversationReferences = {};
var adapter;
async function sendModuleContent(context) {
data = module_text //fetched from Airtable
const currentUser = context.activity.from.id;
conversationReferences[currentUser] = TurnContext.getConversationReference(context.activity);
adapter = context.adapter;
await adapter.continueConversation(conversationReferences[currentUser], async turnContext => {
await turnContext.sendActivity(data);
});
}

How to avoid overriding set data in Cloud Function

I have a function to adding player like so:
inside my external js (player.js)
module.exports.player = function(appPlayer, db){
appPlayer.post('/', async(req, res) => {
const data = req.body
// data
const ipAdress = data.ipAdress
const carslidekey = data.id
const nickname = data.nickname
// random color for
const randomCol = Math.floor(Math.random()*16777215).toString(16)
let player = {
ipAdress: ipAdress,
nickname: nickname,
color: `#${randomCol}`
}
await db.collection('kartesian').doc(carslidekey)
.collection('player')
.doc(ipAdress)
.set(player).then(() => {
let dataResponse = {
status: true,
idDoc: ipAdress,
ipAdress: ipAdress,
color: `#${randomCol}`
}
const jsonStr_1 = JSON.stringify(dataResponse)
res.send(jsonStr_1)
}).catch((err) => {
let dataResponse = {
status: false,
idDoc: ipAdress,
ipAdress: ipAdress,
msg: `Terjadi kesalahan ${err.data}`
}
const jsonStr_1 = JSON.stringify(dataResponse)
res.send(jsonStr_1)
})
})
}
and my index.js (cloud functions) i wrote https request like below:
......
const playerHandler = require('./src/player')
playerHandler.player(app, db)
exports.player = functions.https.onRequest(app)
......
My problem is sometime this function called by different devices and might be concurrently, even though I've created a method set with a different id but why sometimes I find the previous documentation replaced by the document after. how to ensure that there is no overlap? Thanks
Option 1:
I don't think there is a way besides reading the document as a pre-check before writing. I would recommend implementing a queue -> the cloud function would push to the queue, and a separate cloud function would read the queue and execute on it synchronously.
Also, you're mixing await and promises (e.g. .then), you do not need to do that, see code below.
For example:
Note: I didn't run this code, but it should be close to working
Note2: #Nanda Z made a good point, it's probably not a good idea to run background functions without implementing them as cloud tasks and cloud events
let playerQueue = []
var isProcessingQueue = false
async function handlePlayerQueue(appPlayer, db) {
if (isProcessingQueue) return
isProcessingQueue = true
if (playerQueue.length == 0) return
// remove the first item from the queue
player = playerQueue.shift()
let ref = db.collection('kartesian').doc(player.carslidekey)
.collection('player')
.doc(player.ipAdress)
// Check if player exists
var playerExists = false
try {
playerExists = await ref.get().exists
} catch (err) {
console.error("Could not get player record", err)
}
// exit early
if (playerExists) {
isProcessingQueue = false
// recursively handle queue
handlePlayerQueue(appPlayer, db)
return
}
// wrap await in try...catch to catch exceptions.
try {
let setResponse = await ref.set(player)
let dataResponse = {
status: true,
idDoc: player.ipAdress,
...player
}
const jsonStr_1 = JSON.stringify(dataResponse)
console.log(`Player data pushed: ${jsonStr_1}`)
} catch(err) {
let dataResponse = {
status: false,
idDoc: player.ipAdress,
...player
msg: `Terjadi kesalahan ${err.data}`
}
const jsonStr_1 = JSON.stringify(dataResponse)
console.error(`Failed to set player: ${jsonStr_1}`)
}
isProcessingQueue = false
// recursively handle queue
handlePlayerQueue(appPlayer, db)
}
module.exports.player = function(appPlayer, db){
appPlayer.post('/', async(req, res) => {
const data = req.body
// data
const ipAdress = data.ipAdress
const carslidekey = data.id
const nickname = data.nickname
// random color for
const randomCol = Math.floor(Math.random()*16777215).toString(16)
let player = {
carslidekey, // need to add this so that the queueHandler can use it
ipAdress: ipAdress,
nickname: nickname,
color: `#${randomCol}`
}
// if javascript wasn't singlethreaded, this is where there'd be a lock =)
playerQueue.push(player)
// call the queue handler
handlePlayerQueue(appPlayer, db)
// Tell the client we received the request, they won't know the status, but you can set up another endpoint to check if the record exists
res.send({message: "Request received and queue\'d!"})
})
}
Option 2:
You could instead add a Firestore rule to not allow writing to a document if it's already populated:
match /kartesian/{carslidekey}/player/{player} {
// you can use the `resource.data` variable to access the player document:
// resource will be null if the player doesn't exist. resource.data is overkill but I put it there just fyi
allow write: if request.auth != null && (resource == null || resource.data == null);
}
I tested this rule to make sure it behaves as expected. It should let you write only if you're authenticated AND the resource is empty. So writes that come in later will fail.
It's important to note that rules only apply for online writing and will not be applied to offline/cached data.

Single Http Request to get multiple file data Parse.com

I'm using Back4app.
My Profile class schema has 4 File columns containing pictures.
So when I retrieve an object , I have to make an HTTP request for each file URL and get the byte data like this.
const data = await Parse.Cloud.httpRequest({url:profilePhoto.url()});
return data.buffer.toString('base64');
But for all four files I have to do 4 HTTP requests to the server.
Is there anyway to do a batch HTTP request so that with just 1 request I can get data for all 4 files ?
My main aim is to do the least amount of requests to the server as possible.
There is no out-of-the-box way to retrieve multiple files with one request in Parse Server.
You could implement your own Parse Cloud Code function to retrieve multiple files, but you would have to manually combine them server side and separate them client side.
As a starting point you could look at packages like multistream that allow you to combine multiple file streams into one to get some inspiration.
You might be able to do something similar to what I've done in cloud code.
I had to load up a bunch of information at the start of my application, requiring many round trips to the server.
So I wrote a function called getUserData().
This does many unrelated queries, and jams all of the results into one big object. I then return the object from the function.
Here is the entire function:
console.log("startig getUserData");
var callCount = 0;
var lastLoadTime=0;
// Given a user, load all friends. Save the objects to ret.objects,
// and save the objectIds to ret.friends
//
// Note: we always load the exhaustive friend list, because
// otherwise, we would have no way of recognizing
// removed friendships.
//
async function loadFriends(user, ret) {
const friendQuery = user.relation("friends").query();
const friends = await findFully(friendQuery);
for(var i=0;i<friends.length;i++){
ret.friends[friends[i].id]=1;
ret.objects[friends[i].id]=friends[i];
};
}
// Given a user, load all owned cells. Save the objects to ret.owned,
// and save their objectIds to ret.ownedCells.
//
// Also, save the ids of members, which we will use to flesh out ret.objects with
// the objects who are not friends, but share a cell with the current user.
async function loadPublicCells(user, ret, memberIds) {
const ownedCellQ = new Parse.Query('PublicCell');
ownedCellQ.equalTo('owner',user);
const joinedCellQ = new Parse.Query('PublicCell');
joinedCellQ.equalTo('members',user);
const publicCellQ = Parse.Query.or(ownedCellQ,joinedCellQ);
publicCellQ.greaterThan("updatedAt",new Date(lastLoadTime));
const publicCells=await findFully(publicCellQ);
for(var i=0;i<publicCells.length;i++) {
const cell = publicCells[i];
ret.ownedCells[cell.id]=cell;
const owner = cell.get("owner");
if(owner==null)
continue;
ret.objects[cell.id]=cell;
if(owner.id === user.id) {
ret.ownedCells[cell.id]=1;
} else {
ret.joinedCells[cell.id]=1;
};
const memberQ = cell.relation("members").query();
const members = await findFully(memberQ);
if(ret.memberMap[cell.id]==null)
ret.memberMap[cell.id]={};
const map = ret.memberMap[cell.id];
for(var j=0;j<members.length;j++){
const member=members[j];
map[member.id]=1;
ret.objects[member.id]=member;
};
};
};
// given a list of all members of all cells, load those objects and store
// them in ret.objects. We do not have to record which cells they belong
// to, because that information is in ret.memberMap
async function loadMembers(memberIds, ret) {
const memberQ = new Parse.Query(Parse.User);
var partIds;
while(memberIds.length){
partIds = memberIds.splice(0,100);
memberQ.containedIn('objectId',partIds);
const part = await findFully(memberQ);
for(var i=0;i<part.length;i++) {
ret.objects[part[i].id]=part[i];
}
};
};
// given a user, save all of the objectIds of people who have annoyed him with
// spam. We save only the ids, they don't go on ret.objects, because we only
// need to filter them out of things. The objectIds are sufficient.
//
// We always send all spam objects, otherwise we would not recognize deletions
async function loadUserSpams(user, ret) {
const userSpamsQ = new Parse.Query("_User");
userSpamsQ.equalTo("spamUsers",user);
userSpamsQ.greaterThan("updatedAt", new Date(lastLoadTime));
const userSpams = await findFully(userSpamsQ);
for(var i=0;i<userSpams.length;i++){
ret.userSpams[userSpams[i].id]=1;
};
};
// given a user, save all of the objectIds of people who have been annoyed *BY*
// him with spam. We save only the ids, they don't go on ret.objects, because we
// only need to filter them out of things. The objectIds are sufficient.
//
// We always send all spam objects, otherwise we would not recognize deletions
async function loadSpamUsers(user, ret) {
const spamUserR = user.relation('spamUsers');
const spamUserQ = spamUserR.query();
spamUserQ.greaterThan("updatedAt", new Date(lastLoadTime));
const spamUsers = await findFully(spamUserQ);
for(var i=0;i<spamUsers.length;i++){
ret.spamUsers[spamUsers[i].id]=1;
};
};
// given a user, save all of the objectIds of people to whom he has sent a
// friend request which is still pending. We save only the ids, they don't go
// on ret.objects, because we only need to filter them out of things. The
// objectIds are sufficient.
async function loadPendingFriends(user, ret) {
const request1Q = new Parse.Query('Request');
request1Q.equalTo("owner",user);
const request2Q = new Parse.Query('Request');
request2Q.equalTo("sentTo",user);
const requestQ = Parse.Query.or(request1Q,request2Q);
requestQ.equalTo("status",'PENDING');
const requests = await findFully(requestQ);
for(var i=0;i<requests.length;i++){
const request = requests[i];
const sentBy = request.get("owner");
if(sentBy==null){
console.warn("sentBy==null");
continue;
};
const sentTo = request.get("sentTo");
if(sentTo==null){
console.warn("sentTo==null");
continue;
};
console.dump({sentTo,sentBy});
if(sentBy.id==user.id){
ret["pendingFriends"][sentTo.id]=sentTo;
} else if ( sentTo.id==user.id ) {
ret["friendingPends"][sentBy.id]=sentBy;
};
};
};
// given a user, load all of his private cells. We do not store
// the user objects, because only friends will be in your private cells.
async function loadPrivateCells(user, ret) {
const privateCellQ = new Parse.Query('PrivateCell');
privateCellQ.equalTo("owner", user);
privateCellQ.greaterThan("updatedAt", new Date(lastLoadTime));
const privateCells = await findFully(privateCellQ);
for(var i=0;i<privateCells.length;i++) {
const cell = privateCells[i];
ret.objects[cell.id]=cell;
ret.privateCells[cell.id]=cell;
if(ret.memberMap[cell.id]==null)
ret.memberMap[cell.id]={};
const map = ret.memberMap[cell.id];
const memberQ = cell.relation("members").query();
const members = await findFully(memberQ);
for(var j=0;j<members.length;j++){
const member=members[j];
map[member.id]=1;
ret.objects[member.id]=member;
};
};
//});
}
// we use objects as maps to weed out duplicate objects and cells.
// when we are done, we use this function to replace the object
// with an array of objects. we don't need to send the keys, since
// they already exist within the objects.
function objToValueList(k,ret){
const objs = [];
for( var id in ret[k] )
objs.push(ret[k][id]);
ret[k]=objs;
ret.counts[k]=objs.length;
};
// convert the objects which have been used to accumulate key lists
// to arrays of objectIds. k is the name of the list we are working
// on. ret[k] is the list itself.
function objToKeyList(k,ret) {
const objs = [];
for( var id in ret[k] ) {
objs.push(id);
};
ret[k]=objs;
ret.counts[k]=objs.length;
};
async function checkUserConsent(user){
const query = new Parse.Query("PrivacyPolicy");
query.descending("createdAt");
query.limit(1);
const res = await query.find();
if(res.length==0) {
return true;
};
const policy=res[0];
console.dump(policy);
console.log(policy);
const userConsent=user.get("lastConsent");
return userConsent!=null && userConsent.id == policy.id;
};
async function loadAlerts(user,ret) {
const q1 = new Parse.Query("Alert");
q1.equalTo("owner", user);
const q2 = new Parse.Query("Response");
q2.equalTo("owner", user);
const q3 = new Parse.Query("Alert");
q3.matchesKeyInQuery("objectId", "alert", q2);
const q = Parse.Query.or(q1,q3);
const list = await q.find();
var time = new Date().getTime();
time -= 1000*86400;
time=Math.max(lastLoadTime, time);
q.greaterThan("updatedAt",time);
for(var i=0;i<list.length;i++) {
const item=list[i];
ret.alerts[item.id]=1;
ret.objects[item.id]=item;
};
}
async function doGetUserData(user) {
if(!user)
return {fatal: 'not logged in!' };
const ret = {
owner: {},
privateCells: {},
friends: {},
alerts: {},
objects: {},
ownedCells: {},
joinedCells: {},
spamUsers: {},
userSpams: {},
pendingFriends: {},
friendingPends: {},
memberMap: {},
loadTime: lastLoadTime,
counts: {callCount: callCount++},
};
{
user.fetch();
ret.owner=user.id;
const memberIds={};
ret.objects[user.id]=user;
console.log("loadFriends");
await loadFriends(user,ret);
console.log("loadPrivateCells");
await loadPrivateCells(user,ret,memberIds);
console.log("loadPublicCells");
await loadPublicCells(user,ret,memberIds);
console.log("loadPendingFriends");
await loadPendingFriends(user,ret);
console.log("loadUserSpams");
await loadUserSpams(user,ret);
console.log("loadSpamUsers");
await loadSpamUsers(user,ret);
console.log("loadAlerts");
await loadAlerts(user,ret);
const memberList=[];
for( var id in memberIds ) {
console.log(ret.objects[id]);
memberList.push(id);
};
console.log("loadMembers");
await loadMembers(memberList,ret);
}
for(var cell in ret.memberMap) {
var map = ret.memberMap[cell];
var list = [];
ret.memberMap[cell]=list;
for(var member in map) {
list.push(member);
};
}
delete ret.objects[user.id];
[
'friends', "friendingPends", 'pendingFriends',
'privateCells', 'ownedCells', 'joinedCells',
'userSpams', 'spamUsers', "alerts"
].forEach((k)=>{
objToKeyList(k,ret);
});
objToValueList('objects',ret);
delete ret.counts;
return ret;
}
async function getUserData(req) {
try {
var nextLoadTime=new Date().getTime();
const user = req.user;
console.log(user);
lastLoadTime = req.params.lastLoadTime;
if(lastLoadTime==null)
lastLoadTime=0;
lastLoadTime = new Date(lastLoadTime);
const ret = await doGetUserData(user);
ret.loadTime=nextLoadTime;
return ret;
} catch ( err ) {
console.log(err);
try {
console.log(err.stack());
} catch ( xxx ) {
console.log(err);
};
throw (`error getting data: ${err}`);
};
};
Parse.Cloud.define("getUserData", getUserData);
Something like this could easily be done to get your data for you. Like this solution, it is unlikely to be entirely pretty, but it would probably work.

Firebase Authentication causes cloud functions to return empty

I have a firebase function that's supposed to return Items that are sold by a seller. I want to get the seller's profile picture via firebase authentication. But whenever I AWAIT the function
edit: worth noting that mAuth is firebase authentication*
await mAuth.geUser(sellerData.UID);
the application returns me an empty json or []
Here is the full code for the function, the error occurs on line 11 or somewhere around there.
export const getHottestItems = functions.region("asia-east2").https.onRequest(async (data, response) => {
try {
var arrayItem = new Array<Item>();
let itemSeller: Seller;
const sellerSnapshot = await db.collection("users").get();
// this is the list of promises/awaitables for all items
const promises = new Array<Promise<FirebaseFirestore.QuerySnapshot<FirebaseFirestore.DocumentData>>>();
sellerSnapshot.forEach(async (sellerDoc) => {
const sellerData = sellerDoc.data();
// THIS PART CAUSES THE API TO RETURN []
const sellerAuth = await mAuth.getUser(sellerData.UID);
// check for non null / empty strings
if (sellerData.Name as string && sellerData.UID as string) {
// this is all the seller information we need
itemSeller = new Seller(sellerData.Name, sellerData.UID, sellerAuth.photoURL); // placeholder profile picture
const refItem = sellerDoc.ref.collection("Items");
// push all the promises to a list so we can run all our queries in parallel
promises.push(refItem.get());
}
});
// wait for all promises to finish and get a list of snapshots
const itemSnapshots = await Promise.all(promises);
itemSnapshots.forEach((ItemSnapshot) => {
ItemSnapshot.forEach((ItemDoc) => {
// get the data
const itemData = ItemDoc.data();
// if title is not null, the rest of the fields are unlikely to be.
if (itemData.Title as string) {
// the rest of the logic to convert from database to model is in the constructor
arrayItem.push(new Item(ItemDoc.id, itemData.Title, itemSeller, itemData.Likes, itemData.ListedTime, itemData.Rating, itemData.Description, itemData.TransactionInformation, itemData.ProcurementInformation, itemData.Category, itemData.Stock, itemData.Image1, itemData.Image2, itemData.Image3, itemData.Image4, itemData.AdvertisementPoints, itemData.isDiscounted, itemData.isRestocked));
}
});
});
// sort by performance level
arrayItem = arrayItem.sort(x => x.Performance);
if (data.body.userID) {
arrayItem = await markLikedItems(data.body.userID, arrayItem);
}
//send the responseafter all the final modifications
response.send(arrayItem);
} catch (err) {
// log the error
console.log(err);
response.status(500).send(err);
}
});

node.js Get.Request & Pagination & Async

I'm having a tremendously tough time organizing the flow here as I'm self-taught so wondering if someone might be able to assist.
var channelIds = ['XYZ','ABC','QRS']
var playlistIds = [];
var videoIds = [];
ORDER OF PROCESS
1. Get All Playlist IDs: If returning Get Request JSON contains nextPageToken run Get Request again with that page before going to (2)
2. Get All Video IDs: If returning Get Request JSON contains nextPageToken run Get Request again with that page before going to (3)
3. Aggregate into Final Array: I need put all in an array such as:
var ArrFinal = [{channelId,playlistID,videoId},{channelId,playlistID,videoId},{channelId,playlistID,videoId}];
I don't necessarily need someone to write the whole thing. I'm trying to better understand the most efficient way to know when the previous step is done, but also handle the nextPageToken iteration.
i'm not familiar with the youtube api.
But what you basically need is a get function for each endpoint. This function should also care about the "nextPageToken".
Something like that: (not tested)
'use strict';
const Promise = require('bluebird');
const request = Promise.promisifyAll(require('request'));
const playlistEndpoint = '/youtube/v3/playlists';
const baseUrl = 'https://www.googleapis.com'
const channelIds = ['xy', 'ab', 'cd'];
const getPlaylist = async (channelId, pageToken, playlists) => {
const url = `${baseUrl}${playlistEndpoint}`;
const qs = { 
channelId,
maxResults: 25,
pageToken
};
try {
const playlistRequest = await request.getAsync({ url, qs });
const nextPageToken = playlistRequest.body.nextPageToken;
// if we already had items, combine with the new ones
const items = playlists ? playlists.concat(playlistRequest.body.items) : playlistRequest.body.items;
if (nextPageToken) {
// if token, do the same again and pass results to function
return getPlaylist(channelId, nextPageToken, items);
}
// if no token we are finished
return items;
}
catch (e) {
console.log(e.message);
}
};
const getVideos = async (playlistId, pageToken, videos) => {
// pretty much the same as above
}
function awesome(channelIds) {
const fancyArray = [];
await Promise.map(channelIds, async (channelId) => {
const playlists = await getPlaylist(channelId);
const videos = await Promise.map(playlists, async (playlistId) => {
const videos = await getVideos(playlistId);
videos.forEach(videoId => {
fancyArray.push({ channelId, playlistId, videoId })
})
});
});
return fancyArray;
}
awesome(channelIds)
// UPDATE
This may be a lot concurrent requests, you can limit them by using
Promise.map(items, item => { somefunction() }, { concurrency: 5 });

Resources