I have been trying to make a video listing function that makes use of node-js's spawn to spawn a yt-dlp process, whose output gets stored in a database.
Now it works but not as expected (the save order gets messed up even then) when I give it the size of the playlist it must process, but when the submitted playlist size is not known I can't stop the while loop that I have been using to run it.
Here it the function:
const { Sequelize, DataTypes } = require('sequelize'); // including this just in case
const { spawn } = require("child_process");
async function list_background(body_url, start_num, stop_num, chunk_size) {
// sleep just to make it possible to catch
// await sleep(2 * 1000);
console.log('\nlisting in background');
var i = 0;
var dont_stop = true;
// need to find a way to make the loop work only until the time we get a response
// empty response means we should stop
// while (dont_stop) { // this is disastrous as the variable never gets updated
while (i < 10) {
// prepare an empty string to append all the data to
var response = '';
// make the start and stop numbers
start_num = parseInt(start_num) + chunk_size;
stop_num = parseInt(stop_num) + chunk_size;
console.log("\nsupplied data:", "\ni:", i, "\nbody_url:", body_url, "\nstart_num:", start_num, "\nstop_num:", stop_num, "\nchunk_size", chunk_size);
// actually spawn the thing
const yt_list = spawn("yt-dlp", ["--playlist-start", start_num, "--playlist-end", stop_num, "--flat-playlist",
"--print", '%(title)s\t%(id)s\t%(webpage_url)s', body_url]);
yt_list.stdout.on("data", async data => {
response += data;
});
yt_list.stderr.on("data", data => {
response = `stderr: ${data}`;
});
yt_list.on('error', (error) => {
response = `error: ${error.message}`;
});
// apparently await has no effect on this expression
// but then how are we supposed to know when to stop?
// the listing only ends when dont_stop is false
yt_list.on("close", async (code) => {
end = `child process exited with code ${code}`;
response_list = response.split("\n");
// remove the "" from the end of the list
response_list.pop();
// get the status at the end
console.log("\ndata after processing\ni:", i, "response:\n", response, "\nresponse_list:", response_list, "\nresponse_list.length:", response_list.length, "\n");
if (response_list == '') {
// basically when the resonse is empty it means that all
// the items have been listed and the function can just return
// this should then break the outer listing loop
console.log("no vidoes found", "\ni:", i, "\n");
// break wont work as `Jump target cannot cross function boundary.ts(1107)`
// so I am returning false to dont_stop and if dont_stop is is true then the loop
// should stop in the next iteration
dont_stop = false;
} else {
// adding the items to db
console.log("adding items to db", "\ni:", i, "\n");
await Promise.all(response_list.map(async (element) => {
var items = element.split("\t");
// console.log(items, items.length, "\ni:", i, "\n");
// update the vidoes too here by looking for any changes that could have been made
// use find or create here to update the entries
if (items.length == 3) {
try {
if (items[0] == "[Deleted video]" || items[0] == "[Private video]") {
item_available = false;
} else {
item_available = true;
}
const [found, created] = await vid_list.findOrCreate({
where: { url: items[2] },
defaults: {
id: items[1],
reference: body_url,
title: items[0],
downloaded: false,
available: item_available
}
})
//if (created)
//console.log("\nsaved", items[0], "\ni:", i, "\n");
//else
if (found) {
if (!item_available) {
found.available = false;
//console.log("\nfound", items[0], "updated", "\ni:", i, "\n");
}
else {
//console.log("\nfound", items[0], "no changes", "\ni:", i, "\n");
}
found.changed('updatedAt', true);
}
} catch (error) {
// remember to uncomment this later, the sequelize erros are not relevant here now
// console.error(error);
}
}
}));
dont_stop = true;
}
});
console.log('\n\ndont_stop', dont_stop, "\ni:", i, "\n");
i++;
}
console.log('\noutside the loop, and persumably done', "\ni:", i, "\n");
}
this is the test data that I use:
const daft_punk_essentials = { url: "https://www.youtube.com/playlist?list=PLSdoVPM5WnneERBKycA1lhN_vPM6IGiAg", size: 22 }
// first 10 will be listed by the main method so the number of vidoes that we should get here is total-10
list_background(daft_punk_essentials['url'], 1, 10, 10);
I recorded the output of the execution to find out what is happening
can't_stop.log
From my observations I have found out that the spawn doesn't start until after the loop has finished, which I had to limit it 10 as without a limit it just crashes my computer. (see log file for how it happening)
Now I know about await Promise.all() to wait for it's internal stuff to complete but how do i don't get how to implement this for a while loop that need process parts of a list in order to add them to a db.
I am not sure if this is the right approach to do this. I used while loop because there can be up to 5000 videos in a playlist and using a for loop to make chunks would be wasteful if the playlist has like < 500 videos.
The beauty of using promises and async/await is that you can use normal flow of control programming with loops, break, return, etc... because your code isn't running inside of event triggered callback functions which have no control over the higher level scope.
So, the first thing to clean up here is to take all the .on() event handling from the spawn() and wrap it into a promise so that can all be abstracted away in a separate function that you can use await on.
Then, I'd also suggest breaking some of the complication you have into separate functions as that will also allow you to more simply see and control the flow.
I did not follow everything you were trying to do in this loop or how you want to handle all possible error conditions so I'm sure this will need some further tweaking, but here's the general idea.
Synopsis of Changes
Put the spawn operation into a separate function which I called getVideoInfo() that returns a promise that resolves/rejects when its done. This wraps all the .on() event handlers in a promise that the caller can more simply deal with.
Break out the functionality that adds items to the DB into its own function. This is done just to simplify the code and make the main control flow easier to follow and see and write.
Just use a while (true) loop and when you're done, you can simply return. No need for stop loop variables or any of that.
Here's the general idea for how that could look (you will likely have to fix up some details and error handling since I can't run this myself).
const { Sequelize, DataTypes } = require('sequelize'); // including this just in case
const { spawn } = require("child_process");
function getVideoInfo(body_url, start_num, stop_num) {
return new Promise((resolve, reject) => {
// actually spawn the thing
let response = "";
const yt_list = spawn("yt-dlp", [
"--playlist-start",
start_num,
"--playlist-end",
stop_num,
"--flat-playlist",
"--print", '%(title)s\t%(id)s\t%(webpage_url)s',
body_url
]);
yt_list.stdout.on("data", data => {
response += data;
});
yt_list.stderr.on("data", data => {
reject(new Error(`stderr: ${data}`));
});
yt_list.on("close", async (code) => {
resolve(response);
});
yt_list.on("error", reject);
});
}
async function addItemsToDb(response_list) {
// adding the items to db
console.log("adding items to db", "\ni:", i, "\n");
await Promise.all(response_list.map(async (element) => {
const items = element.split("\t");
// update the vidoes too here by looking for any changes that could have been made
// use find or create here to update the entries
if (items.length === 3) {
try {
const item_available = items[0] === "[Deleted video]" || items[0] === "[Private video]";
const [found, created] = await vid_list.findOrCreate({
where: { url: items[2] },
defaults: {
id: items[1],
reference: body_url,
title: items[0],
downloaded: false,
available: item_available
}
});
if (found) {
if (!item_available) {
found.available = false;
//console.log("\nfound", items[0], "updated", "\ni:", i, "\n");
}
else {
//console.log("\nfound", items[0], "no changes", "\ni:", i, "\n");
}
found.changed('updatedAt', true);
}
} catch (error) {
// remember to uncomment this later, the sequelize erros are not relevant here now
// console.error(error);
}
}
}));
}
async function list_background(body_url, start_num, stop_num, chunk_size) {
console.log('\nlisting in background');
start_num = parseInt(start_num);
stop_num = parseInt(stop_num);
while (true) {
// make the start and stop numbers
start_num += chunk_size;
stop_num += chunk_size;
const response = await getVideoInfo(body_url, start_num, stop_num);
const response_list = response.split("\n");
// remove the "" from the end of the list
response_list.pop();
// get the status at the end
if (response_list == '') {
return;
} else {
await addItemsToDb(response_list);
}
}
}
P.S. I don't understand why you're adding chunk_size to start_num before you ever use it. It seems like you'd want to do that after you do the first iteration so you start at start_num, not start at start_num + chunk_size. But, this is how your original code was written so I left it that way.
I'm developing a paypal checkout using the 'basic Smart Payment Buttons integration' and integrating it with server Node installing the 'checkout-server-sdk'.
I followed the documentations:
https://developer.paypal.com/docs/checkout/reference/server-integration/set-up-transaction/
https://github.com/paypal/Checkout-NodeJS-SDK
where they suggest to:
'createOrder' starting from the client and calling the server
generating on the server an orderID and return it to the client
'onApprove' send to the server the orderID and approve it on the server
return back to the client the response
I don't think it is a good flow.
Someone could:
start the payment
so the app create the order on the server taking the shoppingcart from db and elaborate a totalPrice of 100euros.
generete the orderID and send it back to the client
instead of approve this order, a 'bad user' could, in some way, send to the server another orderID that could correspond to a lower price (2euros)
so he could approve the payment of 2 euros
So I don't understand why we need to make the checkout jumping more times from client to server.
Or maybe am i doing something wrong on my checkoutflow ?
unfortunately I feel the Paypal documentation so unclear.
checkout.component.html
<!-- * here there is a form where i get shipment info, invoice info and so on ->
<!-- * PAYPAL SMART BUTTONS -->
<div>
<div #paypal></div>
</div>
checkout.component.ts
onFormSubmit() {
this.isFormSubmitted = true;
// set paypal settings and show the paypal buttons
this.paypalSetting(this.shippmentInfo, this.invoiceRequired, this.invoice, this.addressInvoice);
}
async paypalSetting(shipment, invoiceRequired, invoice, addressInvoice) {
await paypal
.Buttons({
style: {
size: 'responsive',
label: 'pay',
},
experience: {
input_fields: {
no_shipping: 1,
},
},
createOrder: async (data, actions) => {
console.log('CREATE ORDER -->');
var paypalOrderId;
//generate new order
await this.apiService.newOrder().toPromise().then(
(res) => {
console.log('ON CREATE: SUCCESSFULLY CREATED')
paypalOrderId = res.order.paypalOrderId;
// ????? someone here could change 'paypalOrderId' with another value !!!!
//I also would like to return the 'paypalOrderId' only here !!
},
(err) => {
console.log('ON CREATE: ERROR: ' + err);
// how should i manage this error ? i should skip the flow to onError but HOW ?
}
);
return paypalOrderId;
},
onApprove: async (data, actions) => {
console.log('APPROVE ORDER -->');
var paypalOrderId = data.orderID;
console.log('ON APPROVE: save the order on server/DB')
await this.apiService.saveOrder(shipment, invoiceRequired, invoice, addressInvoice, paypalOrderId).toPromise().then(
(res) => {
console.log('ON APPROVE: ORDER APPROVED')
this.isPaid = true;
//if isPaid i can show a 'success page'
},
(err) => {
console.log('ON APPROVE: ERROR: ' + err);
this.isPaid = false;
}
);
},
onError: (err) => {
console.log('ON ERROR: ' + err);
},
})
.render(this.paypalElement.nativeElement);
}
Node api.js
//* paypal
const paypal = require('#paypal/checkout-server-sdk');
const payPalClient = require('../paypalManager');
router.post('/newOrder', tokenManager.verifyAccessToken, async function (req, res, next) {
const idUser = req.userId;
// I get the shoppingcart of the user 'idUser'
// i calculate the total price
var totalPrice;
//* Call PayPal to set up a transaction
let order;
const request = new paypal.orders.OrdersCreateRequest();
request.prefer("return=representation");
request.requestBody({
intent: 'CAPTURE',
purchase_units: [{
description: 'payment ecc..', /
amount: {
currency_code: 'EUR',
value: totalPrice
}
}],
application_context: {
brand_name: "brand",
shipping_preference: 'NO_SHIPPING',
},
});
let response = await payPalClient.client().execute(request);
order = response;
const paypalOrderId = order.result.id;
// return a successful response to the client with the order ID
return res.json({
status: 200,
order: {
paypalOrderId: paypalOrderId,
},
message: "Paypal order sucessfully created",
});
});
router.post('/saveOrder', tokenManager.verifyAccessToken, async function (req, res, next) {
const idUser = req.userId;
var paypalOrderId = req.body.paypalOrderId;
try {
connection.beginTransaction(async () => {
try {
// here i insert all the checkout infos in DB
// confirm the queries executions
connection.commit(async function (err) {
if (err) {
//return connection.rollback(function () {
connection.rollback(function () {
return next(createError.Unauthorized("Sql query error: " + err)); //! or error.message
});
}
//* here i send the Emails to confirm the checkout
//* capture/approve the order
console.log('CAPTURING THE ORDER')
var request = new paypal.orders.OrdersCaptureRequest(paypalOrderId);
request.requestBody({});
// Call API with your client and get a response for your call
let response = await payPalClient.client().execute(request);
//*response
return res.json({
status: 200,
message: "Paypal sucessfully approved",
});
});// end commit
} catch (error) {
connection.rollback(function () {
return next(createError.Unauthorized("Sql query error " + error)); //! or error.message
});
}
});// end transaction
} catch (error) {
return next(error);
}
});
Node paypalManager.js
'use strict';
/**
* PayPal Node JS SDK dependency
*/
const checkoutNodeJssdk = require('#paypal/checkout-server-sdk');
/**
* Returns PayPal HTTP client instance with environment that has access
* credentials context. Use this instance to invoke PayPal APIs, provided the
* credentials have access.
*/
function client() {
return new checkoutNodeJssdk.core.PayPalHttpClient(environment());
}
/**
* Set up and return PayPal JavaScript SDK environment with PayPal access credentials.
* This sample uses SandboxEnvironment. In production, use LiveEnvironment.
*/
function environment() {
let clientId = process.env.PAYPAL_CLIENT_ID;
let clientSecret = process.env.PAYPAL_CLIENT_SECRET;
return new checkoutNodeJssdk.core.SandboxEnvironment(
clientId, clientSecret
);
}
module.exports = {
client: client,
prettyPrint: prettyPrint
};
The reason you are "jumping" between the client and the server, is the approval by the payer has to happen on the client. The payer cannot give their approval on your server, they are not sitting on your server. They are using a client browser.
Regarding:
a 'bad user' could, in some way, send to the server another orderID that could correspond to a lower price (2euros)
If this happens, your server should reject the undesired transaction, and not proceed with it. That's the point of having a server. Nothing happens unless your server OKs it.
Hello I'm using the mssql library to insert some data into an SQL Server DB, the call is being executed inside a queue so if it fails it will retry in 1 min * retryAttemp up until 5 attempts
however I keep getting this weird behavior where I get this error message:
ERROR: Error inserting into BASIS OLAP TransactionError: Can't acquire connection for the request. There is another request in progress.
and I don't know why this is happening... this is my code:
static async insertPresale(req, res) {
logger.info('Inserting new Pre-Sale');
const { CAB, DET } = req.body.basisOlapStructure;
const { country } = req.body;
if (country.toLowerCase() !== 'cl') return res.status(404).json({ success: false, message: 'Other countries are not currently supported' });
let pool;
let transaction;
try {
pool = await Databases.ClSQLServerPreventa;
if (!pool) throw new errors.UNEXPECTED_ERROR({ message: 'Error inserting pre-sale in BASIS OLAP. Could not get a connection pool' });
transaction = new mssql.Transaction(pool);
await new Promise((resolve, reject) => transaction.begin(err => (err ? reject(err) : resolve())));
const request = new mssql.Request(transaction);
logger.info('Inserting data into PVRCabecera');
let queryStr = msql.insert().into(bTables.PVRCabecera.toString()).setFields(CAB);
logger.info(queryStr.toString());
let result = await request.query(queryStr.toString());
if (result.rowsAffected > 0) logger.info('Data successfully inserted into PVRCabecera');
logger.info('Inserting data into PVRDetalle');
const queries = DET.map((d) => {
queryStr = msql.insert().into(bTables.PVRDetalle.toString()).setFields(d);
logger.info(queryStr.toString());
return request.query(queryStr.toString());
});
result = await Promise.all(queries);
if (result[0].rowsAffected > 0) logger.info('Data successfully inserted into PVRDetalle');
await transaction.commit();
logger.info('BASIS_OLAP Transaction completed successfully');
return res.status(201).json({ success: true });
} catch (error) {
logger.error(`Error inserting into BASIS OLAP ${error}`);
if (transaction) {
await transaction.rollback();
}
logger.info('Transaction has been rolled back');
throw new errors.UNEXPECTED_ERROR({ message: `Error inserting pre-sale in BASIS OLAP. ${error.toString()}` });
}
}
what can I do to solve this problem?
I have a firebase (Google) cloud-function as follows
// Initialize the Auth0 client
var AuthenticationClient = require('auth0').AuthenticationClient;
var auth0 = new AuthenticationClient({
domain: 'familybank.auth0.com',
clientID: 'REDACTED'
});
function getAccountBalance(app) {
console.log('accessToken: ' + app.getUser().accessToken);
auth0.getProfile(app.getUser().accessToken, function (err, userInfo) {
if (err) {
console.error('Error getting userProfile from Auth0: ' + err);
}
console.log('getAccountBalance userInfo:' + userInfo)
let accountowner = app.getArgument(PARAM_ACCOUNT_OWNER);
// query firestore based on user
var transactions = db.collection('bank').doc(userInfo.email)
.db.collection('accounts').doc(accountowner)
.collection('transactions');
var accountbalance = transactions.get()
.then( snapshot => {
var workingbalance = 0
snapshot.forEach(doc => {
workingbalance = workingbalance + doc.data().amount;
});
app.tell(accountowner + " has a balance of $" + workingbalance)
})
.catch(err => {
console.log('Error getting transactions', err);
app.tell('I was unable to retrieve your balance at this time.')
});
});
}
actionMap.set(INTENT_ACCOUNT_BALANCE, getAccountBalance);
app.handleRequest(actionMap);
When this executes, I see the following logs
Notice that parts of the function are being executed multiple times, and the second execution is failing. If I close out the auth0.getProfile call after logging userInfo, then the function works, but obviously doesn't have userInfo.
Any idea why parts of this function are executing multiple times and why some calls would fail?
The userInfo is undefined at point (2) because there has been an error (reported on the line right beneath it, which was the previous logged message). Your error block does not leave the function, so it continues to run with an invalid userInfo object.
But that doesn't explain why the callback is getting called twice - once with a valid userInfo and once with an err. The documentation (although not the example) for AuthenticationClient.getProfile() indicates that it returns a Promise (or undefined - although it doesn't say why it might return undefined), so I am wondering if this ends up calling the callback twice.
Since it returns a promise, you can omit the callback function and just handle it with something like this:
function getAccountBalance(app) {
let accountowner = app.getArgument(PARAM_ACCOUNT_OWNER);
console.log('accessToken: ' + app.getUser().accessToken);
var accessToken = app.getUser().accessToken;
auth0.getProfile( accessToken )
.then( userInfo => {
console.log('getAccountBalance userInfo:' + userInfo)
// query firestore based on user
var transactions = db.collection('bank').doc(userInfo.email)
.db.collection('accounts').doc(accountowner)
.collection('transactions');
return transactions.get();
})
.then( snapshot => {
var workingbalance = 0
snapshot.forEach(doc => {
workingbalance = workingbalance + doc.data().amount;
});
app.tell(accountowner + " has a balance of $" + workingbalance)
})
.catch( err => {
console.error('Error:', err );
app.tell('I was unable to retrieve your balance at this time.')
})
});
}
I made simple notification application that subscribes to redis channel and when it will have new message then this application send it to user.
client.on("auth", function(sessId, userId){
console.log('AUTH: userId ' + userId)
var userIdFromRedis;
redis1.get('PHPREDIS_SESSION:' + sessId, function (err , reply) {
if (reply){
reply = reply.toString();
var result = reply.match(/s:2:\"id\";i:(\d+);/);
if (result) {
userIdFromRedis = result[1]
console.log('AUTH: userIdFromRedis ' + userIdFromRedis)
} else {
result = reply.match(/s:7:\"guestId\";i:(\d+);/);
if (result) {
var guestIdFromRedis = result[1]
console.log('AUTH: guestIdFromRedis ' + guestIdFromRedis)
}
}
if (userIdFromRedis == userId) {
client.userId = userId;
subscribe.subscribe(channelPrefix + userId);
clients[userId] = client;
client.emit("auth", {"success":true});
console.log('AUTH: result - ok')
} else if (guestIdFromRedis) {
client.guestId = guestIdFromRedis;
subscribe.subscribe(channelPrefix + guestIdFromRedis);
clients[guestIdFromRedis] = client;
client.emit("auth", {"success":true});
console.log('AUTH: result - ok')
} else {
client.disconnect();
console.log('AUTH: result - fail')
}
} else {
client.disconnect();
}
});
})
subscribe.on("message", function(channel, message) {
var userId = Math.round(channel.substr(channelPrefix.length));
if (client.userId == userId || client.guestId == userId) {
console.log('Subscriber: ' + message)
client.send(message);
}
});
client.on("message", function(text){
client.send(text);
})
And in log file sometimes in top hour I can find error message
(node) warning: possible EventEmitter memory leak detected. 11
listeners added. Use emitter.setMaxListeners() to increase limit.
And node.js application's process uses 90% of CPU.
Please advice how can I resolve this problem?
Whenever your client connects to your server, you add another EventEmitter to the stack. Unfortunatly you are not closing them, so you are hitting the listeners limit of 11. You should track down the piece of code that adds the eventEmitters and then make another statement, that if there is already a client connected, there should not be any other emitters generated.