I'm trying to run an async function that send post request to the server and logs the response in the console. Whenever I run a for loop nothing is happening. I have checked a different similar questions here on stackoverflow but nothing is working in my case.
Note this code is for ionic + angular for android.
Below is the code for the function.
postAPI(suffixUrl, queryString, data, header: any = {}) {
// console.log(suffixUrl + "\n=>", queryString + "\n=>", data + "\n=>", header);
this.isDisabled = true;
data = (header.is == 'yes') ? JSON.parse(data) : data;
let type = this.fileNameSubStringFn(suffixUrl, '?');
return new Promise(resolve => {
this.http.post(this.con.baseUrl + suffixUrl + queryString, data)
.subscribe(res => {
// res = res.json();
// res = JSON.parse(res.data);
console.log("RES:", res);
this.hideLoader();
resolve(this.data);
return;
}, (err) => {
this.logFn("ERROR:" + JSON.stringify(err));
this.isDisabled = false;
// this.isConnected = false;
});
});
}
Below is the code that is accessing the above function in-order to submit the data.
var i;
for (i = 0; i < summeriesArray.length; i++) {
this.postAPI('service_summery', '', JSON.stringify(summeriesArray[i]), { is: 'yes', method: 'post' }).then(data => {
this.hideLoader();
this.toast("Successfully saved");
console.log(data);
});
}
Code inside this.postAPI is not executing which means the data is not being posted and I'm getting no errors so I don't know what is going on here.
I'm expecting the this.postAPI to run successfully in a for lop.
Thank you, posted with Love.
You can use the await keyword, so you wait for the next iteration until the completion of the 'postAPI'.
for (i = 0; i < summeriesArray.length; i++) {
const data = await this.postAPI('service_summery', '', JSON.stringify(summeriesArray[i]), { is: 'yes', method: 'post' })
this.hideLoader();
this.toast("Successfully saved");
console.log(data);
}
Related
I have been trying to make a video listing function that makes use of node-js's spawn to spawn a yt-dlp process, whose output gets stored in a database.
Now it works but not as expected (the save order gets messed up even then) when I give it the size of the playlist it must process, but when the submitted playlist size is not known I can't stop the while loop that I have been using to run it.
Here it the function:
const { Sequelize, DataTypes } = require('sequelize'); // including this just in case
const { spawn } = require("child_process");
async function list_background(body_url, start_num, stop_num, chunk_size) {
// sleep just to make it possible to catch
// await sleep(2 * 1000);
console.log('\nlisting in background');
var i = 0;
var dont_stop = true;
// need to find a way to make the loop work only until the time we get a response
// empty response means we should stop
// while (dont_stop) { // this is disastrous as the variable never gets updated
while (i < 10) {
// prepare an empty string to append all the data to
var response = '';
// make the start and stop numbers
start_num = parseInt(start_num) + chunk_size;
stop_num = parseInt(stop_num) + chunk_size;
console.log("\nsupplied data:", "\ni:", i, "\nbody_url:", body_url, "\nstart_num:", start_num, "\nstop_num:", stop_num, "\nchunk_size", chunk_size);
// actually spawn the thing
const yt_list = spawn("yt-dlp", ["--playlist-start", start_num, "--playlist-end", stop_num, "--flat-playlist",
"--print", '%(title)s\t%(id)s\t%(webpage_url)s', body_url]);
yt_list.stdout.on("data", async data => {
response += data;
});
yt_list.stderr.on("data", data => {
response = `stderr: ${data}`;
});
yt_list.on('error', (error) => {
response = `error: ${error.message}`;
});
// apparently await has no effect on this expression
// but then how are we supposed to know when to stop?
// the listing only ends when dont_stop is false
yt_list.on("close", async (code) => {
end = `child process exited with code ${code}`;
response_list = response.split("\n");
// remove the "" from the end of the list
response_list.pop();
// get the status at the end
console.log("\ndata after processing\ni:", i, "response:\n", response, "\nresponse_list:", response_list, "\nresponse_list.length:", response_list.length, "\n");
if (response_list == '') {
// basically when the resonse is empty it means that all
// the items have been listed and the function can just return
// this should then break the outer listing loop
console.log("no vidoes found", "\ni:", i, "\n");
// break wont work as `Jump target cannot cross function boundary.ts(1107)`
// so I am returning false to dont_stop and if dont_stop is is true then the loop
// should stop in the next iteration
dont_stop = false;
} else {
// adding the items to db
console.log("adding items to db", "\ni:", i, "\n");
await Promise.all(response_list.map(async (element) => {
var items = element.split("\t");
// console.log(items, items.length, "\ni:", i, "\n");
// update the vidoes too here by looking for any changes that could have been made
// use find or create here to update the entries
if (items.length == 3) {
try {
if (items[0] == "[Deleted video]" || items[0] == "[Private video]") {
item_available = false;
} else {
item_available = true;
}
const [found, created] = await vid_list.findOrCreate({
where: { url: items[2] },
defaults: {
id: items[1],
reference: body_url,
title: items[0],
downloaded: false,
available: item_available
}
})
//if (created)
//console.log("\nsaved", items[0], "\ni:", i, "\n");
//else
if (found) {
if (!item_available) {
found.available = false;
//console.log("\nfound", items[0], "updated", "\ni:", i, "\n");
}
else {
//console.log("\nfound", items[0], "no changes", "\ni:", i, "\n");
}
found.changed('updatedAt', true);
}
} catch (error) {
// remember to uncomment this later, the sequelize erros are not relevant here now
// console.error(error);
}
}
}));
dont_stop = true;
}
});
console.log('\n\ndont_stop', dont_stop, "\ni:", i, "\n");
i++;
}
console.log('\noutside the loop, and persumably done', "\ni:", i, "\n");
}
this is the test data that I use:
const daft_punk_essentials = { url: "https://www.youtube.com/playlist?list=PLSdoVPM5WnneERBKycA1lhN_vPM6IGiAg", size: 22 }
// first 10 will be listed by the main method so the number of vidoes that we should get here is total-10
list_background(daft_punk_essentials['url'], 1, 10, 10);
I recorded the output of the execution to find out what is happening
can't_stop.log
From my observations I have found out that the spawn doesn't start until after the loop has finished, which I had to limit it 10 as without a limit it just crashes my computer. (see log file for how it happening)
Now I know about await Promise.all() to wait for it's internal stuff to complete but how do i don't get how to implement this for a while loop that need process parts of a list in order to add them to a db.
I am not sure if this is the right approach to do this. I used while loop because there can be up to 5000 videos in a playlist and using a for loop to make chunks would be wasteful if the playlist has like < 500 videos.
The beauty of using promises and async/await is that you can use normal flow of control programming with loops, break, return, etc... because your code isn't running inside of event triggered callback functions which have no control over the higher level scope.
So, the first thing to clean up here is to take all the .on() event handling from the spawn() and wrap it into a promise so that can all be abstracted away in a separate function that you can use await on.
Then, I'd also suggest breaking some of the complication you have into separate functions as that will also allow you to more simply see and control the flow.
I did not follow everything you were trying to do in this loop or how you want to handle all possible error conditions so I'm sure this will need some further tweaking, but here's the general idea.
Synopsis of Changes
Put the spawn operation into a separate function which I called getVideoInfo() that returns a promise that resolves/rejects when its done. This wraps all the .on() event handlers in a promise that the caller can more simply deal with.
Break out the functionality that adds items to the DB into its own function. This is done just to simplify the code and make the main control flow easier to follow and see and write.
Just use a while (true) loop and when you're done, you can simply return. No need for stop loop variables or any of that.
Here's the general idea for how that could look (you will likely have to fix up some details and error handling since I can't run this myself).
const { Sequelize, DataTypes } = require('sequelize'); // including this just in case
const { spawn } = require("child_process");
function getVideoInfo(body_url, start_num, stop_num) {
return new Promise((resolve, reject) => {
// actually spawn the thing
let response = "";
const yt_list = spawn("yt-dlp", [
"--playlist-start",
start_num,
"--playlist-end",
stop_num,
"--flat-playlist",
"--print", '%(title)s\t%(id)s\t%(webpage_url)s',
body_url
]);
yt_list.stdout.on("data", data => {
response += data;
});
yt_list.stderr.on("data", data => {
reject(new Error(`stderr: ${data}`));
});
yt_list.on("close", async (code) => {
resolve(response);
});
yt_list.on("error", reject);
});
}
async function addItemsToDb(response_list) {
// adding the items to db
console.log("adding items to db", "\ni:", i, "\n");
await Promise.all(response_list.map(async (element) => {
const items = element.split("\t");
// update the vidoes too here by looking for any changes that could have been made
// use find or create here to update the entries
if (items.length === 3) {
try {
const item_available = items[0] === "[Deleted video]" || items[0] === "[Private video]";
const [found, created] = await vid_list.findOrCreate({
where: { url: items[2] },
defaults: {
id: items[1],
reference: body_url,
title: items[0],
downloaded: false,
available: item_available
}
});
if (found) {
if (!item_available) {
found.available = false;
//console.log("\nfound", items[0], "updated", "\ni:", i, "\n");
}
else {
//console.log("\nfound", items[0], "no changes", "\ni:", i, "\n");
}
found.changed('updatedAt', true);
}
} catch (error) {
// remember to uncomment this later, the sequelize erros are not relevant here now
// console.error(error);
}
}
}));
}
async function list_background(body_url, start_num, stop_num, chunk_size) {
console.log('\nlisting in background');
start_num = parseInt(start_num);
stop_num = parseInt(stop_num);
while (true) {
// make the start and stop numbers
start_num += chunk_size;
stop_num += chunk_size;
const response = await getVideoInfo(body_url, start_num, stop_num);
const response_list = response.split("\n");
// remove the "" from the end of the list
response_list.pop();
// get the status at the end
if (response_list == '') {
return;
} else {
await addItemsToDb(response_list);
}
}
}
P.S. I don't understand why you're adding chunk_size to start_num before you ever use it. It seems like you'd want to do that after you do the first iteration so you start at start_num, not start at start_num + chunk_size. But, this is how your original code was written so I left it that way.
This one has had me spinning my tires for about 2 days now, I'm ready to reach out for help :)
I have a google firebase functions app, running as a middle-ware to an angular SPA. Hoping to avoid some of the pay-by-use cost of Azure SQL, I wanted to implement a caching option for the most common queries.
I thought I knew redis, I've worked with it before. There's a simple enough example on the repo: https://www.npmjs.com/package//redis
Everything works fine, if it is top level.
But the way my application is built, i need the ability to set a cache value from within the .then of a Promise, and when I try to do that, all operation just stops, with no identifiable error logging, or even response from redis. Even in Azure insights, i'm not getting much feed-back, only that the 'set' operation isn't being counted in metrics.
So, just to clarify, this works:
// "cache", the object, set globally
export const testCache = functions.https.onRequest(
async (req: any, res) => {
await cache.connect();
var redisKey = 'testing_global';
var result = await cache.get(redisKey);
await cache.set(redisKey, 'testing new class')
console.log("\nDone");
cache.disconnect();
res.send('done');
})
But, this does not:
import { createClient } from 'redis';
const cache = createClient({
url: "rediss://" + process.env.REDIS_HOST_NAME + ":6380",
password: process.env.REDIS_KEY,
});
export const getValues = functions.https.onRequest(
(req: any, response) => {
cors(req, response, async () => {
response.set('Access-Control-Allow-Origin', origin);
var searchText = req.body['search'];
var offset = req.body['offset'];
var fetch = req.body['fetch'];
var x = req.body['x'];
var y = req.body['y'];
var districts = req.body['districtFilter'];
var sort = req.body['sort'];
if (offset) {
if (!/^\d+$/.test(offset))
throw new Error('bad number');
} else {
offset = 0;
}
if (fetch) {
if (!/^\d+$/.test(fetch))
throw new Error('bad number');
} else {
fetch = 200;
}
if (x) {
if (!/^-?\d+$/.test(x))
throw new Error('bad number');
} else {
x = null;
}
if (y) {
if (!/^-?\d+$/.test(y))
throw new Error('bad number');
} else {
y = null;
}
if (districts && districts.length > 0) {
districts = sanitizeStringArray(districts);
} else {
districts = null;
}
if (sort) {
switch (sort) {
case "scoreAsc":
case "scoreDesc":
case "priceAsc":
case "priceDesc":
break;
default:
sort = '';
}
}
var redisKey = `get_values_${searchText}_${offset}_${fetch}_${x}_${y}_${districts}_${sort}`;
var cacheResult: any = null;
await cache.connect();
// because I want to end up in the 'else', for testing
cacheResult = await cache.getFromCache('someOtherKey');
if (null !== cacheResult) {
response.send({
"status": "success",
"totalCount": cacheResult.totalCount,
"data": cacheResult.result
});
cache.disconnect();
} else {
var connection = new Connection(sqlConfig);
var totalCount: number = 0;
connection.on('connect', function (err: any) {
// If no error, then good to proceed.
console.log("Connected");
var sql = `EXEC SomeSPC;`;
const sqlRequest = new Request(sql, function (err: any) {
if (err) {
console.log(err);
}
});
const countRequest = new Request(
`EXEC SomeOtherSPC;`
, function (err: any) {
if (err) {
console.log(err);
}
}
)
sqlRequest.connection = connection;
countRequest.connection = connection;
var result: any[] = [];
sqlRequest.on('row', function (columns: any[]) {
var rowResult: any = {};
columns.forEach(function (column: any) {
rowResult[column['metadata']['colName']] = column['value'];
});
result.push(rowResult);
});
sqlRequest.on("requestCompleted", function (rowCount: any, more: any) {
console.log(rowCount + ' rows returned');
connection.execSql(countRequest);
countRequest.on('row', function (columns: any[]) {
totalCount = columns[0]['value'];
});
countRequest.on('requestCompleted', async function (rowCount: any, more: any) {
connection.close();
cacheResult = {
totalCount: totalCount
, result: result
};
// ******************************************************************
// Does Not Work, Just Fails, Without Much to Go On
await cache.set(redisKey, 'in Promise')
response.send({
"status": "success",
"totalCount": totalCount,
"data": result
});
})
});
connection.execSql(sqlRequest);
});
connection.on('infoMessage', infoError);
connection.on('errorMessage', infoError);
connection.on('end', end);
connection.on('debug', debug);
connection.connect();
console.log("Reading rows from the Table...");
}
})
}
)
There's **update- no longer ** a fair amount of psuedo-code here, so please **update- No Need to ** disregard any inconsistent lines. I went ahead and put in the full function, including all the fluff, since trimming the fat seems to make things difficult for others to understand what is being asked.
The sql stuff all works, if i take out the cache.set() everything is fine, but that line, in the result of the Promise, just fails, and I can't figure out why.
I've tried using cache locally and globally, extracting the cache operations to a function, and then to a separate class, and in all cases, i'm getting the same result.
Is there a known reason this wouldn't work?
As far as I can understand, given you didn't provide a reproducible example, your code is binding to the requestCompleted event for the second request only after it runs execSql(): I would suggest moving the binding block before that, otherwise the event may be skipped.
I'm just learning not to judge strictly. I try to do Rest Client. And I want to do a data search by rows.
At the moment I just want to try to find the given data in the array. And if one of the lines is the same, the message OK will be displayed, and in the others No. But I have all messages displays No. How to fix it?
var data = querystring.stringify({
Name: "CEO"
})
var req = http.request(options, function(res) {
console.log("status: " + res.statusCode);
var content = ''
res.on('data', function(chunk) {
content += chunk
for (var i = 0; i < content.length; i++) {
if (content[i] === data) {
console.log('Ок')
} else {
console.log('No')
}
}
});
}).on('error', function(e) {
console.log("error: " + e.message);
});
req.write(data)
req.end()
-
console.log(`body: ${(chunk)}`):
Answer:
body: {"rowsCount":75,"rows":[..., {"Id":75,"Name":"CEO"}]}
There are some programming / logical problem with your code that prevent you from getting the desired result.
One of the most important parts (sometimes the trickiest one) of programming is to know the types of data you are dealing with then handle them properly.
As in the problem you are resolving, the data returned from the API is in JSON format, and can be treated as object in javascript. You don't have to do a string concat to append them to content variable. By appending chunk to the content variable, you turn the whole set of data return to you to a string, doing a for loop on a string have a very different result compared with on an array.
The second issue is if you look into the data structure that returns to you through the api, the list of records you want to search on is under the rows attribute, therefore, you should loop through the chunk.rows only.
A suggesting fix for your code will be as follows:
var searchQuery = 'SEO';
var req = http.request(options, function(res) {
res.on('data', function(chunk) {
for (var i = 1; i <= chunk.rows.length; i++) {
if (chunk.rows[i].Name === searchQuery) {
console.log('Ок')
} else {
console.log('No')
}
}
});
}).on('error', function(e) {
console.log("error: " + e.message);
});
req.write(data)
req.end()
Your for-loop will skip the first element of the data you get back
You aren't accessing the desired search property correctly.
Your query is a string for which you are testing equality against an object.
You need to access the rows property of your incoming data and then perform your query:
Attempt one:
const rowsToSearch = content.rows; // #2
for (var i = 0; i < rowsToSearch.length; i++) { // #1
if (rowsToSearch[i].Name === 'CEO') { // #3
console.log('Ок')
} else {
console.log('No')
}
}
Attempt two, using lodash:
const query = { Name: 'CEO' };
const rowsToSearch = chunk.rows;
for (var i = 0; i < rowsToSearch.length; i++) {
if (_.isEqual(rowsToSearch[i], query)) {
console.log('Ок')
} else {
console.log('No')
}
}
Attempt 3, using ES6 iteration and lodash:
const query = { Name: 'CEO' };
const rowsToSearch = chunk.rows;
const result = rowsToSearch.find(row => _.isEqual(row, query));
result ? console.log('OK') : console.log('No');
So this is the code I used to crawl my pages (i'm using request and cheerio modules:
for (let j = 1; j < nbRequest; j++)
{
const currentPromise = new Promise((resolve, reject) => {
request(
`https://www.url${j}`,
(error, response, body) => {
if (error || !response) {
console.log("Error: " + error);
}
console.log("Status code: " + response.statusCode + ", Connected to the page");
var $ = cheerio.load(body);
let output = {
ranks: [],
names: [],
numbers: [],
};
$('td.rangCell').each(function( index ) {
if ($(this).text().trim() != "Rang")
{
output.ranks.push($(this).text().trim().slice(0, -1));
nbRanks = nb_ranks+1;
}
});
$('td.nameCell:has(label)').each(function( index ) {
output.names.push($(this).find('label.nameValue > a').text().trim());
});
$('td.numberCell').each(function( index ) {
if ($(this).text().trim() != "Nombre")
{
output.numbers.push($(this).text().trim());
}
});
console.log("HERE 1");
return resolve(output);
}
);
});
promises.push(currentPromise);
}
after that I'm parsing and saving the result in a csv file using a node module.
At this point i've been able to crawl about 100 pages, but when it comes to much bigger numbers (1000+) I'm receiving a 500 response meaning that i'm being kicked i think.
So i think the best solution is to delay requests, but i didn't find the solution.
Do you guys have any idea and how the code would look like ?
what you are looking for is called "Control Flow", you can achieve this by using async.queue for example.
If you add every request to the the queue you can control the amount of parallel requests with the amount of workers. And you could add setTimeouts to the final part of the request's callback to achieve the delaying of requests.
Additionally I'd suggest using a "crawler" package (instead of building your own) e.g. npm-crawler as they ship with build in rate-limiting and have already taken care of other things that you might face next :) e.g. user-agent pool
Update:
const async = require("async");
const delayTime = 1500; //wait 1,5 seconds after every new request
getRequestPromise(csvLine){
return new Promise( make you request here );
}
const asyncQueue = async.queue(function(task, callback) {
getRequestPromise(task).then(_ => {
setTimeout(() => {
callback(null);
}, delayTime);
});
}, 1); //1 one request at a time
for(csv){ //pseudo
asyncQueue.push(csv[i], () => {});
}
asyncQueue.drain = () => {
console.log("finished.");
};
Finding it almost impossible to capture the response of http requests in a loop as an array. I can see the array in console.log but when I pass the array to be the response of http server I get a blank array. What am I doing wrong , or are there better ways to do this?
Code :
router.route('/uprns').post(function(request, response){
response.setHeader('content-type', 'application/text');
console.log('first element from the array is '+request.body.UPRNS[0]);
console.log('Number of items in array is '+request.body.UPRNS.length);
if (request.body.UPRNS.length == 0) {
response.send( 'no UPRNS in request' );
}
var output = [];
var obj = '';
for( var i = 0; i < request.body.UPRNS.length; i++) {
obj = request.body.UPRNS[i];
//Make HTTP calls to
var options = {
host: 'orbisdigital.azure-api.net',
path: '/nosecurity/addresses?uprn='+obj // full URL as path
};
callback = function(res) {
res.on('data', function (chunk) {
output.push(chunk.toString());
});
//the whole response has been recieved
res.on('end', function () {
console.log(output);
});
}
Https.request(options, callback).end();
}
response.send(output);
});
I know there is a lot of talk about blocking process in a for loop , but there is no definitive recommended way to deal with http calls in a loop.
Thank you .
Here is the code. See the code for the added comments. Do some reading on asynchronous programming with node.js, here's a starter.
router.route( '/uprns' ).post( function ( request, response ) {
response.setHeader( 'content-type', 'application/text' );
console.log( 'first element from the array is ' + request.body.UPRNS[ 0 ] ); // your 1st element in JSON array.
console.log( 'Number of items in array is ' + request.body.UPRNS.length );
var output = [];
var obj = '';
for ( var i = 0; i < request.body.UPRNS.length; i++ ) {
obj = request.body.UPRNS[ i ];
console.log( obj );
//Make HTTP calls to
var options = {
host: 'orbisdigital.azure-api.net',
path: '/nosecurity/addresses?uprn=' + obj // full URL as path
};
Https.request( options, callback ).end();
}
var countResponses = 0;
// Don't make functions in a loop, so I moved this function down
// here.
function callback( res ) {
res.on( 'data', function ( chunk ) {
output.push( chunk.toString() );
});
// Handles an error
request.on('error', function(err) {
console.error(err.stack);
response.statusCode = 500; // or what ever.
response.send(500, 'there was an error');
});
//the whole response has been recieved
res.on( 'end', function () {
console.log( output );
countResponses++;
if (countResponses === request.body.UPRNS.length) {
// Previously this code was executed directly
// after the loop finished. It did not wait for
// all the responses, so it sent the empty response.
// However, the other console.log(output) statements
// were called after this.
//
// There is a bug here that if request.body.UPRNS.length
// is zero, then the user will never get a response. I
// let you fix this up :).
response.send( output );
}
} );
}
} );
Better way to handle such scenario is to use async.js instead of for loops. https://github.com/caolan/async