I'm entirely new to NodeJS and this problem has been bugging me for days now. I'm pulling my hairs to find a working solution. I'm trying to get information from the database and pass it to a text file where I later read from it. I cannot do it sequentially. It always reads it first and then creates it. I don't know what way I should take to overcome the issue. Any working solution/ways will help tremendously.
My connection file that retrieves information from the database:
this.getInfo = function() {
return new Promise(function(resolve, reject){
db.query('SELECT ai_code from user_code',
function(err,rows){
if(err)
reject(err);
resolve(rows);
});
});
}
module.exports =
{
getInfo: this.getInfo
}
Functions that calls the method to receive data.
function team1Code(){
db.getInfo().then(function(result){
var code = JSON.stringify(result[0]);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile('./scr1.js', c, { overwrite: true, encoding: 'ascii' },function (err) {
if (err) return console.log(err);
});
});
}
function team2Code(){
db.getInfo().then(function(result){
var code = JSON.stringify(result[1]);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile('./scr2.js', c, { overwrite: true, encoding: 'ascii' },function (err) {
if (err) return console.log(err);
});
});
}
Finally, this is where we try to read the content of the files.
vmHandler.init = function(apiValues) {
team1Code();
team2Code();
// Team 1
try{
vmHandler.team1.scriptCode = fs.readFileSync('./scr1.js');
vmHandler.team1.script = new vm.Script(vmHandler.team1.scriptCode);
vmHandler.team1.sandbox = { api: new Api(apiValues, 1) }
vmHandler.team1.context = new vm.createContext(vmHandler.team1.sandbox);
}catch(err){}
// Team 2
try {
vmHandler.team2.scriptCode = fs.readFileSync('./scr2.js');
vmHandler.team2.script = new vm.Script(vmHandler.team2.scriptCode);
vmHandler.team2.sandbox = { api: new Api(apiValues, 2) }
vmHandler.team2.context = new vm.createContext(vmHandler.team2.sandbox);
} catch(err) {
console.log("ERROR: " + err);
}
};
The approach you are taking is slightly unfavorable since the function calls
team1Code();
team2Code();
doesn't make sure to accomplish before the next try-catch block gets executed. This is because both the calls are asynchronous hence the next lines get executed before they finish even though they are working with promises. promises themselves are asynchronous, what they make easy is all the code inside any then won't be executed until the promises get settled but the rest of the code will be executed as usual. So, here is the way to do your tasks with updated code.
function writeFile(fileName,data){
return new Promise(function(resolve, reject){
var code = JSON.stringify(data);
var c = json2plain(code, options);
c = c.replace('Ai_code:','');
fs.writeFile(fileName, c, { overwrite: true, encoding: 'ascii' },function (err) {
if(err)
reject(err);
resolve();
});
})
}
//Finally, this is where we try to read the content of the files.
vmHandler.init = function(apiValues) {
var files = ['./scr1.js','./scr2.js'];
db.getInfo().then(function(result){
var allPromise = [];
for(var key in files){
allPromise.push(writeFile(files[key], result[key]));
}
return Promise.all(allPromise);
}).then(function(res){
// Team 1
try{
vmHandler.team1.scriptCode = fs.readFileSync('./scr1.js');
vmHandler.team1.script = new vm.Script(vmHandler.team1.scriptCode);
vmHandler.team1.sandbox = { api: new Api(apiValues, 1) }
vmHandler.team1.context = new vm.createContext(vmHandler.team1.sandbox);
}catch(err){}
// Team 2
try {
vmHandler.team2.scriptCode = fs.readFileSync('./scr2.js');
vmHandler.team2.script = new vm.Script(vmHandler.team2.scriptCode);
vmHandler.team2.sandbox = { api: new Api(apiValues, 2) }
vmHandler.team2.context = new vm.createContext(vmHandler.team2.sandbox);
} catch(err) {
console.log("ERROR: " + err);
}
});
};
In the wmHandler.init function, you are starting 2 asynchronous operations (querying and storing) and reading from the files that the aforementioned async operations should write.
However, the file reading is performed right after the 2 async operations are started. Therefore, it is expected that the files are read before written.
To resolve this, make team1Code and team2Code return Promises of their own, and do not read the files until they have been written.
team1Code().
.then(team2Code)
.then(readFiles)
Where readFiles is the function that does the file reading, and team1Code, team2Code return Promises that resolve when the files are written.
This answer explains the asynchronous callbacks in Javascript.
Related
I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})
You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something
JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.
I updated the function to create the CSV file but now I'm getting an error:
In upload function
internal/streams/legacy.js:57
throw er; // Unhandled stream error in pipe.
^
Error: ENOENT: no such file or directory, open 'C:\Users\shiv\WebstormProjects\slackAPIProject\billingData\CSV\1548963844106output.csv'
var csvFilePath = '';
var JSONFilePath = '';
function sendBillingData(){
var message = '';
axios.get(url, {
params: {
token: myToken
}
}).then(function (response) {
message = response.data;
fields = billingDataFields;
// saveFiles(message, fields, 'billingData/');
saveFilesNew(message, fields, 'billingData/');
var file = fs.createReadStream(__dirname + '/' + csvFilePath); // <--make sure this path is correct
console.log(__dirname + '/' + csvFilePath);
uploadFile(file);
})
.catch(function (error) {
console.log(error);
});
}
The saveFilesNew function is:
function saveFilesNew(message, options, folder){
try {
const passedData = message;
var relevantData='';
if (folder == 'accessLogs/'){
const loginsJSON = message.logins;
relevantData = loginsJSON;
console.log(loginsJSON);
}
if(folder == 'billingData/'){
relevantData = passedData.members;
const profile = passedData.members[0].profile;
}
//Save JSON to the output folder
var date = Date.now();
var directoryPath = folder + 'JSON/' + date + "output";
JSONFilePath = directoryPath + '.json';
fs.writeFileSync(JSONFilePath, JSON.stringify(message, null, 4), function(err) {
if (err) {
console.log(err);
}
});
//parse JSON onto the CSV
const json2csvParser = new Json2csvParser({ fields });
const csv = json2csvParser.parse(relevantData);
// console.log(csv);
//function to process the CSV onto the file
var directoryPath = folder + 'CSV/' + date + "output";
csvFilePath = directoryPath + '.csv';
let data = [];
let columns = {
real_name: 'real_name',
display_name: 'display_name',
email: 'email',
account_type: 'account_type'
};
var id = passedData.members[0].real_name;
console.log(id);
console.log("messageLength is" +Object.keys(message.members).length);
for (var i = 0; i < Object.keys(message.members).length; i++) {
console.log("value of i is" + i);
var display_name = passedData.members[i].profile.display_name;
var real_name = passedData.members[i].profile.real_name_normalized;
var email = passedData.members[i].profile.email;
var account_type = 'undefined';
console.log("name: " + real_name);
if(passedData.members[i].is_owner){
account_type = 'Org Owner';
}
else if(passedData.members[i].is_admin){
account_type = 'Org Admin';
}
else if(passedData.members[i].is_bot){
account_type = 'Bot'
}
else account_type = 'User';
data.push([real_name, display_name, email, account_type]);
}
console.log(data);
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err;
fs.writeFileSync(csvFilePath, output, function(err) {
console.log(output);
if (err) {
console.log(err);
}
console.log('my.csv saved.');
});
});
} catch (err) {
console.error(err);
}
}
The upload file function is:
function uploadFile(file){
console.log("In upload function");
const form = new FormData();
form.append('token', botToken);
form.append('channels', 'testing');
form.append('file', file);
axios.post('https://slack.com/api/files.upload', form, {
headers: form.getHeaders()
}).then(function (response) {
var serverMessage = response.data;
console.log(serverMessage);
});
}
So I think the error is getting caused because node is trying to upload the file before its being created. I feel like this has something to do with the asynchronous nature of Node.js but I fail to comprehend how to rectify the code. Please let me know how to correct this and mention any improvements to the code structure/design too.
Thanks!
You don't wait for the callback provided to stringify to be executed, and it's where you create the file. (Assuming this stringify function really does acccept a callback.)
Using callbacks (you can make this cleaner with promises and these neat async/await controls, but let's just stick to callbacks here), it should be more like:
function sendBillingData() {
...
// this callback we'll use to know when the file writing is done, and to get the file path
saveFilesNew(message, fields, 'billingData/', function(err, csvFilePathArgument) {
// this we will execute when saveFilesNew calls it, not when saveFilesNew returns, see below
uploadFile(fs.createReadStream(__dirname + '/' + csvFilePathArgument))
});
}
// let's name this callback... "callback".
function saveFilesNew(message, options, folder, callback) {
...
var csvFilePath = ...; // local variable only instead of your global
...
stringify(data, { header: true, columns: columns }, (err, output) => {
if (err) throw err; // or return callbcack(err);
fs.writeFile(csvFilePath , output, function(err) { // NOT writeFileSync, or no callback needed
console.log(output);
if (err) {
console.log(err);
// callback(err); may be a useful approach for error-handling at a higher level
}
console.log('my.csv saved.'); // yes, NOW the CSV is saved, not before this executes! Hence:
callback(null, csvFilePath); // no error, clean process, pass the file path
});
});
console.log("This line is executed before stringify's callback is called!");
return; // implicitly, yes, yet still synchronous and that's why your version crashes
}
Using callbacks that are called only when the expected events happen (a file is done writing, a buffer/string is done transforming...) allows JS to keep executing code in the meantime. And it does keep executing code, so when you need data from an async code, you need to tell JS you need it done before executing your piece.
Also, since you can pass data when calling back (it's just a function), here I could avoid relying on a global csvFilePath. Using higher level variables makes things monolithic, like you could not transfer saveFilesNew to a dedicated file where you keep your toolkit of file-related functions.
Finally, if your global process is like:
function aDayAtTheOffice() {
sendBillingData();
getCoffee();
}
then you don't need to wait for the billing data to be processed before starting making coffee. However, if your boss told you that you could NOT get a coffee until the billing data was settled, then your process would look like:
function aDayAtTheOffice() {
sendBillingData(function (err) {
// if (err) let's do nothing here: you wanted a coffee anyway, right?
getCoffee();
});
}
(Note that callbacks having potential error as first arg and data as second arg is a convention, nothing mandatory.)
IMHO you should read about scope (the argument callback could be accessed at a time where the call to saveFilesNew was already done and forgotten!), and about the asynchronous nature of No... JavaScript. ;) (Sorry, probably not the best links but they contain the meaningful keywords, and then Google is your buddy, your friend, your Big Brother.)
please bear in mind that I can, at best, be described as a rookie in both node and amazon S3. I have something app that writes to S3 in the background. I want to read from S3 when the file has been written, and only once it's been written. I attempt to check the number of objects and return the result:
function haveFilesBeenWrittenToBucket(bucketName, callback) {
s3.listObjects({ Bucket: bucketName }, function(err, data) {
const items = data.Contents;
callback(items);
});
}
and the readFile function:
OSClient.prototype.readFile = function(params, callback) {
haveFilesBeenWrittenToBucket(params.Bucket, items => {
console.log("Number of items " + items.length);
if (items.length > 0) {
const rl = readline.createInterface({
input: s3.getObject(params).createReadStream()
});
const myArray = [];
rl.on("line", function (line) {
const lineArray = line.split(",");
for (const value of lineArray) {
if (isNaN(value)) {
// line.split creates string elements, adding extraneous quotation marks in a string and converting
// number to string, so there is a need to reverse this process.
const slicedElement = value.slice(1, -1);
myArray.push(slicedElement);
} else {
const valueOfNumber = Number(value);
myArray.push(valueOfNumber);
}
}
})
.on("close", function () {
callback(myArray);
});
}
else{
var myfunction = this.readFile.bind(this, params, callback);
setTimeout(myfunction, 5000);
}
});
};
and lastly:
targetClient.readFile(params, function (arrayResult) {
logger.info("Read file:" + fileName + OS_FILE_SUFFIX);
readArray = arrayResult;
});
If I put a breakpoint on callback(items) (in 'haveFilesBeenWrittenToBucket') everything works fine and I get back the file written in the bucket, but if not, nothing seems to get written to S3. Seems like some race condition, but I'm really clueless and I really would appreciate some help. Is there a conflict between listing objects and writing to S3 (at least not until much later, in some other test, when it shouldn't be (it's part of a mocha test suite - the readFile is in async.waterfall). I have been on this for days and got nowhere. As I said, it's my first exposure to node, so please be patient with me. Thanks.
S3 provides eventual consistency for list after read. So, you might observe the following:
A process writes a new object to Amazon S3 and immediately lists keys within its bucket. Until the change is fully propagated, the object might not appear in the list.
The only situation in which S3 provides immediate consistency is read-after-write for PUTS of new objects (with a minor caveat, documented here).
More details at S3 consistency model.
Here is an example of how you can use async retry to wait for an object and then retrieve its contents (assumed to be text in this example).
var aws = require("aws-sdk");
var async = require("async");
var s3 = new aws.S3();
var bucket = 'mybucket';
var iteration = 0;
function waitForObjects(bucket, callback) {
console.error(`Iteration: ${++iteration}`);
s3.listObjects({Bucket:bucket}, function(err, data) {
if (err) {
callback(err);
} else if (!data.Contents || !data.Contents.length) {
callback(new Error("No objects"))
} else {
callback(null, data);
}
});
}
// Try calling waitForObjects 10 times with exponential backoff
// (intervals of 100, 200, 400, 800, 1600, ... milliseconds)
async.retry({
times: 10,
interval: function(retryCount) {
return 50 * Math.pow(2, retryCount);
}
}, async.apply(waitForObjects, bucket), function(err, data) {
if (err) {
console.error(`Error waitForObjects: ${err}`);
} else {
console.log(`Object count: ${data.Contents.length}`);
data.Contents.forEach(function(item, index) {
console.log(`Object ${index+1} key: ${item.Key}`);
s3.getObject({Bucket:bucket, Key:item.Key}, function(err, data) {
console.log(`Object ${index+1} txt: ${data.Body.toString()}`);
});
});
}
});
Two things. Firstly, it turns out that my issue was not nodeJS related. Sigh
Secondly, the API now provides a 'waitFor' method for polling whether a bucket or objects exists:
http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#waitFor-property
I am writing a program to batch print drawings. They need to be in a certain order ex. drawing A,B,C. The program plots the correct number of prints its just the order is random. I need the first in the array list to complete before next and so on.
startMe(newPaths)
.then(function(result){
})
function startMe(dwgPaths){
return q.all(buildCalls(dwgPaths));
}
var buildCalls = function(dwgPaths) {
var calls = [];
var scFiles = [tmpDir + "425011-fab.scr",tmpDir + "425011-pk.scr",tmpDir + "425011-sc.scr"];
for (var sc in scFiles){
for (var i in dwgPaths) {
calls.push(callAccoreConsole(dwgPaths[i],scFiles[sc]));
}
}
return calls;
};
function callAccoreConsole(dwgPath,scrFile){
var deferred = q.defer();
childProcess.execFile('C:/Program Files/Autodesk/AutoCAD 2015/accoreconsole.exe',['/i',dwgPath,'/s',scrFile], function(err, data) {
if(err)
deferred.resolve({success:false,reason: err});
deferred.resolve({success:true});
});
return deferred.promise;
}
The code below works the way I want. I print 2 files 3 copies each. File A using script 1,then File B using script 1. Then repeats for the other scripts. I have a total of 6 prints "three groups" A,B,A,B,A,B each with the appropriate script ran. With the code above I may get B,A,A,B,A,A.
callAccoreConsole(newPaths[0],scFiles2[0])
.then(function(result){
callAccoreConsole(newPaths[1],scFiles2[0])
.then(function(result){
callAccoreConsole(newPaths[0],scFiles2[1])
.then(function(result){
callAccoreConsole(newPaths[1],scFiles2[1])
.then(function(result){
callAccoreConsole(newPaths[0],scFiles2[2])
.then(function(result){
callAccoreConsole(newPaths[1],scFiles2[2])
.then(function(result){
})
})
})
})
})
});
I have been struggling with this for a while. I found the code below and got it to work for my application, But it doesn't seem the most efficient way to be written. If any one has a more compact way let me know please.Thanks
var itemsToProcess = [];
for (var sc in scFiles){
for (var i in newPaths) {
itemsToProcess.push( {file:newPaths[i],script:scFiles[sc]});
}
}
function getDeferredResult(a) {
return (function (items) {
var deferred;
if (items.length === 0) {
return q.resolve(true);
}
deferred = q.defer();
var payload = {
file:items[0].file,
script:items[0].script
};
callAccoreConsole2(payload)
.then(function(result){
deferred.resolve(items.splice(1));
});
return deferred.promise.then(getDeferredResult);
}(a));
}
q.resolve(itemsToProcess)
.then(getDeferredResult)
.then(function(result){
return res.send({success:true});
})
As you are constructing your array of promises you are simultaneously invoking the execFile method.
function callAccoreConsole(dwgPath,scrFile){
var deferred = q.defer();
childProcess.execFile('C:/Program Files/Autodesk/AutoCAD 2015/accoreconsole.exe',['/i',dwgPath,'/s',scrFile], function(err, data) {
if(err)
deferred.resolve({success:false,reason: err});
deferred.resolve({success:true});
});
return deferred.promise;
}
So, instead using callAccoreConsole to run the process and return a deffered, you need something that calls that method eventually -
calls.push(q.fcall(callAccoreConsole, dwgPaths[i],scFiles[sc]));
I haven't tried this specifically, but the gist is that you are calling your method at the same time you are creating the deferred for it.
I aim to import large amount of data by Mongoose. As a newbie, I fail to setup the flow control properly with various mechanisms by async. Glad if someone could point to an appropriate solution. Thanks.
var async = require('async'),
mongoose = require('mongoose');
mongoose.connect('mongodb://localhost/test');
var Cat = mongoose.model('Cat', { name: String });
// Imagine this is a huge array with a million items.
var content = ['aaa', 'bbb', 'ccc'];
var queries = [];
content.forEach(function(name) {
queries.push(function(cb) {
var obj = new Cat({ name: name });
obj.save(function(err) {
console.log("SAVED: " + name);
console.log(err);
});
return true;
});
});
// FAILED: async.parallel adds all content to db,
// but it would exhaust the resource with too many parallel tasks.
async.parallel(queries, function(err, result) {
if (err)
return console.log(err);
console.log(result);
});
// FAILED: save the first item but not the rest
async.waterfall(queries, function(err, result) {
if (err)
return console.log(err);
console.log(result);
});
// FAILED: same as async.waterfall, async.queue saves the first item only
var q = async.queue(function(name, cb) {
var obj = new Cat({ name: name });
obj.save(function(err) {
console.log("SAVED: " + name);
console.log(err);
});
})
q.push(content, function (err) {
console.log('finished processing queue');
});
I think eachLimit or eachSeries fit your situation best:
var content = ['aaa', 'bbb', 'ccc'];
async.eachLimit(content, 10, function(name, done) {
var obj = new Cat({ name : name });
obj.save(done);
// if you want to print some status info, use this instead:
//
// obj.save(function(err) {
// console.log("SAVED: " + name);
// console.log(err);
// done(err);
// });
//
}, function(err) {
// handle any errors;
});
With eachLimit, you can run an X amount of queries 'in parallel' (10 in the example above) to speed things up without exhausting resources. eachSeries will wait for the previous save before it continues with the next, so effectively saving one object at a time.
Notice that with each*, you won't get a list with (saved) objects back (it's a bit of a fire-and-forget mechanism where you're not interested in the outcome, bar any errors). If you do want a list of saved objects in the end, you can use the equivalent map* functions: mapLimit and mapSeries.