Hi there is some nodejs code as follow:
console.log("Start", new Date().getTime());
var fs = require('fs');
for (var i = 1; i < 10; i++) {
fs.readFile("file1.zip",function(err, data) {
if(err)
console.log("read error: ", err);
else {
fs.writeFile(__dirname + "/file2.zip", data , function(err) {
if(err) {
console.log("write error: ", err);
}
});
}
});
};
console.log("Finished", new Date().getTime());
I want to get the start and finish time when all the work is done, but it seems like the the second log is too early
Instead of console.log("Finished", new Date().getTime()); you might want to try:
process.on('exit', function () {
console.log("Finished", new Date().getTime());
});
The second log function is called too early because of node's async behavior. You have to put it inside the last callback:
console.log("Start", new Date().getTime());
var fs = require('fs');
for (var i = 1; i < 10; i++) {
fs.readFile("file1.zip",function(err, data) {
if(err) {
console.log("read error: ", err);
} else {
fs.writeFile(__dirname + "/file2.zip", data , function(err) {
if(err) {
console.log("write error: ", err);
}
// Put it here
console.log("Finished", new Date().getTime());
});
}
});
};
Related
I am trying to find the best way to write this code. I fetch locations from a remote resource and need to check if there are any new locations present in the data, if there are I should add them to my database, if they are not new I just want to update them.
const http = require('http');
const timeout = 5000; //5 seconds
const MongoClient = require('mongodb').MongoClient;
// Database Name
const dbName = 'weatherApp';
const url = 'mongodb://localhost:27017';
// Connect using MongoClient
MongoClient.connect(url, function(err, client) {
if(err){
console.log(err);
return;
}
const locationsCollection = client.db(dbName).collection('locations');
(function fetchTemperatureLoop(){
console.log('Started http request..');
http.get('remote url..', function(resp){
var data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
if(data.isJson()){
var locations = JSON.parse(data).toArray();
(function locationsLoop(){
var location = locations.pop();
locationsCollection.findOne({location: location.location}, function(err, result){
if(err){
console.log(err);
return;
}
if(result){
//Exists
var measurements = result.measurements;
measurements.push({timestamp: +new Date, temperature: location.temperature})
locationsCollection.update({location: location.location}, {$set: {measurements: measurements}}, function(err){
if(err){
console.log(err);
return;
}
console.log('Added new temperature for location: ' + location.location);
continueLocationsLoop();
});
}else{
//Doesnt exist
location.measurements = [];
location.measurements.push({timestamp: +new Date, temperature: location.temperature});
locationsCollection.insert(location, function(err){
if(err){
console.log(err);
return;
}
console.log('Created new location: ' + location.location);
continueLocationsLoop();
});
}
});
function continueLocationsLoop(){
if(locations.length){
locationsLoop()
}else{
setTimeout(fetchTemperatureLoop, timeout);
}
}
})();
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
console.log("Continue anyways..");
setTimeout(fetchTemperatureLoop, timeout);
});
})();
});
String.prototype.isJson = function(){
try{
JSON.parse(this);
}catch(e){
return false;
}
return true;
}
Object.prototype.toArray = function(){
var arr = [];
for(var key in this){
if(this.hasOwnProperty(key)){
arr.push(this[key]);
}
}
return arr;
}
I really want to avoid using so many closures but I dont want to repeat myself either. Any help rewriting this code in an optimal way is much appriciated.
My main problem was illiterating through the locations and doing the calls to the database.
I am having trouble setting up my promise in my code. This is what I have implemented right now:
let promise = new Promise(function(resolve, reject){
ATVStatement();
let isdone = true;
if(isdone){
resolve();
}else{
reject();
}
})
promise.then(CustomMessage());
The problem I am getting is that the I pretty sure that the promise is not working. If anyone could spot the error that I am doing I would greatly appreciate it.
EDIT
Here is code for the the two functions being used:
function ATVStatement() {
request = new Request("select distinct(\"Product Name\") from SPA_Data_Feeds where \"Strategic Priority\" = 'Accelerate to Value (LD)'",
function(err, rowCount, rows)
{
console.log(rowCount + ' row(s) returned');
}
);
//var result = "";
var count = 0
request.on('row', function(columns) {
columns.forEach(function(column) {
console.log("%s\t", column.value);
result+= column.value + "\t\n"; //result is a global variable
count++;
});
});
connection.execSql(request);
}
function CustomMessage(){
console.log('here')
var customMessage = new builder.Message(session)
.text("### Here is a list of the ATV Compounds: \n" + "> %s ", result)
.textFormat("markdown")
.speak("Here is what I found for \'%s\'.", session.message.text)
.textLocale("en-us");
session.send(customMessage); }
Ok, so now we understand the code a little better (thanks for updating), we need to modify the ATVStatement to return a promise and resolve it when the query is complete.
We change the code above to:
let atvPromise = ATVStatement();
atvPromise.then ((result) => {
CustomMessage();
}).catch ( (err) => {
console.log('An error occurred: ' + err);
});
function ATVStatement() {
return new Promise(function(resolve, reject) {
request = new Request("select distinct(\"Product Name\") from SPA_Data_Feeds where \"Strategic Priority\" = 'Accelerate to Value (LD)'",
function(err, rowCount, rows)
{
console.log(rowCount + ' row(s) returned');
if (err) {
reject(err);
} else {
resolve(rows);
}
}
);
var count = 0
request.on('row', function(columns) {
columns.forEach(function(column) {
console.log("%s\t", column.value);
result+= column.value + "\t\n"; //result is a global variable
count++;
});
});
connection.execSql(request);
});
}
You see, ATVStatement was actually asynchronous.
I have problem with mongoosastic it not update Elasticsearch from first update. I update 1 time and nothing happend and then update 2 time and in elasticsearch make update from what I updated 1st time...
Here is my code:
var User = system.mongoose.model('User', userSchema, 'user');
User.createMapping(function (err, mapping) {
if (err) {
console.log("error creating mapping");
console.log(err);
} else {
console.log("Mapping created");
console.log(mapping);
}
});
//{$in: ['doctor','nurse']}
var stream = User.synchronize();
var count = 0;
stream.on('data', function (data) {
count++;
});
stream.on('close', function () {
console.log("Indexed " + count + " documents");
});
stream.on('error', function (err) {
console.log(err);
});
Anyone know what is problem?
It is not clear from your question what you expected, and what you got.
But I am going to guess that you are experiencing a race condition because you started User.synchronize() before User.createMapping() finished.
If so, this might work better:
var User = system.mongoose.model('User', userSchema, 'user');
User.createMapping(function (err, mapping) {
if (err) {
console.log("error creating mapping");
console.log(err);
} else {
console.log("Mapping created");
console.log(mapping);
doTheNextThing();
}
});
function doTheNextThing () {
//{$in: ['doctor','nurse']}
var stream = User.synchronize();
var count = 0;
stream.on('data', function (data) {
count++;
});
stream.on('close', function () {
console.log("Indexed " + count + " documents");
});
stream.on('error', function (err) {
console.log(err);
});
}
I am trying to combine multiple textfiles,convert them in a single zip file using zip archiver.
exports.downloadFilesInZip = function(req, res, next) {
var respObj = {};
var file_names = [];
var projectId = 111;
var file_ids = 11111;
console.log(projectId);
db.getConnection(function (err, connection) {
if (err) {
debug(err);
next(err);
}
else {
var updateQuery = "select data from file_data where file_id IN (?)";
console.log(updateQuery);
connection.query(updateQuery,[file_ids], function (err, results) {
console.log("inside" + updateQuery);
if (err) {
connection.release();
console.log("error" + JSON.stringify(err));
debug(err);
next(err);
}
else {
async.eachSeries(results,function(item,loopCallBack){
var text = "";
console.log("hllllllll");
console.log(item.data);
console.log(JSON.parse(item.data));
document_text = JSON.parse(item.data);
console.log("dssddssdsdsdsdsd"+document_text);
for(var j=0; j < document_text.length ;j++)
{
text += document_text[j]['text'];
}
//file_names.push(convertStringToTextFile(text));
convertStringToTextFile(text,function(err,file_name){
if(err){
console.log(err);
loopCallBack(err);
}
else {
file_names.push(file_name);
loopCallBack();
}
})
},function(err){
if(err){
console.log(err);
next(err);
}
else {
var updateQuery = "select name from project where id in (?)";
console.log(updateQuery);
connection.query(updateQuery,[projectId], function (err, results) {
console.log("inside" + updateQuery);
connection.release();
if (err) {
console.log("error" + JSON.stringify(err));
debug(err);
next(err);
}
else {
var fileName_link = JSON.stringify(results[0].name);
console.log("projectname"+fileName_link);
convertTextFilesToZip(file_names,fileName_link, function (err, filename) {
if (err) {
console.log(err);
next(err);
}
else {
console.log("filename link" + filename);
res.json({
status: 0,
file_link: filename
});
}
});
}
});
}
});
}
});
}
});
}
}
convertStringToTextFile = function(text,cb){
var json_filename = 'tmp/file_'+uuid.v4().replace('-','')+'.txt';
fs.writeFile(json_filename, text , function (err) {
if (err) {
debug(err);
cb(err);
}
else{
cb(null,json_filename);
}
});
};
convertTextFilesToZip = function(textFiles,file_link,cb){
console.log("textfiles"+textFiles);
var filename = 'reports/'+JSON.parse(file_link)+'_extractedText.zip';
var output = fs.createWriteStream(filename);
output.on('close', function() {
console.log(zipArchive.pointer() + ' total bytes');
console.log('archiver has been finalized and the output file descriptor has closed.');
});
zipArchive.on('error', function(err) {
cb(err);
});
zipArchive.pipe(output);
zipArchive.bulk([
{ expand: true, src: textFiles }
]);
zipArchive.finalize();
cb(null,filename);
}
It works okay the first time and after that it throws this error.I have checked other posts in which res is returned twice but i couldn't find it.It says that can't set headers after they are sent.I think the problem is in the convertTextFilesToZip function but i cant seem to pinpoint the exact location which is generating the error.ANy help is appreciated.
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (_http_outgoing.js:350:11)
at ServerResponse.header (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:700:10)
at ServerResponse.send (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:154:12)
at fn (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:934:10)
at View.exports.renderFile [as engine] (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/jade/lib/index.js:374:12)
at View.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/view.js:93:8)
at EventEmitter.app.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/application.js:566:10)
at ServerResponse.res.render (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/lib/response.js:938:7)
at /Users/zeeshandar/Desktop/Agreements_info/agreements_info/app.js:207:13
at Layer.handle_error (/Users/zeeshandar/Desktop/Agreements_info/agreements_info/node_modules/express/li b/router/layer.js:58:5)
Making my comment into an answer since it appears to have led to the solution.
The variable zipArchive is not initialized in convertTextFilesToZip() therefore you are reusing that variable from one function call to the next and that seems unlikely to be the right implementation.
Also, I would expect your method calls to zipArchive to be asynchronous and it doesn't look like your are coding for that since the callback is called before you have any sort of completion notification.
I am new to async library. I have used async.eachSeries and async.waterfall for each iteration. I see, the async.waterfall runs only once.
Here is my code :
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});
Output:
124
JSON saved to debuginfo.json
async completed
Any help is really appreciated.
I found my mistake. I missed calling the callback after each iteration just after async is completed.
var fs = require('fs'),
async = require('async'),
Client = require('node-rest-client').Client;
// REST API Call and output in jsonOutput.results
console.log(jsonOutput.results.length); // jsonOutput.results has 124 records.
async.eachSeries(jsonOutput.results, function(account, callback) {
var dataObject = {};
dataObject.updatetime = new Date();
var setAccountInfoURL = ""; // Data Update REST API Request
async.waterfall([
function setAccountInfo(updateCallback) {
// client.get(setAccountInfoURL, function (data, response) {
// var jsonOutput = JSON.parse(data.toString('utf8'));
updateCallback(null, "output", account)
// });
},
function saveAccountInfo(jsonOutput, account, updateCallback) {
var debuglog = JSON.stringify(account) + "\n" + jsonOutput;
fs.appendFile("debuginfo.json", debuglog + "\n", function (err) {
if(err) {
console.log(err);
}
console.log("JSON saved to " + "debuginfo.json");
updateCallback(null);
});
}
],function asyncComplete(err) {
if (err) {
console.warn('Error setting account info.', err);
}
console.log('async completed');
callback(null); // this is the change.
});
}, function(err){
if (err) {
console.log('error in loop');
}
console.log('loop completed');
});