Node.js - undefined variable after assignment in fs functions - node.js

stackoverflow!
I am fairly new to node.js, and this seems to be node-specific. I keep defining a variable and I get errors that fileName is undefined. I have no idea why this is happening because, from my perspective, I'm just assigning a global variable inside a function, and that should work in all other programming languages I've worked in. Does the argument function in fs.readFile() somehow differ from a normal function? I honestly have no idea. Anyways, this is my full code:
var fs = require('fs');
var dateObject = new Date();
var fileName;
function Start() {
fs.readFile('./counter.txt', function (err, data) {
if (err) throw err;
var current = parseInt(data);
current++;
fs.writeFile('./counter.txt', current.toString(), function(err) {
if (err) throw err;
console.log("Written!");
});
var fullDate = `${dateObject.getDate()}-${dateObject.getMonth() + 1}-${dateObject.getFullYear()}`;
fileName = `./logs/${fullDate} ${current}.txt`;
console.log(fileName);
fs.appendFile(fileName, "Logger Initiated!", function(err){
if (err) throw err;
})
});
}
function PAL (text) {
if (fileName === undefined) {
console.log("...");
return "500";
}
console.log(fileName);
fs.appendFile(fileName, text, function(err) {
if (err) throw err;
})
}
module.exports = {Start, PAL};
Another file:
const logger = require('./logger')
// ....
app.listen(port, () => {
logger.Start();
logger.PAL("test");
})

You got an asynchronous trap in code, fs.readFile is an async function, so when you run .PAL, I think you expected .Start function done, but it's not true. Start function:
async function Start() {
return new Promise((resolve, reject) => {
fs.readFile('./counter.txt', function (err, data) {
if (err) throw reject(err);
var current = parseInt(data);
current++;
fs.writeFile('./counter.txt', current.toString(), function(err) {
if (err) reject(err);
console.log("Written!");
var fullDate = `${dateObject.getDate()}-${dateObject.getMonth() + 1}-${dateObject.getFullYear()}`;
fileName = `./logs/${fullDate} ${current}.txt`;
console.log("FILENAME: ", fileName);
fs.appendFile(fileName, "Logger Initiated!", function(err){
if (err) reject(err);
resolve();
})
});
});
})
}
Call: logger.Start().then(() => logger.PAL("test"));

Related

Not able to return value from promise in Nodejs

I have written the following code in Nodejs which is saving data in MongoDB:
function insertDoc(db,data){
return new Promise(resolve => {
callback=db.collection('AnalysisCollection').insertOne(data).then(function(response,obj){
console.log("Inserted record");
resolve(obj);
//console.log(obj);
// response.on('end',function(){
// resolve(obj);
// });
//return resolve(obj);
}).then(() => { return obj }
).catch(function(error){
throw new Error(error);
});
})
}
I am calling the above function from the main function like this:
async function cosmosDBConnect(nluResultJSON){
try{
//console.log("Inserting to cosmos DB");
console.log(nluResultJSON);
var url = config.cosmos_endpoint;
var result="";
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
// insertDoc(db, data, function() {
result=insertDoc(db, data, function() {
console.log(result);
client.close();
//return data._id;
});
});
}
catch (e) {
console.log(e);
}
}
module.exports = { cosmosDBConnect };
But in cosmosDBConnect, I am getting 'undefined' for the result, though in insertDoc I am getting the output for'obj' with _id for the inserted record.
Please help me to return this _id to cosmosDBConnect.
You are use callbacks inside of async function, which creates internal scopes. So your return aplies to them instead of whole function. You should use Promise-based methods inside of async function using await (without callbacks) or wrap whole function into own Promise otherwise.
Example:
function cosmosDBConnect(nluResultJSON) {
return new Promise((resolve, reject) => {
var url = config.cosmos_endpoint;
var result = '';
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
if (err) return reject(err);
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
insertDoc(db, data).then(obj => {
console.log(obj);
client.close();
return resolve(data._id);
});
});
});
}
Also you need to understand that your insertDoc return Promise and do not accept callback you tried to pass.
Ref: async function
result = insertDoc(db, data).then((data) => {
console.log(data);
}).catch(err => console.error(err));

How to convert readFileSync code to readFile?

I'm trying to convert the following code to fs.readFile? I'm running this on a nodeserver and need it to be async. I read the node documentation and I found it very difficult to follow.
I'm reading a file and setting it to a variable, im then saving it to the database (mongo). I can't find a way to asynchronously use fs.readFile and set it to a variable.
Here is my current undesired synchronous code:
a.img.data = fs.readFileSync(path.resolve(__dirname + '/imgTest/image.png'));
a.img.contentType = 'image/jpg';
a._id = accountId;
a.save(function (err, a) {
if (err) throw err;
console.log('saved img to mongo');
})
Here is my attempt but it doesn't work:
await fs.readFile(path.resolve(__dirname + '/imgTest/image.jpg'), function (err, data) {
if (err) return console.error(err);
a.img.data = data;
});
a.img.contentType = 'image/jpg';
a._id = accountId;
a.save(function (err, a) {
if (err) throw err;
console.log('saved img to mongo');
})
You are mixing syntaxes. The normal fs library does not deal in promises so you can't use await, and you do all the work in the callback:
fs.readFile(path.resolve(__dirname + '/imgTest/image.jpg'), function (err, data) {
if (err) return console.error(err);
a.img.data = data;
a.img.contentType = 'image/jpg';
a._id = accountId;
a.save(function (err, a) {
if (err) throw err;
console.log('saved img to mongo');
})
});
or you go the Promises route
const fsp = require("fs/promises");
try {
const data = await fsp.readFile(path.resolve(__dirname + '/imgTest/image.jpg'));
a.img.data = data;
a.img.contentType = 'image/jpg';
a._id = accountId;
a.save(function (err, a) {
if (err) throw err;
console.log('saved img to mongo');
})
} catch(err) {
console.log(err);
}

Instructions not executed into the right order with nodejs

I am new to nodejs and trying to cat multiple css files on-the-fly while coding. The package chokidar allow me to call a function when a file is modified, however I have a problem with the execution.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
});
});
});
I want to first delete the previous file, then read the directory and then write a new "concat.css". However I have an error;
Error: ENOENT: no such file or directory, open 'public/css/concat.css'
at error (native)
It appears that the function concat() is executed before the directory update and not after, and therefore it is trying to cat a file that just have been deleted. Why ?
I know that nodejs is executing functions in a synchronous way but I can't find a way to solve this problem. I tried async but I can't declare a variable between two functions and I couldn't manage to make it work.
If it cannot exist in a callback, using the setTimeout(fn, 0) trick may help make sure it's executed after the variable assignment.
var goconcat =
fs.readdir(paths, function (err, files) {
if (err) {console.log(err);}
fs.unlink(paths + 'concat.css', function (err) {
if (err) throw err;
var list = files.map(function (files) {
return path.join(paths, files);
});
setTimeout(function() {
concat(list, paths + 'concat.css', function(err) {
if (err) throw err
})}, 0);
});
});
The problem you're having is that your concat function is being invoked before the file is deleted by invoking unlink. You can prevent this by having nested callbacks; however, you can probably have better control flow if you use a module like async, and prevent yourself from dealing with Callback Hell.
Below is an example on how you can use the async module.
var fs = require('fs');
var async = require('async');
var myDir = __dirname + '/data';
async.waterfall([function(callback) {
fs.readdir(myDir, 'utf-8', function(error, files) {
if (error) {
return callback(error);
}
return callback(null, files);
});
}, function(files, callback) {
fs.open(myDir + '/myFile', 'wx', function(error, f) {
if (error && error.code === 'EEXIST') {
return callback(null, 'EEXIST');
}
return callback(null, 'CREATE');
});
}, function(fileStatus, callback) {
if (fileStatus === 'EEXIST') {
console.log('File exists. Deleting file...');
fs.unlink(myDir + '/myFile', function(error) {
if (error) {
return callback(error);
} else {
return callback(null);
}
});
} else {
console.log('File does not exist...');
return callback(null);
}
}, function(callback) {
fs.writeFile(myDir + '/myFile', "Hello World", function(err) {
if(err) {
return callback(error);
}
return callback(null, 'File Created');
});
}], function(error, results) {
console.error(error);
console.log(results);
});
The waterfall function runs the tasks array of functions in series,
each passing their results to the next in the array. However, if any
of the tasks pass an error to their own callback, the next function is
not executed, and the main callback is immediately called with the
error.

Calculate a file hash and save the file

Users upload files into my express app. I need to calc hash of the uploaded file and then write file to disk using calculated hash as a filename. I try to do it using the following code:
function storeFileStream(file, next) {
createFileHash(file, function(err, hash) {
if (err) {
return next(err);
}
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
return next(err);
});
stream.on('finish', function() {
return next();
});
file.pipe(stream);
});
}
function createFileHash(file, next) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
return next(err);
});
file.on('end', function(data) {
hash.end();
return next(null, hash.read());
});
file.pipe(hash);
}
The problem is that after I calc file hash the writed file size is 0. What is the best way do solve this task?
Update
According #poke suggestion I try to duplicate my stream. Now my code is:
function storeFileStream(file, next) {
var s1 = new pass;
var s2 = new pass;
file.pipe(s1);
file.pipe(s2);
createFileHash(s1, function(err, hash) {
if (err) {
return next(err);
}
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
return next(err);
});
stream.on('finish', function() {
return next();
});
s2.pipe(stream);
});
}
function createFileHash(file, next) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
return next(err);
});
file.on('end', function(data) {
hash.end();
return next(null, hash.read());
});
file.pipe(hash);
}
The problem of this code is that events end and finish are not emited. If I comment file.pipe(s2); events are emited, but I again get my origin problem.
This code fix the problem:
var s1 = new passThrough,
s2 = new passThrough;
file.on('data', function(data) {
s1.write(data);
s2.write(data);
});
file.on('end', function() {
s1.end();
s2.end();
});
The correct and simple way should be as follow:
we should resume the passthroughed stream
function storeFileStream(file, directory, version, reject, resolve) {
const fileHashSource = new PassThrough();
const writeSource = new PassThrough();
file.pipe(fileHashSource);
file.pipe(writeSource);
// this is the key point, see https://nodejs.org/api/stream.html#stream_three_states
fileHashSource.resume();
writeSource.resume();
createFileHash(fileHashSource, function(err, hash) {
if (err) {
return reject(err);
}
const fileName = path.join(directory, version + '_' + hash.slice(0, 8) + '.zip');
const writeStream = fs.createWriteStream(fileName);
writeStream.on('error', function(err) {
return reject(err);
});
writeStream.on('finish', function() {
return resolve();
});
writeSource.pipe(writeStream);
});
}
function createFileHash(readStream, next) {
const hash = crypto.createHash('sha1');
hash.setEncoding('hex');
hash.on('error', function(err) {
return next(err);
});
hash.on('finish', function(data) {
return next(null, hash.read());
});
readStream.pipe(hash);
}
You could use the async module (not tested but should work):
async.waterfall([
function(done) {
var hash = crypto.createHash('sha1');
hash.setEncoding('hex');
file.on('error', function(err) {
done(err);
});
file.on('end', function(data) {
done(null, hash.read);
});
file.pipe(hash);
},
function(hash, done) {
var fileName = path.join(config.storagePath, hash),
stream = fs.createWriteStream(fileName);
stream.on('error', function(err) {
done(err);
});
stream.on('finish', function() {
done(null);
});
file.pipe(stream);
}
], function (err) {
console.log("Everything is done!");
});

"object is not a function" error during MongoDB document insertion from a CSV stream using async.queue

I'm trying MongoDB document insertion from a CSV stream using async.queue.
But I face this following error. I've tried all the remedies given in similar SO posts.
Exact error message is:
C:\Users\admin\node_modules\mongodb\lib\mongo_client.js:406
throw err
TypeError:object is not a function
at C:\Users\admin\Desktop\mynodefile.js:13:2
at C:\Users\admin\node_modules\mongodb\lib\mongo_client.js:403:11
at process._tickCallback(node.js:355:11)
node.js code I used:
var csv = require('csv');
var async = require('async');
var fs = require('fs');
var MongoClient = require('mongodb').MongoClient;
MongoClient.connect('mongodb://localhost:27017', function(err, db) {
if (err) throw err;
var collection = db.collection('myCSVs');
var queue = async.queue(collection.insert.bind(collection), 5);
csv()
.from.path('./input.csv', { columns: true })
.transform(function (data, index, cb) {
queue.push(data, function (err, res) {
if (err) return cb(err);
cb(null, res[0]);
});
})
.on('error', function (err) {
console.log('ERROR: ' + err.message);
})
.on('end', function () {
queue.drain = function() {
collection.count(function(err, count) {
console.log('Number of documents:', count);
db.close();
});
};
});
});
You haven't mentioned the database name in your MongoClient.connect function call. You can do so like this:
MongoClient.connect('mongodb://localhost:27017/database_name',function(err, db) {
Then you can do:
var collection = db.collection('myCSVs');
If myCSVs is a collection inside database_name
Or you can also do:
MongoClient.connect('mongodb://localhost:27017',function(err, mongoclient) {
var db = mongoclient.db('database_name');
var collection = db.collection('myCSVs');
});
You have to change
var queue = async.queue(collection.insert.bind(collection), 5);
Into:
var q = async.queue(function (task, callback) {
console.log('hello ' + task.name);
callback();
}, 2);
IN this line :
queue.push(data, function (err, res) {
if (err) return cb(err);
cb(null, res[0]);
});
you are calling push with data and with a callback, but its not implemented in your
var queue = async.queue(collection.insert.bind(collection), 5);

Resources