Recursive function in node.js giving unexpected results - node.js

Well the results are unexpected for me at least. I'm trying to search through a directory and get all files and subfolders, however I'm having some problems with working with subdirectories of subdirectories. The function itself is designed to find all the contents of all the folders and all the files within a given folder, and call itself when one of the things inside a folder is a folder.
However, it never finishes, it keeps feeding the same folder into itself and never doing anything with it, eventually just crashing.
var fs = require('fs');
var array = fs.readdirSync(__dirname);
function getAllSub(array){
for (i = 0; i < array.length; i++){
if (array[i].indexOf(".") == (-1))
{
array = array.concat(array[i] + "/" + fs.readdirSync(__dirname + "/" + array[i]));
}
if (array[i].indexOf("/") != (-1)){
var foldcon = array[i].substr(array[i].indexOf("/") + 1);
var folder = array[i].substr(0, array[i].indexOf("/"));
foldcon = foldcon.split(",");
for (n = 0; n < foldcon.length; n++){
foldcon[n] = folder + "/" + foldcon[n]
if (foldcon[n].indexOf(".") == (-1)){
console.log([foldcon[n]]);
foldcon[n] = getAllSub([foldcon[n]]);
}
}
array.splice(i, 1, foldcon);
}
}
return array;
}
array = getAllSub(array);
console.log(array);

You have a couple of problems with your code. You should be using the fs.stat call to get information about files to determine if they are directories or files, parsing the paths themselves is really error prone. Also, readdir just returns the names of the file in the directory, not the full path to the files. Finally, I'd suggest you use the async versions of readdir so that you don't lock up the node.js thread while you're doing this.
Here's some sample code for getting all of the folders and files from a start path:
var parseDirectory = function (startPath) {
fs.readdir(startPath, function (err, files) {
for(var i = 0, len = files.length; i < len; i++) {
checkFile(startPath + '/' + files[i]);
}
});
};
var checkFile = function (path) {
fs.stat(path, function (err, stats) {
if (stats.isFile()) {
console.log(path + ' is a file.');
}
else if (stats.isDirectory()) {
console.log(path + ' is a directory.');
parseDirectory(path);
}
});
};
parseDirectory(__dirname + '/../public');

Related

Asynchronously writing files in a loop, how to manage streams

I am trying to write over 100 png files via node-canvas in a loop. Only 40 files get generated and then the process completes.
I have tried creating a png stream via createPNGStream() and piping the results to a write stream created by fs.createWriteStream().
Write function:
function writeFile(row, col) {
// canvas code ...
const stream = canvas.createPNGStream();
let path = __dirname + '/imgs/heatmapRow' + row + "Col" + col + '.png';
const out = fs.createWriteStream(path);
stream.pipe(out);
out.on('finish', () => console.log('The PNG file was created.'))
}
Calling function:
function generateImages(){
var numRows = 20;
var numCols = 5;
for(let row = 0; row < numRows; ++row) {
for (let col = 0; col < numCols; ++col) {
writeFile(row, col);
}
}
}
The loop runs and completes, and at the end I get a bunch of the following lines all at once:
The PNG file was created.
The PNG file was created.
The PNG file was created.
The PNG file was created.
I'm thinking that on each loop a writeStream is being created and is asynchronous. The process is terminating because I can have only so many streams open.
How can I write all of my files and do so asynchronously to speed up the process time? (I prefer not to write the files synchronously) Do I need to add each writeFile to a queue? How do I determine limit the number of streams I have open and how do I manage them?
You have to use Promises to make your asynchronous calls.
I give you a solution:
function writeFile(row, col) {
// canvas code ...
const stream = canvas.createPNGStream();
let path = __dirname + "/imgs/heatmapRow" + row + "Col" + col + ".png";
return new Promise(resolve => {
const out = fs.createWriteStream(path);
stream.pipe(out);
out.on("finish", () => {
console.log("The PNG file was created.");
resolve();
});
});
}
function generateImages() {
var numRows = 20;
var numCols = 5;
var imagesToGenerate = [];
for (let row = 0; row < numRows; ++row) {
for (let col = 0; col < numCols; ++col) {
imagesToGenerate.push(writeFile(row, col));
}
}
Promise.all(imagesToGenerate).then(() => {
console.log("All images generated");
});
}
Take a look at Promise.all docs if you don't clearly understand how it is working

getFileAsync causing Excel to crash

I'm building an office-js add-in for Excel. I need to upload the current workbook to a back end server. I've implemented an example from the Micrsoft Documentation, which seems to work fine the first time I call it, but on subsequent calls, it causes Excel to crash. I'm using Excel 365 version 1812 (build 11126.20132)
Here is the link to the example in the MS docs:
https://learn.microsoft.com/en-us/javascript/api/office/office.document
There are many examples on this page, to find the one I'm working from search for "The following example gets the document in Office Open XML" I've included the example below for ease of reference.
The code just get's the current file and dumps the characters to the console's log. It works fine the first but crashes Excel the second time--after it has shown the length of FileContent.
export function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
console.log("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}else {
console.log("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
// console.log("file part",sliceResult.value.data)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
console.log("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
console.log("fileContent.length",fileContent.length)
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
Here is the result
File size:21489 #Slices: 1
fileContent.length 21489
Original example from Microsoft documentation (https://learn.microsoft.com/en-us/javascript/api/office/office.document)
// The following example gets the document in Office Open XML ("compressed") format in 65536 bytes (64 KB) slices.
// Note: The implementation of app.showNotification in this example is from the Visual Studio template for Office Add-ins.
function getDocumentAsCompressed() {
Office.context.document.getFileAsync(Office.FileType.Compressed, { sliceSize: 65536 /*64 KB*/ },
function (result) {
if (result.status == "succeeded") {
// If the getFileAsync call succeeded, then
// result.value will return a valid File Object.
var myFile = result.value;
var sliceCount = myFile.sliceCount;
var slicesReceived = 0, gotAllSlices = true, docdataSlices = [];
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Get the file slices.
getSliceAsync(myFile, 0, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
else {
app.showNotification("Error:", result.error.message);
}
});
}
function getSliceAsync(file, nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived) {
file.getSliceAsync(nextSlice, function (sliceResult) {
if (sliceResult.status == "succeeded") {
if (!gotAllSlices) { // Failed to get all slices, no need to continue.
return;
}
// Got one slice, store it in a temporary array.
// (Or you can do something else, such as
// send it to a third-party server.)
docdataSlices[sliceResult.value.index] = sliceResult.value.data;
if (++slicesReceived == sliceCount) {
// All slices have been received.
file.closeAsync();
onGotAllSlices(docdataSlices);
}
else {
getSliceAsync(file, ++nextSlice, sliceCount, gotAllSlices, docdataSlices, slicesReceived);
}
}
else {
gotAllSlices = false;
file.closeAsync();
app.showNotification("getSliceAsync Error:", sliceResult.error.message);
}
});
}
function onGotAllSlices(docdataSlices) {
var docdata = [];
for (var i = 0; i < docdataSlices.length; i++) {
docdata = docdata.concat(docdataSlices[i]);
}
var fileContent = new String();
for (var j = 0; j < docdata.length; j++) {
fileContent += String.fromCharCode(docdata[j]);
}
// Now all the file content is stored in 'fileContent' variable,
// you can do something with it, such as print, fax...
}
// The following example gets the document in PDF format.
Office.context.document.getFileAsync(Office.FileType.Pdf,
function(result) {
if (result.status == "succeeded") {
var myFile = result.value;
var sliceCount = myFile.sliceCount;
app.showNotification("File size:" + myFile.size + " #Slices: " + sliceCount);
// Now, you can call getSliceAsync to download the files,
// as described in the previous code segment (compressed format).
myFile.closeAsync();
}
else {
app.showNotification("Error:", result.error.message);
}
}
);
Since you're using Excel, have you tried the CreateWorkbork API? Might be a good workaround if the Document API has a bug, like Xuanzhou indicated earlier.
Here's a CreateDocument snippet that you can load into Script Lab. It shows how to create a Workbook copy based on an existing file.
Hope all that is helpful.
We already have a fix for it now. But the fix still need some time to go to production. Please try it several days later and let me know if the issue still exists. Thanks.

Are multiple async file appends on the same file safe?

Looking at the pseudo-code below, is it possible for the writes to the file to become mangled?
for(var i=0;i<5;i++)
fs.appendFile("myfile.txt", "myline"+i+'\n', somecallback)
fs is found here
Possibility I'd expect:
myline3
myline4
myline1
myline2
myline0
But would this be possible?
mylimyline4
ne3
myline1
myline2
myline0
In which case the second append would have occurred in the middle of the first. Because if this can happen I'll have to queue the writes manually.
I wrote a programm to test that and was unable to make it mix different appends.
var fs = require('fs')
var filename = __dirname + '/file.bin'
var bytes_per_buff = parseInt(process.argv[2]) || 4096
var num_buffs = parseInt(process.argv[3]) || 256
var buffs = []
for (var i=0; i<num_buffs; i++) {
buffs[i] = new Buffer(bytes_per_buff)
for (var j=0; j<bytes_per_buff; j++) {
buffs[i][j] = i
}
}
fs.writeFile(filename, '', ()=>console.log('file created'))
for (var i=0; i<num_buffs; i++) {
(function(buff_num) { //closure to log buff_num
fs.appendFile(filename, buffs[buff_num], ()=>console.log(buff_num))
}(i))
}

I'm working on creating a static node.js server

I am working on creating a static node.js server that just serves up the plain html, css, and javascript that is in the specified directory. I am trying to get the server to read every subdirectory and route the url to the file it specifies. However it only reads the root directory.
var fs = require('fs');
var array = fs.readdirSync(__dirname);
function getAllSub(array){
for (i = 0; i < array.length; i++){
if (array[i].indexOf(".") == (-1))
{
array = array.concat(array[i] + "/" + fs.readdirSync(__dirname + "/" + array[i]));
}
if (array[i].indexOf("/") != (-1)){
var foldcon = array[i].substr(array[i].indexOf("/") + 1);
var folder = array[i].substr(0, array[i].indexOf("/"));
foldcon = foldcon.split(",");
for (n = 0; n < foldcon.length; n++){
foldcon[n] = folder + "/" + foldcon[n]
if (foldcon[n].indexOf(".") == (-1)){
console.log([foldcon[n]]);
foldcon[n] = getAllSub([foldcon[n]]);
}
}
array.splice(i, 1, foldcon);
}
}
return array;
}
array = getAllSub(array);
console.log(array);
Right now this code reads the directory and it recognizes if an item in the array of files is a folder, however it doesn't add the files from the subdirectories into the array properly. Right now it kinda goes all infinite recursion, and I can't really figure out how to stop it.
This isn't meant to be something I am actually going to use, I just thought it would be a good project to work on to introduce myself to the basics of node.js
edited^
I know it's late, but this is the right answer for a recursive solution to reading file paths in sub-folders:
var fs = require("fs");
/**
* Recurse through a directory and populate an array with all the file paths beneath
* #param {string} path The path to start searching
* #param {array} allFiles Modified to contain all the file paths
*/
function readdirSyncRecursive(path, allFiles) {
var stats = fs.statSync(path);
if (stats.isFile()) {
// base case
allFiles.push(path);
} else if (stats.isDirectory()) {
// induction step
fs.readdirSync(path).forEach(function(fileName) {
readdirSyncRecursive(path + "/" + fileName, allFiles);
});
}
}
var allFiles = [];
readdirSyncRecursive("/path/to/search", allFiles);
console.info(allFiles);
var fs = require('fs');
var array = fs.readdirSync(__dirname);
for (i = 0; i < array.length; i++){
if (array[i].indexOf(".") == (-1))
{
// you need to use the return value from concat
array = array.concat(array[i] + "/" + fs.readdirSync(__dirname + "/" + array[i]));
console.log('if executed');
}
}
console.log(array);

Getting NaN for variables in Node.js.. Arg?

ok, I have a homework assignment where I have to read in files and calculate the distance between a bunch of numbers in the files and then print out the mean and standard deviation of each set of numbers. The end of the script, where the console.log stuff is, is giving all NaN for the variables. Can anyone help me out?
*I've omitted repeating parts of the script to make it shorter (their are more arrays than just the lHipJoint array and the calculations for them but I left them out).
var fs = require('fs');
var lHipJoint = new Array();
//open the first text file
fs.readFile('file.txt','utf8', function (err, data)
{
if (err) throw err;
//split the data into an array with each line as an element
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
//function that processes each line into an array
//with each number as an element and does the euclidean dis.
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//do the same for the next file
fs.readFile('file2.txt','utf8', function (err, data)
{
if (err) throw err;
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//and again
fs.readFile('file3.txt','utf8', function (err, data)
{
if (err) throw err;
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//and again
fs.readFile('file4.txt','utf8', function (err, data)
{
if (err) throw err;
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//and again
fs.readFile('file5.txt','utf8', function (err, data)
{
if (err) throw err;
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//and again
fs.readFile('file6.txt','utf8', function (err, data)
{
if (err) throw err;
stuff=data.split('\n');
for (var i = 0; i < stuff.length; i++)
{
processLine(stuff[i]);
}
data.length = 0;
stuff.length = 0;
});
//function to split each line into an array with each number as an element
//then parse the number strings into floats and do the euclidean distances,
//storing the values in arrays for each bone.
function processLine(line)
{
var line1 = line
var numbers = line1.split(" ");
line1.length = 0;
for (var i = 0; i < numbers.length; i++)
{
var number = parseFloat(numbers[i]);
line1[i] = number[i];
}
lHipJoint = Math.sqrt((line1[6] - line1[9])*(line1[6] - line1[9]) + (line1[7] - line1[10])*(line1[7] - line1[10]) + (line1[8] - line1[11])*(line1[8] - line1[11]));
//reset the arrays so they can be reused
line1.length = 0;
numbers.length = 0;
number.length = 0;
}
//calculations and output for the mean and SD of each bone's distance from the root bone.
for(var i = 0; i < lHipJoint.length; i++)
{
var lHipJointTotal = lHipJointTotal + lHipJoint[i];
}
var lHipJointMean = lHipJointTotal/lHipJoint.length;
for(var i = 0; i < lHipJoint.length; i++)
{
var lHipJointSDSum = lHipJointSDSum + (lHipJoint[i] - lHipJointMean)*(lHipJoint[i] - lHipJointMean);
}
var lHipJointSD = Math.sqrt(lHipJointSDSum/lHipJoint.length);
console.log("The mean distance of the left hip joint from the root bone is " +lHipJointMean+ " and the standard deviation is " +lHipJointSD+ ".\n");
You are doing a lot of strange things here in your script i will try to
bring upp as manny as i can.
So first of all dont reset arrays.
your in a garbage collected language just reallocate new ones.
Also in the processLine function you are assigning numbers to the indexes of a string
i asume you think its an array but its not the same thing.
strings are immutable (cant be changed) in javascript.
In the aggregating for loops att the bottom of the file you are
declaring the variable in every iteration. you want to declare it before the loop like this.
var x = 0;
for(var i = 0; i < list.length; i++) {
x = x + ......
}
Your cals to read the files all do the same thing.
So you want to use the same function for that.
write it ones.
You are assigning to the lHipJoint array in the
processLine function my understanding is that you want to add
the calculated value to the array.
You can do this with the push method like this
lHipJoint.push(Math.sqr(........
Also theres a problem with using the async file reading
sins your printing the result before you even read the files.
if you want to use the async ones you need to coordinate so that.
you only print the result when you done all the file reading.
but a tips is to use the non async ones in this case.
I understand this is an assignment so you might not want to read my
attempt to correct the program beneath.
Maybe read it after you handed in yours, but im leaving it here
for the q&a reference for others reading this.
var fs = require("fs");
var filePaths = ["file.txt", "file2.txt",
"file3.txt", "file4.txt",
"file5.txt", "file6.txt"];
var lHipJoint = [];
filePaths.forEach(function(path) {
var content = fs.readFileSync(path, "utf-8");
var lines = content.split("\n");
lines.forEach(function(line) {
if(line.trim() === "") return;
var numbers = line.split("\t").map(parseFloat);
// im not touching your calculation :D
lHipJoint.push(Math.sqrt((numbers[6] - numbers[9])*(numbers[6] - numbers[9])
+ (numbers[7] - numbers[10])*(numbers[7] - numbers[10]) + (numbers[8] - numbers[11])
* (numbers[8] - numbers[11])));
});
});
var lHipJointTotal = lHipJoint.reduce(function(p, c) {
return p + c;
});
var lHipJointMean = lHipJointTotal / lHipJoint.length;
var lHipJointSDSum = lHipJoint.reduce(function(p, c) {
return p + (c - lHipJointMean) * (c - lHipJointMean);
}, 0);
var lHipJointSD = Math.sqrt(lHipJointSDSum/lHipJoint.length);
console.log("The mean distance of the left hip joint from the root bone is "
+ lHipJointMean + " and the standard deviation is " + lHipJointSD + ".\n");
there might be some error in this program i dont know how the data looks but i hope this helps
you.

Resources