After running the following code, the file handle is still opened on Windows, how to close it?
var Name="1.mp4";
var inp = fs.createReadStream("Temp/" + Name);
var out = fs.createWriteStream("Video/" + Name);
inp.pipe(out);
inp.on("end", function() {
fs.unlink("Temp/" + Name, function (){
out.close();
console.log("unlink this file:",Name );
});
});
screen shot: file handle not colsed on Windows
You need to close inp too not just out. You can do this by calling inp.close(); at the same point you already call out.close();.
Also you could simply move the file with
fs.rename("Temp/" + Name, "Video/" + Name, function() {
console.log("Renamed:", Name)
});
rather than rewriting the file.
Related
So I use this API that helps me turn a .docx file into a .pdf. I placed the code that converts the file into a function. :
function conv(){
convertapi.convert('pdf', { File: final_path })
.then(function(result) {
// get converted file url
console.log("Converted file url: " + result.file.url);
finp = path + file_name.slice(0, file_name.length - 5) + ".pdf";
console.log(finp);
// save to file
return result.file.save(finp);
})
.then(function(file) {
console.log("File saved: " + file);
process.exit(1);
})
.catch(function(e) {
console.log("numele si/sau extensia fisierului sunt gresite");
process.exit(1);
});
}
The code above works only for one file at a time. I made a loop that goes through every file (.docx) in my folder and save its name into an array. I go through every item of the array and call the function :
for(var j = 0; j<=i ; j++){
file_name = toate_nume[j];
final_path = path + file_name;
conv();
}
The file names are stored correctly, but when I run my project, the function is called after the loop itself ends ( is called the correct number of times for each and every file). So if I have 2 files : test1.docx and test2.docx the output shows me that the conv() is called 2 times for the test2.docx, instead of one time for each file. What should I do?
The reason might be this:
The API is slow so your program is executing the loop faster than the API the can handle the requests. So what ends up happening is that you have modified the final_path variable twice before convertapi gets called, and then it gets called twice with the same final_path. Try to modify your conv function so that it accepts a parameter, e.g. path and uses that. Then call conv with the current final_path parameter:
conv(final_path)
And:
function conv(path) {
convertapi.convert('pdf', { File: path })
...
So you are calling n Promise in a serial. And you want to wait for the end?
You can use Promise. all
const toate_nume = ['fileName1', 'fileName2'];
const spawn = toate_nume.map(x => {
const final_path = path + x;
return conv(final_path);
});
Promise.all(spawn).then(results => {
console.log('All operation done successfully %o', results);
});
or use await:
const results = await Promise.all(spawn);
the results is an array, an entry for each call.
NOTE** I pass the path as an argument instead of a global var
I'm setting up a Google Cloud Functions (GCF) function that gets triggered often enough that there are multiple instances running at the same time.
I am getting errors from a readStream the source file of the stream does not exist, but at this point in my program I've actually just created it.
I've made sure the file exists before the start of the stream by console.log()-ing the file JSON, so the file does actually exist. I've also made sure that the file I'm trying to access has finished being written by a previous stream with an await, but no dice.
EDIT: The code now contains the entire script. The section that seems to be throwing the error is the function columnDelete().
var parse = require('fast-csv');
var Storage = require('#google-cloud/storage');
var Transform = require('readable-stream').Transform;
var storage = new Storage();
var bucket = storage.bucket('<BUCKET>');
const DMSs = ['PBS','CDK','One_Eighty','InfoBahn'];
class DeleteColumns extends Transform{
constructor(){
super({objectMode:true})
}
_transform(row, enc, done){
//create an array 2 elements shorter than received
let newRow = new Array(row.length - 2);
//write all data but the first two columns
for(let i = 0; i < newRow.length; i++){
newRow[i] = row[i+2];
}
this.push(newRow.toString() + '\n');
done();
}
}
function rename(file, originalFile, DMS){
return new Promise((resolve, reject) => {
var dealer;
var date;
var header = true;
var parser = parse({delimiter : ",", quote:'\\'});
//for each row of data
var stream = originalFile.createReadStream();
stream.pipe(parser)
.on('data', (row)=>{
//if this is the first line do nothing
if(header){
header = false;
}
//otherwise record the contents of the first two columns and then destroy the stream
else {
dealer = row[0].toString().replace('"', '').replace('"', '');
date = row[1].toString().replace('"', '').replace('"', '');
stream.end();
}
})
.on('finish', function(){
var newName = dealer + ' ' + date + '_' + DMS + 'temp.csv';
//if this was not triggered by the renaming of a file
if(!file.name.includes(dealer)&&!file.name.includes(':')){
console.log('Renamed ' + file.name);
originalFile.copy(newName);
originalFile.copy(newName.replace('temp',''));
}else{
newName = 'Not Renamed';
console.log('Oops, triggered by the rename');
}
resolve(newName);
});
});
}
function columnDelete(fileName){
return new Promise((resolve, reject) =>{
console.log('Deleting Columns...');
console.log(bucket.file(fileName));
var parser = parse({delimiter : ",", quote:'\\'});
var del = new DeleteColumns();
var temp = bucket.file(fileName);
var final = bucket.file(fileName.replace('temp', ''));
//for each row of data
temp.createReadStream()
//parse the csv
.pipe(parser)
//delete first two columns
.pipe(del)
//write to new file
.pipe(final.createWriteStream()
.on('finish', function(){
console.log('Columns Deleted');
temp.delete();
resolve();
})
);
});
}
exports.triggerRename = async(data, context) => {
var DMS = 'Triple';
var file = data;
//if not a temporary file
if(!file.name.includes('temp')){
//create a new File object from the name of the data passed
const originalFile = bucket.file(file.name);
//identify which database this data is from
DMSs.forEach(function(database){
if(file.name.includes(database)){
DMS = database;
}
});
//rename the file
var tempName = await rename(file, originalFile, DMS);
//if it was renamed, delete the extra columns
if (!tempName.includes('Not Renamed')){
await columnDelete(tempName);
}
} else if(file.name.includes('undefined')){
console.log(file.name + ' is invalid. Deleted.');
bucket.file(file.name).delete();
}
else {
console.log( file.name + ' is a temporary file. Did not rename.');
}
};
What I expect to be output is as below:
Deleting Columns...
Columns Deleted
Nice and simple, letting us know when it has started and finished.
However, I get this instead:
Deleting Columns...
ApiError: No such object: <file> at at Object.parseHttpRespMessage(......)
finished with status: 'crash'
Which is not wanted for obvious reasons. My next thought is to make sure that the file hasn't been deleted by another instance of the script midway through, but to do that I would have to check to see if the file is being used by another stream, which is, to my knowledge, not possible.
Any ideas out there?
When I was creating the file I called the asynchronous function copy() and moved on, meaning that when trying to access the file it was not finished copying. Unknown to me, the File Object is a reference variable, and did not actually contain the file itself. While the file was copying, the pointer was present but it was pointing to an unfinished file.
Thus, "No Such Object". To fix this, I simply used a callback to make sure that the copying was finished before I was accessing the file.
Thanks to Doug Stevenson for letting me know about the pointer!
var promisePipe = require("promisepipe");
var fs = require("fs");
var crypt = require("crypto");
var // ....
files = ['/mnt/Storage/test.txt', '/mnt/Storage/test2.txt', '/mnt/Storage/test3.txt']
var promises = files.map(function(file_enc) {
return new Promise(function(resolve, reject) {
var file_out = file_enc + '.locked';
promisePipe(
fs.createReadStream(file_enc),
crypt.createCipheriv(alg, genhashsub, iv),
fs.createWriteStream(file_out),
).then(function(streams){
console.log('File written: ' + file_out);
// Promise.resolve(file_out); // tried but doesnt seem to do anything
}, function(err) {
if(err.message.substring(0, 7) === 'EACCES:') {
console.log('Error (file ' + file_out + '): Insufficient rights on file or folder');
} else {
console.log('Error (file ' + file_out + '): ' + err);
}
// Promise.reject(new Error(err)); // tried but doesnt seem to do anything
});
})
});
Promise.all(promises).then(final_function(argument));
I'm trying to encrypt files contained in an array named files.
For the sake of simplicity I added them manually in this example.
What I want to happen:
Create promises array to call with promises.all on completion
Iterate through the array
Create promise for this IO operation
Read file \
Encrypt file -- all done using streams, due to large files (+3GB)
Write file /
On finish write, resolve promise for this IO operation
Run finishing script once all promises have resolved (or rejected)
What happens:
Encryption of first file starts
.then(final_function(argument)) is called
Encryption of first file ends
The files all get encrypted correctly and they can be decrypted afterwards.
Also, errors are displayed, as well as the write confirmations.
I've searched both Stack as well as Google and I have found some similar questions (with answers). But they don't help because many are outdated. They work, until I rewrite them into promises, and then I'm back to where I started.
I could also place 8 different ways to achieve this job, using npm modules or vanilla code, but all of them are also failing in one way or another.
If you already have a promise at your disposal (and promisepipe appears to create a promise), then you generally should not use new Promise(). It looks like your main problem is that you are creating promises that you never resolve.
The other problem is that you are calling final_function in the last line instead of passing a function that will call final_function.
I suggest giving this a try:
var promises = files.map(function(file_enc) {
var file_out = file_enc + '.locked';
return promisePipe(
fs.createReadStream(file_enc),
crypt.createCipheriv(alg, genhashsub, iv),
fs.createWriteStream(file_out),
).then(function(streams){
console.log('File written: ' + file_out);
return file_out;
}, function(err) {
if(err.message.substring(0, 7) === 'EACCES:') {
console.log('Error (file ' + file_out + '): Insufficient rights on file or folder');
} else {
console.log('Error (file ' + file_out + '): ' + err);
}
throw new Error(err);
});
});
Promise.all(promises).then(() => final_function(argument));
an analog of short , file-based process wrapped in a promise all . You could do your encrypt in the 'encrypt' which is wrapped in file handler. encrypt() returns a promise.
segments passes your array of files needing work.
var filHndlr = function(segment){
var uri = segment.uri;
var path = '/tmp/' + uri;
return that.getFile(path)
.then (datatss => {
return that.encrypt(uri, datatss);
});
}
...
Promise.all(segments.map(filHndlr))
.then(resp => { ... });
I have an Electron app which uploads a dropped file to a predefined server with node-ftp. The upload works like a charm, but despite reading a couple of suggestions I cannot figure out how to get information on the actual progress for a progress-bar.
My upload-code so far:
var ftp = new Client();
let uploadfile = fs.createReadStream(f.path);
let newname = uuid(); //some function I use for renaming
ftp.on('ready', function () {
ftp.put(uploadfile, newname, function (err) {
if (err) throw err;
ftp.end();
});
});
c.connect({user: 'test', password: 'test'});
I always stumble across monitoring the 'data' event, but could not find out how or where to access it (as you can see I'm quite new to JavaScript).
Got it. I found the answer in streams with percentage complete
With my code changed to
var ftp = new Client();
let uploadfile = fs.createReadStream(f.path);
let newname = uuid(); //some function I use for renaming
ftp.on('ready', function() {
uploadfile.on('data', function(buffer) {
var segmentLength = buffer.length;
uploadedSize += segmentLength;
console.log("Progress:\t" + ((uploadedSize/f.size*100).toFixed(2) + "%"));
});
ftp.put(uploadfile, newname, function(err) {
if (err) throw err;
ftp.end();
});
});
c.connect({user: 'test', password: 'test'});
I get the percentage uploaded in console. From here it's only a small step to a graphical output.
on client side you can create a byte count for your upload stream (http://www.experts.exchange.com/questions/24041115/upload-file-on-ftp-with-progressbar-and-time-left.html)
set lower limit of the progressbar to 0
set upper limit to file length of upload file
feed the progress bar with the byte count
(http://www.stackoverflow.com/questions/24608048/how-do-i-count-bytecount-in-read-method-of-inputstream)
maybe you can use npm like stream-meter (https://www.npmjs.com/package/stream-meter) or progress-stream (https://www.npmjs.com/package/progress-stream) and pipe your file stream through to feed the progressbar. i am not sure about that because i do not know the internals of the npms. in progress-stream is a function transferred() that would fit exactly
a very accurate way is to have code on the server that gives feedback to the browser (http://www.stackoverflow.com/questions/8480240/progress-bar-for-iframe-uploads)
I'm building an app for windows using Electron. To package and distribute it I'm using electron-builder. Electron-builder relies on many packages, and for auto-updates it uses Squirrel-windows.
I've been battling with auto-update on Windows for 3 days and at the end I've come up with a working solution that seems to give no problems.
I wont' go into the details of what I've tried, and failed. Instead, I'll post here the solution with which I've come up.
I'm sharing it with you guys, to see if you may point out to me any flaws that will make my system fail, or, if it truly is a solid solution, to help those who are struggling as I was. For this latter reason, I'm posting some more code than it would be necessary, hoping it will help others.
The logic is as follows:
if the sub-folder fullupdate inside the path of the current executable does not exists (see later, it will be clarified), we connect with an online server and check if there is an update by sending the current app version;
if there is no update, do nothing.
if there is an update, we instruct the server to return a json string that contains the url from which we can download of the .exe installer produced by electron-builder. NB: not the .nupkg (server code not provided :-)).
we download the file and save it inside a sub folder fullupdate in the local folder in which our app is currently saved. This should be "safe" as electron-builder saves the app in the current user folder AppData, so we should not have permissions issues.
at the end of the download, we create a new file update inside the folder fullupdate to be sure the download has finished successfully. We could also rename the file, but I prefer this way.
next time the app opens:
if the folder fullupdate exists we check if the file update exists. If it does not exists, the download was not finished, so we delete the folder fullupdate and call the remote server again to start all over again.
else, if the file update exists, we launch the .exe file we have downloaded, and return true. This will prevent the app from opening the main window. The cool thing is that the updater will delete the whole old version of the app saved in AppData (while leaving local user data) and replace it with the new version. In this way we will get rid also of the folder fullupdate.
Now the code:
// we want to run this only on windows
var handleStartupEvent = function() {
if (process.platform !== 'win32') {
return false;
}
/////////////////
// MANUAL UPDATER
/////////////////
var appFolder = 'app-' + appVersion;
var pathApp = path.dirname(process.execPath);
var pathUpdate = pathApp + '\\fullupdate';
var checkupdateurl = 'https://api.mysite.com/getjson/' + appVersion.split('.').join('-');
function checkIfDownloaded(){
if (!fs.existsSync(pathUpdate)) checkUpdate();
else return checkIfInstallLocal();
}
function checkIfInstallLocal(){
if(fileExists('fullupdate\\update')) return installLocal();
else {
deleteFolderRecursive(pathUpdate);
checkUpdate();
}
}
function installLocal(){
cp.exec('fullupdate\\Update.exe', function( error, stdout, stderr){
if ( error != null ) {
console.log(stderr);
}
});
return true;
}
// from http://www.geedew.com/remove-a-directory-that-is-not-empty-in-nodejs/
var deleteFolderRecursive = function(path) {
if( fs.existsSync(path) ) {
fs.readdirSync(path).forEach(function(file,index){
var curPath = path + "/" + file;
if(fs.lstatSync(curPath).isDirectory()) deleteFolderRecursive(curPath);
else fs.unlinkSync(curPath);
});
fs.rmdirSync(path);
}
};
// from http://stackoverflow.com/questions/4482686/check-synchronously-if-file-directory-exists-in-node-js
function fileExists(path) {
try {
return fs.statSync(path).isFile();
}
catch (e) {
if (e.code == 'ENOENT') { // no such file or directory. File really does not exist
return false;
}
throw e; // something else went wrong, we don't have rights, ...
}
}
function checkUpdate(){
https.get('https://api.mysite.com/getjson/' + app.getVersion().split('.').join('-'), (res) => {
res.setEncoding('utf8');
res.on('data', function(chunk) {
if(chunk) thereIsUpdate(chunk);
});
}).on('error', (e) => {
console.log(e);
});
}
function thereIsUpdate(chunk){
var data = JSON.parse(chunk);
if(data && data.url) getNewUpdate(data.urlsetup);
}
function getNewUpdate(url){
fs.mkdirSync(pathUpdate);
var file = fs.createWriteStream(pathUpdate + '/Update.exe');
var responseSent = false; // flag to make sure that response is sent only once.
var request = https.get(url, function(response) {
response.pipe(file);
file.on('finish', () =>{
file.close(() => {
if(responseSent) return;
responseSent = true;
});
fs.closeSync(fs.openSync(pathUpdate + '/update', 'w'));
});
});
}
if(checkIfDownloaded()) return true;
/////////////////////////
// SQUIRREL EVENTS HANDLER
//////////////////////////
// see http://stackoverflow.com/questions/30105150/handle-squirrels-event-on-an-electron-app
};
// here we call the function. It is before the opening of the window, so that we prevent the opening if we are updating, or if there is a Squirrel event going on (see SO question, link above)
if (handleStartupEvent()) {
return;
}