There is a function called musicPromise(). What this function does is
It gets all mp4 files and loop through it.
then it tries to convert each mp4 to mp3, using fluent-ffmpeg
The problem I am facing is
It only converts 1 file, no matter how many mp4 files I have.
And it seems never reach to proc.on('end', (x) => {
Full code here:
// search
const glob = require('glob');
// wait for
const Promise = require('bluebird');
// fs
const fs = require('fs');
// mp3
const ffmpeg = require('fluent-ffmpeg');
// video source file path
const videoPath = '/home/kenpeter/Videos/4K\ Video\ Downloader';
// audio source file path
const audioPath = __dirname + "/audio";
// child process, exec
const exec = require('child_process').exec;
// now rename promise
function renamePromise() { return new Promise((resolve, reject) => {
glob(videoPath + "/**/*.mp4", (er, files) => {
Promise.each(files, (singleClipFile) => {
return new Promise((resolve1, reject1) => {
let arr = singleClipFile.split("/");
let lastElement = arr[arr.length - 1];
let tmpFileName = lastElement.replace(/[&\/\\#,+()$~%'":*?<>{}\ ]/g, "_");
let tmpFullFile = videoPath + "/"+ tmpFileName;
// rename it
fs.rename(singleClipFile, tmpFullFile, function(err) {
if ( err ) console.log('ERROR: ' + err);
console.log("-- Rename one file --");
console.log(tmpFullFile);
resolve1();
}); // end rename
});
})
.then(() => {
console.log('--- rename all files done ---');
resolve();
});
});
}); // end promise
};
// music promise
function musicPromise() { new Promise((resolve, reject) => {
glob(videoPath + "/**/*.mp4", (er, files) => {
Promise.each(files, (singleClipFile) => {
return new Promise((resolve1, reject1) => {
// test
console.log('-- music promise --');
console.log(singleClipFile);
// split
let arr = singleClipFile.split("/");
// e.g. xxxx.mp4
let clipFile = arr[arr.length - 1];
// e.g. xxxx no mp4
let fileName = clipFile.replace(/\.[^/.]+$/, "");
// music file name
let musicFile = fileName + '.mp3';
// set source
let proc = new ffmpeg({source: singleClipFile});
// set ffmpeg path
proc.setFfmpegPath('/usr/bin/ffmpeg');
// save mp3
proc.output("./audio/" + musicFile);
// proc on error
proc.on('error', (err) => {
console.log(err);
});
// done mp3 conversion
proc.on('end', (x) => {
console.log("single mp3 done!");
console.log(x);
// it is resolve1..............
resolve1();
});
// Run !!!!!!!!!!!!!
proc.run();
});
})
.then(() => {
console.log('--------- all mp3 conversion done --------');
resolve();
});
}); // end glob
});
};
// adb kill
function adbKillPromise() { return new Promise((resolve, reject) => {
exec("adb kill-server", (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
console.log('---adb kill---');
resolve();
});
});
};
// adb start
function adbStartPromise() { return new Promise((resolve, reject) => {
exec("adb start-server", (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
console.log('---adb start---');
resolve();
});
});
};
// adb push promise
function adbPushPromise() { return new Promise((resolve, reject) => {
glob(audioPath + "/**/*.mp3", (er, files) => {
Promise.each(files, (singleMusicFile) => {
return new Promise((resolve1, reject1) => {
let cmd = "adb push" + " " + singleMusicFile + " " + "/sdcard/Music";
exec(cmd, (err, stdout, stderr) => {
console.log(cmd);
resolve1();
});
});
})
.then(() => {
console.log('---- done push all music ---');
resolve();
});
});
});
};
// Run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
renamePromise()
.then(musicPromise)
.then(adbKillPromise)
.then(adbStartPromise)
.then(adbPushPromise)
.then(() => {
console.log('---- all done----');
process.exit(0);
})
.catch(err => {
console.log('Error', err);
process.exit(1);
});
A very stupid mistake
function musicPromise() { new Promise((resolve, reject) => {
should be
function musicPromise() { return new Promise((resolve, reject) => {
Related
Im trying to make a quick node script to download MP3s from a RSS feed. At the moment I have this :
const https = require('https');
const xml2js = require('xml2js');
const parser = new xml2js.Parser();
const fs = require('fs');
const URL_TO_PARSE = 'https://some-rss.feed.xml';
const req = https.get(URL_TO_PARSE, async (res) => {
let xml = '';
res.on('data', (stream) => {
xml = xml + stream;
});
res.on('end', () => {
parser.parseString(xml, async (err, result) => {
if (err) {
console.log(err);
} else {
let items = result.rss.channel[0].item;
await Promise.all(items.map(async (item) => {
let title = item.title[0];
let enclosure = item.enclosure[0];
let url = enclosure.$.url;
let filepath = `./${title}`;
console.log(`Downloading ${title} to ${filepath}`);
await download_audio_file(url, filepath);
}));
}
});
});
});
const download_audio_file = async (url, filepath) => {
https.get(url, (res) => {
const writeStream = fs.createWriteStream(filepath);
res.pipe(writeStream);
writeStream.on('finish', () => {
writeStream.close();
console.log('File downloaded');
Promise.resolve();
});
writeStream.on('error', (err) => {
console.log(err);
Promise.reject(err);
});
})
But it currently tried to download each one at the same time. Is there a better way to write this to download just one at a time - and possibly also track the % progress?
I see 2 problems with your code.
The first one is that download_audio_file is not returning a promise that resolves when the file is fully downloaded.
You can fix that with this refactored version:
const download_audio_file = async (url, filepath) => {
const promise = new Promise((resolve, reject) => {
https.get(url, (res) => {
const writeStream = fs.createWriteStream(filepath);
res.pipe(writeStream);
writeStream.on("finish", () => {
writeStream.close();
console.log("File downloaded");
resolve();
});
writeStream.on("error", (err) => {
console.log(err);
reject(err);
});
});
});
return promise;
};
Secondly, you are using Promise.all which awaits for all the promises in parallel.
You can replace that code snippet with:
const req = https.get(URL_TO_PARSE, async (res) => {
let xml = '';
res.on('data', (stream) => {
xml = xml + stream;
});
res.on('end', () => {
parser.parseString(xml, async (err, result) => {
if (err) {
console.log(err);
} else {
let items = result.rss.channel[0].item;
for(const item of items) {
let title = item.title[0];
let enclosure = item.enclosure[0];
let url = enclosure.$.url;
let filepath = `./${title}`;
console.log(`Downloading ${title} to ${filepath}`);
await download_audio_file(url, filepath);
}
}
});
});
});
Notice how I replaced the Promise.all with for(const item of items)
I'm trying to setup an endpoint that takes a file through a multipart post request, and saves it into a specific directory using formidable and https://github.com/andrewrk/node-mv. And then upon completion of saving all of the files, I want to respond with a list of all of the files in that directory for rendering. the thing is the response seems to be sent before the directory listing is updated. I tried wrapping the mv operations into a promise and then responding in a then block to no avail. Any help would be much appreciated!
app.post("/api/v1/vendor/:id/menu", (req, res, next) => {
const id = req.params.id;
const form = formidable({ multiples: true, keepExtensions: true });
form.parse(req, (err, fields, files) => {
if (err) {
next(err);
return;
}
if (!Array.isArray(files.image)) {
files = [files.image];
}
let filelist;
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});
Promise.all(proms).then(() => {
console.log('now reading dir...');
filelist = fs.readdirSync("pics/" + id);
res.send(filelist);
});
});
});
I think we're missing the return keywork before new Promise. You can check the proms variable if it contains the list of promises or not.
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});
For me, it should be :
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
return new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});
I have a simple HTML form with a file input. I need to save the file with Postgres Large Object but I'm having issues with the fileStream.pipe. It should be in a Promise itself like so:
return new Promise((resolve, reject) => {
stream.on('finish', resolve(oid));
stream.on('error', reject);
});
But it wouldn't work since it would result into a Promise in a Promise. The complete functions are:
function storeLargeObject(path) {
return new Promise((resolve, reject) => {
pgp.db.tx(tx => {
const man = new LargeObjectManager({pgPromise: tx});
return man.createAndWritableStreamAsync(bufferSize);
}).then(([oid, stream]) => {
const fileStream = createReadStream(path);
fileStream.pipe(stream);
stream.on('finish', resolve(oid));
stream.on('error', reject);
}).catch(err => {
console.log(err);
return err;
});
});
}
const addBinary = (req, res) => {
pgp.db.oneOrNone({
text: insertQuery,
values: values
}).then(f => {
let explFile = req.files.bin;
let uploadPath = __dirname + '/' + f.id;
if (!req.files || Object.keys(req.files).length === 0) {
res.status(400).send('No File was uploaded.');
return;
}
explFile.mv(uploadPath, async function (err) {
if (err)
return res.status(500).send(err);
let newOid = await storeLargeObject(uploadPath);
fs.unlink(uploadPath, err => console.log(err));
coupleIds(exp.id, newOid);
return res.status(200).send({
status: "success",
message: "File successfully uploaded!"
})
});
}).catch(err => {
return res.status(401).send({
status: 'error',
errorCode: 102,
message: 'Error! ' + err
});
});
}
I need to convert & write each base64 string from an array to a different file by using fs.writeFile but only the last array element is saved. Have a look at my code below :
let receivedData =["iVBORw0KGgoAAAANSUhEUgAABM...", "4AAANiCAYAAACZ8IVOAAAKqGlDQ1BJQ0MgU..."]:
//decode base 64 to buffer
let decodeBase64 = (dataArray) => {
let result = [];
dataArray.forEach((element) => {
result.push(Buffer.from(element, 'base64'));
})
return result; //array
};
let writeFileAsync = (data) => {
let filename = Date.now() + '.jpeg';
return new Promise((resolve, reject) => {
data.forEach((value) => {
fs.writeFile(filename, value, (err) => {
if(err){
reject(err);
} else {
console.log('file is saved');
resolve(value);
}
})
})
}
let decoded = decodeBase64(receivedData);
writeFileAsync(decoded)
.then((result) => {
console.log('file is saved');
})
.catch((error) => {
console.log(error);
})
the last value of receivedData is saved to filesystem but not the first element
I know it has something to do with doing async method in a loop and maybe something else. The loop just imcrement before writefile finishes.
Any helps/tips would be appreciated thx
Try this, It will upload your file one by one, it will take time but git you the appropriate result:
let fileUpload = async (FILES , PATH, CB)=>{
/** image Path **/
let lengthOfFiles = FILES.length, RETURN_IMAGEPATHS = [], count = 0,
uploadFunc= [];
for(let index= 0; index < lengthOfFiles; index++){
let name = Date.now() + ".png";
RETURN_IMAGEPATHS.push('http://localhost:3000/' + <PATH_FOR_FILE> + name);
uploadImage(<Image save path>, <Imagte file path>, FILES[index]);
}
CB(null, RETURN_IMAGEPATHS);
};
function uploadImage(imageSavePath, filePath, FILE){
return new Promise(async (resolve, reject)=> {
await fs.writeFileSync(imageSavePath, FILE, "base64", function (err) {
if (err)
reject(err);
else {
filePath ='http://localhost:3000/' + filePath;
resolve(filePath);
}
});
})
}
fileUpload(FILES , PATH, (err)=>{
if(err){
console.log(err);
}
else
console.log('Success');
})
How to send "End" message after resolving promise? Sometimes I can send 2 "end" messages out of 4, sometimes 3. Files from FTP are being downloaded and it's ok. The only thing that doesn't work is sending "end" message after downloading a file. Do you have any idea why this code doesn't work properly?
This code was updated:
const ftp = require("jsftp");
const fs = require("fs");
const net = require("net");
const mkdirp = require("mkdirp");
class ftpCredentials {
constructor(host) {
this.user = "xxx";
this.pass = "xxx";
this.host = host;
}
}
const downloadFromFTP = (credentials, file) => {
const client = new ftpCredentials(credentials.host);
const ftpClient = new ftp(client);
return new Promise((res, rej) => {
let buf = null;
ftpClient.get(file, (err, stream) => {
if (!err && typeof stream !== "undefined") {
// Events
stream.on("data", (data) => {
if (buf === null) buf = new Buffer(data);
else buf = Buffer.concat([buf, data]);
});
stream.on("close", (err) => {
if (err) rej("FILE_ERROR");
const actualPath = `${credentials.path}/${file}`;
fs.writeFile(actualPath, buf, "binary", (err) => {
if (err) rej(err);
ftpClient.raw("quit", (err, data) => {
if (err) rej(err)
res(file);
});
});
});
// Resume stream
stream.resume();
} else {
rej("STREAM_ERROR");
}
});
})
}
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host, () => {
Promise.all(filesOnFTP.map((file) => {
return new Promise((resolve, reject) => {
// The problem is here:
const socketWrite = socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
resolve(socketWrite);
// Events
socketForEndMsg.on("error", () => {
console.log("Problem with sending End message!");
reject();
});
});
})).then(() => {
socketForEndMsg.end();
}).catch((err) => {
console.log(err);
});
});
})
})
}
const getScheme = (credentials) => {
const socketForData = net.createConnection(18005, credentials.host, () => socketForData.write("Scheme", "UTF16LE"));
// Events
socketForData.on("close", () => console.log("TCP Connection closed"));
socketForData.on("error", err => console.log(err));
socketForData.on("data", (data) => {
socketForData.end();
const toUTF16Format = Buffer.from(data).toString("UTF16LE");
const arrayFromTCPMessage = toUTF16Format.split(/\||;/);
const filteredImages = arrayFromTCPMessage.filter(item => item.startsWith("scheme"))
const isOK = arrayFromTCPMessage[0] === "OK";
if (isOK) {
handleSavingFile(credentials, filteredImages);
}
})
}
module.exports = getScheme;
Error message: Error: This socket is closed
at Socket._writeGeneric (net.js:722:18)
at Socket._write (net.js:782:8)
at doWrite (_stream_writable.js:407:12)
at writeOrBuffer (_stream_writable.js:393:5)
at Socket.Writable.write (_stream_writable.js:290:11)
at Promise (xxx\getScheme.js:56:29)
at new Promise (<anonymous>)
at Promise.all.filesOnFTP.map (xxx\getScheme.js:54:18)
at Array.map (<anonymous>)
at Socket.net.createConnection (xxx\getScheme.js:52:32)
I see that, you like to listen to error event & that made you to use Promise to catch the error. But, the placement of the error event handler registration is wrong, as it is inside .map function call. So, error event will be registered number of times of filesOnFTP length.
I've moved that error handler to next line & using writable flag to see, if the socket is still writable before writing to it. I have also added few more event handlers, which will give you more information about the socket status(for debugging, you can remove them later).
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host, () => {
for(let file of filesOnFTP) {
// Before write to socket, check if it is writable still!
if(socketForEndMsg.writable) {
socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
}
else {
console.log('Socket is not writable! May be closed already?');
}
}
});
// This is the correct place for the error handler!
socketForEndMsg.on("error", (error) => {
console.log("Problem with sending End message!", error);
});
socketForEndMsg.on("close", () => {
console.log("Socket is fully closed!");
});
socketForEndMsg.on("end", () => {
console.log("The other end of the socket has sent FIN packet!");
});
});
});
}
Let me know if this works!
Thanks!
You can try to wait for connection event on socketForEndMsg and then start sending your data
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host);
socketForEndMsg.on('connect', () => {
Promise.all(filesOnFTP.map((file) => {
return new Promise((resolve, reject) => {
// The problem is here:
const socketWrite = socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
resolve(socketWrite);
// Events
socketForEndMsg.on("error", () => {
console.log("Problem with sending End message!");
reject();
});
});
})).then(() => {
socketForEndMsg.end();
}).catch((err) => {
console.log(err);
});
})
})
})
}