How to send "End" message after resolving promise? Sometimes I can send 2 "end" messages out of 4, sometimes 3. Files from FTP are being downloaded and it's ok. The only thing that doesn't work is sending "end" message after downloading a file. Do you have any idea why this code doesn't work properly?
This code was updated:
const ftp = require("jsftp");
const fs = require("fs");
const net = require("net");
const mkdirp = require("mkdirp");
class ftpCredentials {
constructor(host) {
this.user = "xxx";
this.pass = "xxx";
this.host = host;
}
}
const downloadFromFTP = (credentials, file) => {
const client = new ftpCredentials(credentials.host);
const ftpClient = new ftp(client);
return new Promise((res, rej) => {
let buf = null;
ftpClient.get(file, (err, stream) => {
if (!err && typeof stream !== "undefined") {
// Events
stream.on("data", (data) => {
if (buf === null) buf = new Buffer(data);
else buf = Buffer.concat([buf, data]);
});
stream.on("close", (err) => {
if (err) rej("FILE_ERROR");
const actualPath = `${credentials.path}/${file}`;
fs.writeFile(actualPath, buf, "binary", (err) => {
if (err) rej(err);
ftpClient.raw("quit", (err, data) => {
if (err) rej(err)
res(file);
});
});
});
// Resume stream
stream.resume();
} else {
rej("STREAM_ERROR");
}
});
})
}
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host, () => {
Promise.all(filesOnFTP.map((file) => {
return new Promise((resolve, reject) => {
// The problem is here:
const socketWrite = socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
resolve(socketWrite);
// Events
socketForEndMsg.on("error", () => {
console.log("Problem with sending End message!");
reject();
});
});
})).then(() => {
socketForEndMsg.end();
}).catch((err) => {
console.log(err);
});
});
})
})
}
const getScheme = (credentials) => {
const socketForData = net.createConnection(18005, credentials.host, () => socketForData.write("Scheme", "UTF16LE"));
// Events
socketForData.on("close", () => console.log("TCP Connection closed"));
socketForData.on("error", err => console.log(err));
socketForData.on("data", (data) => {
socketForData.end();
const toUTF16Format = Buffer.from(data).toString("UTF16LE");
const arrayFromTCPMessage = toUTF16Format.split(/\||;/);
const filteredImages = arrayFromTCPMessage.filter(item => item.startsWith("scheme"))
const isOK = arrayFromTCPMessage[0] === "OK";
if (isOK) {
handleSavingFile(credentials, filteredImages);
}
})
}
module.exports = getScheme;
Error message: Error: This socket is closed
at Socket._writeGeneric (net.js:722:18)
at Socket._write (net.js:782:8)
at doWrite (_stream_writable.js:407:12)
at writeOrBuffer (_stream_writable.js:393:5)
at Socket.Writable.write (_stream_writable.js:290:11)
at Promise (xxx\getScheme.js:56:29)
at new Promise (<anonymous>)
at Promise.all.filesOnFTP.map (xxx\getScheme.js:54:18)
at Array.map (<anonymous>)
at Socket.net.createConnection (xxx\getScheme.js:52:32)
I see that, you like to listen to error event & that made you to use Promise to catch the error. But, the placement of the error event handler registration is wrong, as it is inside .map function call. So, error event will be registered number of times of filesOnFTP length.
I've moved that error handler to next line & using writable flag to see, if the socket is still writable before writing to it. I have also added few more event handlers, which will give you more information about the socket status(for debugging, you can remove them later).
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host, () => {
for(let file of filesOnFTP) {
// Before write to socket, check if it is writable still!
if(socketForEndMsg.writable) {
socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
}
else {
console.log('Socket is not writable! May be closed already?');
}
}
});
// This is the correct place for the error handler!
socketForEndMsg.on("error", (error) => {
console.log("Problem with sending End message!", error);
});
socketForEndMsg.on("close", () => {
console.log("Socket is fully closed!");
});
socketForEndMsg.on("end", () => {
console.log("The other end of the socket has sent FIN packet!");
});
});
});
}
Let me know if this works!
Thanks!
You can try to wait for connection event on socketForEndMsg and then start sending your data
const handleSavingFile = (credentials, filesOnFTP) => {
mkdirp(credentials.path, () => {
fs.readdir(credentials.path, (err, fileNames) => {
if (err) return err;
const needToConnectToFTP = filesOnFTP.filter(name => fileNames.indexOf(name) !== -1).length === 0;
const socketForEndMsg = net.createConnection(18005, credentials.host);
socketForEndMsg.on('connect', () => {
Promise.all(filesOnFTP.map((file) => {
return new Promise((resolve, reject) => {
// The problem is here:
const socketWrite = socketForEndMsg.write(`End|ftp://${credentials.host}/${file}`, "UTF16LE");
resolve(socketWrite);
// Events
socketForEndMsg.on("error", () => {
console.log("Problem with sending End message!");
reject();
});
});
})).then(() => {
socketForEndMsg.end();
}).catch((err) => {
console.log(err);
});
})
})
})
}
Related
Is there a way to cancel a WriteFile operation?
example:
await promises.writeFile(path, data)
Yes you need to create an AbortController and pass it into your writeFile call:
const fs = require('fs')
// Some stub values
const tempFilePath = () => `/tmp/foo`
data = 'bar'
const myAbortController = new AbortController()
// With callback
fs.writeFile(tempFilePath(), data, { signal: myAbortController.signal }, (err) => {
if(err) {
if(err.name === "AbortError") {
console.warn("write aborted")
return
}
throw err
}
})
// Or with promises
fs.promises.writeFile(tempFilePath(), data, { signal: myAbortController.signal })
.catch((err) => {
if(err.name === "AbortError") {
console.warn("write aborted")
return
}
throw err
})
// Or Async/Await
;(async () => {
try {
await fs.promises.writeFile(tempFilePath(), data, { signal: myAbortController.signal })
} catch (err) {
if(err.name === "AbortError") {
console.warn("write aborted")
return
}
throw err
}
})()
// Somewhere else...
myAbortController.abort()
The code is tested and runs in v18, v16, and v14 with the --experimental-abortcontroller flag
I am trying to set up a route that sends data from a nested promise to my vue app.
But i'm having trouble with the getting data from the nested promises.
i tried using a callback with no success
app.get('/notification', (req, res) => {
const getData = (data) => {
console.log(data)
}
scheduler(data)
})
const scheduler = (callback) => {
sftp
.connect({ credentials })
.then(() => {
return sftp.list(root);
})
.then(async data =>
{
const filteredFile = data.filter(file => {
let currentDate = moment();
let CurrentTimeMinusFive = moment().subtract(5, "minutes");
let allAccessTimes = file.accessTime;
let parsedAccessTimes = moment(allAccessTimes);
let filteredTime = moment(parsedAccessTimes).isBetween(
CurrentTimeMinusFive,
currentDate
);
return filteredTime;
});
for (const file of filteredFile) {
let name = file.name;
let filteredThing;
await sftp
.get(`Inbound/${name}`)
.then(data => {
csv()
.fromString(data.toString())
.subscribe(function (jsonObj) {
return new Promise(function (resolve, reject) {
filteredThing = new Notification(jsonObj);
filteredThing.save()
.then(result => {
console.log(result);
callback(result) **// THIS IS THE RESULT I NEED IN MY FRONT END**
})
.catch(err => {
console.log(err);
});
resolve();
});
});
});
}
})
When i go to localhost/notification i get:
ReferenceError: data is not defined
Thanks in advance!
My code is as shown below :
rabbitmq.js
const connectRabbitMq = () => {
amqp.connect(process.env.CLOUDAMQP_MQTT_URL, function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
channel = ch;
connection = conn;
}
});
conn.on("error", function (err) {
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.on("close", function () {
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
const sendMessage = () => {
let data = {
user_id: 1,
test_id: 2
};
if (channel) {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
}
else {
connectRabbitMq(() => {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
})
}
};
const receiveMessage = () => {
if (channel) {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
}
else {
connectRabbitMq(() => {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
})
}
}
scheduler.js
let cron = require('node-cron');
const callMethodForeverRabbitMq = () => {
cron.schedule('*/1 * * * * *', function () {
rabbitMqClientPipeline.receiveMessage();
});
};
app.js
rabbitmq.sendMessage();
now what happens here is , the code is not able to maintain the connection alive forever . so is there any way I can keep it alive forever ?
I am not sure if you are using Promise api or callback API.
With Promise API you can do it like this:
const amqp = require('amqplib');
const delay = (ms) => new Promise((resolve => setTimeout(resolve, ms)));
const connectRabbitMq = () => amqp.connect('amqp://127.0.0.1:5672')
.then((conn) => {
conn.on('error', function (err) {
if (err.message !== 'Connection closing') {
console.error('[AMQP] conn error', err.message);
}
});
conn.on('close', function () {
console.error('[AMQP] reconnecting');
connectRabbitMq();
});
//connection = conn;
return conn.createChannel();
})
.then(ch => {
console.log('Channel created');
//channel = ch;
})
.catch((error) => {
console.error(error);
console.log('[AMQP] reconnecting in 1s');
return delay(1000).then(() => connectRabbitMq())
});
connectRabbitMq();
With callback API like this:
const amqp = require('amqplib/callback_api');
const connectRabbitMq = () => {
amqp.connect('amqp://127.0.0.1:5672', function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1000);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
//channel = ch;
//connection = conn;
}
});
conn.on("error", function (err) {
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.on("close", function () {
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
connectRabbitMq();
UPDATE new code with request buffering
const buffer = [];
let connection = null;
let channel = null;
const connectRabbitMq = () => {
amqp.connect('amqp://127.0.0.1:5672', function (err, conn) {
if (err) {
console.error(err);
console.log('[AMQP] reconnecting in 1s');
setTimeout(connectRabbitMq, 1000);
return;
}
conn.createChannel((err, ch) => {
if (!err) {
console.log('Channel created');
channel = ch;
connection = conn;
while (buffer.length > 0) {
const request = buffer.pop();
request();
}
}
});
conn.once("error", function (err) {
channel = null;
connection = null;
if (err.message !== "Connection closing") {
console.error("[AMQP] conn error", err.message);
}
});
conn.once("close", function () {
channel = null;
connection = null;
console.error("[AMQP] reconnecting");
connectRabbitMq();
});
})
};
const sendMessage = () => {
let data = {
user_id: 1,
test_id: 2
};
if (channel) {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
}
else {
buffer.push(() => {
channel.sendToQueue(q, new Buffer(JSON.stringify(data)), {
persistent: true
});
});
}
};
const receiveMessage = () => {
if (channel) {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
}
else {
buffer.push(() => {
channel.consume(q, function (msg) {
// ch.ack(msg);
console.log(" [x] Received %s", msg.content.toString());
});
})
}
};
There are edge cases where this code won't work - for example it won't reestablish queue.consume unless it's called explicitly. But overall this hopefully gives you idea on how to implement proper recovery...
There is a function called musicPromise(). What this function does is
It gets all mp4 files and loop through it.
then it tries to convert each mp4 to mp3, using fluent-ffmpeg
The problem I am facing is
It only converts 1 file, no matter how many mp4 files I have.
And it seems never reach to proc.on('end', (x) => {
Full code here:
// search
const glob = require('glob');
// wait for
const Promise = require('bluebird');
// fs
const fs = require('fs');
// mp3
const ffmpeg = require('fluent-ffmpeg');
// video source file path
const videoPath = '/home/kenpeter/Videos/4K\ Video\ Downloader';
// audio source file path
const audioPath = __dirname + "/audio";
// child process, exec
const exec = require('child_process').exec;
// now rename promise
function renamePromise() { return new Promise((resolve, reject) => {
glob(videoPath + "/**/*.mp4", (er, files) => {
Promise.each(files, (singleClipFile) => {
return new Promise((resolve1, reject1) => {
let arr = singleClipFile.split("/");
let lastElement = arr[arr.length - 1];
let tmpFileName = lastElement.replace(/[&\/\\#,+()$~%'":*?<>{}\ ]/g, "_");
let tmpFullFile = videoPath + "/"+ tmpFileName;
// rename it
fs.rename(singleClipFile, tmpFullFile, function(err) {
if ( err ) console.log('ERROR: ' + err);
console.log("-- Rename one file --");
console.log(tmpFullFile);
resolve1();
}); // end rename
});
})
.then(() => {
console.log('--- rename all files done ---');
resolve();
});
});
}); // end promise
};
// music promise
function musicPromise() { new Promise((resolve, reject) => {
glob(videoPath + "/**/*.mp4", (er, files) => {
Promise.each(files, (singleClipFile) => {
return new Promise((resolve1, reject1) => {
// test
console.log('-- music promise --');
console.log(singleClipFile);
// split
let arr = singleClipFile.split("/");
// e.g. xxxx.mp4
let clipFile = arr[arr.length - 1];
// e.g. xxxx no mp4
let fileName = clipFile.replace(/\.[^/.]+$/, "");
// music file name
let musicFile = fileName + '.mp3';
// set source
let proc = new ffmpeg({source: singleClipFile});
// set ffmpeg path
proc.setFfmpegPath('/usr/bin/ffmpeg');
// save mp3
proc.output("./audio/" + musicFile);
// proc on error
proc.on('error', (err) => {
console.log(err);
});
// done mp3 conversion
proc.on('end', (x) => {
console.log("single mp3 done!");
console.log(x);
// it is resolve1..............
resolve1();
});
// Run !!!!!!!!!!!!!
proc.run();
});
})
.then(() => {
console.log('--------- all mp3 conversion done --------');
resolve();
});
}); // end glob
});
};
// adb kill
function adbKillPromise() { return new Promise((resolve, reject) => {
exec("adb kill-server", (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
console.log('---adb kill---');
resolve();
});
});
};
// adb start
function adbStartPromise() { return new Promise((resolve, reject) => {
exec("adb start-server", (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
console.log('---adb start---');
resolve();
});
});
};
// adb push promise
function adbPushPromise() { return new Promise((resolve, reject) => {
glob(audioPath + "/**/*.mp3", (er, files) => {
Promise.each(files, (singleMusicFile) => {
return new Promise((resolve1, reject1) => {
let cmd = "adb push" + " " + singleMusicFile + " " + "/sdcard/Music";
exec(cmd, (err, stdout, stderr) => {
console.log(cmd);
resolve1();
});
});
})
.then(() => {
console.log('---- done push all music ---');
resolve();
});
});
});
};
// Run !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
renamePromise()
.then(musicPromise)
.then(adbKillPromise)
.then(adbStartPromise)
.then(adbPushPromise)
.then(() => {
console.log('---- all done----');
process.exit(0);
})
.catch(err => {
console.log('Error', err);
process.exit(1);
});
A very stupid mistake
function musicPromise() { new Promise((resolve, reject) => {
should be
function musicPromise() { return new Promise((resolve, reject) => {
I am trying to move a file from a location abc to location xyz if the file already exists in location xyz delete it then save the new one.
Here is my code
const promises = {
deleteFile: path => {
return new Promise((resolve, reject) => {
const fs = require('fs');
fs.stat(path, (err, stat) => {
if (err === null) {
fs.unlink(path, err => {
if (err) { return reject(err) }
resolve();
})
} else if(err.code == 'ENOENT') {
console.log('File does not exist');
resolve();
} else {
reject(err);
}
});
});
},
copyFile: (from, to) => {
return new Promise((resolve, reject)=> {
copyFile(from, to, (err) => {
if (err) { return reject(err); }
console.log(`Finished writing file ${to}`);
resolve();
})
})
}
}
const copyFile = (from, to, overallCb) => {
const fs = require('fs');
const rs = fs.createReadStream(from)
const ws = fs.createWriteStream(to)
let cbCalled = false;
function done (err) {
overallCb(err);
cbCalled = true;
}
rs.on('error', (err) => {
done(err);
})
ws.on('error', (err) => {
done(err);
})
rs.pipe(ws);
}
;
const OUTPUT_PATH = `./js/libs/`
, _NODE_MODULES = './node_modules/'
, filePath = `${_NODE_MODULES}somePathToAFile`
;
promises.deleteFile(`${OUTPUT_PATH}someFile.min.js`)
.then(promises.copyFile(filePath, `${OUTPUT_PATH}someFile.min.js`))
.then(words => {
console.log('**** done doing things ****');
})
.catch(error => { console.log(`ERROR`, error); })
If the file exists it just simply deletes the file and does nothing else.
If the file DOES NOT exist everything works fine.
Any idea on what im doing wrong?
I had some promise errors in my code... For future me here is the working version of the above code
'use strict';
const promises = {
deleteFile: path => {
return new Promise((resolve, reject) => {
const fs = require('fs');
fs.stat(path, (err, stat) => {
if (err === null) {
fs.unlink(path, err => {
if (err) { return reject(err) }
resolve(`Removing document at ${path}`);
})
} else if(err.code === 'ENOENT') {
resolve('File does not exist');
} else {
reject(err);
}
});
});
},
copyFile: (from, to) => {
return new Promise((resolve, reject) => {
copyFile(from, to, (err) => {
if (err) { return reject(err); }
resolve(`Finished writing file ${to}`);
})
})
}
}
const copyFile = (from, to, overallCb) => {
const fs = require('fs');
const rs = fs.createReadStream(from)
const ws = fs.createWriteStream(to)
let cbCalled = false;
function done (err) {
overallCb(err);
cbCalled = true;
}
rs.on('error', done)
ws.on('error', done)
.on('close', done)
rs.pipe(ws);
}
;
const OUTPUT_PATH = './js/libs/'
, _NODE_MODULES = './node_modules/'
, filePath = `${_NODE_MODULES}somePathToAFile`
;
promises.deleteFile(`${OUTPUT_PATH}someFile.min.js`)
.then(msg => {
console.log(msg)
return promises.copyFile(filePath, `${OUTPUT_PATH}someFile.js`)
})
.then(msg => {
console.log(msg)
})
.catch(err => { // Do errory things}