What I need to do is,
to check if fileFullPath exist
if not, in the end of successfull file download, to call saveInfo.
When I execute the application, what I observe is, it calls saveInfo before finishing file write operation. And I get error message:
(node:20224) UnhandledPromiseRejectionWarning: Error: BatchCluster has ended, cannot enqueue -charset
What am I doing wrong?
async function dl(url, path, data = null) {
await request.get({
url: url,
})
.on("error", async function (error) {
console.log(error);
return false;
})
.on('response', async function (res) {
var fileExt = res.headers['content-type'].split('/')[1];
var fileFullPath = `${path}.${fileExt}`;
await res.pipe(fs.createWriteStream(fileFullPath));
console.log("file downloaded");
if (data) {
await saveInfo(fileFullPath, data);
}
});
return true;
}
async function saveInfo(filePath, data) {
await exiftool.write(filePath, {
Keywords: data.keywords,
Copyright: data.copyright,
});
console.log("Tags are saved");
exiftool.end();
}
OK, I found a way to do this. piping to streams is not very friendly to promises so I ended up doing some manual promise manipulations. I think better promise support for streams is coming to node.js as we already have some async iterators. Anyway, here's a way to make things work by watching for the right events on your streams:
function dl(url, path, data = null) {
return new Promise((resolve, reject) => {
request.get({
url: url,
}).on("error", function (error) {
console.log(error);
reject(error);
}).on('response', function (res) {
let fileExt = res.headers['content-type'].split('/')[1];
let fileFullPath = `${path}.${fileExt}`;
let writeStream = fs.createWriteStream(fileFullPath);
// set up event handlers to monitor the writeStream for error or completion
writeStream.on('error', reject).on('close', async () => {
if (data) {
try {
await saveInfo(fileFullPath, data);
} catch(e) {
reject(e);
return;
}
}
console.log("file downloaded");
resolve(true);
});
// send the response stream to our file
res.pipe(writeStream).on('error', reject);
});
});
}
async function saveInfo(filePath, data) {
await exiftool.write(filePath, {
Keywords: data.keywords,
Copyright: data.copyright,
});
console.log("Tags are saved");
exiftool.end();
}
Related
In Node.js how can you delete the file a WriteStream was writing to and then recreate the stream without error?
With this code
clear(){
return new Promise((resolve, reject) => {
this.stream.once('close', () => {
Fs.unlinkSync(this.filepath);
this.stream = Fs.createWriteStream(this.filepath, {flags: 'a'});
resolve();
});
this.stream.end();
});
}
This error occurs. I've tried everything.
Uncaught Error: EPERM: operation not permitted, open 'C:\test\test.log'
The calling code
it('erases the log file', async () => {
const file = new File("C:\test\test.log");
await file.clear();
const exists = Fs.existsSync(testfile);
Assert.strictEqual(exists, false);
});
I can only avoid the error if I remove the this.stream = Fs.createWriteStream(this.filepath, {flags: 'a'}); in clear()
When I try to duplicate your issue here, I only get that error if you are not properly waiting for the clear() method to finish before you attempt to use the stream again.
For example, this code works:
const fs = require('fs');
class MyFile {
constructor(filepath) {
this.filepath = filepath;
this.open();
}
open() {
this.stream = fs.createWriteStream(this.filepath, { flags: 'a' });
this.stream.on('error', err => {
console.log(err);
});
}
log(str) {
this.stream.write(str);
}
clear() {
return new Promise((resolve, reject) => {
this.stream.once('close', () => {
console.log("got clear close event");
try {
fs.unlinkSync(this.filepath);
} catch (e) {
console.log(e);
}
this.open();
resolve();
});
this.stream.end();
});
}
close() {
return new Promise((resolve, reject) => {
this.stream.once('close', () => {
console.log("got final close event");
resolve();
});
this.stream.end();
});
}
}
async function run() {
const f = new MyFile("./temp.xxx");
f.log("hello\n");
f.log("goodbye\n");
await f.clear();
f.log("hello\n");
f.log("goodbye\n");
await f.close();
}
run().then(result => {
console.log("done");
}).catch(err => {
console.log(err);
});
But, if I remove the await in front of await f.clear(), then I get your exact error because the code proceeds trying to use the stream after you've called this.stream.end(), but before the new stream is in place which is an EPERM error.
If this doesn't illustrate for you what your problem is, then you need to show us all the rest of the code that uses this class and calls the clear() method. The problem is likely in that code.
Yes, I have seen many other questions and answers. I know I need to use a callback response. However, I still don't get how to do this particular example. Most examples involve a callback response that logs something or the post has hundreds of different answers.
How do I return the request response from getPageData?
var url = "myurl";
var name = await getPageData(url);
// wait until I get name and then do stuff with name
function getPageData(url)
{
const https = require('https');
https.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
var name = JSON.parse(data);
// what do I do here to get name out?
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
}
await can only be used in async functions. You can however return a promise from getPageData and "await" using chained then:
Use the Promise object:
const https = require('https');
var url = "myurl";
var name;
getPageData(url)
.then(data => { name = data; /*This is the scope in which you would use name*/ })
.catch(err => { console.log('Error occured', err); });
async function getPageData(url) {
return new Promise((resolve, reject) => {
https
.get(url, resp => {
let data = '';
resp.on('data', chunk => {
data += chunk;
});
resp.on('end', () => {
const name = JSON.parse(data);
// what do I do here to get name out?
resolve(name);
});
})
.on('error', err => {
console.log(`Error: ${err.message}`);
reject(err);
});
});
}
The higher level solution here is to use a module for making http requests that already supported promises. You can see a list of many of them here. My favorite from that list is got() and you can use it to solve your problem like this:
function getPageData(url) {
return got(url);
}
// can only use await inside a function declared as async
async function someFunction() {
try {
let name = await getPageData(url);
console.log(name);
} catch(e) {
console.log(e);
}
}
The got() library does a whole bunch of things for you.
It is entirely based on promises so you can directly use await on the promise it returns.
It collects the whole response for you (you don't have to write your own code to do that).
If the response is JSON, it automatically parses that for you and resolves to the parsed Javascript object.
It automatically detects http or https URL and uses the right low level module.
And, it has dozens of other useful features (not needed in this example).
Or, if you want the lower level solution where you make your own promisified function for doing an https request, you can do this:
const https = require('https');
// can only use await inside a function declared as async
async function someFunction() {
const url = "myurl";
try {
let name = await getPageData(url);
console.log(name);
} catch(e) {
console.log(e);
}
}
function getPageData(url) {
return new Promise((resolve, reject) => {
https.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
try {
const name = JSON.parse(data);
resolve(name);
} catch(e) {
// JSON parsing error
reject(e);
}
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(err);
});
}).on('error', (err) => {
console.log("Error: " + err.message);
reject(err);
});
}
So i encountered a problem while doing my project.The problem is that when i try to write my data to csv file,it only write half of the data ,even sometimes only less than half of my data.I don't know what the problem is because there is no error shown in the terminal.
Below is my code
async function getFile(req, res, next) {
try {
let URI;
const listOfKeys = await listAllKeys();
let temp = []
await Promise.all(listOfKeys.map(async function (data) {
let response = await writeFile(data.Key);
temp.push(response)
}))
.then(async _ => {
fs.writeFileSync(FILE_PATH, Buffer.concat(temp));
})
.catch(err => {
console.log(err)
})
return res.json({ message: 'halo' });
} catch (err) {
console.log('hmm.... error', err);
return next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
};
};
And this is the writeFile function
function writeFile(key) {
return new Promise((resolve, reject) => {
s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
if (err) reject(err)
else resolve(data.Body)
})
});
};
If possible, i would like to know the detail of my problem and how to fix it.Thanks.
It looks to me like you can do it like this (function names have been modified to make sense to me):
const fsp = require('fs').promises;
async function getDataAndWriteFile(req, res, next) {
try {
let URI;
const listOfKeys = await listAllKeys();
let responses = await Promise.all(listOfKeys.map(function (data) {
return getData(data.Key);
}));
await fsp.writeFile(FILE_PATH, Buffer.concat(responses);
res.json({ message: 'halo' });
} catch(err) {
console.log('hmm.... error', err);
next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
}
}
function getData(key) {
return new Promise((resolve, reject) => {
s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
if (err) reject(err)
else resolve(data.Body)
})
});
}
Summary of changes:
Change function names to better describe what they do
Use let responses = await Promise.all() to get the data from the promise array.
Use the promise interface in the fs module with await fsp.writeFile() to write the data out to your file.
Use try/catch to catch all the promise rejections in one place
Possible Open Issues:
Writing this Buffer.concat(responses) to disk seems kind of odd. Is that really what you want in this file?
I am unable to get the value of the response outside the callback code. It returns undefined outside whereas in the callback it is giving proper result.
function doCall(urlString, callback) {
request.get(
urlString,
null,
null,
(err, data, result) => {
var statusCode = result.statusCode;
return callback(data);
}
);
}
const apiResponse = doCall(urlString, function(response) {
console.log('***************************' + response); //Prints correct result
return JSON.parse(response);
});
console.log('+++++++++++++++++++++++++' + apiResponse); //Prints undefined
function doCall(urlString) {
return new Promise((resolve, reject) => {
request.get(
urlString,
null,
null,
(err, data, result) => {
if (error) reject(error);
var statusCode = result.statusCode;
resolve(data);
});
});
}
async function myBackEndLogic() {
try {
const result = await doCall(urlString);
console.log(result);
//return JSON.parse(result) if you want
} catch (error) {
console.error('ERROR:');
console.error(error);
}
}
myBackEndLogic();
Read this for more explanations
If you want synchronous looking code, wrap everything in an async function:
(async (){
async function doCall(urlString, callback) {
return await request.get(urlString, null, null); // or store in a variable and return modified response
}
const apiResponse= await doCall(urlString, (response) => {
console.log('response', response);
return JSON.parse(response);
});
console.log('apiResponse', apiResponse);
})()
My Codes below;
I've a then-catch block. My responseArray is a global variable. i got response from functionName function; but i can't use result out of then block. How can i use then response out of block?
My Codes below;
I've a then-catch block. My responseArray is a global variable. i got response from functionName function; but i can't use result out of then block. How can i use then response out of block?
module.exports = {
foo1: function(param){
return new Promise((resolve,reject) => {
var result = //some code here
resolve(result);
});
},
foo2: function(param){
return new Promise((resolve,reject) => {
this.foo1('abc').then(function(res){
let response = {
'item':'ok',
'result':res.some_field
};
console.log(response); // its ok here.
responseArray.push(response); //its ok here too
}).catch(err =>{
console.log(err);
reject(err);
});
console.log(responseArray); //nothing in array here
resolve(responseArray);
});
}
};
First thing to remember is that promises are asynchronous. Promises are doing exactly what they say, you are essentially signing a contract (promise) that you will get your data (or error) but not synchronously, but at some time in the future when the computations have finished.
In order to access your responseArray you will need to resolve your foo2 promise (inside of .then) and continue the promise chain by calling it, i.e.
module.exports = {
foo1: function(param){
return new Promise((resolve,reject) => {
var result = //some code here
resolve(result);
});
},
foo2: function(param){
return new Promise((resolve,reject) => {
this.foo1('abc').then(function(res){
let response = {
'item':'ok',
'result':res.some_field
};
console.log(response); // its ok here.
responseArray.push(response); //its ok here too
resolve(responseArray) // resolve the promise inside of .then
}).catch(err =>{
console.log(err);
reject(err);
});
});
}
};
foo2('someValue').then(response => {
console.log(response) // this will be your array
})
Also, as a side note, ensure you are not falling into the trap of the promise constructor anti-pattern. This is where you unnecessarily turn synchronous code into asynchronous code just for the sake of using "promises"
For example, a valid use of a promise would be to convert a callback, like so:
const getFile = filename => {
return new Promise((resolve, reject) => {
fs.readFile(filename, 'utf8', (err, data) => {
if (err) reject(err)
resolve(data)
})
})
}
whereas this is unnecessary:
const printData = data => {
return new Promise((resolve, reject) => {
resolve(console.log(data))
})
}
vs
const printData = data => {
console.log(data)
}
Read more here: What is the explicit promise construction antipattern and how do I avoid it?