My problem is I'm trying to await on some promises, but my await call seems to return to the calling function right away.
In the code below (part of a controller function handling a POST), the "files coming out" log after my function call is executed prior to the promises being executed. My async function seemingly returns at the "await" call.
However the "await" still does wait for all the promises to execute, so the "I just finished waiting" log does have the magic var inserted into the req.files correctly.
const upload = multer({
storage: multer.diskStorage ({
destination: (req, file, cb) => { cb(null, config.DESTINATION) }
})
}).array(config.FIELD_NAME);
exports.upload_image = function(req, res) {
upload(req, res, (err) => {
if (err) {
return res.end("error uploading file");
}
// got file ok, do validation
checkAreDicom2( req.files );
console.log("files coming out: ", req.files);
return res.end("OK");
});
}
async function checkAreDicom2(inFiles) {
let promises = inFiles.map(function (file) {
return new Promise( function (resolve, reject) {
fs.createReadStream(file.path,{start: 128, end: 131, autoClose: true})
.on('error',reject)
.on('data', (chunk) => file.magic = file.magic ? file.magic += chunk : chunk)
.on('end', resolve);
});
});
await Promise.all(promises);
console.log("i just finished waiting: ", inFiles);
return;
}
The await is waiting for the Promise.all, and the log after it is correctly delayed. However it does of course not block the caller of function.
So the callback in upload_image does not wait for the asynchronous processing, it just receives the promise that checkAreDicom2(…) returns - and discards it, and immediately logs something. You would need to await the asynchronous result explicitly here again.
exports.upload_image = async function(req, res) {
try {
await new Promise((resolve, reject) => {
upload(req, res, (err) => {
if (err) reject(err);
else resolve();
});
});
// got file ok, do validation
await checkAreDicom2( req.files );
// ^^^^^
console.log("files coming out: ", req.files);
return res.end("OK");
} catch(err) {
return res.end("error uploading file");
}
};
Related
I want to generate multiple pdf files and attach to the email. But await seems not working on res.app.render.
route.get('/:id/receipts', async function (req, res) {
...
let attachments = [];
for await(let item of items){
res.view.item = item;
console.log(1)
await res.app.render('pdfs/receipt', res.view, async function(err, html){
console.log(2)
if (err) return res.end(err.stack)
return await pdf.create(html).toBuffer(async function(err, buffer){
console.log(3)
attachments.push({
content: buffer,
filename: 'receipt.pdf',
})
});
});
}
console.log(4)
...
})
Expect Result:
1
2
3
4
Actually Result:
1
4
2
3
I think res.app.render is not returning a promise that's why you are facing this issue. You have to make a custom promise. I hope following code will help you.
oute.get('/:id/receipts', async function (req, res) {
...
let attachments = [];
for await(let item of items){
res.view.item = item;
console.log(1)
const customPromise = new Promise((resolve, reject) => {
res.app.render('pdfs/receipt', res.view, async function(err, html){
console.log(2)
if (err) { res.end(err.stack);reject()}
else {
await pdf.create(html).toBuffer(async function(err, buffer){
console.log(3)
attachments.push({
content: buffer,
filename: 'receipt.pdf',
})
});
resolve();
}
});
})
}
console.log(4)
...
})
The async function below is supposed to check if a url is a legit url
let CheckUrl = function (url, done) {
dns.lookup(url, function(err, address) {
if (err) return done(err);
done(null, true); //return true because I don't care what the address is, only that it works
});
}
The express.js code below gets the url but I'm having trouble understanding how to write the if statement so that it returns true or false.
// Gets URL
app.post("/api/shorturl/new", function(req, res) {
if (CheckUrl(req.body.url)) {
// do something
}
});
I'm not sure what to pass as the second argument in CheckUrl() in this if statement. Or maybe I wrote the first async function incorrectly to begin with?
Please use the async await
I have written a test code for you as below:
const express = require('express');
const app = express();
const dns = require('dns');
let CheckUrl = function (url, done) {
return new Promise((resolve, reject) => {
dns.lookup(url, function(err, address) {
console.log("err " , err)
if (err) {
resolve(false)
} else {
resolve(true)
}
});
});
}
app.post("/api/shorturl/new", async function(req, res) {
try {
let result = await CheckUrl(req.body.url);
console.log("result " , result)
res.send(result)
}
catch (error) {
console.log("in catch error " , error)
res.send(error)
}
});
app.listen(3000)
you can get the knowledge to know about the Promise here. The Promise object represents the eventual completion (or failure) of an asynchronous operation and its resulting value.
As mentioned by DeepKakkar, this was what I was looking for:
app.post("/api/shorturl/new", async (req, res) => {
try {
let result = await CheckUrl(req.body.url);
res.send(result)
}
catch (error) {
return new Error('Could not receive post');
}
});
What I need to do is,
to check if fileFullPath exist
if not, in the end of successfull file download, to call saveInfo.
When I execute the application, what I observe is, it calls saveInfo before finishing file write operation. And I get error message:
(node:20224) UnhandledPromiseRejectionWarning: Error: BatchCluster has ended, cannot enqueue -charset
What am I doing wrong?
async function dl(url, path, data = null) {
await request.get({
url: url,
})
.on("error", async function (error) {
console.log(error);
return false;
})
.on('response', async function (res) {
var fileExt = res.headers['content-type'].split('/')[1];
var fileFullPath = `${path}.${fileExt}`;
await res.pipe(fs.createWriteStream(fileFullPath));
console.log("file downloaded");
if (data) {
await saveInfo(fileFullPath, data);
}
});
return true;
}
async function saveInfo(filePath, data) {
await exiftool.write(filePath, {
Keywords: data.keywords,
Copyright: data.copyright,
});
console.log("Tags are saved");
exiftool.end();
}
OK, I found a way to do this. piping to streams is not very friendly to promises so I ended up doing some manual promise manipulations. I think better promise support for streams is coming to node.js as we already have some async iterators. Anyway, here's a way to make things work by watching for the right events on your streams:
function dl(url, path, data = null) {
return new Promise((resolve, reject) => {
request.get({
url: url,
}).on("error", function (error) {
console.log(error);
reject(error);
}).on('response', function (res) {
let fileExt = res.headers['content-type'].split('/')[1];
let fileFullPath = `${path}.${fileExt}`;
let writeStream = fs.createWriteStream(fileFullPath);
// set up event handlers to monitor the writeStream for error or completion
writeStream.on('error', reject).on('close', async () => {
if (data) {
try {
await saveInfo(fileFullPath, data);
} catch(e) {
reject(e);
return;
}
}
console.log("file downloaded");
resolve(true);
});
// send the response stream to our file
res.pipe(writeStream).on('error', reject);
});
});
}
async function saveInfo(filePath, data) {
await exiftool.write(filePath, {
Keywords: data.keywords,
Copyright: data.copyright,
});
console.log("Tags are saved");
exiftool.end();
}
So i encountered a problem while doing my project.The problem is that when i try to write my data to csv file,it only write half of the data ,even sometimes only less than half of my data.I don't know what the problem is because there is no error shown in the terminal.
Below is my code
async function getFile(req, res, next) {
try {
let URI;
const listOfKeys = await listAllKeys();
let temp = []
await Promise.all(listOfKeys.map(async function (data) {
let response = await writeFile(data.Key);
temp.push(response)
}))
.then(async _ => {
fs.writeFileSync(FILE_PATH, Buffer.concat(temp));
})
.catch(err => {
console.log(err)
})
return res.json({ message: 'halo' });
} catch (err) {
console.log('hmm.... error', err);
return next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
};
};
And this is the writeFile function
function writeFile(key) {
return new Promise((resolve, reject) => {
s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
if (err) reject(err)
else resolve(data.Body)
})
});
};
If possible, i would like to know the detail of my problem and how to fix it.Thanks.
It looks to me like you can do it like this (function names have been modified to make sense to me):
const fsp = require('fs').promises;
async function getDataAndWriteFile(req, res, next) {
try {
let URI;
const listOfKeys = await listAllKeys();
let responses = await Promise.all(listOfKeys.map(function (data) {
return getData(data.Key);
}));
await fsp.writeFile(FILE_PATH, Buffer.concat(responses);
res.json({ message: 'halo' });
} catch(err) {
console.log('hmm.... error', err);
next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
}
}
function getData(key) {
return new Promise((resolve, reject) => {
s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
if (err) reject(err)
else resolve(data.Body)
})
});
}
Summary of changes:
Change function names to better describe what they do
Use let responses = await Promise.all() to get the data from the promise array.
Use the promise interface in the fs module with await fsp.writeFile() to write the data out to your file.
Use try/catch to catch all the promise rejections in one place
Possible Open Issues:
Writing this Buffer.concat(responses) to disk seems kind of odd. Is that really what you want in this file?
I'm trying to learn Asynchronous programming with NodeJS and I'm having trouble understanding how to create usable functions.
I'm trying to compare the results of a HTTP get request and a file read all inside an "express" callback. What is the best way to split out two different async operations into their own functions so that they can be used again together in a different callback?
I Have it working when I write everything inside the express callback
app.get('/', (req, res) => {
axios.get('http://127.0.0.1:8080')
.then(function(response) {
var http_data = response.data
// Do more stuff with data
fs.readFile('fwversion_current', 'utf8', function(err, contents) {
var file_data = contents.trim()
// Do more stuff with data
if (http_data == file_data) {
res.send("Match")
}
else {
res.send("No Match")
}
});
});
But I'm hoping for something more like this so I can use these same operations in other places. I'm not sure the right node way to get there.
function getHttpData() {
axios.get('http://127.0.0.1:8080')
.then(function(response) {
var http_data = response.data
// Do more stuff with data
return http_data
});
}
function getFileData() {
fs.readFile('fwversion_current', 'utf8', function(err, contents) {
var file_data = contents.trim()
// Do more stuff with data
return file_data
});
}
app.get('/', (req, res) => {
let http_data = await getHttpData()
let file_data = await getFileData()
if (http_data == file_data) {
res.send("Match")
}
else {
res.send("No Match")
}
});
You will need to wrap those functions inside a function that returns a Promise, this will let you the ability to await for them to complete before continuing.
function getHttpData(url) {
// axios.get already returns a Promise so no need to wrap it
return axios.get(url)
.then(function(response) {
let http_data = response.data;
// Do more stuff with data
return http_data;
});
}
function getFileData(path) {
return new Promise(function(resolve, reject) {
fs.readFile(path, function(err, contents) {
if (err) {
reject(err);
return;
}
let file_data = contents.trim();
// Do more stuff with data
resolve(file_data);
});
});
}
Now when both functions returns a Promise we can await for them to complete.
Make the handler an async function because it's needed to use the await keyword, I'm using Promise.all to fire both requests simultaneously and not wait for one to complete before we fire the other.
Wrap it in a try catch to handle errors and send status 500
app.get('/', async (req, res) => {
try {
const [http_data, file_data] = await Promise.all([
getHttpData(url),
getFileData(path),
]);
http_data == file_data
? res.send('Match')
: res.send('No Match');
} catch (err) {
console.error(err);
res.status(500).send('Something went wrong');
}
});