Using fs.read inside promise does not work - node.js

I am trying to do a fs.read after the promise job is done by using the .then()
Here is how my code looks like
(async () => {
const feed = await parser.parseURL('https://www.nasa.gov/rss/dyn/breaking_news.rss');
console.log(feed.title);
const items = [];
await Promise.all(feed.items.map(async (currentItem) => {
// some code here to create data
items.push(data);
})).then(
items.forEach((element) => {
const file = downloadFile(element.url);
let checksumValue;
try {
fs.readFileSync(file, (_err, data) => {
checksumValue = generateChecksum(data);
console.log(`The checksum is: ${checksumValue}`);
// Delete the downloaded file
deleteFile(file);
});
} catch (error) {
console.error(error);
// expected output: ReferenceError: nonExistentFunction is not defined
// Note - error messages will vary depending on browse
}
})(),
);
})();
But it doesn't operate this piece of code :
fs.readFileSync(file, (_err, data) => {
checksumValue = generateChecksum(data);
console.log(`The checksum is: ${checksumValue}`);
// Delete the downloaded file
deleteFile(file);
});
How should I read the file?

fs.readFileSync is sync, so it doesn't take a callback.
Either use the non-sync version:
fs.readFile(file, (_err, data) => {
checksumValue = generateChecksum(data);
console.log(`The checksum is: ${checksumValue}`);
// Delete the downloaded file
deleteFile(file);
});
or use it as intended:
const data = fs.readFileSync(file);
checksumValue = generateChecksum(data);
console.log(`The checksum is: ${checksumValue}`);
// Delete the downloaded file
deleteFile(file);

Related

Node JS - createWriteStream

I am going crazy trying to fix this bug so please help :-)
I am using https://pdfkit.org/
This creates a stream that when finished is piped to fs.createWriteStream
My issue is the first time the code runs this works and the PDF is generated.
The next time the Code runs a file with Zero Bytes is created.
I am calling the function from an API running on express.
The issue appears to be the async nature of fs.createWriteStream.
The stream finishes after the API has returned. I cannnot seem to find a way to block while confirming the file has been created.
What is odd is that the first time the code works run again it fails:
Here is the Pipe Function;
async function _writeFile(fileObj) {
let fileStream = fs.createWriteStream(fileObj.fileName)
pipeline(
doc,
fileStream,
async (err) => {
if (err) {
console.error('PDF failed', err);
return ('Pipeline failed', err)
} else {
console.log('PDF succeeded');
}
}
)
}
This is called from:
exports.drawReport = async (payload) => {
var date = new Date();
const timeStamp = date.toJSON();
let path = './controllers/tmp/'
var fileName = path + timeStamp + '.' + payload.type + '.pdf'
try {
// Start Report
await _startReport(payload)
// Check Starting position on page & add status box header
if (device_card_reference == 260) {
await _deviceTitle(payload);
}
// Add Devices
await _reportDevice(payload);
// Call Footer for final page
await _reportFooter()
console.log("PDF Done - Writing File")
// File Meta Data
let fileObj = {
type: payload.type,
siteId: payload.siteId,
fileName: fileName,
timeStamp: timeStamp
}
// Create file to store PDF
await _writeFile(fileObj)
doc.end()
console.log("PDF MADE?")
return (fileObj)
} catch (err) {
console.error('MakePDF ERROR: ' + err.message);
return (err.message)
}
}
pipeline runs asynchronously, so it's not awaited, which is why doc.end() runs before the file is done
try wrapping pipeline in a promise, and then resolve when the stream is done:
// function that returns a promise
function _writeFile(fileObj) {
return new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(fileObj.fileName);
pipeline(
doc,
fileStream,
async(err) => {
if (err) {
console.error('PDF failed', err);
// err, handle in `.catch`
reject({res:'Pipeline failed', err});
} else {
console.log('PDF succeeded');
// done, resolve, to move to doc.end
resolve('PDF succeeded');
}
}
)
});
}
add .catch() to handle error:
// Create file to store PDF
await _writeFile(fileObj).catch(err => console.log(err));
or even better, use stream promises API
const {pipeline } = require('stream/promises');
async function _writeFile(fileObj) {
const fileStream = fs.createWriteStream(fileObj.fileName);
await pipeline(doc, fileStream);
console.log('PDF succeeded');
}

How to make fs.readFile async await?

I have this nodejs code here which read a folder and process the file. The code works. But it is still printing all the file name first, then only read the file. How do I get a file and then read a content of the file first and not getting all the files first?
async function readingDirectory(directory) {
try {
fileNames = await fs.readdir(directory);
fileNames.map(file => {
const absolutePath = path.resolve(folder, file);
log(absolutePath);
fs.readFile(absolutePath, (err, data) => {
log(data); // How to make it async await here?
});
});
} catch {
console.log('Directory Reading Error');
}
}
readingDirectory(folder);
To use await, you need to use promise versions of fs.readFile() and fs.readdir() which you can get on fs.promises and if you want these to run sequentially, then use a for loop instead of .map():
async function readingDirectory(directory) {
const fileNames = await fs.promises.readdir(directory);
for (let file of fileNames) {
const absolutePath = path.join(directory, file);
log(absolutePath);
const data = await fs.promises.readFile(absolutePath);
log(data);
}
}
readingDirectory(folder).then(() => {
log("all done");
}).catch(err => {
log(err);
});

Nodejs download binary octet-stream

I am trying to download (meaning create an instance of the file on the server) a .pdf file from a server that returns it to me in binary format, with:
Content-Type = application / octet-stream.
After a bit of online research I came to write:
http.get(url.parse(pdfURL), res => {
let data = [];
console.log(res.statusCode);
res.on('data', chunk => {
data.push(chunk);
}).on('end', () => {
let buffer = Buffer.concat(data);
console.log(buffer.toString('base64'));
fs.open(path, 'w', (e, fd) => {
if (e) throw e;
fs.write(fd, buffer, 0, buffer.length, null, e => {
if (e) throw e;
fs.close(fd, () => console.log('Wrote successfully'));
});
});
});
});
Everything works properly, but when I try to open the generated pdf, it tells me that the file is corrupt and not readable.
Any idea what might have been wrong?
Thanks
Edit:
I noticed that with postman everything works as it should, so I think the way I treat the binary is wrong
Ok, i got it,
I wasn't de-gzipping the response, now works properly
This didn't work for me, tried so many different ways until I found got, an npm library that handles http requests, here's what worked for me:
const stream = require('stream');
const { promisify } = require('util');
const fs = require('fs');
const got = require('got');
const pipeline = promisify(stream.pipeline);
async function downloadImage(url, name) {
await pipeline(
got.stream(url),
fs.createWriteStream(name)
);
}
More info here: https://bleext.com/post/downloading-images-with-nodejs

Node download and save gif

I want to download a gif from the net and save it in local drive.
So far I have:
1) getFile function that returns the gif as Buffer:
import fs from 'fs'
import https from 'https'
let url = '//some gif url'
const getFile = function(url) {
return new Promise((res, rej) =>{
https.get(url, (resp) => {
let data = [];
resp.on('data', (chunk) => {
data.push(chunk);
});
resp.on('end', () => {
var buffer = Buffer.concat(data);
res(buffer)
});
}).on("error", (err) => {
rej(err);
});
})
}
2) Saving function:
async function save() {
let data = await getFile(url)
fs.writeFile('./test.gif', data, (e, r) => {
console.log(e,r) // null, undefined
})
}
save()
The file is saved and have contents but when i try to open it it says: Could not load image....
I tried to convert the output as utf8 encoding but it gives me the same result.
NOTE: I want to make it work with raw nodejs without 3-party tools or "pipe" to understand the working parts.
I guess the problem is with encoding but i could't figure what exactly it is.
Thanks

Upload synthesized speech from firebase function node.js server's tmp directory

I am trying to upload the audio returned by Google's Text-to-Speech API in a Firebase Function and having trouble writing the audio file to the Node.js server's temp directory. I receive the following error in my functions log:
Write ERROR: { Error: ENOENT: no such file or directory, open '/tmp/synthesized/output.mp3' at Error (native) errno: -2, code: 'ENOENT', syscall: 'open', path: '/tmp/synthesized/output.mp3' }
Here's my imports:
// Cloud Storage
import * as Storage from '#google-cloud/storage';
const gcs = new Storage();
import { tmpdir } from 'os';
import { join, dirname } from 'path';
import * as fs from 'fs';
import * as fse from 'fs-extra';
// Cloud Text to Speech
import * as textToSpeech from '#google-cloud/text-to-speech';
const client = new textToSpeech.TextToSpeechClient();
...and the part of my function I'm having trouble with:
// Construct the text-to-speech request
const request = {
input: { text: text },
voice: { languageCode: 'en-US', ssmlGender: 'NEUTRAL' },
audioConfig: { audioEncoding: 'MP3' },
};
// Creat temp directory
const workingDir = join(tmpdir(), 'synthesized');
const tmpFilePath = join(workingDir, 'output.mp3');
// Ensure temp directory exists
await fse.ensureDir(workingDir);
// Performs the Text-to-Speech request
client.synthesizeSpeech(request)
.then(responses => {
const response = responses[0];
// Write the binary audio content to a local file in temp directory
fs.writeFile(tmpFilePath, response.audioContent, 'binary', writeErr => {
if (writeErr) {
console.error('Write ERROR:', writeErr);
return;
}
// Upload audio to Firebase Storage
gcs.bucket(fileBucket).upload(tmpFilePath, {
destination: join(bucketDir, pageName)
})
.then(() => { console.log('audio uploaded successfully') })
.catch((error) => { console.log(error) });
});
})
.catch(err => {
console.error('Synthesize ERROR:', err);
});
What is wrong with my temp directory creation or fs.writeFile() function?
(Answer edited in response to question edit...)
In your original question, you invoked
client.synthesizeSpeech(request, (err, response) => {...})
following Node's http callback pattern, in which the callback function may initiate before the response is complete. Your subsequent code calls methods that assume response content; if the response is still empty, fs.writeFile() writes nothing initially, and subsequent methods cannot find the non-existent file. (Because fs.writeFile() follows the same callback pattern, you might even discover that output.mp3 file after the program exits, because fs will stream the input. But I bet your Firebase methods aren't waiting.)
The solution is to use Promises or async/await. Looking at the Google TextToSpeechClient class docs, it looks like the synthesizeSpeech method supports this:
Returns: Promise -> Array. The first element of the array is an object representing SynthesizeSpeechResponse.
Example:
client.synthesizeSpeech(request)
.then(responses => {
var response = responses[0];
// doThingsWith(response)
})
.catch(err => {
console.error(err);
});
That should solve the problem with client.synthesizeSpeech, but unfortunately fs.writeFile is still synchronous. If you were using Node >10 you could use a native fsPromise.writeFile method, and if you were using Node >8 you could use util.promisify() to convert fs.writeFile to promises. But you've indicated in comments that you are using Node 6, so we'll have to do things manually. Thieving from this reference:
const writeFilePromise = (file, data, option) => {
return new Promise((resolve, reject) => {
fs.writeFile(file, data, option, error => {
if (error) reject(error);
resolve("File created! Time for the next step!");
});
});
};
client.synthesizeSpeech(request)
.then(responses => {
const response = responses[0];
return writeFilePromise(tmpFilePath, response.audioContent, 'binary');
})
.then(() => {
return gcs.bucket(fileBucket).upload(tmpFilePath, {
destination: join(bucketDir, pageName)
});
})
.then(() => {
console.log('audio uploaded successfully');
return null;
})
.catch((error) => { console.log(error) });
I've written all of this using .then constructs, but naturally, you could also use async/await if you would rather do that. I hope this fixes things--it will force your Firebase code to wait until fs.writeFile has completed its job. I have also, unfortunately, smooshed all of the error checking into one final .catch block. And made things a bit verbose for clarity. I'm sure you can do better.

Resources