storing file with Large Object Postgres with Nodejs - node.js

I have a simple HTML form with a file input. I need to save the file with Postgres Large Object but I'm having issues with the fileStream.pipe. It should be in a Promise itself like so:
return new Promise((resolve, reject) => {
stream.on('finish', resolve(oid));
stream.on('error', reject);
});
But it wouldn't work since it would result into a Promise in a Promise. The complete functions are:
function storeLargeObject(path) {
return new Promise((resolve, reject) => {
pgp.db.tx(tx => {
const man = new LargeObjectManager({pgPromise: tx});
return man.createAndWritableStreamAsync(bufferSize);
}).then(([oid, stream]) => {
const fileStream = createReadStream(path);
fileStream.pipe(stream);
stream.on('finish', resolve(oid));
stream.on('error', reject);
}).catch(err => {
console.log(err);
return err;
});
});
}
const addBinary = (req, res) => {
pgp.db.oneOrNone({
text: insertQuery,
values: values
}).then(f => {
let explFile = req.files.bin;
let uploadPath = __dirname + '/' + f.id;
if (!req.files || Object.keys(req.files).length === 0) {
res.status(400).send('No File was uploaded.');
return;
}
explFile.mv(uploadPath, async function (err) {
if (err)
return res.status(500).send(err);
let newOid = await storeLargeObject(uploadPath);
fs.unlink(uploadPath, err => console.log(err));
coupleIds(exp.id, newOid);
return res.status(200).send({
status: "success",
message: "File successfully uploaded!"
})
});
}).catch(err => {
return res.status(401).send({
status: 'error',
errorCode: 102,
message: 'Error! ' + err
});
});
}

Related

how to send response only after mv library on nodejs completes. Wrapping in promise doesn't work

I'm trying to setup an endpoint that takes a file through a multipart post request, and saves it into a specific directory using formidable and https://github.com/andrewrk/node-mv. And then upon completion of saving all of the files, I want to respond with a list of all of the files in that directory for rendering. the thing is the response seems to be sent before the directory listing is updated. I tried wrapping the mv operations into a promise and then responding in a then block to no avail. Any help would be much appreciated!
app.post("/api/v1/vendor/:id/menu", (req, res, next) => {
const id = req.params.id;
const form = formidable({ multiples: true, keepExtensions: true });
form.parse(req, (err, fields, files) => {
if (err) {
next(err);
return;
}
if (!Array.isArray(files.image)) {
files = [files.image];
}
let filelist;
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});
Promise.all(proms).then(() => {
console.log('now reading dir...');
filelist = fs.readdirSync("pics/" + id);
res.send(filelist);
});
});
});
I think we're missing the return keywork before new Promise. You can check the proms variable if it contains the list of promises or not.
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});
For me, it should be :
const proms = files.map((file) => {
const dst = `pics/${id}/${file.name}`;
return new Promise((resolve, reject) => {
mv(file.path, dst, { mkdirp: true }, (err) => {
if (err) {
console.error("error: ", err.status);
reject(err);
}
console.log("done moving");
resolve();
});
});
});

Send KnexJS error message to the frontend

I am having trouble sending an error to the front end when a csv file is uploaded and the numbers already exist in the database. The backend is logging an error that the primary key value already exist, but the code I have written tells the front end that the file uploaded just fine.
Code snippet:
router.post('/:program/upload', upload.single('testUpload'), (req, res, next) => {
try {
CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json({
message: 'File Uploaded Just fine :)'
});
} catch (error) {
return res.status(500).json({
message: error
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', () => {
csvData.forEach(item => {
queries.upload(program, item)
.then(() => {
console.log('QR Code Added: ' + item);
}).catch((err) => {
console.log(`oopsie: ${err}`);
});
})
});
stream.pipe(csvStream);
}
Pretty confident the issue is with my poor understanding of promises.
As expected, I wasn't handling my promises correctly. I've updated the code a bit and now it responds with 2 arrays of successful uploads and errored uploads.
router.post('/:program/upload', upload.single('testUpload'), async (req, res, next) => {
try {
const result = await CSVtoPSQL(req.params.program, req.file.filename)
return res.status(201).json(result);
}
catch (error) {
return res.status(500).json({
message: error,
})
}
});
const CSVtoPSQL = (program, filePath) => {
let stream = fs.createReadStream(path.resolve(__dirname, '../files', filePath));
let csvData = [];
return new Promise((resolve) => {
const results = {
seccess: [],
error: [],
}
let csvStream = csv
.parse({ headers: false })
.on('error', error => console.error(error))
.on('data', (data) => {
csvData.push(data.toString());
})
.on('end', async () => {
await Promise.all(
csvData.map(async (item) => {
try{
await queries.upload(program, item);
results.success.push(item);
console.log('QR Code Added: ' + item);
}
catch (error) {
console.log(`oopsie: ${error}`)
results.error.push(item);
}
})
)
resolve(results);
});
stream.pipe(csvStream);
})
}

My program writes the data to file successfully, but the URL data needs to be checked by my link checker. Node.js

https://github.com/strawberries73/OSD600-Journey-on-rocky-trails
I need the data converted to show URLs only for my Link Check to work. I have tried another solution but the new code solution deletes the conversion before it gets read. If there is a way to pull the URL's and get them to be written to a different file or the same file, my link checker would work.
// ExitCode
process.on("SIGTERM", () => {
server.close(() => {
console.log("Program is terminated");
});
});
//Flag labeling each URL as a good or bad
const _label = ({
good: "GOOD",
bad: "BAD"
});
//const getTelescope = fetch('http:/') data
async function getTelescopeData(){
//fetching data from local host
fetch("http://localhost:3000/posts").then(response => {
return response.json();
}).then(data => {
console.log(data);
//truncate the data and write to file
fs.truncate("telescopeData.txt", 0, function() {
for(i = 0; i < data.length; i++) {
fetch(`http://localhost:3000${data[i].url}`)
.then(res => {
return res.json();
}).then(telescopeData => {
//Append to telescopeData.txt
fs.appendFile("telescopeData.txt", telescopeData.html,
(err) => {
if(err) {
console.log(err)
.then.process(); //Termination
}
});
});
}
});
}
);
}
getTelescopeData(); //Data fetched
if(process.argv.length==2){
greetingMessage();
console.log("Hello");
}
else{
const filePath=path.join(__dirname,process.argv[2]);
fs.readFile(filePath,"utf-8",(err,data)=>{
if(err){
console.log("Fail to read file",err);
}
else{
const validUrl=data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
validUrl.forEach((url)=>{
fetch(url,{method:"HEAD",timeout:2000})
.then((res)=>{
if(res.status==200)
console.log(res.status,url.green,_label.good.rainbow);
else if(res.status==400||res.status==404)
console.log(res.status,url);
else console.log(res.status,url)
.then.process(); //Termination
})
.catch((error)=>{
console.log("404",url.red, _label.bad.bgRed);
});
});
}
});
}
Output of Data in the console
New Code tried, only it deletes the input after it has been written to file.
// ExitCode
process.on("SIGTERM", () => {
server.close(() => {
console.log("Program is terminated");
});
});
//Flag labeling each URL as a good or bad
const _label = ({
good: "GOOD",
bad: "BAD"
});
//const getTelescope = fetch('http:/') data
async function getTelescopeData() {
//fetching data from local host
fetch("http://localhost:3000/posts").then(response => {
return response.json();
}).then(data => {
console.log(data);
//truncate the data and write to file
fs.truncate("telescopeData.txt", 0, function () {
for (i = 0; i < data.length; i++) {
fetch(`http://localhost:3000${data[i].url}`)
.then(res => {
return res.json();
}).then(telescopeData => {
//Append to telescopeData.txt
fs.appendFile("telescopeData.txt", telescopeData.html,
(err) => {
if (err) {
console.log(err)
.then.process(); //Termination
}
});
});
}
});
// its not reading the file for some reason. File is empty when I add this.
fs.readFile("telescopeData.txt", "utf-8", (err, data) => {
if (err) {
console.log("Fail to read file", err);
}
else {
console.log(data);
console.log("^^^");
const validUrl = data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
console.log(validUrl);
console.log("^^^");
validUrl.forEach((url) => {
fetch(url, { method: "HEAD", timeout: 2000 })
.then((res) => {
if (res.status == 200)
console.log(res.status, url.green, _label.good.rainbow);
else if (res.status == 400 || res.status == 404)
console.log(res.status, url);
else console.log(res.status, url)
.then.process(); //Termination
})
.catch((error) => {
console.log("404", url.red, _label.bad.bgRed);
});
});
}
});
} // then ends
);
}
if (process.argv.length == 2) {
greetingMessage();
console.log("Hello");
} else if (process.argv[2] == "--telescope") {
console.log("Telescope");
getTelescopeData();
}
else {
const filePath = path.join(__dirname, process.argv[2]);
fs.readFile(filePath, "utf-8", (err, data) => {
if (err) {
console.log("Fail to read file", err);
}
else {
const validUrl = data.match(/(http|https)(:\/\/)([\w+\-&#`~#$%^*.=/?:]+)/gi);
validUrl.forEach((url) => {
fetch(url, { method: "HEAD", timeout: 2000 })
.then((res) => {
if (res.status == 200)
console.log(res.status, url.green, _label.good.rainbow);
else if (res.status == 400 || res.status == 404)
console.log(res.status, url);
else console.log(res.status, url)
.then.process(); //Termination
})
.catch((error) => {
console.log("404", url.red, _label.bad.bgRed);
});
});
}
});
}
async function getTelescopeData(body, url){
//fetching data from local host
const localRegex= /https?:\/\/localhost:[0-9]*/;
const baseurl=url.match(localRegex);
return JSON.parse(body).map((i)=>{return baseurl[0]+i.url;
})
}

Error [ERR_STREAM_CANNOT_PIPE]: Cannot pipe, not readable nodejs

function unzipCode() {
console.log('Unzipping contents...');
return new Promise((resolve, reject) => {
const files = [];
unzip.open(filePath, { autoclose: false, lazyEntries: true }, (err, zipfile) => {
if (err) reject;
zipfile.readEntry();
zipfile.on('entry', (entry) => {
if (!sourceDirectoryRegEx.test(entry.fileName)) {
console.log(" [X] Skipping: " + entry.fileName);
zipfile.readEntry();
}
else {
console.log(" [+] Unzipping: " + entry.fileName);
if (/\/$/.test(entry.fileName)) {
// directory file names end with '/'
mkdirp(path.join(cwd, entry.fileName), (err) => {
if (err) reject;
zipfile.readEntry();
});
}
else {
zipfile.openReadStream(entry, (err, readStream) => {
if (err) reject;
// ensure parent directory exists
mkdirp(path.join(cwd, path.dirname(entry.fileName)), (err) => {
if (err) reject;
let stream = fs.createWriteStream(path.join(cwd, entry.fileName));
readStream.pipe(stream, { end: false });
// console.log(readStream.pipe);
readStream.on('end', () => {
console.log('After write', entry.fileName);
// add file details to files array
files.push({
key: entry.fileName,
body: stream,
});
zipfile.readEntry();
readStream.unpipe(stream);
stream.end();
});
});
});
}
}
});
zipfile.once('end', () => {
zipfile.close();
resolve(files);
});
});
});
}
I am trying to unzip some files in aws-lambda console using the function. The enviornment I've chosen is nodejs 12.x. It was running with nodejs 8.10.
The exact error I'm encountering is Cannot pipe, not readable.
How to solve it?

Move file in Nodejs if file exists otherwise create file

I am trying to move a file from a location abc to location xyz if the file already exists in location xyz delete it then save the new one.
Here is my code
const promises = {
deleteFile: path => {
return new Promise((resolve, reject) => {
const fs = require('fs');
fs.stat(path, (err, stat) => {
if (err === null) {
fs.unlink(path, err => {
if (err) { return reject(err) }
resolve();
})
} else if(err.code == 'ENOENT') {
console.log('File does not exist');
resolve();
} else {
reject(err);
}
});
});
},
copyFile: (from, to) => {
return new Promise((resolve, reject)=> {
copyFile(from, to, (err) => {
if (err) { return reject(err); }
console.log(`Finished writing file ${to}`);
resolve();
})
})
}
}
const copyFile = (from, to, overallCb) => {
const fs = require('fs');
const rs = fs.createReadStream(from)
const ws = fs.createWriteStream(to)
let cbCalled = false;
function done (err) {
overallCb(err);
cbCalled = true;
}
rs.on('error', (err) => {
done(err);
})
ws.on('error', (err) => {
done(err);
})
rs.pipe(ws);
}
;
const OUTPUT_PATH = `./js/libs/`
, _NODE_MODULES = './node_modules/'
, filePath = `${_NODE_MODULES}somePathToAFile`
;
promises.deleteFile(`${OUTPUT_PATH}someFile.min.js`)
.then(promises.copyFile(filePath, `${OUTPUT_PATH}someFile.min.js`))
.then(words => {
console.log('**** done doing things ****');
})
.catch(error => { console.log(`ERROR`, error); })
If the file exists it just simply deletes the file and does nothing else.
If the file DOES NOT exist everything works fine.
Any idea on what im doing wrong?
I had some promise errors in my code... For future me here is the working version of the above code
'use strict';
const promises = {
deleteFile: path => {
return new Promise((resolve, reject) => {
const fs = require('fs');
fs.stat(path, (err, stat) => {
if (err === null) {
fs.unlink(path, err => {
if (err) { return reject(err) }
resolve(`Removing document at ${path}`);
})
} else if(err.code === 'ENOENT') {
resolve('File does not exist');
} else {
reject(err);
}
});
});
},
copyFile: (from, to) => {
return new Promise((resolve, reject) => {
copyFile(from, to, (err) => {
if (err) { return reject(err); }
resolve(`Finished writing file ${to}`);
})
})
}
}
const copyFile = (from, to, overallCb) => {
const fs = require('fs');
const rs = fs.createReadStream(from)
const ws = fs.createWriteStream(to)
let cbCalled = false;
function done (err) {
overallCb(err);
cbCalled = true;
}
rs.on('error', done)
ws.on('error', done)
.on('close', done)
rs.pipe(ws);
}
;
const OUTPUT_PATH = './js/libs/'
, _NODE_MODULES = './node_modules/'
, filePath = `${_NODE_MODULES}somePathToAFile`
;
promises.deleteFile(`${OUTPUT_PATH}someFile.min.js`)
.then(msg => {
console.log(msg)
return promises.copyFile(filePath, `${OUTPUT_PATH}someFile.js`)
})
.then(msg => {
console.log(msg)
})
.catch(err => { // Do errory things}

Resources