It worked yesterday, and now it stopped without any changes made to the code. What is going on?
Client
async function uploadFile(file) {
let formData = new FormData();
formData.append("recordUid", recordUid);
formData.append("fieldUid", fieldUid);
formData.append("file", file);
await fetchPostFormData("/api/files", formData);
}
async function fetchPostFormData(url, formData) {);
try {
let result = await (
await fetch(url, {
method: "POST",
withCredentials: true,
credentials: "include",
headers: {
Authorization: localStorage.getItem("token"),
},
body: formData,
})
).json();
return result;
} catch (err) {
return err;
}
}
Server
router.post("/api/files", async (req, res, next) => {
try {
console.log("starting upload..."); // <------------------- THIS ONE IS LOGGED
let bb = busboy({
headers: req.headers,
limits: {
fileSize: 20 * 1024 * 1024, // 20 mb
},
});
let fields = {};
// Get any text values
bb.on("field", (fieldname, val, fieldnameTruncated, valTruncated) => {
console.log("on.field", fieldname, val); // <------------------ NOT FIRING
fields[fieldname] = val;
});
bb.on("file", (fieldname, file, filename, encoding, mimetype) => {
console.log("on.file"); // <----------------------------------- NOT FIRING
let parts = filename.filename.split(".");
let name = parts[0];
let extension = parts[parts.length - 1];
let finalName = `${+new Date()}-${name}.${extension}`;
let filePath = `${filesFolderPath}${finalName}`;
// Open writeable stream to path
let writeStream = fs.createWriteStream(filePath);
// Pipe the file to the opened stream
file.pipe(writeStream);
// Check for errors
writeStream.on("error", (err) => {
console.log(err);
});
writeStream.on("close", async (err) => {
let sizeBytes = fs.statSync(filePath).size;
});
});
bb.on("finish", () => {
res.status(200).send({ success: true });
});
} catch (err) {
next(err);
}
});
Managed to solve it.
The problem was the missing req.pipe(bb) at the very end.
// previous code... ^^^^^
bb.on("finish", () => {
res.status(200).send({ success: true });
});
req.pipe(bb) // <------------- THIS SHIT RIGHT HERE
} catch (err) {
next(err);
}
});
Related
I read Pipe a stream to s3.upload()
but im having difficulty with I am not sure if that actually solves and I have tried.
What I am doing is a get call to www.example.com. this returns a stream, I want to upload that stream to s3.
heres my try.
fetch('https://www.example.com',fileName{
method: 'GET',
headers: {
'Authorization': "Bearer " + myAccessToken,
},
})
.then(function(response) {
return response.text();
})
.then(function(data) {
uploadToS3(data)
});
const uploadToS3 = (data) => {
// Setting up S3 upload parameters
const params = {
Bucket:myBucket,
Key: "fileName",
Body: data
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
output: ///File uploaded successfully. https://exampleBucket.s3.amazonaws.com/fileName.pdf
however this is blank.
I figured it out, but i did not keep using fetch.
and I actually download the file, then upload it. then delete the file.
function getNewFilesFromExampleDotCom(myAccessToken, fileName, fileKey) {
let url2 = 'https://example.com' + fileKey;
axios
.get(url2, {
headers: { 'Authorization': "Bearer " + myAccessToken },
responseType: 'stream',
})
.then(response => {
let file = fileName;
response.data.pipe(fs.createWriteStream(file))
let myFileInfo = [];
if( myFileInfo.length > 0){
myFileInfo.splice(0, myFileInfo.length)
}
myFileInfo.push(file)
processArray(myFileInfo)
console.log(file + " saved")
})
.catch(error => console.log(error));
}
async function processArray(array) {
for (const item of array) {
await delayedLog(item);
}
console.log('Downloaded!');
console.log('Uploading to s3!');
}
function delay() {
return new Promise(resolve => setTimeout(resolve, 300));
}
async function delayedLog(item) {
await delay();
uploadFiles(item)
}
async function uploadFiles(file){
uploadToS3List(file)
await new Promise((resolve, reject) => setTimeout(resolve, 1000));
deleteMyFiles(file)
}
const uploadToS3List = (fileName) => {
// Read content from the file
const fileContent = fs.readFileSync(fileName);
// Setting up S3 upload parameters
const params = {
Bucket:"myBucketName",
Key: fileName,
Body: fileContent
};
// Uploading files to the bucket
s3.upload(params, function(err, data) {
if (err) {
throw err;
}
console.log(`File uploaded successfully. ${data.Location}`);
});
};
function deleteMyFiles(path){
fs.unlink(path, (err) => {
console.log(path + " has been deleted")
if (err) {
console.error(err)
return
}
})
}
When I POST data to my app, I write it to a file and share it with the other instances via PUT. I want to return from POST with the status of each stream (the file and the PUTs).
putResults is an array that's part of the enclosing class, meant to hold the results of each request.
How do I collect the responses? I could return an array of Promises from createWriteStreams but then how could I req.pipe to them? Can you stream to a Promise?
post(req, res, next) {
let listeners = this.getWriteStreams();
let c = listeners.length;
for (i = 0; i < c; i++) {
req.pipe(listeners[i]);
}
/* What do I put here to return after all requests finish? */
}
put(req, res, next) {
var fstream = fs.createWriteStream(this.path);
req.pipe(fstream);
req.on('end', () => {
fstream.close();
res.status(201).send("OK");
});
req.on('error', (err) => {
res.status(500).send(err);
});
}
createWriteStreams() {
let listeners = [];
// We always want to save to a file
listeners.push(fs.createWriteStream(this.path).on('close', ()=>{
this.putResults.push({ host: hutil.myHost, status: 201 });
}));
// If there are other servers in current env, send to them, too!
let otherGuys = hostutil.otherServers();
if (otherGuys.length > 0) {
for (i = 0; i < otherGuys.length; i++) {
let opts = {
hostname: hutil.fq(otherGuys[i]),
port: this.port,
path: this.endpoint,
method: 'PUT',
};
let req = https.request(opts, res => {
this.putResults.push({ host: opts.hostname, status: res.statusCode});
});
req.on('error', (e) => {
this.putResults.push({ host: opts.hostname, status: e });
});
listeners.push(req);
}
}
return listeners;
}
Well, in case anyone ever has this question, the key point of knowledge is that an input stream can be passed around as if it has multiple custom spigots on it - opening one doesn't seem to prematurely spray data all over the other places you're placing the hose!
So, since you cannot stream to a Promise, you still stream to the streams, and you can apparently take your time setting them up. Here's my solution: pass the request to the streams wrapped in the promises.
function post(req, res, next) {
let promises = this.streamPromises(req);
Promise.allSettled(promises).then((results) => {
// Remove the Promise container junk - results come in 2 completely different object structures. Great design, jeez. :-\
let noContainer = results.map(item => item.value).filter(i => i != undefined);
noContainer.push(...results.map(item => item.reason).filter(i => i != undefined));
res.status(200).json(noContainer);
}).catch(err => {
log.warn(`POST request for ${this.filename} failed, at least in part: ${err}`)
res.status(200).json({ host: hutil.myHost, status: err });
});
}
function put(req, res, next) {
var fstream = fs.createWriteStream(this.fqFile);
req.pipe(fstream);
req.on('end', () => {
fstream.close();
log.info(`${req.transID} Saved data to ${this.fqFile} sent by ${req.referer}`);
res.status(201).send("OK");
});
req.on('error', (err) => {
log.warn(`${req.transID} Error receiving/saving PUT file ${this.fqFile} sent by ${req.referer}`);
res.status(500).send(err);
});
}
function streamPromises(postReq) {
let listeners = [];
listeners.push(this.streamLocalFrom(postReq)); // add file first
// Send to other servers in the environment
let otherGuys = hosts.peerServers();
if (otherGuys.length > 0) {
for (i = 0; i < otherGuys.length; i++) {
let opts = {
hostname: hosts.fq(otherGuys[i]),
thatHost: otherGuys[i], // ducked this into the object to avoid parsing fq hostname
port: appconfig.port, // we are all listening here
path: this.endpoint,
method: 'PUT',
timeout: 1000,
ca: fs.readFileSync(appconfig.authorityFile)
};
let p = new Promise((resolve, reject) => {
let req = https.request(opts, res => {
log.info(`${this.filename}: Response from ${opts.hostname}:${opts.port}: ${res.statusCode}`);
// let hn = opts.hostname.match(/(.*?)\./)[1] || opts.hostname;
resolve({ host: opts.thatHost, status: res.statusCode });
});
req.on('error', (e) => {
log.warn(`Error piping ${this.filename} to ${opts.hostname}:${opts.port}: ${e}`);
reject({ host: opts.thatHost, status: e });
});
postReq.pipe(req);
});
listeners.push(p);
}
}
return listeners;
}
function streamLocalFrom(postReq) {
return new Promise((resolve, reject) => {
let fileError = false;
let fstream = fs.createWriteStream(this.fqFile);
fstream.on('close', (err) => {
if (!fileError) {
log.info(`Saved data to file at ${this.fqFile}`);
resolve({ host: me, status: 201 });
}
});
fstream.on('error', (err) => {
log.warn(`Could not save ${this.fqFile} because ${err}`);
reject({ host: hutil.myHost, status: 500 });
fileError = true;
fstream.close();
});
postReq.pipe(fstream);
});
}
I have a AWS Lambda that first gets data from an external API endpoint and then will loop the json data and will send email to each of the records. Here is the code:
var aws = require("aws-sdk");
var ses = new aws.SES({ region: "us-east-1" });
const https = require('https');
exports.handler = async function(event, context) {
return httprequest().then((data) => {
const response = {
statusCode: 200,
body: JSON.stringify(data),
};
data.forEach(function(item) {
// send email
sendEmailAsync(item);
});
return response;
});
};
function sendEmailAsync(item) {
return new Promise((resolve, reject) => {
console.log(`Sending email to: ${item.fullName} `);
let emailFrom = process.env.emailFrom;
const htmlBody = `
<!DOCTYPE html>
<html>
<head>
</head>
<body>
<p>Hi ${item.fullName},</p>
<p>...</p>
</body>
</html>
`;
const textBody = `
Hi ${item.fullName},
...
`;
var params = {
Destination: {
ToAddresses: [item.email],
},
Message: {
Body: {
Html: {
Charset: "UTF-8",
Data: htmlBody
},
Text: {
Charset: "UTF-8",
Data: textBody
}
},
Subject: {
Charset: "UTF-8",
Data: "Test Email"
}
},
Source: emailFrom,
};
const sendPromise = ses
.sendEmail(params)
.promise();
sendPromise.then(data => {
console.log(data.MessageId);
context.done(null, "Success");
resolve(data.MessageId);
})
.catch(err => {
console.error(err, err.stack);
context.done(null, "Failed");
reject(err.message);
});
});
}
function httprequest() {
let host = process.env.endPointHost;
let path = process.env.endPointPath;
let apiKey = process.env.apiKey;
return new Promise((resolve, reject) => {
const options = {
host: host,
path: path + '?APIKey=' + apiKey,
method: 'GET',
headers: {
'Content-Type': 'application/json',
}
};
const req = https.request(options, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
return reject(new Error('statusCode=' + res.statusCode));
}
var body = [];
res.on('data', function(chunk) {
body.push(chunk);
});
res.on('end', function() {
try {
body = JSON.parse(Buffer.concat(body).toString());
}
catch (e) {
reject(e);
}
resolve(body);
});
});
req.on('error', (e) => {
reject(e.message);
});
// send the request
req.end();
});
}
Everything works fine except that when the sendEmailAsync(item); function is being called I get the console log but never get any result about the email being sent or not. Of course, I don't get any email but I'm pretty sure that the problem is that I have to await maybe the function inside the foreach statement but have tried some code and didn't work.
Any clue?
As the forEach won't give the expected result with the await function, we shall use Promise.all or for..of way of looping.
Let me give you the example by taking only the loop from your snippet.
Promise.all
await Promise.all(data.map(async (item) => {
try {
let res = await sendEmailAsync(item);
console.log(res)
} catch (error) {
console.log('Error: '+ error);
}
}))
for..of
for(let item of data) {
try {
let res = await sendEmailAsync(item);
console.log(res);
} catch (error) {
console.log('Error: ' + error);
}
}
I was trying to send .xlsx file using exceljs and nodejs to the client as a downloadable file. So far I'm able to create employees.xlsx file but it's not triggering save file dialog in the browser. Here's what I've tried in my backend API:
My Controller:
exports.exportEmployees = async (req, res) => {
try {
const employees = await Employee.find();
let workbook = new Excel.Workbook();
let worksheet = workbook.addWorksheet("Employees");
worksheet.columns = [
{ header: "Employee Name", key: "fullName", width: 40 },
{ header: "Department", key: "departmentName", width: 25 },
{ header: "Position", key: "positionName", width: 25 },
];
worksheet.addRows(employees);
res.setHeader(
"Content-Type",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
);
res.setHeader("Content-Disposition", "attachment; filename=employees.xlsx");
// This doesn't work either
// workbook.xlsx.write(res).then(function () {
// res.status(200).end();
// });
workbook.xlsx.writeFile("./employees.xlsx").then(
(response) => {
console.log("File is created"); // I'm able to see this in my console
console.log(path.join(__dirname, "../employees.xlsx"));
res.sendFile(path.join(__dirname, "../employees.xlsx"));
},
(err) => {
console.log("ERROR: ", err);
}
);
} catch (err) {
res.status(500).json({ errors: err });
}
};
And in my route.js
router.get("/employee-excel", auth, exportExcelController.exportEmployees);
module.exports = router;
So, the problem I'm facing now is whenever I call the api from my Angular app the browser response is a binary code.
Angular service to call the API
generateEmployeeExcel(query: any): Observable<any> {
return this.http.get(`${this.serverReportAPI}/employee-excel`, {
params: query,
headers: this.tokenHelperService.getAuthToken().headers,
});
}
My component
this.reportService.generateEmployeeExcel(query).subscribe(
(response) => {
console.log("Are we getting here?", response); // not able to get here
this.isLoading = false;
},
(err) => {
console.log("Is there any error?", err); // This is displayed in console
this.isLoading = false;
}
);
Any help would be appreciated.
The response in my browser
Change http code to :-
generateEmployeeExcel(query: any): Observable<any> {
return this.http.get(`${this.serverReportAPI}/employee-excel`, {
params: query,
headers: this.tokenHelperService.getAuthToken().headers,
responseType: 'blob'
});
}
For subscribe and download :-
this.reportService.generateEmployeeExcel(query).subscribe(
(res) => {
console.log("Are we getting here?", res); // not able to get here
this.isLoading = false;
let url = window.URL.createObjectURL(res.data);
let a = document.createElement('a');
document.body.appendChild(a);
a.setAttribute('style', 'display: none');
a.href = url;
a.download = "employee.xlsx";
a.click();
window.URL.revokeObjectURL(url);
a.remove();
},
(err) => {
console.log("Is there any error?", err); // This is displayed in console
this.isLoading = false;
}
);
If support for IE is required :-
this.reportService.generateEmployeeExcel(query).subscribe(
(res) => {
console.log("Are we getting here?", res); // not able to get here
if(window.navigator.msSaveBlob) {
window.nagigator.msSaveBlob(res, "employee.xlsx");
return;
}
this.isLoading = false;
let url = window.URL.createObjectURL(res.data);
let a = document.createElement('a');
document.body.appendChild(a);
a.setAttribute('style', 'display: none');
a.href = url;
a.download = "employee.xlsx";
a.click();
window.URL.revokeObjectURL(url);
a.remove();
},
(err) => {
console.log("Is there any error?", err); // This is displayed in console
this.isLoading = false;
}
);
I'm uploading a PDF file using AJAX and express and sending the data to external REST API for some file processing.
Whenever I upload more than 3MB file I will get an error in AJAX before the REST API response.
I get correct results from the external REST API, but processing takes more time so AJAX is not receiving the success msg.
How do I handle this?
Tried with AXIOS and other promise handler but same result
module.exports = (req, res) =>
{
var pdf = [],
form = new formidable.IncomingForm(),
pdfFile,
dirname = process.cwd();
form.multiples = false;
// Upload directory for the images
form.uploadDir = path.join(dirname, 'tmp_uploads');
form.on('file', function(name, file) {
let token;
console.log('FORM ON')
if (req.cookies.userData.token) {
token = req.cookies.userData.token;
let buffer = null,
type = null,
filename = '',
renameFile = '';
// Read a chunk of the file.
buffer = readChunk.sync(file.path, 0, 262);
// filename = Date.now() + '-' + file.name;
filename = file.name;
renameFile = path.join(dirname, 'uploads/' + filename);
fs.rename(file.path, renameFile, function(err) {
if (err) throw err;
console.log('renamed complete');
pdfFile = path.join(dirname, 'uploads/' + filename);
let readFileStream = fs.createReadStream(pdfFile),
data = '',
formData = {};
formData = {
name: filename,
my_file: readFileStream,
Width: 1024,
Height: 768
};
function postData() {
// Setting URL and headers for request
var options = {
url: EXTERNALURL,
headers: {
"Authorization": 'bearer ' + token,
"content-type": "multipart/form-data"
},
formData: formData
};
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.post(options, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
function getData(url) {
// Return new promise
return new Promise(function(resolve, reject) {
// Do async job
request.get({ url: url, encoding: null }, function(err, resp, body) {
if (err) {
reject(err);
} else {
resolve(body);
}
})
})
}
var errHandler = function(err) {
console.log(err);
}
var filePath;
function main() {
var dataPromise = postData();
// Get user details after that get followers from URL
dataPromise.then(JSON.parse, errHandler)
.then(function(result) {
fData = result;
var fileName = fData.Id,
file_url = fData.PresentationZipUrl;
filePath = path.join(dirname, fData.HtmlPath);
// Do one more async operation here
var downloadPromise = getData(file_url).then();
return downloadPromise;
}, errHandler)
.then(function(data) {
//console.log(data);
var zip = new AdmZip(data);
if (!fs.existsSync(filePath)) {
fs.mkdirSync(filePath);
zip.extractAllTo(filePath, true);
}
}, errHandler)
.then(function(data) {
console.log('Done');
res.status(200).json({ 'status': "success" });
}, errHandler);
}
// console.log('before');
main();
}); // END RENAME FILE
} else { //
console.log('ERROR')
res.redirect('/');
}
});
form.on('error', function(err) {
console.log('Error occurred during processing - ' + err);
});
// Invoked when all the fields have been processed.
form.on('end', function() {
console.log('All the request fields have been processed.');
});
// Parse the incoming form fields.
form.parse(req, function(err, fields, files) {})}
AJAX:
$.ajax({
enter code here
url: '/presentation/uploadFiles',
method: 'post',
data: formData,
processData: false,
contentType: false,
timeout: 0}).done(function(d) {
console.log(d);
if (d.status == 'success') {
location.reload()
}}).fail(function(e, t) {
console.log(e);
console.log(t)})
.always(function() {
});