var uploadMedia = (String route, Map<String, String> stuff, XFile media) async {
var url = Uri.parse('$baseUrl$route');
final request = http.MultipartRequest("POST", url);
request.headers.addAll({
'Content-Type': 'application/json',
});
request.fields['fileName'] = 'test';
request.files.add(await http.MultipartFile.fromPath("job", media.path));
final response = await request.send();
print(response.statusCode); // Use the response data here
};
this is my node
const upload = multer({ dest: "uploads/" });
router.post("/media", upload.single("job"), async (req, res) => {
const fileName = req.body.fileName;
const filePath = req.file?.size;
console.log(`Received file ${fileName} at ${filePath}`);
res.send("File uploaded successfully");
res.send("hi :)");
});
I see the request has been received but console.log returns undefined and I get errors. How can I upload the file to my node.js directory?
Related
I've been trying to save uploaded image files to IPFS in NodeJs , while it seems Pinata saves them, the files are pretty much gibberish (after downloading the images are broken).
My code :
// Nodejs route.
exports.postImage = async (req, res, next) => {
// Using multer to get the file.
fileUploadMiddleware(req, res, async (err) => {
// getting bunch of data from query string.
let meta = {
origin,
originid,
context,
ownerid,
format
} = req.query;
if(!meta.format || !req.files) {
return next(new ErrorResponse("File format not specified", 404));
}
if(!meta.originid) {
meta.originid = uuidv4();
}
// NOTE: is this the right way to get the data of the file ?
const buffer = req.files[0].buffer;
const filename = `${metadata.origin}_${metadata.originid}.${ metadata.format }`;
let stream;
try {
stream = Readable.from(buffer);
// HACK to make PINATA WORK.
stream.path = filename;
}
catch(e) {
logger.logError(e);
return false;
}
const options = {
pinataMetadata: {
name: filename,
keyvalues: {
context: metadata.context,
ownerid: metadata.ownerid
}
},
pinataOptions: {
cidVersion: 0
}
};
try {
var result = await pinata.pinFileToIPFS(stream, options);
console.log("SUCCESS ", result);
return result;
}
catch(e) {
logger.logError(e);
return null;
}
res.status(200).json({
success: true,
data: 'You got access'
})
});
}
So basically creating the stream based on the uploaded file buffer and sending it away to Pinata. Where do I go wrong?
const buffer = req.files[0].buffer;
If you used MemoryStorage. buffer property would be available. It is not available for diskStorage because it will save the file locally.:
const storage = multer.memoryStorage()
const upload = multer({ storage: storage })
Also I think it not req.files[0]
const buffer = req.file.buffer;
after I get the buffer, I convert it to FormData using form-data npm package:
import FormData from "form-data";
const formData = new FormData();
formData.append("file", buffer, {
contentType,
filename: fileName + "-" + uuidv4(),
});
then you send a post request to pinata
const url = `https://api.pinata.cloud/pinning/pinFileToIPFS`;
const fileRes = await axios.post(url, formData, {
maxBodyLength: Infinity,
headers: {
// formData.getBoundary() is specific to npm package. native javascript FormData does not have this method
"Content-Type": `multipart/form-data: boundary=${formData.getBoundary()}`,
pinata_api_key: pinataApiKey,
pinata_secret_api_key: pinataSecretApiKey,
},
});
var title="this is title";
var content="this is content";
const config = { headers: {'Accept': 'application/json', 'Content-Type': 'multipart/form-data' } };
const form = new FormData()
let file =event.target.files[0]
form.append('file', file)
form.append('title', title)
form.append('content', content)`enter code here`
Axios.post("http://localhost:3001/article/get/123", form,config ).then((res)=>{
console.log(res.data)
})
in node I have used multer for upload image or anything.
Below is the code for upload which I have used as a middleware.
const util = require("util");
const path = require("path");
const multer = require("multer");
const storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, "./Uploads") // folder path where to upload
},
filename: function (req, file, cb) {
cb(null, file.originalname + "-" + Date.now() + path.extname(file.originalname))
}
});
const maxSize = 1 * 20000 * 20000; // file size validation
const uploadFiles = multer({ storage: storage, limits: { fileSize: maxSize } }).array("myfiles", 10); // key name should be myfiles in postman while upload
const uploadFilesMiddleware = util.promisify(uploadFiles);
module.exports = uploadFilesMiddleware;
Below is the function which I have created in controller for upload and file check.
fileUpload = async (req, res) => {
try {
let userCode = req.headers.user_code;
await upload(req, res);
if (req.files.length <= 0) {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Kindly select a file to upload..!!', true));
}
let response = [];
for (const element of req.files) {
let data = await service.addFileData(element, userCode);
response.push(data); // for file path to be stored in database
}
if (response && response.length > 0) {
return res.status(httpStatusCode.OK).send(responseGenerators(response, httpStatusCode.OK, 'File uploaded sucessfully..!!', false));
} else {
return res.status(httpStatusCode.OK).send(responseGenerators({}, httpStatusCode.OK, 'Failed to upload file kindly try later..!!', true));
}
} catch (error) {
logger.warn(`Error while fetch post data. Error: %j %s`, error, error)
return res.status(httpStatusCode.INTERNAL_SERVER_ERROR).send(responseGenerators({}, httpStatusCode.INTERNAL_SERVER_ERROR, 'Error while uploading file data', true))
}
}
and the route will go like this.
router.post('/upload/file', fileUploadController.fileUpload);
And be sure to keep same name in postman while file upload as in middleware.
The above code is in react.js. I want to do same work in node.js and the file will be upload from the public folder. main issue is how to upload image file in format like we have in frontend event.target.files[0]
Hey i work with multer package in node js to upload file.
I make it dynamic but the request is in get and i have problem to change it to post.
Its only work to me in get request.
What it doing now?
It save the file in new folder that call flow and take the id from the client.
I need to replace it to post and get the id in post but i want to move the logic to the route and make it stay same. thanks
My Code:
fileupload service:
const path = require("path");
const multer = require("multer");
const crypto = require("crypto");
const models = require("../models");
const fsex = require("fs-extra");
const today = new Date();
const date =
today.getFullYear() + "-" + (today.getMonth() + 1) + "-" + today.getDate();
const storage = multer.diskStorage({
destination: (req, file, callback) => {
let flowID = req.params.flowID;
let path = `./uploads/flow/${flowID}`;
fsex.mkdirsSync(path);
callback(null, path);
},
filename: (req, file, cb) => {
crypto.pseudoRandomBytes(8, (err, raw) => {
if (err) return cb(err);
cb(
null,
date + "_" + raw.toString("hex") + "_" + path.extname(file.originalname)
);
});
}
});
const upload = multer({ storage: storage }).single("upload");
module.exports = {
upload: upload,
};
my route:
router.get("/uploadfile/:flowID", upload, (req, res, next) => {
const file = req.file;
const flowID = req.params.flowID;
if (!file) {
const error = new Error("Please upload a file");
error.httpStatusCode = 400;
return next(error);
} else {
createFileInDB(file.originalname, flowID, file.filename)
.then(() => {
console.log("File Created");
res.json(file);
})
.catch(err => {
res.status(500).send(err);
});
}
});
I want to stream a file upload request in multipart/form-data to another server and change some fields name at the same time.
I don't want to store temporarily a file on disk and don't want to store the file completely in memory either.
I tried to use multer, busboy and multiparty. I think I got closer by using custom Transform streams but it is not working yet.
const express = require('express');
const request = require('request');
const { Transform } = require('stream');
const router = express.Router();
class TransformStream extends Transform {
_transform(chunk, encoding, callback) {
// here I tried to manipulate the chunk
this.push(chunk);
callback();
}
_flush(callback) {
callback();
}
}
router.post('/', function pipeFile(req, res) {
const transformStream = new TransformStream();
req.pipe(transformStream).pipe(request.post('http://somewhere.com'));
res.sendStatus(204);
});
I tried to manipulate chunks in _transform without success (EPIPE). It sounds quit hacky, are they any better solutions ?
Here is a solution using replacestream along with content-disposition.
const replaceStream = require('replacestream');
const contentDisposition = require('content-disposition');
router.post('/', function pipeFile(req, res) {
let changeFields = replaceStream(/Content-Disposition:\s+(.+)/g, (match, p1) => {
// Parse header
let {type, parameters} = contentDisposition.parse(p1);
// Change the desired field
parameters.name = "foo";
// Prepare replacement
let ret = `Content-Disposition: ${type}`;
for(let key in parameters) {
ret += `; ${key}="${parameters[key]}"`;
}
return ret;
})
req.pipe(changeFields)
.pipe(request.post('http://somewhere.com'))
.on('end', () => {
res.sendStatus(204);
});
});
This worked for a single file multipart upload using express, multiparty, form-data, pump and got.
const stream = require('stream');
const express = require('express');
const multiparty = require("multiparty");
const got = require("got");
const FormData = require('form-data');
const pump = require('pump');
const app = express();
app.post('/upload', (req, res) => {
const url = "<<multipart image upload endpoint>>";
var form = new multiparty.Form();
form.on("part", function(formPart) {
var contentType = formPart.headers['content-type'];
var formData = new FormData();
formData.append("file", formPart, {
filename: formPart.filename,
contentType: contentType,
knownLength: formPart.byteCount
});
const resultStream = new stream.PassThrough();
try {
// Pipe the formdata to the image upload endpoint stream and the result to the result stream
pump(formData, got.stream.post(url, {headers: formData.getHeaders(), https:{rejectUnauthorized: false}}), resultStream, (err) =>{
if(err) {
res.send(error);
}
else {
// Pipe the result of the image upload endpoint to the response when there are no errors.
resultStream.pipe(res);
}
resultStream.destroy();
});
}
catch(err) {
resultStream.destroy();
console.log(err);
}
});
form.on("error", function(error){
console.log(error);
})
form.parse(req);
});
My goal:
Display a dialog box prompting the user to save a file being downloaded from aws.
My problem:
I am currently using awssum-amazon-s3 to create a download stream. However I've only managed to save the file to my server or stream it to the command line... As you can see from my code my last attempt was to try and manually set the content disposition headers which failed. I cannot use res.download() as the headers have already been set?
How can I achieve my goal?
My code for node:
app.post('/dls/:dlKey', function(req, res, next){
// download the file via aws s3 here
var dlKey = req.param('dlKey');
Dl.findOne({key:dlKey}, function(err, dl){
if (err) return next(err);
var files = dl.dlFile;
var options = {
BucketName : 'xxxx',
ObjectName : files,
};
s3.GetObject(options, { stream : true }, function(err, data) {
// stream this file to stdout
fmt.sep();
data.Headers['Content-Disposition'] = 'attachment';
console.log(data.Headers);
data.Stream.pipe(fs.createWriteStream('test.pdf'));
data.Stream.on('end', function() {
console.log('File Downloaded!');
});
});
});
res.end('Successful Download Post!');
});
My code for angular:
$scope.dlComplete = function (dl) {
$scope.procDownload = true;
$http({
method: 'POST',
url: '/dls/' + dl.dlKey
}).success(function(data/*, status, headers, config*/) {
console.log(data);
$location.path('/#!/success');
}).error(function(/*data, status, headers, config*/) {
console.log('File download failed!');
});
};
The purpose of this code it to let users use a generated key to download a file once.
This is the entire code using streaming on the latest version of aws-sdk
var express = require('express');
var app = express();
var fs = require('fs');
app.get('/', function(req, res, next){
res.send('You did not say the magic word');
});
app.get('/s3Proxy', function(req, res, next){
// download the file via aws s3 here
var fileKey = req.query['fileKey'];
console.log('Trying to download file', fileKey);
var AWS = require('aws-sdk');
AWS.config.update(
{
accessKeyId: "....",
secretAccessKey: "...",
region: 'ap-southeast-1'
}
);
var s3 = new AWS.S3();
var options = {
Bucket : '/bucket-url',
Key : fileKey,
};
res.attachment(fileKey);
var fileStream = s3.getObject(options).createReadStream();
fileStream.pipe(res);
});
var server = app.listen(3000, function () {
var host = server.address().address;
var port = server.address().port;
console.log('S3 Proxy app listening at http://%s:%s', host, port);
});
This code worked for me with the most recent library:
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'your bucket',
Key: 'path/to/the/file.ext'
};
s3.getObject(s3Params, function(err, res) {
if (err === null) {
res.attachment('file.ext'); // or whatever your logic needs
res.send(data.Body);
} else {
res.status(500).send(err);
}
});
Simply create a ReadStream from S3 and WriteStream to the location were u want to download. Find the code below. Works perfectly for me:
var AWS = require('aws-sdk');
var path = require('path');
var fs = require('fs');
AWS.config.loadFromPath(path.resolve(__dirname, 'config.json'));
AWS.config.update({
accessKeyId: AWS.config.credentials.accessKeyId,
secretAccessKey: AWS.config.credentials.secretAccessKey,
region: AWS.config.region
});
var s3 = new AWS.S3();
var params = {
Bucket: '<your-bucket>',
Key: '<path-to-your-file>'
};
let readStream = s3.getObject(params).createReadStream();
let writeStream = fs.createWriteStream(path.join(__dirname, 's3data.txt'));
readStream.pipe(writeStream);
You've already figured what's most important to solve your issue: you can pipe the file stream coming from S3 to any writable stream, be it a filestream… or the response stream that will be sent to the client!
s3.GetObject(options, { stream : true }, function(err, data) {
res.attachment('test.pdf');
data.Stream.pipe(res);
});
Note the use of res.attachment that will set the correct headers. You can also check out this answer regarding streams and S3.
Using aws SDK v3
npm install #aws-sdk/client-s3
download code
import { GetObjectCommand } from "#aws-sdk/client-s3";
/**
* download a file from AWS and send to your rest client
*/
app.get('/download', function(req, res, next){
var fileKey = req.query['fileKey'];
var bucketParams = {
Bucket: 'my-bucket-name',
Key: fileKey,
};
res.attachment(fileKey);
var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
// for TS you can add: if (fileStream.Body instanceof Readable)
fileStream.Body.pipe(res)
});
For this I use React frontend and node js backend. Frontend I use Axios. I used this click the button download file.
==== Node js backend code (AWS S3) ======
//inside GET method I called this function
public download = (req: Request, res: Response) => {
const keyName = req.query.keyName as string;
if (!keyName) {
throw new Error('key is undefined');
}
const downloadParams: AWS.S3.GetObjectRequest = {
Bucket: this.BUCKET_NAME,
Key: keyName
};
this.s3.getObject(downloadParams, (error, data) => {
if (error) {
return error;
}
res.send(data.Body);
res.end();
});
};
====== React js frontend code ========
//this function handle download button onClick
const downloadHandler = async (keyName: string) => {
const response = await axiosInstance.get( //here use axios interceptors
`papers/paper/download?keyName=${keyName}`,{
responseType:'blob', //very very important dont miss (if not downloaded file unsupported to view)
}
);
const url = window.URL.createObjectURL(new Blob([response.data]));
const link = document.createElement("a");
link.href = url;
link.setAttribute("download", "file.pdf"); //change "file.pdf" according to saved name you want, give extension according to filetype
document.body.appendChild(link);
link.click();
link.remove();
};
------ OR (if you are using normal axios and not axios interceptors) -----
axios({
url: 'http://localhost:5000/static/example.pdf',
method: 'GET',
responseType: 'blob', // very very important
}).then((response) => {
const url = window.URL.createObjectURL(new Blob([response.data]));
const link = document.createElement('a');
link.href = url;
link.setAttribute('download', 'file.pdf');
document.body.appendChild(link);
link.click();
});
For more refer below articles
1. article 1
2. article 2
Using express, based on Jushua's answer and https://docs.aws.amazon.com/AmazonS3/latest/userguide/example_s3_GetObject_section.html
public downloadFeedFile = (req: IFeedUrlRequest, res: Response) => {
const downloadParams: GetObjectCommandInput = parseS3Url(req.s3FileUrl.replace(/\s/g, ''));
logger.info("requesting S3 file " + JSON.stringify(downloadParams));
const run = async () => {
try {
const fileStream = await this.s3Client.send(new GetObjectCommand(downloadParams));
if (fileStream.Body instanceof Readable){
fileStream.Body.once('error', err => {
console.error("Error downloading s3 file")
console.error(err);
});
fileStream.Body.pipe(res);
}
} catch (err) {
logger.error("Error", err);
}
};
run();
};