is it possible for the client to directly push a file onto our servers via API? How about pushing a file from our server to another again via API?
Right now my servers go out and grab files from locations via 'get'. This isn't the most efficient potentiality and there are reasons to believe I won't be able to access all files directly via get.
For sending you can use the request module's form feature to upload files.
var formData = {
my_file: fs.createReadStream(__dirname + '/unicycle.jpg'),
};
request.post({url:'http://service.com/upload', formData: formData}, function optionalCallback(err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
});
For receiving files in node you can use something like express with multer:
var multer = require('multer')
var upload = multer({ dest: 'uploads/' })
app.post('/profile', upload.single('avatar'), function (req, res, next) {
// req.file is the `avatar` file
// req.body will hold the text fields, if there were any
})
Related
I have 2 nodeJS services and I would want to upload file in a dir, from one NodeJS (backend) to another NodeJS(backend). The receiver nodeJS is an express app.
Looking for some working code sample.
PS: Couldn't find any code samples in search, since everywhere it was Multer from client to server uploads that receives multipart/form-data.
Uploading file using POST request in Node.js
Receive the file first as you correctly said using Multer. Then, you may either save the file to a temporary directory before uploading it again or just send the file as-is.
You need to setup a server running with Multer on the 2nd server that wishes to receive the file.
const express = require('express');
const app = express();
const upload = multer({ dest: 'files/' });
app.post('/upload', upload.single('file'), (req, res) => {
res.sendStatus(200);
});
app.listen(3001);
Then on the server you wish to send the file from, do something like this:
const request = require('request');
const req = request.post('localhost:3001/upload', (err, res, body) => {
if (err) throw new Error(err);
if (res && res.statusCode == 200) {
console.log('Success');
} else {
console.log('Error');
};
});
const form = req.form();
form.append('file', fs.createReadStream('./location/to/file'));
does anyone know how to use tinyPNG's API with multer? The docs seem deceptively simple:
var source = tinify.fromFile("unoptimized.jpg");
source.toFile("optimized.jpg");
though there's no clear indication of where this is meant to go, especially in something as convoluted as this:
var storage = multer.diskStorage(
{
destination: function (req, file, callback) {
callback(null, './uploads');
},
filename: function (req, file, callback) {
//use date to guarantee name uniqueness
callback(null, file.originalname + '-' + Date.now());
}
}
);
//.any() allows multiple file uploads
var upload = multer({ storage : storage}).any()
app.post('/api/photo', function(req,res){
upload(req,res,function(err) {
if(err) {
return res.end("Error uploading file.");
}
res.end("File is uploaded");
});
});
Where am I meant to "intercept" the file uploaded by multer so that I can compress it with tinyPNG?
Thanks in advance for the help!
Use following basic sample that changes uploaded photo/gallery files:
// Import express and multer.
var express = require('express');
var multer = require('multer');
// Setup upload.
var upload = multer({ dest: 'uploads/' });
var multipleFiles = upload.fields([{ name: 'photo', maxCount: 1 },
{ name: 'gallery', maxCount: 8 }]);
// Setup tinify.
var tinify = require("tinify");
tinify.key = "YOUR_API_KEY";
// Get request handler for '/' path.
var app = express();
app.get('/', function (req, res) {
res.setHeader("Content-Type", "text/html");
res.end(
"<form action='/api/photo' method='post' enctype='multipart/form-data'>" +
"<input type='file' name='photo' />" +
"<input type='file' name='gallery' multiple/>" +
"<input type='submit' />" +
"</form>"
);
});
// Upload file handler with '/api/photo' path.
app.post('/api/photo', multipleFiles, function (req, res) {
req.files['gallery'].forEach(function(file) {
// Your logic with tinify here.
var source = tinify.fromFile(file.path);
source.toFile(file.path + "_optimized.jpg");
});
res.end("UPLOAD COMPLETED!");
});
Feel free to change express middleware how you need it, just make sure you use upload.fields and authenticate using tinify.key = "YOUR_API_KEY";
https://github.com/expressjs/multer
https://tinypng.com/developers/reference/nodejs#compressing-images
I recently worked out a similar problem for myself using the tinify package and found the docs to be somewhat lacking.
I have a Vue front end collecting file uploads from the user using vue2dropzone. These are sent to a node / Express back end.
I have a need to compress the file and upload it to an S3 instance without storing on disk. That means using multer memory storage.
As a result there won’t be an ability to use tinify.fromFile() as there is no file stored locally.
In my images middleware:
Const multer = require(“multer”);
const tinify = require("tinify");
tinify.key = "your_key";
exports.singleFile = multer({ storage: multer.memoryStorage() }).fields([{ name: "file", maxCount: 1 }]);
exports.uploadCompImage = async (req, res, next) => {
try {
const fileName = `${req.params.name}${path.extname(req.files.file[0].originalname)}`;
const source = tinify.fromBuffer(req.files.file[0].buffer);
source.store({
service: "s3",
aws_access_key_id: "your_id",
aws_secret_access_key: "your_key
region: "your_region",
headers: {
"Cache-Control": "public"
},
path: `your_bucket/your_folder/${fileName}`
});
return res.status(200).send(`path_to_file/${fileName}`)
} catch (err) {
console.log(err);
next(err);
}
}
Then in my routes file:
Const images = require(“../middleware/images”);
// skipped several lines for brevity
productRouter
.route("/images/:name")
.post(images.singleFile, images.uploadCompImage)
This process creates a multer singleFile upload to memoryStorage, making the file available at req.files.file[0] (req.files[“file”] because I specified “file” as the name in multer fields, loop through this array if uploading multiple).
After setting that up I get the file name, set the source by using tinify to read from req.files.file[0].buffer as a buffer.
Then I set the source to my s3 instance and send back a public link to the file.
Hopefully this answer helps you. I could definitely see altering the process to change where the file goes or even write it to disk by altering the multer options.
I have a xmlhttprequest which uploads the file and I am trying to receive it in my node-express server. but for some reason I am not able to retrieve the file content in the server. Not sure where I am missing it.
app.post('/api/uploadfiles', function(req, res) {
console.log("apicalled");
console.log(req);
console.log(req.body);
console.log(req.files);
console.log(JSON.stringify(req.files));
});
In order for you to see the files, you will need to add another middleware that parses multi-part request.
Try using connect-multiparty module like so:
var multipart = require('connect-multiparty'); //for files upload
var multipartMiddleware = multipart();//for files upload
app.post('/api/uploadfiles', multipartMiddleware, function(req, res) {
console.log("apicalled");
console.log(req);
console.log(req.body);
console.log(req.files);
console.log(JSON.stringify(req.files));
});
Using Express with Node, I can upload a file successfully and pass it to Azure storage in the following block of code.
app.get('/upload', function (req, res) {
res.send(
'<form action="/upload" method="post" enctype="multipart/form-data">' +
'<input type="file" name="snapshot" />' +
'<input type="submit" value="Upload" />' +
'</form>'
);
});
app.post('/upload', function (req, res) {
var path = req.files.snapshot.path;
var bs= azure.createBlobService();
bs.createBlockBlobFromFile('c', 'test.png', path, function (error) { });
res.send("OK");
});
This works just fine, but Express creates a temporary file and stores the image first, then I upload it to Azure from the file. This seems like an inefficient and unnecessary step in the process and I end up having to manage cleanup of the temp file directory.
I should be able to stream the file directly to Azure storage using the blobService.createBlockBlobFromStream method in the Azure SDK, but I am not familiar enough with Node or Express to understand how to access the stream data.
app.post('/upload', function (req, res) {
var stream = /// WHAT GOES HERE ?? ///
var bs= azure.createBlobService();
bs.createBlockBlobFromStream('c', 'test.png', stream, function (error) { });
res.send("OK");
});
I have found the following blog which indicates that there may be a way to do so, and certainly Express is grabbing the stream data and parsing and saving it to the file system as well. http://blog.valeryjacobs.com/index.php/streaming-media-from-url-to-blob-storage/
vjacobs code is actually downloading a file from another site and passing that stream to Azure, so I'm not sure if it can be adapted to work in my situation.
How can I access and pass the uploaded files stream directly to Azure using Node?
SOLUTION (based on discussion with #danielepolencic)
Using Multiparty(npm install multiparty), a fork of Formidable, we can access the multipart data if we disable the bodyparser() middleware from Express (see their notes on doing this for more information). Unlike Formidable, Multiparty will not stream the file to disk unless you tell it to.
app.post('/upload', function (req, res) {
var blobService = azure.createBlobService();
var form = new multiparty.Form();
form.on('part', function(part) {
if (part.filename) {
var size = part.byteCount - part.byteOffset;
var name = part.filename;
blobService.createBlockBlobFromStream('c', name, part, size, function(error) {
if (error) {
res.send({ Grrr: error });
}
});
} else {
form.handlePart(part);
}
});
form.parse(req);
res.send('OK');
});
Props to #danielepolencic for helping to find the solution to this.
As you can read from the connect middleware documentation, bodyparser automagically handles the form for you. In your particular case, it parses the incoming multipart data and store it somewhere else then exposes the saved file in a nice format (i.e. req.files).
Unfortunately, we do not need (and necessary like) black magic primarily because we want to be able to stream the incoming data to azure directly without hitting the disk (i.e. req.pipe(res)). Therefore, we can turn off bodyparser middleware and handle the incoming request ourselves. Under the hood, bodyparser uses node-formidable, so it may be a good idea to reuse it in our implementation.
var express = require('express');
var formidable = require('formidable');
var app = express();
// app.use(express.bodyParser({ uploadDir: 'temp' }));
app.get('/', function(req, res){
res.send('hello world');
});
app.get('/upload', function (req, res) {
res.send(
'<form action="/upload" method="post" enctype="multipart/form-data">' +
'<input type="file" name="snapshot" />' +
'<input type="submit" value="Upload" />' +
'</form>'
);
});
app.post('/upload', function (req, res) {
var bs = azure.createBlobService();
var form = new formidable.IncomingForm();
form.onPart = function(part){
bs.createBlockBlobFromStream('taskcontainer', 'task1', part, 11, function(error){
if(!error){
// Blob uploaded
}
});
};
form.parse(req);
res.send('OK');
});
app.listen(3000);
The core idea is that we can leverage node streams so that we don't need to load in memory the full file before we can send it to azure, but we can transfer it as it comes along. The node-formidable module supports streams, hence piping the stream to azure will achieve our objective.
You can easily test the code locally without hitting azure by replacing the post route with:
app.post('/upload', function (req, res) {
var form = new formidable.IncomingForm();
form.onPart = function(part){
part.pipe(res);
};
form.parse(req);
});
Here, we're simply piping the request from the input to the output. You can read more about bodyParser here.
There are different options for uploading binary data (e.g. images) via Azure Storage SDK for Node, not using multipart.
Based on the Buffer and Stream definitions in Node and manipulating them, these could be handled using almost all the methods for BLOB upload: createWriteStreamToBlockBlob, createBlockBlobFromStream, createBlockBlobFromText.
References could be found here: Upload a binary data from request body to Azure BLOB storage in Node.js [restify]
People having trouble with .createBlockBlobFromStream trying to implement the solutions, note that this method has been changed slightly in newer versions
Old version:
createBlockBlobFromStream(containerName, blobName, part, size, callback)
New version
createBlockBlobFromStream(containerName, blobName, part, size, options, callback)
(if you don't care about options, try an empty array) for the parameter.
Oddly enough, "options" is supposed to be optional, but for whatever reason, mine fails if I leave it out.
I am trying to upload files using express and formidable (eventualy forwarding to MongoDB and GridFS). I am starting by creating a form with a field of type file. On the action of that field I use the following route....
exports.addItem = function(req, res, next){
var form = new formidable.IncomingForm(),
files = [],
fields = [];
form
.on('file', function(field, file) {
console.log(field, file);
})
.on('end', function() {
console.log('-> upload done');
});
}
Everything runs fine but when I post I don't see anything in the console and it hangs.
The route looks like the following...
app.post('/item/add', routes.addItem, routes.getPlaylist, routes.index)
Any ideas?
UPDATE
Here is an example of grabbing the file, however, this still doesn't include formidable...
https://gist.github.com/2963261
The reason it is hanging is because you need to call next() to tell Express to continue.
Also use the bodyParser() middleware in express (included by default) to get the files. Something like this:
exports.addItem = function(req, res, next){
if(req.files.length > 0)
{
// process upload
console.log(req.files);
}
next();
}