I have a http request as follow(python code):
data = open("/tmp/ibus.tar.gz", 'rb').read()
resp = requests.post(url="http://192.168.1.156:3000/upload", data=data, headers={'Content-Type': 'application/octet-stream'})
print resp.text
now I implement this request with nodejs as follow:
router.post('/', function(req, res, next) {
var b = req.body.file;
fs.writeFile("/tmp/upload.tgz", b, "binary", function(err) {
if (err) return console.log(err);
console.log("file is saved");
return res.send({"status": 200});
})
});
but I test the /tmp/upload.tgz with tar zxf /tmp/upload.tgz, got error as follow:
tar: Unrecognized archive format
tar: Error exit delayed from previous errors.
req.body.file is undefined because express doesn't handle file upload. You have to use another package to handle the upload. You're also using requests the wrong way:
# use an object
data = { 'test': open(path + "test.gz", 'rb') }
# use files as identifier, not data
resp = requests.post("http://localhost:3000/upload", files = data)
print resp.text
For example, using multer
var fs = require('fs');
var multer = require('multer');
// specify a folder to put the temporary files
var upload = multer({ dest: __dirname + '/uploads' });
// 'test' is the identifier used in the python side, it's not random
router.post('/upload', upload.single('test'), function (req, res, next) {
var file = req.file;
console.log(file);
// at this point, the file is inside the tmp folder, now you can:
// make a copy
var src = fs.createReadStream(file.path);
var dest = fs.createWriteStream(__dirname + '/upload.gz');
src.pipe(dest);
// or move it
fs.renameSync(file.path, __dirname + '/upload.gz');
res.send('ok');
});
Related
I am building an API that needs to accept file uploads. So a user can POST a file to an endpoint, the file will be sent to a virus scan, then if it's clean will be sent to storage (probably S3). So far I have achieved this with one issue: The files are temporarily saved in the applications file system. I need to design an app that doesn't store things in memory. Here is my currently working code:
app.js
const express = require('express');
const bb = require('express-busboy');
const app = express();
// Busboy modules extends the express app to handle incoming files
bb.extend(app, {
upload: true,
path: './tmp'
});
Routes.js
const express = require('express');
const router = express.Router();
const fileManagementService = require('./file-management-service')();
router
.route('/:fileId')
.post(async (req, res, next) => {
try {
const {fileId} = req.params;
const {files} = req;
const response = await fileManagementService.postFile(files, fileId);
res.status(201).json(response);
} catch (err) {
next(err);
}
})
file-management-service.js
const fs = require('fs');
function createUploader() {
// POST /:fileId
async function postFile(data, fileId) {
const {file} = data.file;
const fileStream = fs.createReadStream(file);
const scanOutput = await scanFile(fileStream); // Function scans file for viruses
const status = scanOutput.status === 'OK';
let upload = 'NOT UPLOADED';
if (status) {
upload = await postS3Object({file}); // Some function that sends the file to S3 or other storage
}
fs.unlinkSync(file);
return {
fileId,
scanned: scanOutput,
upload
};
}
return Object.freeze({
postFile
});
}
module.exports = createUploader;
As mentioned, the above works as expected, the file is sent to be scanned, then sent to an S3 bucket before returning a response to the poster to that effect. However my implementation of express-busboy is storing the file in the ./tmp folder, then I'm converting this into a readable stream using fs.createReadStream(filePath); before sending it to the AV and again in the function that sends the file to S3.
This API is being hosted in a kubernetes cluster and I need to avoid creating states. How can I achieve the above without actually saving the file? I'm guessing busboy receives this file as some sort of stream, so without sounding dense, can it not just remain a stream and be piped through these functions to achieve the same outcome?
You can use busboy at a bit lower level and get access to it's translated readstream. Here's an example from the busboy doc that can be adapted for your situation:
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
file.pipe(fs.createWriteStream(saveTo));
});
busboy.on('finish', function() {
res.writeHead(200, { 'Connection': 'close' });
res.end("That's all folks!");
});
return req.pipe(busboy);
}
res.writeHead(404);
res.end();
}).listen(8000, function() {
console.log('Listening for requests');
});
The key part is this which I've annotated:
// create a new busboy instance on each incoming request that has files with it
var busboy = new Busboy({ headers: req.headers });
// register for the file event
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
// at this point the file argument is a readstream for the data of an uploaded file
// you can do whatever you want with this readstream such as
// feed it directly to your anti-virus
// this example code saves it to a tempfile
// you would replace this with code that sends the stream to your anti-virus
var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
file.pipe(fs.createWriteStream(saveTo));
});
// this recognizes the end of the upload stream and sends
// whatever you want the final http response to be
busboy.on('finish', function() {
res.writeHead(200, { 'Connection': 'close' });
res.end("That's all folks!");
});
// this gets busboy started, feeding the incoming request to busboy
// so it can start reading it and parsing it and will eventually trigger
// one or more "file" events
return req.pipe(busboy);
When you've identified an incoming request that you want to do this custom busboy operation in, you create an instance of Busboy, pass it the headers and register for the file event. That file event gives you a new file readstream that is the converted file as a readstream. You could then pipe that stream directly to your anti-virus without ever going through the file system.
While uploading a file from ajax request Multer is giving an error that is given below.
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be one of
type string or Buffer. Received type object
at rite_ (_http_outgoing.js:595:11)
// code block for multer start
var Storage = multer.diskStorage({
destination: function(req, file, callback) {
callback(null, "./uploads/posts");
},
filename: function(req, file, callback) {
callback(null, file.fieldname + "_" + Date.now() + "_" + file.originalname);
}
});
var upload = multer({
storage: Storage
}).single('imgData');
//route Ajax Rquest URL Start
router.post('/blog/saveUploadImage',urlencoderParser,(req,res)=>{
upload(req, res, function(err) {
if (err) {
return res.end({UplaodStatus:true,type:'success',text:' š· Image Uploaded Now Saving Your Data It will take just a sec.'});
}
return res.end({UplaodStatus:false,type:'error',text:' ā¹ Sorry There was some Problem Uploading Image '});
});
});
//route Ajax Rquest URL End
//JS code
// code for geting file
let fileUpload = document.getElementById('uploadFile').files;
//appending the file to formdata
var formData = new FormData();
formData.append('imgData', fileUpload);
//AJAX Request
$.ajax({
enctype:'multipart/form-data',
data:formData,
url:'/admin/blog/saveUploadImage',
type:'POST',
cache:false,
contentType:false,
processData:false,
timeout:10000,
});
You're passing an array of files to formData.append(...), instead you should pick just the first element from this array:
let fileUpload = document.getElementById('uploadFile').files[0];
The issue was I Imported this package ( Look Below ) because of this multer was not working.
const fileUpload = require('express-fileupload');
So I removed it now it works fine.
Thank you for ur help.
The issue was I Imported this package ( Look Below ) because of this multer was not working.
const fileUpload = require('express-fileupload');
So I removed it now it works fine.
I intended to use multer to upload multiple file and then rename them back to their original names. The below are the sample code:
var express = require('express');
var app = express();
var fs = require("fs");
var multer = require('multer');
app.use(express.static('public'));
var upload = multer({ dest: './upload/' });
app.get('/index.html', function (req, res) {
res.sendFile(__dirname + "/" + "index.html");
})
app.post('/file_upload', upload.array('theFile', 2), function (req, res, next) {
var errorcode = 0;
for (var i = 0; i < req.files.length; i++) {
fs.rename(req.files[i].path, req.files[i].destination + req.files[i].originalname, function (err) {
errorcode = err;
}(i));
}
if (errorcode != 0) {
console.log("errorcode is " + errorcode);
res.sendStatus(500);
return;
} else {
res.json({
message: 'File uploaded successfully',
});
}
})
var server = app.listen(8089, function () {
var host = server.address().address
var port = server.address().port
console.log("Example app listening at http://%s:%s", host, port)
})
I'm testing the above code on a windows server. And my observation is that the files can be uploaded successfully but the fs.rename() keeps returning error "1". And the renamed files in the targeted folder are always 1Kb. It seems that the rename function intends to fetch the files which might be still uploading. I'm not sure whether my understanding is correct. If so, is there a way to determine whether the files have been uploaded completely? Any suggestion for my problem?
Why not use Multer's built-in renaming functionality?
Adapted from the documentation:
var storage = multer.diskStorage({
destination: '/path/to/uploads/folder',
filename: function (req, file, cb) {
// Here we specify the file name to save it as
cb(null, file.originalname);
}
})
// And we can use it for example like this:
app.post('/upload', upload.single('image'), function (req, res, next) {
// req.file is the `image` file
// req.body will hold the text fields, if there were any
})
However, there are a couple things you should be aware of if you take this approach:
The client can send any type of file, with any (potentially incorrect) extension. This is a potential security risk.
If two files are uploaded with the same name, the second file will overwrite the first.
If you serve these files to other users, the security risk greatly increases. An attacker could create a script or HTML page and upload it, possibly giving it a different file name extension. There are several ways it could be run, such as if the user tries to open it in a new tab because an image didn't show up. The full implications of this, and how to deal with it, are a topic of their own.
Finally, make very, very, sure that the user cannot write to a directory other than the uploads folder. What happens when file.originalname is something like ../../index.js? It may be better to convert the file name to a slug.
does anyone know how to use tinyPNG's API with multer? The docs seem deceptively simple:
var source = tinify.fromFile("unoptimized.jpg");
source.toFile("optimized.jpg");
though there's no clear indication of where this is meant to go, especially in something as convoluted as this:
var storage = multer.diskStorage(
{
destination: function (req, file, callback) {
callback(null, './uploads');
},
filename: function (req, file, callback) {
//use date to guarantee name uniqueness
callback(null, file.originalname + '-' + Date.now());
}
}
);
//.any() allows multiple file uploads
var upload = multer({ storage : storage}).any()
app.post('/api/photo', function(req,res){
upload(req,res,function(err) {
if(err) {
return res.end("Error uploading file.");
}
res.end("File is uploaded");
});
});
Where am I meant to "intercept" the file uploaded by multer so that I can compress it with tinyPNG?
Thanks in advance for the help!
Use following basic sample that changes uploaded photo/gallery files:
// Import express and multer.
var express = require('express');
var multer = require('multer');
// Setup upload.
var upload = multer({ dest: 'uploads/' });
var multipleFiles = upload.fields([{ name: 'photo', maxCount: 1 },
{ name: 'gallery', maxCount: 8 }]);
// Setup tinify.
var tinify = require("tinify");
tinify.key = "YOUR_API_KEY";
// Get request handler for '/' path.
var app = express();
app.get('/', function (req, res) {
res.setHeader("Content-Type", "text/html");
res.end(
"<form action='/api/photo' method='post' enctype='multipart/form-data'>" +
"<input type='file' name='photo' />" +
"<input type='file' name='gallery' multiple/>" +
"<input type='submit' />" +
"</form>"
);
});
// Upload file handler with '/api/photo' path.
app.post('/api/photo', multipleFiles, function (req, res) {
req.files['gallery'].forEach(function(file) {
// Your logic with tinify here.
var source = tinify.fromFile(file.path);
source.toFile(file.path + "_optimized.jpg");
});
res.end("UPLOAD COMPLETED!");
});
Feel free to change express middleware how you need it, just make sure you use upload.fields and authenticate using tinify.key = "YOUR_API_KEY";
https://github.com/expressjs/multer
https://tinypng.com/developers/reference/nodejs#compressing-images
I recently worked out a similar problem for myself using the tinify package and found the docs to be somewhat lacking.
I have a Vue front end collecting file uploads from the user using vue2dropzone. These are sent to a node / Express back end.
I have a need to compress the file and upload it to an S3 instance without storing on disk. That means using multer memory storage.
As a result there wonāt be an ability to use tinify.fromFile() as there is no file stored locally.
In my images middleware:
Const multer = require(āmulterā);
const tinify = require("tinify");
tinify.key = "your_key";
exports.singleFile = multer({ storage: multer.memoryStorage() }).fields([{ name: "file", maxCount: 1 }]);
exports.uploadCompImage = async (req, res, next) => {
try {
const fileName = `${req.params.name}${path.extname(req.files.file[0].originalname)}`;
const source = tinify.fromBuffer(req.files.file[0].buffer);
source.store({
service: "s3",
aws_access_key_id: "your_id",
aws_secret_access_key: "your_key
region: "your_region",
headers: {
"Cache-Control": "public"
},
path: `your_bucket/your_folder/${fileName}`
});
return res.status(200).send(`path_to_file/${fileName}`)
} catch (err) {
console.log(err);
next(err);
}
}
Then in my routes file:
Const images = require(ā../middleware/imagesā);
// skipped several lines for brevity
productRouter
.route("/images/:name")
.post(images.singleFile, images.uploadCompImage)
This process creates a multer singleFile upload to memoryStorage, making the file available at req.files.file[0] (req.files[āfileā] because I specified āfileā as the name in multer fields, loop through this array if uploading multiple).
After setting that up I get the file name, set the source by using tinify to read from req.files.file[0].buffer as a buffer.
Then I set the source to my s3 instance and send back a public link to the file.
Hopefully this answer helps you. I could definitely see altering the process to change where the file goes or even write it to disk by altering the multer options.
I am trying to get rename and onFileUploadComplete to call their console log but it never does, it does however call my console in post, which is giving me a string of a file that is not there.
var express = require("express");
var multer = require("multer");
var Excel = require("exceljs");
var router = express.Router();
var fs = require('fs');
var doneUpload = false;
var displayResults = [];
var Excelfile;
router.use(multer({ dest: "./uploads/",
rename: function (fieldname, filename) {
console.log("Filename: "+filename);
return new Date().getTime() + filename;
},
onFileUploadComplete: function (file) {
saveFile = file;
console.log("onFileUploadComplete ");
doneUpload = true;
}
}).single("excelFile"));
router.get("/", function(req, res){
res.sendFile(__dirname + "/index.html");
});
router.post("/api/fileupload" ,function(req, res){
Excelfile = req.file.path;
console.log(typeof Excelfile);
res.render("results", {
displayResults: displayResults
});
});
module.exports = router;
Thanks in advance.My ultimate goal is to have a file I can pass into excel JS and currently it is saying the file is not in the correct string format. I suspect because the file is being renamed to the long string of numbers "uploads/7ab48192a1c548a3dce3b9d74cad6592" which is never pointing to an actual file. I further suspect this because I ran typeof and it is does give me back a string - just not one pointing to a file, especially if onFileUploadComplete never gets fired telling me the file is uploaded?
The usage for Multer has changed.
These callbacks would not work. Have a look at these links:
https://github.com/expressjs/multer
multer callbacks not working ?