Working with express to create a file transfer tool, and I've almost gotten everything completed. Just need to figure out how to get the data from the request written to file.
My issue appears to be stemming from not knowing where the file contents are placed in the request object.
My code to process sending the request
let file = watcher.getOneFile(config.thisLocation);
console.dir(file);
let contents = fs.readFileSync(file.fullPath, 'utf-8');
console.log(contents);
let form = {
attachments: [
contents
]
}
rq.post({
url: `http://${homeAddress}:${port}/files/?loc=${config.thisLocation}&file=${file.fileName}`,
headers: {'content-type':'application/x-www-form-urlencoded'},
formData: form
}, (err, res, body) => {
// body = JSON.parse(body);
console.log(body);
});
and when I get the request on the server, I'm not sure where the file contents actually are.
Code for handling the request
app.post('/files', (req, res) => {
console.log(req.query.loc);
// console.dir(req);
let incoming = watcher.getOutputPath(req.query.loc, config.locations);
console.log(incoming);
console.dir(req.body);
// console.log(req.body);
// let body = JSON.parse(req.body);
console.log(req.query);
let filename = path.join(incoming, req.query.file);
console.log(filename);
fs.writeFile(filename, req.body, (err) => {
if(err){
console.error(err);
}
console.log(`Successfully wrote file: ${path.join(incoming, req.query.file)}`);
});
res.sendStatus(200);
});
Where on the Request Object is the file contents?
Unfortunately you can't access the file content in any straightforward way. I recommend you to use busboy or similar package to parse form-data requests.
Here is how can you read file content using busboy and write it to the file system:
const Busboy = require('busboy');
app.post('/files', (req, res) => {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldname, file, filename, encoding, mime) => {
const newFilename = `${Date.now()}_${filename}`,
newFile = fs.createWriteStream(newFilename);
file.pipe(newFile);
file.on('end', () => {
console.log(`Finished reading ${filename}`);
});
});
busboy.on('finish', () => {
console.log('Finished parsing form');
res.sendStatus(200);
});
req.pipe(busboy);
});
Related
I want to create an API which takes the form data as input and pass it to another API, but I am not able fetch that formdata from the request. please help me with this.
Here is the code of my API
router.post('/file', async (req, res) => {
const { file } = req.body;
try {
const url = `${API_URL}/api/file/upload`;
const response = await axios.get(url, file);
res.json({ status: true, data: response.data });
} catch (err) {
res.status(404).json({ status: false, error: 'Error in uploading' });
}
});
Here is the screenshot of data I want to sent by the postman
The key is how the variable in body is named.
If you are trying to make file upload i would recommend express-fileupload
You can get the file buffer from that.
Then you can use form-data to upload it somewhere else.
router.use(require(`express-fileupload`)());
router.post('/upload', async (req, res) => {
var FormData = require('form-data');
const buffer = req.files.foo.data // data is the file buffer // foo is the key/file name
var form = new FormData();
form.append('file', buffer);
form.submit('example.org/upload', function(err, res) {
res.send("done");
//Do something here
});
});
I'm trying to upload a file to nanosets API. I uploaded the following node js function to firebase and trying to excess it with following URL for example with a file in body (trying to test this with postman)
Node js function looks like:
exports.uploadFile = functions.https.onRequest((req, res) => {
cors(req, res, () => {
if (req.method !== "POST") {
return res.status(500).json({
message: "Not allowed"
});
}
const busboy = new Busboy({headers: req.headers});
let uploadData = null;
busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
const filepath = path.join(os.tmpdir(), filename);
uploadData = {modelId: '4bc54977-60cf-4415-a417-c39f1c18b83f', file: fs.createReadStream(filename), type: mimetype};
const options = {
url: 'https://app.nanonets.com/api/v2/OCR/Model/XXXXXXX-60cf-4415-a417-c39f1c18b83f/LabelFile/',
formData: uploadData,
headers: {
'Authorization': 'Basic ' + Buffer.from('tiOJNxuDbdl40lXXXXXXXXXXFTYbY' + ':').toString('base64')
}
};
request.post(options, function (err, httpResponse, body) {
if (err) {
console.log(err);
}
console.log(body)
});
});
busboy.on("finish", () => {
res.status(200).json({
message: "It worked!"
});
});
busboy.end(req.rawBody);
});
});
Why I check the logs with firebase functions:log I get the following results:
2020-06-06T09:35:06.168774140Z D uploadFile: Function execution started
2020-06-06T09:35:06.344Z I uploadFile: invoice_4.pdf
2020-06-06T09:35:06.432Z I uploadFile: FileStream {
2020-06-06T09:35:06.439Z E uploadFile: TypeError: source.pause is not a function
Anyone an idea? How to pass the file to nanosets?
In order to make outgoing requests from Cloud Functions, your project must be on the Blaze payment plan.
I'm trying to create a system that would read a file from another site and serve this as a file download to the user. The file size could vary from Mbs to Gbs.
I have have already created a proof of concept using vanilla nodejs.
const app = express();
app.get('/', (req, res) => {
res.setHeader('Content-disposition', 'attachment; filename=FILENAME');
res.setHeader('Content-type', 'application/octet-stream');
http.get('URL_TO_FILE', data => {
data.pipe(res);
});
});
I would like to do this using loopback 4. Is this possible? Thanks!
You can do this easily with nodejs request and using the loopback 4 controllers.
You need to import this packages:
var request = require('request');
var fs = require('fs');
And put this code in the endpoint method:
#get('/getImage')
async retrieveImage(): Promise<any> {
let file = fs.createWriteStream(__dirname + `/file.jpg`);
console.log('the file is here: ' + file.path);
return await new Promise((resolve, reject) => {
request({
url: 'URL_TO_FILE',
headers: {
'Content-disposition': 'attachment; filename=FILENAME',
'Content-type': 'application/octet-stream',
},
gzip: true,
})
.pipe(file)
.on('finish', () => {
console.log(`The file is finished downloading.`);
resolve();
})
.on('error', (error: any) => {
reject(error);
})
}).catch(error => {
console.log(`something happened: ${error}`)
});
}
From this you need to go to this url:
http://localhost:3000/getImage
And the file would be in your controllers folder with the "file.jpg" name.
I am working on an ios app which sends images and text to my firebase server using mutipart/form-data URLRequest. In order to process the data in my cloud function, I am using the method mentioned in documentation to parse the mutipart/form-data into JSON format, and here is my code:
const Busboy = require('busboy');
exports.test = functions.https.onRequest((req, res) => {
console.log("start");
console.log(req.rawBody.toString());
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers});
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
console.log('field');
});
busboy.on('finish', function() {
console.log('finish');
res.json({
data: null,
error: null
});
});
req.pipe(busboy);
} else {
console.log('else...');
}
});
However, the above code doesn't seem to work, and here is the output from console:
Function execution started
start
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09
Content-Disposition: form-data; name="json"
{"name":"Alex","age":"24","friends":["John","Tom","Sam"]}
--Boundary-43F22E06-B123-4575-A7A3-6C144C213D09--
finish
Function execution took 517 ms, finished with status code: 200
As you can see, the on('field') function never execute. What did I miss?
Also, here is the code in swift for sending httpRequest:
var request = URLRequest(url: myCloudFunctionURL)
request.httpMethod = "POST"
request.setValue("multipart/form-data; boundary=myBoundary", forHTTPHeaderField: "Content-Type")
request.addValue(userToken, forHTTPHeaderField: "Authorization")
request.httpBody = myHttpBody
let session = URLSession.shared
session.dataTask(with: request) { (data, response, requestError) in
// callback
}.resume()
You will have to call busboy.end(req.rawBody); instead of req.pipe(busboy) as described in the example of the documentation. I dont know why .pipe doesnt work. Calling .end will produce the same result but with a different way.
const Busboy = require('busboy');
exports.helloWorld = functions.https.onRequest((req, res) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on('field', (fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
// We're just going to capture the form data in a JSON document.
formData[fieldname] = val;
console.log('Field [' + fieldname + ']: value: ' + val)
});
busboy.on('finish', () => {
res.send(formData);
});
// The raw bytes of the upload will be in req.rawBody.
busboy.end(req.rawBody);
});
Enjoy this simple express middleware which converts all the Content-Type: multipart/form-data into you req.body in json format :)
const Busboy = require('busboy');
const expressJsMiddleware = (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
let formData = {};
busboy.on(
"field",
(fieldname, val, fieldnameTruncated, valTruncated, encoding, mimetype) => {
formData = { ...formData, [fieldname]: val };
},
);
busboy.on("finish", () => {
req.body = formData;
next();
});
req.pipe(busboy);
};
I have a service, what receive a post request with a file and json data too. I use the body-parser package either in the app.js. I want to send the file to a "filer" service, and process the answer from that, but I don't want to pipe the request, because I need to process the json content too and make some actions after the filer answered.
const Busboy = require('busboy');
const request = require('request');
const sendFile = (req, file, callback) => {
return request.post({
uri: 'http://localhost:5000/stream',
headers: req.headers,
formData: { value: file }
}, (err, resp, body) => {
if (err) return callback(err);
return callback();
});
};
app.post('/route', (req, res, next) {
const busboy = new Busboy({ headers: req.headers });
busboy.on('file', (fieldName, file) => {
file.on('error', err => reject(err));
return sendFile(req, file, (err, link) => {
file.resume();
if (err) return reject(err);
});
});
busboy.on('field', (fieldName, val) => {
// process the json here...
});
busboy.on('finish', () => {
console.log('busboy.on(finish)');
return next();
});
req.pipe(busboy);
}
The filer service the following:
app.post('/stream', (req, res, next) => {
const busboy = new Busboy({ headers: req.headers });
// here we are ok
busboy.on('file', function (fieldName, file, name) {
// but this part never run
res.send(200, { fileId: fileDoc._id });
});
return req.pipe(busboy);
});
Unfortunatelly the filer service never answer, and I don't know, where is a missing part. Tried to put the file.resume() to some places inside the busboy.on('file'), but doesn't helped.
Its probably because the file stream from busboy is never processed properly by request formData.
Another way is:
Temporarily write the stream to a local file in main service.(using fs.createWriteStream)
Create a stream from that file and pipe it to filer service.(using fs.createReadStream)
Do whatever processing you need to do in main service.
Wait for response from filer service and call next()
This way you can even use the file in main service if you need to or send the file to another service.