const xlsx = require("xlsx");
const requestretry = require("requestretry");
let response = await requestretry({
url: "https://dummy.com/filename",
json: false,
headers: { Authorization: token },
maxAttempts: 5,
retryDelay: 5000,
retryStrategy: requestretry.RetryStrategies.HTTPOrNetworkError
})
const workbook = xlsx.read(response.body, { type: "buffer" });
exceldata = await xlsx.utils.sheet_to_json(
workbook.Sheets[workbook.SheetNames[0]], { range: 4, raw: false })
console.log(exceldata);// didn't get my excel data
when I use Postman's Send button to call that external api I receive a unicode chracters in respose tab
But when I use Postman's Send and Download button to call that external api I can get the xlsx file to save.
Now, I am able to replicate same unicode as in I am receiving with Postman's Send button in my NodeJS code using requestretry module but don't know what to do next.
I thought its a buffer and tried xlsx.read(response.body, { type: "buffer" }) but got wrong output.
xlsx file
Related
I'm creating a web application that generates a pdf on a server then sends it to the client for display within the browser.
The client is using Vuejs / Axios to send a POST request. Afterwards, The server is receiving it with Expressjs, generating a unique PDF, converting the file to a base64 value then sending it back as a response.
I cannot seem to get the response correct. When I attempt to display response.data.pdfData within the client I get undefined in the console. I can see that there is indeed a response with the key and value pair using inspection tools within the Network tab under the Preview section but cannot seem to access it.
// FILE: ./Client/src/App.vue
submit(personalInfo) {
this.cardInfo.personalInfo = personalInfo;
console.log('Sending POST preview_card...');
axios({
url: 'http://localhost:5000/api/preview_card',
method: 'POST',
responseType: 'blob',
data: {
cardInfo: this.cardInfo,
},
}).then((response) => {
console.log(response.data.pdfData);
});
},
// FILE: ./Server/app.js
app.post('/api/preview_card', (req, res) => {
// Generate pdf
const doc = new jsPDF('p');
doc.setFontSize(40);
doc.text(req.body.cardInfo.templateInfo, 100, 100);
doc.save('response.pdf');
// Convert pdf to base64
var tempFile = path.resolve(__dirname, './response.pdf');
var pdfBase64 = fs.readFileSync(tempFile).toString('base64');
res.setHeader('Content-Type', 'application/json');
return res.send(JSON.stringify({ pdfData: pdfBase64 }));
});
I find it necessary to serve the pdf this way due to my client's compnents as well as to enforce a level of data coherency between concurrent users.
The Scenario
I am running a VueJs client, a NodeJs Restify API Server, and a Tika-server out of the official Docker Image. A user makes a POST call with formData containing a PDF file to be parsed. The API server receives the POST call and I save the PDF on the server. The API server should PUT the file to the unpack/all endpoint on the Tika-server and receive a zip containing a text file, a metadata file, and the set of images in the PDF. I would then process the zip and pass some data back to the client.
The Problem
I create a buffer containing the file to be parsed using let parsingData = fs.createReadStream(requestFilename); or let parsingData = fs.readFileSync(requestFilename);, set the axios data field to parsingData, then make my request. When I get the response from the Tika-server, it seems the Tika-server has treated the request as empty; within the zip, there are no images, the TEXT file is empty, the METADATA.
When I make the following request to the Tika-server via CURL curl -T pdf_w_images_and_text.pdf http://localhost:9998/unpack/all -H "X-Tika-PDFExtractInlineImages: true" -H "X-Tika-PDFExtractUniqueInlineImagesOnly: true"> tika-response.zip, I get a response zip file containing accurate text, metadata, stripped images.
The Code
let parsingData = fs.createReadStream('pdf_w_images_and_text.pdf');
axios({
method: 'PUT',
url: 'http://localhost:9998/unpack/all',
data: parsingData,
responseType: 'arraybuffer',
headers: {
'X-Tika-PDFExtractInlineImages': 'true',
'X-Tika-PDFExtractUniqueInlineImagesOnly': 'true'
},
})
.then((response) => {
console.log('Tika-server response recieved');
const outputFilename = __dirname+'\\output.zip';
console.log('Attempting to convert Tika-server response data to ' + outputFilename);
fs.writeFileSync(outputFilename, response.data);
if (fs.existsSync(outputFilename)) {
console.log('Tika-server response data saved at ' + outputFilename);
}
})
.catch(function (error) {
console.error(error);
});
The Question
How do I encode and attach my file to my PUT request in NodeJs such that the Tika-server treats it as it does when I make the request through CURL?
Axios is sending the request with a content type of application/x-www-form-urlencoded and therefore the file content isn't being detected and parsed.
You can change this by passing either the known content type of the file, or a content type of application/octet-stream to allow Apache Tika Server to auto-detect.
Below is a sample based on your question's code that illustrates this:
#!/usr/bin/env node
const fs = require('fs')
const axios = require('axios')
let parsingData = fs.createReadStream('test.pdf');
axios({
method: 'PUT',
url: 'http://localhost:9998/unpack/all',
data: parsingData,
responseType: 'arraybuffer',
headers: {
'X-Tika-PDFExtractInlineImages': 'true',
'X-Tika-PDFExtractUniqueInlineImagesOnly': 'true',
'Content-Type': 'application/octet-stream'
},
})
.then((response) => {
console.log('Tika-server response recieved');
const outputFilename = __dirname+'/output.zip';
console.log('Attempting to convert Tika-server response data to ' + outputFilename);
fs.writeFileSync(outputFilename, response.data);
if (fs.existsSync(outputFilename)) {
console.log('Tika-server response data saved at ' + outputFilename);
}
})
.catch(function (error) {
console.error(error);
});
I'm trying to send an image and some data from an API to another. The image is stored in memory with multer. But when I want to send it, the body is just empty. I tried the same request with postman and it worked perfectly.
postman test
postman test image
server test
server test image
Here is some code. I removed some of it so you can read it better
export const saveImage = async ({ image, name, folder, options }: { image: any, name?: any, folder: string, options?: any }) => {
try {
const fd = new FormData();
fd.append("image", image.buffer, image.originalname);
if(options) {
fd.append("options[resize][height]", options?.resize?.height);
fd.append("options[resize][width]", options?.resize?.width);
}
if(name) fd.append("name", name);
fd.append("folder", folder);
fd.append("servideId", IMAGES_ID);
fd.append("serviceSecret", IMAGES_SECRET);
console.log(fd)
const formHeaders = fd.getHeaders();
const request = await axios.post(`${IMAGES_URL}/api/images`, {
headers: formHeaders,
body: fd
});
return request.data.id;
} catch (error) {
const { response } = error;
console.log(response.request.data)
if(error?.response?.data?.error) {
throw { statusCode: error.response.status, message: error.response.data.error }
}
console.error("Images API", error);
throw new InternalError("Something gone wrong");
}
}
When I log the FormData, I can see in _streams, the data that I'm sending, but the Images API receives an empty body.
FormData screenshot
If you need more information tell me, please! Thank you
The axios API for the post method is: axios.post(url[, data[, config]]). The second argument must always be the data you send along.
In your case axios thinks { headers: formHeaders, body: fd } is the body and the request ends up being application/json. To send a file with data using axios in Node.js, do the following:
const response = await axios.post(`${IMAGES_URL}/api/images`, fd, {
headers: {
...formHeaders,
'X-Custom-Header': 'lala', // optional
},
});
Your question inspired me to turn this answer into an article — Send a File With Axios in Node.js. It covers a few common pitfalls and you'll learn how to send files that are stored as a Buffer or coming from a Stream.
With Axios, you can directly use the form data without having to deal with headers.
axios.post("/api/images", fd)
If you wish to modify headers at some point in the future, you should pass the formData to the `data` field instead of `body`.
axios.post("/api/images", { headers: formHeaders, data: fd })
Correction in comments.
It can also be done using the Axios API syntax.
axios({method: 'post', url: 'url', data: fd, headers: {} })
In the backend, multer will add your files to req.file instead of req.body, if you have properly configured it to do so.
I have two microservices: 1) in which I am generating pdf using Puppeteer, which is essentially a Buffer object. From this service I want to send the pdf to another microservice, 2) which receives the pdf in request and attaches it in email using mailgun (once I am able to send pdf from one service to another, attaching as an email wont be difficult).
The way I m sending the pdf in requestpromise is this:
import requestPromise from "request-promise";
import {Readable} from "stream";
//pdfBuffer is result of 'await page.pdf({format: "a4"});' (Puppeteer method).
const stream = Readable.from(pdfBuffer);
/*also tried DUPLEX and Readable.from(pdfBuffer.toString()) and this code too.
const readable = new Readable();
readable._read = () => {}
readable.push(pdf);
readable.push(null);
*/
requestPromise({
method: "POST",
url: `${anotherServiceUrl}`,
body: {data},
formData: {
media: {
value: stream,
options: {
filename: "file.pdf",
knownLength: pdfBuffer.length,
contentType: "application/pdf"
}
}
},
json: true
}
});
But doing so, I get "ERR_STREAM_WRITE_AFTER_END" error. How can I send this pdf from one service to another, since the other service sends the email to the user?
I've done it from the frontend with:
fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/pdf'
},
body: pdfData
}
In this case pdfData is a blob so you would need a polyfill for that plus node-fetch
const buffer = Buffer.from(pdfBuffer).toString("base64").toString();
sending the buffer in the body.
On Receiving service:
const pdf = Buffer.from(body.buffer, "base64");
fs.write("file.pdf", pdf, ()=> {});
I have this payload format in my hapi route that accepts only multipart/form-data data and the output is set as a stream:
payload: {
maxBytes: 1024,
output: 'stream',
parse: true,
allow: 'multipart/form-data',
}
I want to test that route and my payload is this:
const FD = new FormData();
FD.append('field1', 'value');
FD.append('field2', 'value');
The hapi inject method looks like this:
const res = await server.inject({
method,
url,
headers: {
...
'Content-Type': 'multipart/form-data; boundary=--SEPARATION--',
},
payload: FD,
});
I am getting
Invalid multipart payload format
I tried to set a stream using a Steam object too but it doesn't work. Also I tried to send a File object.
At this point I just want to send something to the route that will not result in an error. It's not necessary to be a FormData. Anything that will be accepted by the route is fine as long as I can add some custom fields to test it further.
Thanks in advance.
As far as I know, FormData doesn’t exist in node.js, so I assume you’re using the form-data module?
In that case, you need to ask form-data for the buffer and headers separately, like this:
const FormData = require('form-data');
const FD = new FormData();
FD.append('field1', 'value');
FD.append('field2', 'value');
const response = await server.inject({
method,
url,
headers: {
<your other headers>
...FD.getHeaders(),
},
payload: FD.getBuffer(),
});
If you want to test sending files, you also need to provide the filename property to append(…) for the binary data to be decoded correctly by the server:
const fileContent = <any Buffer>;
FD.append('form_file_property', fileContent, {filename: 'a testfile'});