Handle API Response in .gzip with XML file inside - node.js

I'm trying to handle this API interaction with node.js (using express and request) but i'm having a lot of trouble dealing with the data.
Here's my current code:
// Requirements
const express = require("express");
const bodyParser = require("body-parser");
const request = require("request");
const fs = require("fs");
const zlib = require("zlib");
const gunzip = require("gunzip-file");
const decompressResponse = require("decompress-response");
// Setting Up App
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
// Routes
app.get("/", (req, res) => {
res.send("App Running");
});
// API Integration
let responseXML = "";
let bodyXML =
'<?xml version="1.0" encoding="UTF-8"?><RequestMensagemCB><login>14087809000107</login><senha>xxxx</senha><mId>1</mId></RequestMensagemCB>';
const options = {
url: "http://webservice.newrastreamentoonline.com.br/",
method: "POST",
body: bodyXML
};
app.get("/onix", function(req, res) {
request(options, function(error, response, body) {
// body is the decompressed response body
console.log(
"server encoded the data as: " +
(response.headers["content-encoding"] || "identity")
);
console.log("the decoded data is: " + body);
})
.on("data", function(data) {
// decompressed data as it is received
console.log("decoded chunk: " + data);
})
.on("response", function(response) {
// unmodified http.IncomingMessage object
response.on("data", function(data) {
// compressed data as it is received
console.log("received " + data.length + " bytes of compressed data");
});
});
});
// Server Listening
app.listen(process.env.PORT || 3000, () => {
console.log("Server Online Listening to port 3000");
});
This is the console.log response i get:
Using postman i can reach the XML through the following route:
I first make the post request with the XML needed to validate the API Access, then i send and download the response giving it the extension of .gz and inside the .gz there's a compressed version of the file that when opened shows the XML response:
This is my first time working with an API that returns the data in .gz that way. I've tried piping the data using zLib and now was thinking on the following route: download the response in .gz, decompress the resulting file, then opening it to reach the XML. I imagine there's a better way of doing this!

I Could fix the code through another approach.
I've added 2 more params for the const options:
const options = {
url: "http://webservice.newrastreamentoonline.com.br/",
method: "POST",
body: bodyXML,
headers: {
"Accept-Encoding": "gzip"
},
encoding: null,
gzip: true
};
The key value is the following: encoding: null, , that way the stream comes without corruption.
Then i've been able to print out the XML response. Using xml2json NPM i've been able to convert it on a JSON object, and now i'll work normally with the data.

Related

Multer fails when using https on express. Works when using only http

I can't receive files when using https on express but works well when using only http. I just updated my express app to use a self-signed certificate for testing.
Here's what I am trying:
upload function
exports.upload = async (req, res) => {
console.log(req.file); //returns undefined
console.log(req.body.file); //returns [Object: null prototype] {}
...REST OF CODE...
}
client function
const formData = new FormData();
formData.append('file', pdfFile);
formData.append('user', 'test');
const params = {
body: formData,
method: 'POST',
credentials: 'include'
}
const result = await fetch('https://localhost:3000/service/upload', params);
I also noticed that the content-length is set to "0" when inspecting the network tab. Payload tab is also missing.
Request headers

How to get the name of the file uploaded?

I made a research and found only forms and formidable tutorials. However, I couldn't find how to get the original file name.
I am using Postman to send a file to http://localhost:8081/file. The file is sent in binary as body. The file sent is xxx.json.
In Node I created an HTTP server:
const http = require("http");
const fs = require("fs");
const server = http.createServer(async (req, res) => {
if (req.method === "POST" && req.url === "/file") {
req.on("data", (chunk) => {
console.log(`Data chunk available: ${chunk}`);
// fs.createWriteStream("./finalFolder");
});
}
res.end();
});
I want to save the file to /finalFolder preserving the original filename xxx.json.
Where do I get the name of the file uploaded?

Node js Stream file without saving to memory

I am building an API that needs to accept file uploads. So a user can POST a file to an endpoint, the file will be sent to a virus scan, then if it's clean will be sent to storage (probably S3). So far I have achieved this with one issue: The files are temporarily saved in the applications file system. I need to design an app that doesn't store things in memory. Here is my currently working code:
app.js
const express = require('express');
const bb = require('express-busboy');
const app = express();
// Busboy modules extends the express app to handle incoming files
bb.extend(app, {
upload: true,
path: './tmp'
});
Routes.js
const express = require('express');
const router = express.Router();
const fileManagementService = require('./file-management-service')();
router
.route('/:fileId')
.post(async (req, res, next) => {
try {
const {fileId} = req.params;
const {files} = req;
const response = await fileManagementService.postFile(files, fileId);
res.status(201).json(response);
} catch (err) {
next(err);
}
})
file-management-service.js
const fs = require('fs');
function createUploader() {
// POST /:fileId
async function postFile(data, fileId) {
const {file} = data.file;
const fileStream = fs.createReadStream(file);
const scanOutput = await scanFile(fileStream); // Function scans file for viruses
const status = scanOutput.status === 'OK';
let upload = 'NOT UPLOADED';
if (status) {
upload = await postS3Object({file}); // Some function that sends the file to S3 or other storage
}
fs.unlinkSync(file);
return {
fileId,
scanned: scanOutput,
upload
};
}
return Object.freeze({
postFile
});
}
module.exports = createUploader;
As mentioned, the above works as expected, the file is sent to be scanned, then sent to an S3 bucket before returning a response to the poster to that effect. However my implementation of express-busboy is storing the file in the ./tmp folder, then I'm converting this into a readable stream using fs.createReadStream(filePath); before sending it to the AV and again in the function that sends the file to S3.
This API is being hosted in a kubernetes cluster and I need to avoid creating states. How can I achieve the above without actually saving the file? I'm guessing busboy receives this file as some sort of stream, so without sounding dense, can it not just remain a stream and be piped through these functions to achieve the same outcome?
You can use busboy at a bit lower level and get access to it's translated readstream. Here's an example from the busboy doc that can be adapted for your situation:
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
file.pipe(fs.createWriteStream(saveTo));
});
busboy.on('finish', function() {
res.writeHead(200, { 'Connection': 'close' });
res.end("That's all folks!");
});
return req.pipe(busboy);
}
res.writeHead(404);
res.end();
}).listen(8000, function() {
console.log('Listening for requests');
});
The key part is this which I've annotated:
// create a new busboy instance on each incoming request that has files with it
var busboy = new Busboy({ headers: req.headers });
// register for the file event
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
// at this point the file argument is a readstream for the data of an uploaded file
// you can do whatever you want with this readstream such as
// feed it directly to your anti-virus
// this example code saves it to a tempfile
// you would replace this with code that sends the stream to your anti-virus
var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
file.pipe(fs.createWriteStream(saveTo));
});
// this recognizes the end of the upload stream and sends
// whatever you want the final http response to be
busboy.on('finish', function() {
res.writeHead(200, { 'Connection': 'close' });
res.end("That's all folks!");
});
// this gets busboy started, feeding the incoming request to busboy
// so it can start reading it and parsing it and will eventually trigger
// one or more "file" events
return req.pipe(busboy);
When you've identified an incoming request that you want to do this custom busboy operation in, you create an instance of Busboy, pass it the headers and register for the file event. That file event gives you a new file readstream that is the converted file as a readstream. You could then pipe that stream directly to your anti-virus without ever going through the file system.

Write on file of another server

I have a file called Sitemap.xml on Server1 and I want to write to this file from another server Server2.
File Structure of Server1
Server1:
app
views
public
sitemap.xml
app.js
Sitemap can be accessed by Server1/sitemap as I have used below code in my express file
app.use('/sitemap', express.static(__dirname + '/sitemap.xml'));
You should to protect route with secret token to avoid exposing. Hope this will help you:
// Server 1
const fs = require('fs');
app.use(function(req, res, next) {
var secret = req.headers.hasOwnProperty('authorization')
? req.headers.authorization
: false;
if (! secret || secret !== 'token [your-secret-token]') {
res.status(403).send('Access forbidden');
}
// Create write stream to sitemap file
var stream = fs.createWriteStream('sitemap.xml');
// Redirect request body to stream which writes to sitemap file
req.pipe(stream)
.on('end', () => res.send('ok'));
});
// Server 2
const http = require('http');
const fs = require('fs');
var stream = fs.createReadStream('new-sitemap.xml');
var req = http.request({
host: 'server1',
headers: {
authorization: 'token [your-secret-token]',
},
});
req.on('response', (res) => {
if (res.status === 200) {
console.log('File uploaded');
}
else {
console.error('File not loaded');
}
});
// Write data from file into request body
stream.pipe(req);
Note that token should be minimum 32 chars length to be strong enough. And don't forget to update it from time to time. And also it's a concept.

Sending/handling GET requests in typescript using Express, Request and Node.js

I'm using a combination of Express and Request (installed using npm) to try to send a get request to get some json from the server. However no matter what I do the body that is returned is "undefined".
This is the code in my server.js file. The json isn't actually what I'm sending, it's just an example as I can't post what I'm actually sending.
import express = require("express");
import bodyParser = require("body-parser");
let app = express();
app.use(bodyParser.json());
app.get('/config', function(req, res){
res.json('{name: test}');
})
app.listen(3000);
I've tried both of the following but both of them say that body is undefined.
import request = require("request");
let req = {
url: `http://localhost:3000/config`,
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}
request(req, function(error, response, body){
this.config = JSON.parse(body);
})
request(`/config`, function(err, res, body) {
this.config = JSON.parse(body);
});
Does anyone know what I'm doing wrong? I've never used express or request before so any tips would be greatly appreciated.
UPDATE
If I change the request code to the following, the inside of the function is never run. Does anyone know why this would be?
let req = {
url: `http://localhost:3000/config`,
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}
request(req, function(error, response, body){
console.log("response => "+JSON.parse(body));
return JSON.parse(body);
})
Since OP hasn't got it working and I believe the code he got up there is correct. I may as well post my working solution here to help him get started.
Hopefully this will save you hours of debugging...
Client:
"use strict";
let request = require("request");
let req = {
url: `localhost:4444/config`,
proxy: 'http://localhost:4444',
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
};
request(req, function (err, res, body) {
this.config = JSON.parse(body);
console.log("response => " + this.config);
});
Server:
"use strict";
var express = require("express");
var bodyParser = require("body-parser");
var app = express();
var config = require('config');
app.use(bodyParser.json());
app.get('/config', function(req, res){
res.json('{name: test}');
});
// Start the server
app.set('port', 4444);
app.listen(app.get('port'), "0.0.0.0", function() {
console.log('started');
});
Output:
response => {name: test}
I dont't know if you have posted whole of your server's code, it seems like you missed app.listen(port) so that your server cannot be started up correctly.
Also, if you added if (error) { console.log(error); } at the first line of the callback function of request, you'll find it print an error: [Error: Invalid URI "/config"]
And that's why the body is always undefined: You have to give the full url such like http://localhost:xxxx to request.
In short:
Your server didn't listen to a specific port. app.listen(5678)
Your client didn't know the complete url. request('http://localhost:5678/config', (...)=>{...})

Resources