I currently have a node.js script sitting on Azure that gets a file (via a download URL link to that file) and base64 encodes it, and then sends this base64 encoded file back to the request source. The problem I am running into is performance based. The script below, in some instances, is timing out a separate application by having a run time over 30 seconds. The file in question on one of these timeouts was under a MB in size. Any ideas?
The script:
const express = require('express');
const bodyParser = require('body-parser');
const https = require('https');
const fs = require('fs');
const request = require('request');
const util = require('util');
const port = process.env.PORT || 3000;
var app = express();
app.use(bodyParser.json());
app.post('/base64file', (req, res) => {
var fileURL = req.body.fileURL;
var listenerToken = req.body.listenerToken;
var testingData = {
fileURL: fileURL,
listenerToken: listenerToken
};
/*
Make sure correct token is used to access endpoint..
*/
if( listenerToken !== <removedforprivacy> ) {
res.status(401);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'You are not authorized'}));
} else if ( !fileURL ){
res.status(400);
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({ error: 'The request could not be understood by the server due to malformed syntax.'}));
} else {
https.get(fileURL, function(response) {
var data = [];
response.on('data', function(chunk) {
data.push(chunk);
}).on('end', function() {
//build the base64 endoded file
var buffer = Buffer.concat(data).toString('base64');
//data to return
var returnData = {
base64File: buffer
};
if( buffer.length > 0 ) {
res.setHeader('Content-Type', 'application/json');
res.status(200);
res.send(JSON.stringify(returnData));
} else {
res.setHeader('Content-Type', 'application/json');
res.status(404);
res.send(JSON.stringify({ error: 'File URL not found.'}));
}
});
});
}
});
app.listen(port, () => {
console.log('Server is up and running ' + port);
});
One idea: you are missing error handling.
If you get an error on the https.get(), you will just never send a response and the original request will timeout.
Related
I have a task where I am given a URL such as https://xyz.json. This URL prompts the downloading of the JSON file into the local. I am now required to read the use this JSON data for further processing. Since I am new to NodeJS and express, I find myself confused about how to achieve this in ExpressJS.
This is what I've tried :
const https = require("https");
const fs = require("fs");
const file = fs.createWriteStream("outputFile.json");
const request = https.get(
"https://xyz.json",
function (response) {
response.pipe(file);
// after download completed close filestream
file.on("finish", () => {
file.close();
console.log("Download Completed");
});
}
);
Here, in the outputFile.json, no data is present
Qn2) Can I periodically download using setTimeOut(). Would it be efficient or is there any better way of caching data to make the application faster?
Thanks in advance!
Here's a sample app that downloads a json triggered when you hit an API route hosted as ExpressJS sever.
const express = require('express');
const cors = require('cors');
const morgan = require('morgan');
const bodyParser = require('body-parser');
const axios = require('axios');
const fs = require('fs');
const app = express();
app.use(cors());
app.use(morgan(':method :url :status :user-agent - :response-time ms'));
app.use(bodyParser.json());
app.get('/', async (req, res) => {
try {
const { status, data } = await axios.get('http://52.87.135.24/json-files/events.json'); // Can be replaced by your json url
if (status === 200) {
fs.writeFileSync('data.json', JSON.stringify(data));
res.status(200).json({
success: 'Downloaded file.',
data: data // Comment it if you don't want to send the data back
})
} else {
res.status(404).json({ 'Failed': 'File not found.' })
}
} catch (err) {
console.log(err);
res.status(500).json({ 'Error': 'Internal Server Error' });
}
});
app.listen(process.env.PORT || 3000, function () {
console.log('Express app running on port ' + (process.env.PORT || 3000))
});
And as I mentioned that this download gets triggered every time you make a request on http://localhost:3000 in this case, you can create a client script that acts like a cron job in which you can use the setTimeout or actually, setInterval to download your file periodically.
const axios = require('axios');
setInterval(async () => {
await axios.get('http://localhost:3000/');
}, 5000);
Here's such a script along! :)
I'm trying to retrieve data from KEEPA about Amazon's products.
I'm straggling to receive the data in proper JSON format, as KEEPA sending the data as gzip.
I tried to used 'decompressResponse' module which helped to get the data as JSON but it was received multiple times on each call.
As the code appears below I'm just getting a huge Gibberish to my console.
Let me know what am I missing here, or if you have a better suggestion please let me know.
Thanks
const express = require("express");
const https = require("https");
const bodyParser = require("body-parser");
const app = express();
app.use(bodyParser.urlencoded({extended: true}));
app.get("/", function(req, res) {
res.sendFile(__dirname + "/index.html");
});
app.post("/", function(req, res) {
const query = req.body.asinId;
const apiKey = "MY_API_KEY";
const url = "https://api.keepa.com/product?key=" + apiKey + "&domain=1&asin=" + query;
const options = {
methode: "GET",
headers: {
"Content-Type": "application/json;charset=UTF-8",
"Accept-Encoding":"gzip"
}
}
https.get(url,options,function(res) {
console.log(res.statusCode);
console.log(res.headers);
var data;
res.on("data", function(chunk){
if(data){
data = chunk;
} else {
data += chunk;
}
console.log(data);
});
});
res.send("server is running");
});
app.listen(3000, function() {
console.log("server is running on port 3000");
});
Have you tried using the built-in zlib module with gunzip()?
zlib.gunzip(data, (error, buff) => {
if (error != null) {
// An error occured while unzipping the .gz file.
} else {
// Use the buff which contains the unzipped JSON.
console.log(buff)
}
});
Full example with your code: https://www.napkin.io/n/7c6bc48d989b4727
well the output function was wrong .. the correct one below
https.get(url,options,function(response) {
response = decompressResponse(response);
console.log(res.statusCode);
console.log(res.headers);
let data = '';
response.on("data", function(chunk){
data += chunk;
});
response.on("end",function(){
console.log(data);
});
});
I am making an API for my minecraft server and have been able to get as far as getting the JSON file to update what I send it in a POST request. I would like to know if it is possible to only update on key of the JSON file.
This is my current code:
var fs = require('fs');
var fileName = './serverStatus.json';
var file = require(fileName);
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
const cors = require('cors');
const { fileURLToPath } = require('url');
app.get('/status', alldata);
function alldata(request, response) {
response.send(file);
}
app.post('/status', (req, res) => {
if (!req.is('application/json')) {
res.status(500);
res.send('500 - Server Error');
} else {
res.status(201);
fs.writeFile(
fileName,
JSON.stringify(req.body, null, 4),
function writeJSON(err) {
if (err) return console.error(err);
console.log(JSON.stringify(file));
console.log('writing to ' + fileName);
}
);
res.send(file);
}
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () =>
console.log(`Server running on: http://localhost:${PORT}`)
);
and my JSON file:
{
"lobby": "offline",
"survival": "offline",
"creative": "offline"
}
Thanks in advance!
You could use fs.readFileSync or to read file content.
Then update your JSON content such as jsonData["survival"] = "online".
Final, write content back to file with fs.writeFile. (See note-1)
You could see the following example code.
const fs = require("fs");
// 1. get the json data
// This is string data
const fileData = fs.readFileSync("./serverStatus.json", "utf8")
// Use JSON.parse to convert string to JSON Object
const jsonData = JSON.parse(fileData)
// 2. update the value of one key
jsonData["survival"] = "online"
// 3. write it back to your json file
fs.writeFile("./serverStatus.json", JSON.stringify(jsonData))
Note-1: Because you save data in file, you need to write the whole data when you want to update file content.
But, if you want to get the latest file content after you write your new data into file, you should fs.readFileSync your file again like following code to avoiding any modified which are forgot to save.
app.get('/status', alldata);
function alldata(request, response) {
const fileContent = fs.readFileSync(fileName, "utf8");
const fileJsonContent = JSON.parse(fileContent)
// do other stuff
response.send(fileContent);
}
var fs = require('fs');
const express = require('express');
const bodyParser = require('body-parser');
var fileName = './serverStatus.json';
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
// maybe use this instead of bodyParser:
//app.use(express.json());
const cors = require('cors');
const { fileURLToPath } = require('url');
app.get('/status', alldata);
function alldata(request, response) {
response.send(file);
}
app.post('/status', (req, res) => {
if (!req.is('application/json')) {
res.status(500);
res.send('500 - Server Error');
} else {
// read full config file:
var src = fs.readFileSync(fileName);
// convert src json text to js object
var srcObj = JSON.parse(src);
// convert req json text to js object
var reqObj = JSON.parse(req.body);
// update the src with the new stuff in the req
for(var prop in reqObj){
srcObj[prop] = reqObj[prop];
}
// update any additional things you want to do manually like this
srcObj.bob = "creep";
// convert the updated src object back to JSON text
var updatedJson = JSON.stringify(srcObj, null, 4);
// write the updated src back down to the file system
fs.writeFile(
fileName,
updatedJson,
function (err) {
if (err) {
return console.error(err);
}
console.log(updatedJson);
console.log('updated ' + fileName);
}
);
res.send(updatedJson);
}
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () =>
console.log(`Server running on: http://localhost:${PORT}`)
);
//res.status(201);
I am currently learning NodeJs, thus I am building a simple NodeJs server which is called server.js
const http = require('http');
const port = 3000;
const fs = require('fs');
const sourceFile = './client/src/account.json';
var data = []
const service = http.createServer(
(req, res) => {
var receive = "";
var result = "";
req.on('data', (chunk)=>{ receive += chunk })
req.on('end', () =>{
data = fs.readFileSync(sourceFile, 'UTF8');
result = JSON.stringify(data);
var data_receive = receive;
console.log(data)
res.setHeader('Access-Control-Allow-Origin', '*');
res.write(data);
res.end()
})
})
service.listen(port)
Why does every time I request to the server, the console.log returns the data 2 times. It seems like it is looped somewhere.
This is my json file account.json
[
{
"id":"account_1",
"pass":"abc123",
"name":"Account 1"
}
]
Thank you for you help!
When performing the request from the browser, you will get an extra request for favicon.ico.
To avoid those double requests, handle the favicon.
if (req.url === '/favicon.ico') {
res.writeHead(200, { 'Content-Type': 'image/x-icon' });
console.log('favicon...');
return res.end();
}
/* The rest of your code */
I am trying to call a REST API from node using node-rest-client. In the event that my call returns an error, I want to catch the error and report it to the caller.
I am trying this with Postman, unfortunately this only works once. When I press send the second time, my node.js program crashes with the error "Can't set headers after they are sent."
I am new at node.js, so any help is highly appreciated!
//app stuff
const client_id = "x";
const client_secret = "y";
const callback = "https://myurl;
// Basic Setup
var http = require('http'),
express = require('express'),
mysql = require('mysql'),
parser = require('body-parser'),
Client = require('node-rest-client').Client;
var client = new Client();
// Setup express
var app = express();
app.use(parser.json());
app.use(parser.urlencoded({ extended: true }));
app.set('port', process.env.PORT || 5000);
// Set default route
app.get('/', function (req, res) {
res.send('<html><body><p>Welcome to Bank API Wrapper</p></body></html>');
});
app.post('/authorize', function (req,res) {
var response = [];
if (typeof req.body.code !== 'undefined' && typeof req.body.state !== 'undefined' ){
var code = req.body.code, state = req.body.state;
//conversion to base64 because citi api wants it this way
var authorization = "Basic " + Buffer.from(client_id + ":" + client_secret).toString('base64');
var args = {
data:{"grant_type":"authorization_code","code":code,"redirect_uri":callback},
headers:{"Authorization":authorization,"Content-Type":"application/x-www-form-urlencoded"}
};
//get access and refresh token
client.post("https://sandbox.apihub.citi.com/gcb/api/authCode/oauth2/token/sg/gcb", args, function (citidata, citiresponse) {
//console.log(citidata);
//console.log(citiresponse);
});
client.on('error', function (err) {
response.push({'result' : 'error', 'msg' : 'unauthorized access'});
res.setHeader('Content-Type', 'application/json');
res.status(200).send(JSON.stringify(response));
});
}
else {
response.push({'result' : 'error', 'msg' : 'Please fill required details'});
res.setHeader('Content-Type', 'application/json');
res.status(200).send(JSON.stringify(response));
}
});
// Create server
http.createServer(app).listen(app.get('port'), function(){
console.log('Server listening on port ' + app.get('port'));
});
You've got this:
client.on('error', function (err) {
This register an error handler on the client but it never gets removed. The client is shared between requests so any errors on subsequent requests will still fire the old error handlers.
Instead you can listen for an error on the request. Something like this:
var request = client.post("...
request.on('error', function(err) {
// handle error
});
See https://www.npmjs.com/package/node-rest-client#error-handling