I have a task where I am given a URL such as https://xyz.json. This URL prompts the downloading of the JSON file into the local. I am now required to read the use this JSON data for further processing. Since I am new to NodeJS and express, I find myself confused about how to achieve this in ExpressJS.
This is what I've tried :
const https = require("https");
const fs = require("fs");
const file = fs.createWriteStream("outputFile.json");
const request = https.get(
"https://xyz.json",
function (response) {
response.pipe(file);
// after download completed close filestream
file.on("finish", () => {
file.close();
console.log("Download Completed");
});
}
);
Here, in the outputFile.json, no data is present
Qn2) Can I periodically download using setTimeOut(). Would it be efficient or is there any better way of caching data to make the application faster?
Thanks in advance!
Here's a sample app that downloads a json triggered when you hit an API route hosted as ExpressJS sever.
const express = require('express');
const cors = require('cors');
const morgan = require('morgan');
const bodyParser = require('body-parser');
const axios = require('axios');
const fs = require('fs');
const app = express();
app.use(cors());
app.use(morgan(':method :url :status :user-agent - :response-time ms'));
app.use(bodyParser.json());
app.get('/', async (req, res) => {
try {
const { status, data } = await axios.get('http://52.87.135.24/json-files/events.json'); // Can be replaced by your json url
if (status === 200) {
fs.writeFileSync('data.json', JSON.stringify(data));
res.status(200).json({
success: 'Downloaded file.',
data: data // Comment it if you don't want to send the data back
})
} else {
res.status(404).json({ 'Failed': 'File not found.' })
}
} catch (err) {
console.log(err);
res.status(500).json({ 'Error': 'Internal Server Error' });
}
});
app.listen(process.env.PORT || 3000, function () {
console.log('Express app running on port ' + (process.env.PORT || 3000))
});
And as I mentioned that this download gets triggered every time you make a request on http://localhost:3000 in this case, you can create a client script that acts like a cron job in which you can use the setTimeout or actually, setInterval to download your file periodically.
const axios = require('axios');
setInterval(async () => {
await axios.get('http://localhost:3000/');
}, 5000);
Here's such a script along! :)
Related
Right now I have a front end react application using axios and and a backend server using node.js and express. I cannot for the life of me get my serp api data to post so that my front end can get it through axios and display the json data. I know how to get data to the front end but I am not a backend developer so this is proving to be incredibly difficult at the moment. I'm able to get the data from the the external api, I just don't know how to post it once I get it. Also I would not like to have all these request running on server.js so I created a controller but I think that is where it is messing up. Any help is appreciated
//pictures controller
const SerpApi = require('google-search-results-nodejs');
const {json} = require("express");
const search = new SerpApi.GoogleSearch("674d023b72e91fcdf3da14c730387dcbdb611f548e094bfeab2fff5bd86493fe");
const handlePictures = async (req, res) => {
const params = {
q: "Coffee",
location: "Austin, Texas, United States",
hl: "en",
gl: "us",
google_domain: "google.com"
};
const callback = function(data) {
console.log(data);
return res.send(data);
};
// Show result as JSON
search.json(params, callback);
//res.end();
}
// the above code works. how do i then post it to the server so that i can retrieve it to the backend?
module.exports = {handlePictures};
//server.js
const express = require('express');
const app = express();
const path = require('path');
const cors = require('cors');
const corsOptions = require('./config/corsOptions');
const { logger } = require('./middleware/logEvents');
const errorHandler = require('./middleware/errorHandler');
const cookieParser = require('cookie-parser');
const credentials = require('./middleware/credentials');
const PORT = process.env.PORT || 3500;
// custom middleware logger
app.use(logger);
// Handle options credentials check - before CORS!
// and fetch cookies credentials requirement
app.use(credentials);
// Cross Origin Resource Sharing
app.use(cors(corsOptions));
// built-in middleware to handle urlencoded form data
app.use(express.urlencoded({ extended: false }));
// built-in middleware for json
app.use(express.json());
//middleware for cookies
app.use(cookieParser());
//serve static files
app.use('/', express.static(path.join(__dirname, '/public')));
// routes
app.use('/', require('./routes/root'));
app.use('/pictures', require('./routes/api/pictures'));
app.all('*', (req, res) => {
res.status(404);
if (req.accepts('html')) {
res.sendFile(path.join(__dirname, 'views', '404.html'));
} else if (req.accepts('json')) {
res.json({ "error": "404 Not Found" });
} else {
res.type('txt').send("404 Not Found");
}
});
app.use(errorHandler);
app.listen(PORT, () => console.log(`Server running on port ${PORT}`));
//api/pictures.js
const picturesController= require('../../controllers/picturesController');
const express = require('express')
const router = express.Router();
// for POST request use app.post
router.route('/')
.post( async (req, res) => {
// use the controller to request external API
const response = await picturesController.handlePictures()
// send the response back to client
res.json(response)
})
module.exports = router;
You just need to return the result from SerpApi in your handlePictures function. To do this make a new Promise and when search.json runs callback do what you need with the results and pass it in resolve.
Your picturesController.js with an example of returning all results.
//pictures controller
const SerpApi = require("google-search-results-nodejs");
const { json } = require("express");
const search = new SerpApi.GoogleSearch(process.env.API_KEY); //your API key from serpapi.com
const handlePictures = async (req, res) => {
return new Promise((resolve) => {
const params = {
q: "Coffee",
location: "Austin, Texas, United States",
hl: "en",
gl: "us",
google_domain: "google.com",
};
const callback = function(data) {
resolve(data);
};
search.json(params, callback);
});
};
module.exports = { handlePictures };
Output:
And I advise you to change your API key to SerpApi to prevent it from being used by outsiders.
Since I don't have the full context of your App I can just assume the context. But given the fact that you already have wrapped the logic of calling the external API into a dedicated controller you can use it in the following way in an express app (used the hello world example from express):
// import your controller here
const express = require('express')
const app = express()
const port = 3000
// for POST request use app.post
app.get('/', async (req, res) => {
// use the controller to request external API
const response = await yourController.method()
// send the response back to client
res.json(response)
})
app.listen(port, () => {
console.log(`Example app listening on port ${port}`)
})
Here's an example how to execute the http request from the frontend:
const response = await fetch('http://localhost:3000') // result from res.json(response)
I am looking for a solution to directly download a file in the Firebase Storage when hitting an API endpoint. I tried initializing a Google-Cloud Storage and downloading the file from the bucket.
const app = require('express')();
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({keyFilename: keyPath});
app.get("/download", (req, res) => {
storage.bucket(bucketName).file("file.txt").download({destination: './file.txt'});
});
app.listen(8080);
But this does not work!
I simply get:
UnhandledPromiseRejectionWarning: Error: Not Found
Could someone help me, please?
Where did you initialize the app
Original answer:
// Dependencies
const express = require('express')
const PORT = process.env.PORT || 3002;
// Initialize the App
const app = express();
// Start the app
app.listen(PORT, () => {
console.info(`Server is listening on port ${PORT}`);
});
Update:
Making HTTP requests to download files is an asynchronous operation. You need to wait for the file to be downloaded from the Google Cloud Storage before sending it to the client
const app = require('express')();
const { Storage } = require("#google-cloud/storage");
const storage = new Storage({keyFilename: keyPath});
// I am using async/await here
app.get("/download", async (req, res) => {
// You have to wait till the file is downloaded
await storage.bucket(bucketName).file("file.txt").download({destination: './file.txt'});
// Send the file to the client
res.download('./file.txt')
});
app.listen(8080);
If the intention is to stream the file to the requesting client, you can pipe the data from Cloud Storage through to the response. It will look similar to the following:
const {Storage} = require('#google-cloud/storage');
const express = require('express');
const BUCKET_NAME = 'my-bucket';
const app = express();
const storage = new Storage({keyFilename: './path/to/service/key.json'});
app.get("/download", (req, res) => {
storage.bucket(bucketName).file("path/in/bucket/to/file.txt").createReadStream()
.on('error', (err) => {
res.status(500).send('Internal Server Error');
console.log(err);
})
.on('response', (storageResponse) => {
// make sure to check storageResponse.status
res.setHeader('content-type', storageResponse.headers['Content-Type']);
res.setHeader('content-length', storageResponse.headers['Content-Length']);
res.status(storageResponse.status);
// other headers may be necessary
// if status != 200, make sure to end the response as appropriate. (as it won't reach the below 'end' event)
})
.on('end', () => {
console.log('Piped file successfully.');
res.end();
}).pipe(res);
});
app.listen(8080);
I am making an API for my minecraft server and have been able to get as far as getting the JSON file to update what I send it in a POST request. I would like to know if it is possible to only update on key of the JSON file.
This is my current code:
var fs = require('fs');
var fileName = './serverStatus.json';
var file = require(fileName);
const express = require('express');
const bodyParser = require('body-parser');
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
const cors = require('cors');
const { fileURLToPath } = require('url');
app.get('/status', alldata);
function alldata(request, response) {
response.send(file);
}
app.post('/status', (req, res) => {
if (!req.is('application/json')) {
res.status(500);
res.send('500 - Server Error');
} else {
res.status(201);
fs.writeFile(
fileName,
JSON.stringify(req.body, null, 4),
function writeJSON(err) {
if (err) return console.error(err);
console.log(JSON.stringify(file));
console.log('writing to ' + fileName);
}
);
res.send(file);
}
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () =>
console.log(`Server running on: http://localhost:${PORT}`)
);
and my JSON file:
{
"lobby": "offline",
"survival": "offline",
"creative": "offline"
}
Thanks in advance!
You could use fs.readFileSync or to read file content.
Then update your JSON content such as jsonData["survival"] = "online".
Final, write content back to file with fs.writeFile. (See note-1)
You could see the following example code.
const fs = require("fs");
// 1. get the json data
// This is string data
const fileData = fs.readFileSync("./serverStatus.json", "utf8")
// Use JSON.parse to convert string to JSON Object
const jsonData = JSON.parse(fileData)
// 2. update the value of one key
jsonData["survival"] = "online"
// 3. write it back to your json file
fs.writeFile("./serverStatus.json", JSON.stringify(jsonData))
Note-1: Because you save data in file, you need to write the whole data when you want to update file content.
But, if you want to get the latest file content after you write your new data into file, you should fs.readFileSync your file again like following code to avoiding any modified which are forgot to save.
app.get('/status', alldata);
function alldata(request, response) {
const fileContent = fs.readFileSync(fileName, "utf8");
const fileJsonContent = JSON.parse(fileContent)
// do other stuff
response.send(fileContent);
}
var fs = require('fs');
const express = require('express');
const bodyParser = require('body-parser');
var fileName = './serverStatus.json';
const app = express();
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
// maybe use this instead of bodyParser:
//app.use(express.json());
const cors = require('cors');
const { fileURLToPath } = require('url');
app.get('/status', alldata);
function alldata(request, response) {
response.send(file);
}
app.post('/status', (req, res) => {
if (!req.is('application/json')) {
res.status(500);
res.send('500 - Server Error');
} else {
// read full config file:
var src = fs.readFileSync(fileName);
// convert src json text to js object
var srcObj = JSON.parse(src);
// convert req json text to js object
var reqObj = JSON.parse(req.body);
// update the src with the new stuff in the req
for(var prop in reqObj){
srcObj[prop] = reqObj[prop];
}
// update any additional things you want to do manually like this
srcObj.bob = "creep";
// convert the updated src object back to JSON text
var updatedJson = JSON.stringify(srcObj, null, 4);
// write the updated src back down to the file system
fs.writeFile(
fileName,
updatedJson,
function (err) {
if (err) {
return console.error(err);
}
console.log(updatedJson);
console.log('updated ' + fileName);
}
);
res.send(updatedJson);
}
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () =>
console.log(`Server running on: http://localhost:${PORT}`)
);
//res.status(201);
i have a simple node node express server in which i get data from an api,it works fine on the first request but fails when i try to make a second request
const express=require("express");
const axios =require("axios");
const cors = require('cors');
const app=express();
app.use(cors());
app.get("/devices",(req,res)=>{
axios.get(
'http://ipaddress/api/reports',
).then((response) => {
res.status(200);
res.json(response.data);
}).catch((error) => {
res.status(400)
res.send("error")
});
});
app.listen(3002,()=>{
console.log("started on port 3002");
});
The problem i found here is you have initialize the server in your get route. The app.listen.. code should be outside the get route implementation. I doubt if your code even runs for the first time. Change your code like:
const express = require("express");
const axios = require("axios");
const cors = require('cors');
const app = express();
app.use(cors());
app.get("/devices", (req,res) => {
axios.get(
'http://ipaddress/api/reports',
).then((response) => {
res.status(200).json(response.data);
}).catch((error) => {
res.status(400).send("error")
});
});
app.listen(3002,() => {
console.log("started on port 3002");
});
Hope this helps :)
I'm trying to upload images. It's reaching the backend, but the request body and req.image are coming out empty.
I have the submission:
const handleSubmit = async () => {
try {
const data = createFormData();
console.log(data); // prints the correct request object
const response = await axios.post(
`http://${GATEWAY}:5000/api/uploads/single`,
JSON.stringify(data)
);
alert("Upload success!");
console.log("response.data", response.data);
} catch (err) {
console.log("err caught --> ", err);
}
};
const createFormData = () => {
const data = new FormData();
data.append("title", title); // coming from
data.append("body", body); // react hooks state (useState)
data.append("image", {
height: image.height,
width: image.width,
type: image.type,
uri:
Platform.OS === "android" ? image.uri : image.uri.replace("file:/", "")
});
return data;
};
My endpoint:
const express = require("express");
const multer = require("multer");
const bodyParser = require("body-parser");
express().use(bodyParser.json());
const router = express.Router();
// middleware
const auth = require("../../middleware/auth");
const storage = multer.diskStorage({
destination(req, file, callback) {
callback(null, "./images");
},
filename(req, file, callback) {
callback(null, `${file.fieldname}_${Date.now()}_${file.originalname}`);
}
});
const upload = multer({ storage });
// #route POST api/uploads/single
// #desc Upload single image
// #access Private
router.post(
"/single",
// upload.array("photo", 3),
auth,
upload.single("image"),
(req, res) => {
console.log("req", req.body); // {}
console.log("req", req.image); // undefined
return res.status(200).json({
message: "Response from backend"
});
}
);
module.exports = router;
And my server.js
const express = require("express");
const connectDb = require("./config/db");
const app = express();
// connect to db
connectDb();
// Define routes (some omitted for brevity)
app.use("/api/uploads", require("./routes/api/uploads"));
const PORT = process.env.PORT || 5000;
app.listen(PORT, () => console.log(`Listening on port ${PORT}`));
For some reason, in the first snippet, if I do not stringify the data: FormData object that is built from createFormData(), my backend is never even reached.
I've tried so many things, and I'm not starting to think that maybe my backend isn't setup properly. The line where I'm doing express().use(bodyParser.json()); exists because I can't do app.use(bodyParser.json()); (or at least I think), because the app object is in the main server.js file. I'm including other API routes in other files.
For example, my server.js has these, amongst others:
// Define routes
app.use("/api/auth", require("./routes/api/auth"));
app.use("/api/users", require("./routes/api/users"));
app.use("/api/profile", require("./routes/api/profile"));
app.use("/api/uploads", require("./routes/api/uploads"));
And I was following this tutorial to use multer with react-native. A little lost at this point, not sure what I'm doing wrong.
Edit:
I'm making the request like this now,
const config = {
headers: {
"Content-Type": "multipart/form-data"
}
};
const response = await axios.post(
`http://${GATEWAY}:5000/api/uploads/single`,
data,
config
);
But It's failing with a
If I stringify it, it hits the backend but not in the way I need it to:
Edit:
I got it working by specifying the image type, as per the suggestion here
I think this is because you are not setting your content-type to multipart/form-data. Try adding this to your request options:
const response = await axios.post(
`http://${GATEWAY}:5000/api/uploads/single`,
data,
headers: {
`Content-Type`: `multipart/form-data`
}
);
Because of multipart/form-data, do not stringify the data you are sending. Stringifying the data will cause it to only be read as text by the server but is expecting a file to be attached.
Got it working by specifying the image type as per the suggestion here
You're trying to access the file from req.image and req.body, but as mentioned in the https://www.npmjs.com/package/multer, you can access it from :
req.file
req.body will hold the text fields only, on the other hand, if you only uploaded a single file you can find it in req.file, but if you uploaded multiple files you will find them in req.files.
I used the following line to get images:-
concole.log(req.files);