I'm developing a simple app which consumes a third-party API (Shopify REST API), the Shopify API is throttled to allow unlimited requests over time though a leaky bucket algorithm which is detailed in their documentation.
I'm using the Shopify-api-node module to make my API calls, it supports an autoLimit option out of the box which automatically limits the rate in which requests are made so you never have to think about the API call limits.
This works for most Node.js scripts, but in you run into trouble in the context of an Express app where you need to share the API instance across different routes through a session because the autoLimit option is only reliable with one single Shopify instance.
Here's my code (notice that I'm creating a new Shopify instance for each route):
var express = require("express");
var session = require("express-session");
var Shopify = require("shopify-api-node");
var app = express();
var port = 3000;
app.use(session({
secret: "secret",
resave: false,
saveUninitialized: true
}));
app.get("/", (req, res) => {
var shopName = req.session.shopName = req.session.shopName || req.query.shopName;
var apiKey = req.session.apiKey = req.session.apiKey || req.query.apiKey;
var password = req.session.password = req.session.password || req.query.password;
var shopify = new Shopify({ shopName, apiKey, password, autoLimit: true });
shopify.on("callLimits", (limits) => console.log(limits));
var requests = Array.from({ length: 100 }, () => {
return shopify.shop.get()
});
Promise.all(requests)
.then((requestsRes) => {
return res.send(JSON.stringify(requestsRes));
})
.catch((err) => {
return res.status(500).send(err);
});
});
app.get("/other-route", (req, res) => {
var shopName = req.session.shopName = req.session.shopName || req.query.shopName;
var apiKey = req.session.apiKey = req.session.apiKey || req.query.apiKey;
var password = req.session.password = req.session.password || req.query.password;
var shopify = new Shopify({ shopName, apiKey, password, autoLimit: true });
shopify.on("callLimits", (limits) => console.log(limits));
var requests = Array.from({ length: 100 }, () => {
return shopify.product.list()
});
Promise.all(requests)
.then((requestsRes) => {
return res.send(JSON.stringify(requestsRes));
})
.catch((err) => {
return res.status(500).send(err);
});
});
app.listen(port, () => {
console.log(`Example app listening on port ${port}!`)
});
Here's my package.json:
{
"name": "third-party-api-limit-issue",
"scripts": {
"start": "node index.js"
},
"dependencies": {
"express": "^4.16.4",
"express-session": "^1.15.6",
"shopify-api-node": "^2.18.1"
}
}
When this script is run, go to this URL:
http://localhost:3000?shopName=liquify-app.myshopify.com&apiKey=9d1fae34bf670cc71230fee001486e82&password=dc6f650d5921eb40e7ab1e612e9dae7e
This'll make 100 requests to the Shopify API, store the API credentials in the session and display the shop JSON data.
This breaks if you open the same URL in two tabs at the same time, or open http://localhost:3000/other-route at the same time. This is happening because I can't figure out how to share the same Shopify API instance across my routes.
I need to be able to make as many calls to the API across my routes without running into the "Too many requests" error.
Any help would be greatly appreciated.
I think you have an impedance mismatch here. Shopify has a limit. It tells you exactly where you are at. So you make a request. The response header you get back tells you how many more requests you can make. That is the number you use.
Your approach of saying, my code will limit itself, so as not to tickle the monster over there, is going to bomb out as that is not the way to really architect the request-response cycle.
Maybe try looking at the response you get before issuing another request. If you have hit the limits, pause while you wait for the bucket to recharge a little. Standard stuff. Works well.
Related
I created a simple API using express, and deployed it to Heroku, this is the code for it:
const express = require("express");
const app = express();
const cors = require("cors");
app.use(express.json());
app.use(cors());
app.use(express.static("build"));
let notes = [
{
id: 1,
content: "HTML is easy",
date: "2022-05-30T17:30:31.098Z",
important: true,
},
{
id: 2,
content: "Browser can execute only Javascript",
date: "2022-05-30T18:39:34.091Z",
important: false,
},
{
id: 3,
content: "GET and POST are the most important methods of HTTP protocol",
date: "2022-05-30T19:20:14.298Z",
important: true,
},
];
const generateId = (arr) => {
const maxId = arr.length < 0 ? 0 : Math.max(...arr.map((item) => item.id));
return maxId + 1;
};
app.get("/", (req, res) => {
res.send(`<h1>Hello World!</h1>`);
});
app.get("/api/notes", (req, res) => {
res.json(notes);
});
app.get("/api/notes/:id", (req, res) => {
const id = Number(req.params.id);
const note = notes.find((note) => note.id === id);
if (note) {
res.json(note);
} else {
res.status(404).end();
}
});
app.delete("/api/notes/:id", (req, res) => {
const { id } = Number(req.params);
notes = notes.filter((note) => note.id !== id);
res.status(204).end();
});
app.post("/api/notes", (req, res) => {
const body = req.body;
if (!body.content) {
return res.status(400).json({
error: "Content Missing",
});
}
const note = {
content: body.content,
important: body.important || false,
date: new Date(),
id: generateId(notes),
};
notes = notes.concat(note);
res.json(note);
});
app.put("/api/notes/:id", (req, res) => {
const newNote = req.body;
notes = notes.map((note) => (note.id !== newNote.id ? note : newNote));
res.json(newNote);
});
const PORT = process.env.PORT || 3001;
app.listen(PORT, () => {
console.log(`Server running on port ${PORT}`);
});
as you can see, the data served to the frontend (A React app) comes from the '/api/notes' endpoint, this endpoint returns a response with the notes array.
After deploying to Heroku (https://fierce-chamber-07494.herokuapp.com/) the functionality of adding notes, and setting importance all work perfectly normal, but what I wasn't expecting was for the data to be persistent even after refreshing the page, visiting it in another device, etc. The data only comes from a variable, not a database, nothing. So why is it persistent? does Heroku modify the variable itself? how does this work?
The top-level code of an Express server often runs once, when you start up the server. Variables declared at that top level are then persistent if there are any handlers that reference them.
Consider how a client-side page with JavaScript works - the page loads, and then the JavaScript runs. If you keep the tab open for a couple hours and then come back to it, you'll see that variables declared on pageload still exist when you come back. The same sort of thing is happening here, except that the persistent environment is on your server, rather than on a client's page.
The code that starts up the Express server - that is, your
const express = require("express");
const app = express();
const cors = require("cors");
app.use(express.json());
app.use(cors());
...
and everything below it - doesn't run every time a request is made to the server. Rather, it runs once, when the server starts up, and then when requests are made, request handlers get called - such as the callback inside
app.get("/", (req, res) => {
res.send(`<h1>Hello World!</h1>`);
});
So, the variables declared at the top-level are persistent (even across different requests) because that server environment is persistent.
That said - something to keep in mind with Heroku is that with their free and cheap tiers, if no request is made to your app for a period of time (maybe 30 minutes), Heroku will essentially turn your server off by spinning down the dyno until another request is made, at which point they'll start your server up again, which will run the top-level code again. So while you'll sometimes see a top-level variable that appears to have its mutated values persist over multiple requests, that's not something to count on if your Heroku plan doesn't guarantee 100% uptime for your server.
For my senior capstone, my group and I have developed a web-based application to simulate Bitcoin - using react.js for the front-end and node.js/express for the back-end. Up until recently, we've had all of simulation-creating-code (javascript files) inside the src directory, meaning it was being built client-side. Due to high waiting times to create a simulation from all the hashing necessary in transactions, we decided that our simulation-creating-code would be better suited for the back-end rather than the front end. Taking the load off the client and putting it on the server drastically improved the speed of creating a simulation, so 'Great success!'.
When we made this change, we ended up having some issues with require and import statements. Reactjs only supports import statements and Express uses require statements. We had to use some js functions that we developed in our API's so we imported them with require statements, and we thought we thought it was resolved because on our development environment, everything runs as smooth as butter, but once it's deployed, our login page is unable to make an API call. The error is: Failed to load resource: the server responded with a status of 500 (Internal Server Error).
It's interesting because this route in the API worked prior to making this big switch from require to import, and those changes were in other files/routes. The login API remains completely unchanged.
Either way, I'll drop some code in case it's helpful in troubleshooting.
server.js
const express = require("express");
const app = express();
const router = express.Router();
const path = require("path");
var cors = require("cors");
require("dotenv").config();
app.use(express.json({ limit: "50mb" }));
app.use(express.urlencoded({ limit: "50mb" }));
app.use(function (req, res, next) {
res.header("Access-Control-Allow-Origin", "*");
res.header(
"Access-Control-Allow-Headers",
"Origin, X-Requested-With, Content-Type, Accept"
);
next();
});
// List of routes
router.use("/api/users", require("./api/users"));
router.use("/api/data", require("./api/data"));
router.use("/api/share", require("./api/share"));
router.use("/api/addresses", require("./api/addresses"));
const root = path.join(__dirname, "client/build");
app.use(express.static(root));
app.use(router);
app.use(cors({ origin: true, credentials: true }));
app.listen(
process.env.PORT,
() => `Server running on port ${process.env.PORT}`
);
api/users.js login route
const express = require("express");
const app = express();
const db = require("../dbConn");
const bcrypt = require("bcrypt-nodejs");
const cors = require("cors");
const router = express.Router();
const jwt = require("jwt-simple");
const config = require("../configuration/config.json");
// to parse JSON
app.use(express.json());
router.post("/login", (req, res) => {
//check if email and password are sent
if (!req.body.email || !req.body.password) {
return res.status(401).json({ error: "Missing username and/or password" });
}
// go into mysql and get info
let qry = `select * from user where email = "${req.body.email}"`;
db.query(qry, (err, rows) => {
if (err) {
return res.status(500).json({ error: err });
}
// assert: no error - process the result set
if (rows.length == 0) {
// no users found
res.status(400).json({ msg: "No users found" });
} else {
// process the user records
let users = [];
rows.forEach((row) => {
let user = {
uid: row.uid,
email: row.email,
role: row.role,
dateCreated: row.created_date,
password: row.password,
};
users.push(user);
});
if (users[0]) {
// Does given password hash match the database password hash?
bcrypt.compare(req.body.password, users[0].password, (err, result) => {
// Send back a token that contains the user's username
const token = jwt.encode({ email: req.body.email }, config.secret);
if (result == true) {
res.status(200).json({
msg: "user authenticated",
fname: users[0].fname,
lname: users[0].lname,
role: users[0].role,
token: token,
});
} else {
res.sendStatus(401);
}
});
}
}
});
});
router.post("/auth", cors(), (req, res) => {
try {
let user = jwt.decode(req.body.token, config.secret);
res.status(200).send(user);
} catch (err) {
res.sendStatus(401);
}
});
SignIn.js client/src/components. This is wrapped in a react.useEffect() arrow function, but again I don't believe the issue is here because this page remains unchanged from a working version.
const handleSubmit = (e) => {
e.preventDefault();
const credentials = { email, password };
// API call to login to account
// if successful, redirect to landing page
// if not, display error message
fetch(`http://${process.env.REACT_APP_API_URL}/api/users/login`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(credentials),
})
.then(async (res) => {
if (res.status == 200) {
return res.json();
} else {
throw new Error("Failed to Login!");
}
})
.then(async (res) => {
// Store token in cookie
setCookie("token", res.token, { path: "/${path}", maxAge: 3600 * 24 });
// Toggle state of sign in
toggleSignIn();
// Feedback
setFeedback(true);
setFeedbackObj({ message: "Signed in!", severity: "success" });
//redirect
history.push(`${process.env.PUBLIC_URL}/simulation`);
})
.catch(async (err) => {
// Feedback
setFeedback(true);
setFeedbackObj({ message: "Sign In Error", severity: "error" });
console.error(err);
});
};
If there are any other files that are of interest please let me know.
I've tried to mess with the proxy in package.json, but I don't think thats the answer because it was working previously. I've had a really difficult time finding others with similar issues or resources other than how to build a simple app with Express backend and React.js front end. This is not our issue because our application was working perfectly before this big switch. I believe the issue is stemming from require statements in our API and the running of JS functions in the API. I have no way to confirm this because in production (deployment), the errors are super uninformative, and in development, it runs perfectly fine.
I have been trying to solve this issue for a couple of weeks now, and I've made very little progress. If anyone has suggestions or tips on troubleshooting deployment, I would greatly appreciate it.
Thanks!
I am building a single page application with a Vuejs frontend and a Nodejs backend. Been reading tons about single sign on and oidc, and managed to implement authentication using Oidc for the frontend, where I get a token from my identity provider.
Not sure however, now, how to also implement this for the backend and where/when/how.
So currently, when a user accesses the page, in my router.js file, this happens:
router.beforeEach(vuexOidcCreateRouterMiddleware(store));
In the store then, I do this:
Vue.use(vuex);
const store = new Vuex.Store({
state: {
// holds current list of products
products: [],
},
getters,
mutations,
actions,
modules: {
// initialize PING-OIDC module
oidcStore: vuexOidcCreateStoreModule(
oidcSettings,
{ namespaced: false },
{
userLoaded: (oidcUser) => {
axios.defaults.headers.common.Authorization = `${oidcUser.token_type} ${oidcUser.access_token}`;
},
},
),
},
});
export default store;
So I set the authorization header, but now I am not sure where and how to proceed in the backend to also add + validate authentication there.
Very new to all this and there seem to be so many different way to proceed, so appreciate all the hints.
Currently, in server.js, I just do the following:
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
app.use(bodyParser.json())
const cors = require('cors')
const corsOptions = {
origin: 'http://localhost:5002',
optionsSuccessStatus: 200
}
app.use(cors(corsOptions))
//get mysql db here
const data = require('./app/config/db.config.js');
const db = data.MySQL_DB;
// show all products
app.get('/api/productlist',(req, res) => {
const sql = "SELECT ID FROM Product_Table";
const query = db.query(sql, (err, results) => {
if(err) throw err;
console.log("productIds ", results);
res.send(JSON.stringify({"status": 200, "error": null, "response": results}));
});
});
// Create a Server
var server = app.listen(8080, function () {
var host = server.address().address
var port = server.address().port
console.log("App listening at http://%s:%s", host, port)
})
So should I create a post request in there as well? Just not sure on how to validate the request there.. Thanks a lot for the help!
You need to clarify which data/routes/pages you want to protect.
You log in from your client -> send to the server (ex : api/login) -> respond to your client with credential -> store the user and credential
see passport.js for express
see accesscontrol for your data server side
Note that routes in client can be easily hack
Im working with a React App where I present a list top Podcasts. I'm using iTunes Search API to dynamically present data to the user. For now, I working with a Node Express server to setup my custom endpoints. The problem is that the API has a request limit, so I tought that I could save what I get from the response to Firebase and present the data from firebase instead.
To my question;
Can in some way save the response I get from iTunes Search API to Firebase?
For now my code for fetching data from my API Endpoints looks like this in my Node+Express server:
const express = require('express');
const unirest = require('unirest');
const app = express();
const port = process.env.PORT || 5000;
// Get all Episodes from a specific podcast
app.get('/api/podcast/episodes', (req, res) => {
const feedurl = req.query.feedurl
unirest.get(feedurl)
.end((response) => {
res.status(200).send(response.body)
});
});
// Get Podcast by ID
app.get('/api/podcast/:id', (req, res) => {
const podID = req.params.id;
unirest.get(`https://itunes.apple.com/lookup?id=${podID}&country=se`)
.end((response) => {
res.status(200).send(response.body)
});
});
// Get Podcast Categorys
app.get('/api/podcast/:category/:amount', (req, res) => {
const categoryID = req.params.category;
const amount = req.params.amount;
unirest.get(`https://itunes.apple.com/se/rss/toppodcasts/limit=${amount}/genre=${categoryID}/explicit=true/json`)
.end((response) => {
res.status(200).send(response.body)
});
});
// Get Podcast Categorys
app.get('/api/categorys', (req, res) => {
unirest.get('https://itunes.apple.com/WebObjects/MZStoreServices.woa/ws/genres?id=26&cc=se')
.end((response) => {
res.status(200).send(response.body)
});
});
app.listen(port, () => console.log(`Listening on port ${port}`));
Im just looking for someone who could point me in the right direction how to proceed. Cause for now I'm stuck, big time.
Depending on how long you want to cache the response, you can use a whole different things - a physical database like MySql, Sqlite, MongoDB etc to locally persist data.
If you only want to keep the cached result for a short period of time, you can use in-memory cache or just any other tool that offers you same functionality. Redis is also a good contender as a temporary store, especially when you expect to scale to more than one node instance for your application.
Below, I have modified a part of your code to cache result for 10mins, using memory-cache npm module
const express = require('express');
const unirest = require('unirest');
const cache = require('memory-cache');
const CACHE_DURATION = 10 * 60 * 1000; //10mins
const app = express();
const port = process.env.PORT || 5000;
// Get all Episodes from a specific podcast
app.get('/api/podcast/episodes', (req, res) => {
const cacheKey = req.query.feedurl; //Or anything unique to this route
const cachedData = cache.get(cacheKey);
if(cachedData) {
return res.json(cachedData);
}
const feedurl = req.query.feedurl
unirest.get(feedurl)
.end((response) => {
res.status(200).send(response.body);
cache.put(cacheKey, response.body, CACHE_DURATION);
});
});
---- the rest of your code ----
You can hit the route as many times as you want and be guaranteed that data will be fetched from iTunes only once in 10mins.
The second and subsequent requests will be served a lot faster from cache.
Let me know if this is what you are looking for.
I'm developing an app with node JS, the app generates a report calling the endpoint api.example.com/generate-report
But this report takes around 1 minute on be generated, then I want to implement something like this:
User click on generate report
System return response {generating:"ok"}
After the system generate the report send a notification (this I what I know how to do)
User get the report
Is this possible with nodejs?
After I do some research, this can be easily done using Promises.
To run the following code it's necessary to install express and node uuid
npm install --save express
npm install --save uuid
node index.js
The source code of index is:
//index.js
const express = require("express");
const app = express();
const PORT = process.env.PORT || 5000;
const uuidV1 = require('uuid/v1');
// this is where we'll store the results of our jobs by uuid once they're done
const JOBS = {};
app.get("/", (req, res) => {
res.send("It works!");
});
app.get("/startjob", (req, res) => {
let times = [100, 1000, 10000, 20000];
let promises = [];
for (let time of times) {
promises.push(new Promise((resolve, reject) => {
setTimeout(resolve, time, `${time} is done.`);
}));
}
// obviously, you'd want to generate a real uuid here to avoid collisions
let uuid = uuidV1();
console.log(uuid);
Promise.all(promises).then(values => { JOBS[uuid] = values; });
res.redirect(`progress/${uuid}`);
});
app.get("/progress/:uuid", (req, res) => {
if (JOBS[req.params.uuid] === undefined) {
res.send("Still processing your request.");
} else {
res.send(`Here's your result: ${JOBS[req.params.uuid]}.`);
// instead of immediately deleting the result of the job (and making it impossible for the user
// to fetch it a second time if they e.g. accidentally cancel the download), it would be better
// to run a periodic cleanup task on `JOBS`
delete JOBS[req.params.uuid];
}
});
app.listen(PORT, () => {
console.log(`Listening on localhost:${PORT}.`);
});
When the code runs you will be redirected to /process/uuid and I get the status of the process.
This needs some improvements because I want the response like "{process:uuid}" and I can store this on my Local Storage to use after.
Well, I hope this help to someone.