I am trying to make a site for crypto data using coin-gecko's API.
They have 2 different end points for what i require and as such require 2 different URLs.
I had no problem using into the globalUrl to get data such as the total Market cap, volume, etc. which i was able to render into my ejs.
My problem is now i cannot use the other URL for this, seeing as I cannot make another get request, what would be the best way to get data from the topCoinsUrl such as say the "id" of bitcoin from the 2nd url please
const https = require('https');
const app = express();
app.get("/", function(req, res) {
const globalUrl = "https://api.coingecko.com/api/v3/global";
const topCoinsUrl = "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=100&page=1&sparkline=false&price_change_percentage=1h"
https.get(globalUrl , function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const globalMarket = JSON.parse(data);
const totalCryptocurrencies = globalMarket.data.active_cryptocurrencies
let totalMarketCap = globalMarket.data.total_market_cap.usd
let totalMarketCapUsd = totalMarketCap.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let totalVolume = globalMarket.data.total_volume.usd
let total24hVolume = totalVolume.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let markets = globalMarket.data.markets
let bitcoinMarketShare = Math.round(globalMarket.data.market_cap_percentage.btc);
res.render("home", {
totalCryptocurrencies: totalCryptocurrencies,
totalMarketCap: totalMarketCapUsd,
total24hVolume: total24hVolume,
markets: markets,
bitcoinMarketShare: bitcoinMarketShare
});
})
}).on("error", function(error) {
console.error(error)
});
});
// Ideally i would like to add this to get the ID of bitcoin, but I get an error when i try to use the 2 get requests:
https.get(topCoinsUrl, function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const topCoinsUrl = JSON.parse(data);
let bitcoinId = topCoinsUrl[0].symbol
res.render("home", {
bitcoinId: bitcoinId
})
})
// Error handler
}).on("error", function(error) {
console.error(error)
});
});
If you wish to make 2 simultaneous requests, you should use something like Promise.all() . Create two network requests and fire them at the same time using Promise.all & collect their result.
You can use Blurebird as well... http://bluebirdjs.com/docs/api/promise.all.html
Related
const { response } = require("response");
const express = require("express");
const https = require("https")
const app = express ();
app.get("/", function(req,res) {
const url = "https://api.openweathermap.org/data/2.5/weather?q=London&applied=536bcef96b2f01cd9b9f076db90807fe&unit=metric";
https.get(url, function(response) {
console.log(response.statusCode);
})
response.on("data", function(data) {
const weatherData = JSON.parse(data)
console.log(weatherData);
})
res.send("Welcome to the future");
})
app.listen(3000, function() {
console.log("listening on port 3000");
})
The problem here is that when I type
response.on to get data from the url
to print it in the command line, it
brings const { response } = require
("express") as shown above which is
very alien to me.
Please, how do I fix it so I can get
my weatherData printed in the CMD?
There are quite a few things you'll need to change.
First, this section is wrong:
https.get(url, function(response) {
console.log(response.statusCode);
})
response.on("data", function(data) {
const weatherData = JSON.parse(data)
console.log(weatherData);
})
Since "response" is a parameter you receive from the callback on the "get" function, you need to declare the "response.on" inside the funcion scope, like this:
https.get(url, function(response) {
console.log(response.statusCode);
response.on("data", function(data) {
const weatherData = JSON.parse(data)
console.log(weatherData);
})
})
Also, the "data" event only delivers a chunk of data. You should be listening for an "end" event aswell, and only parse the data when you receive the "end" event
https.get(url, function(response) {
console.log(response.statusCode);
const result = []
response.on("data", function(data) {
result.push(data);
})
.on("end", function() {
const weatherData = JSON.parse(result.join(""));
console.log(weatherData);
})
})
And since you're not using the module named "response", you also need to remove this:
const { response } = require("response");
And then correct all the typos that were already mentioned in the comments, which were:
Add the missing quote " at require("express) on line 2
Remove the extra backsting at console.log("listening on port 3000")`; on line 17
Change the second query parameter on your URL on line 6 from "applied" to "appid"
First confirm your url is valid
https://api.openweathermap.org/data/2.5/weather?q=London&appid=536bcef96b2f01cd9b9f076db90807fe&unit=metric
If you are using windows 10 or 11 you don't need all those responses simply try this at cmd line (NOTE you need for each & within the url to escape with ^ like this ^&)
curl -o current.txt https://api.openweathermap.org/data/2.5/weather?q=London^&appid=536bcef96b2f01cd9b9f076db90807fe^&unit=metric
type current.txt
you could include both in one line but for the &type that last & does not need ^escape
curl -o current.txt https://api.openweathermap.org/data/2.5/weather?q=London^&appid=536bcef96b2f01cd9b9f076db90807fe^&unit=metric&type current.txt
after the download you should see the response in the console.
so you can call that command any way you wish (hidden or not) and or read the text file on the screen or in any application you choose.
Current.txt
{"coord":{"lon":-0.1257,"lat":51.5085},"weather":[{"id":800,"main":"Clear","description":"clear sky","icon":"01n"}],"base":"stations","main":{"temp":276.78,"feels_like":272.23,"temp_min":275.76,"temp_max":278.14,"pressure":999,"humidity":84},"visibility":10000,"wind":{"speed":6.17,"deg":250},"clouds":{"all":9},"dt":1641696366,"sys":{"type":2,"id":2019646,"country":"GB","sunrise":1641715415,"sunset":1641744666},"timezone":0,"id":2643743,"name":"London","cod":200}
Here's a fairly simple version using the got() library for making the http request. It's a promise-based, higher level library that is just a lot easier to use than the https library which works at a lower level and requires more code to work properly and handle errors.
Here's how you would do this with the got() library:
const got = require("got");
const express = require("express");
const app = express();
app.get("/", async function(req, res) {
try {
const url = "https://api.openweathermap.org/data/2.5/weather?q=London&appid=536bcef96b2f01cd9b9f076db90807fe&unit=metric";
const result = await got(url).json();
console.log(result);
res.json(result);
} catch (e) {
console.log(e);
res.sendStatus(500);
}
});
app.listen(3000, function() {
console.log("listening on port 3000");
});
Changes:
Fixed URL (change applied to appid).
Switch to got() library for the http request and built in JSON parsing
Add error handling
Send result as JSON
This generates the following output:
{
coord: { lon: -0.1257, lat: 51.5085 },
weather: [ { id: 800, main: 'Clear', description: 'clear sky', icon: '01n' } ],
base: 'stations',
main: {
temp: 276,
feels_like: 271.47,
temp_min: 274.33,
temp_max: 277.49,
pressure: 1000,
humidity: 86
},
visibility: 10000,
wind: { speed: 5.66, deg: 250 },
clouds: { all: 8 },
dt: 1641707384,
sys: {
type: 2,
id: 2019646,
country: 'GB',
sunrise: 1641715415,
sunset: 1641744666
},
timezone: 0,
id: 2643743,
name: 'London',
cod: 200
}
So I am making a kind of API middleware for my company that will grab information from the NOAA API and then store in in my database. It does more then but that a separate part. I have set it up so that it works it will get the information and store it in my sql database perfectly The issue is the information I get is based off of zipcode. One request is the information for one zipcode. I need to be able to 'loop" through a list of zipcode one at a time and store the information in the database. I am not sure how to properly get it to work. I have tested a couple of ways but have not been able to get it to work so if someone can get me pointed in the right direction it would be appreciated.
Sorry in advance my code is not cleaned up.
Everything below apiRequest.end() has little function for the question. I keep it for context.
let mysql = require('mysql');
let config = require('./config.js');
var https = require("https");
var express = require("express");
var app = express();
const port = 3000;
var fs= require('fs');
var csv = require('fast-csv');
//last test
//array will replace this zip variable
let zip = '90012';
api(zip);
function api(zips){
//All of the parts for building the get requests url
app.get("/", function(req, response) {
var apiKey = "gPaEVizejLlbRVbXexyWtXYkfkWkoBhd";
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid="datasetid=GHCND";
let datatypeid="&datatypeid=TMAX";
let location="&locationid=ZIP:";
const zipcode = zips;
let startdate="&startdate=2019-01-01";
let enddate="&enddate=2020-01-01";
let units = "&units=standard";
let limit="&limit=1000";
let url = webapi + datasetid + datatypeid + location + zipcode + startdate + enddate + units + limit;
var options = {
port: 443,
method: "GET",
headers: {
"token": apiKey
}
};
let data = "";
//request to grab from NOAA api
let apiRequest = https.request(url, options, function(res) {
console.log("Connected");
//grabing all data
res.on("data", chunk => {
data += chunk;
});
res.on("end", () => {
console.log("data collected");
//Format JSON data
response.send(JSON.parse(data));
var getData = JSON.parse(data);
if(isEmpty(getData)){
emptyCorrect();
}
dataFormat(getData);
});
});
apiRequest.end();
});
//fix format for date Can add more formating if needed here
function dataFormat(formData){
for(x in formData.results){
let date = formData.results[x].date;
formData.results[x].date = date.slice(0,10);
}
jsonToSQL(formData.results);
}
//test function is going to be used for inserting the zip
function test(){
var content = "";
console.log("your test worked see ***************");
return "92507";
}
//function to add grabed JSON data into the SQL database
function jsonToSQL(datafin){
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let values = [];
for(let x in datafin){
values.push([zipcode,datafin[x].date,datafin[x].value]);
}
// execute the insert statment
connection.query(stmt, [values], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function emptyCorrect(){
console.log("Eror correction");
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let valueE = [];
valueE.push([zipcode,"0","No Data"]);
// execute the insert statment
connection.query(stmt, [valueE], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function isEmpty(obj) {
for(var key in obj) {
if(obj.hasOwnProperty(key))
return false;
}
return true;
}
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
}
As I understand your problem can roughly be summarized as "How to loop through asynchronous evaluations in Nodejs".
There are some options for you. I would recommend wrapping call to the NOAA API with a promise and then chain those promises. This can be done as follows:
app.get('/', async function(req, response) {
var apiKey = 'some value';
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid = 'datasetid=GHCND';
let datatypeid = '&datatypeid=TMAX';
let location = '&locationid=ZIP:';
let startdate = '&startdate=2019-01-01';
let enddate = '&enddate=2020-01-01';
let units = '&units=standard';
let limit = '&limit=1000';
var options = {
port: 443,
method: 'GET',
headers: {
token: apiKey
}
};
const zipCodes = ['90012', '90013']; // Place a call to your function for fetching zip codes here
let datas = [];
prom = Promise.resolve();
zipCodes.forEach(zipcode => {
prom = prom.then(() =>
new Promise((resolve, reject) => {
let url =
webapi +
datasetid +
datatypeid +
location +
zipcode +
startdate +
enddate +
units +
limit;
let apiRequest = https.request(url, options, function(res) {
console.log('Connected');
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
console.log('data collected for zip ' + zipcode);
datas.push(data);
resolve();
});
});
apiRequest.end();
})
);
});
prom.then(() => {
// All requests have now been handled sequentially
response.send(/* You'll need to figure out what to do here */);
});
});
An alternative is to use something like the async library for dealing with sequentially calling callbacks. The async library (https://github.com/caolan/async) describes itself as:
Async is a utility module which provides straight-forward, powerful functions for working with asynchronous JavaScript.
See e.g. Node.js: How do you handle callbacks in a loop? for a similar problem (not with regards to callign an API, but dealing with asynchronous function in a loop).
I am having difficulty getting my LatLon look up to work - I have read
Get Google Maps Geocoding JSON from Express - but that just says use HTTP...and I have read the docs on http/https - but I'm still getting an error.
Here is my code - so calling myUrl/LatLon should give me the Google API response - or at least that is what I want...
const https = require('https');
router.get( '/LatLon', ( res ) => {console.log('Here getting https');
const googleKey = '---';
const address = '1600 Amphitheatre Parkway, Mountain View, CA';
const options = new URL('https://maps.googleapis.com/maps/api/geocode/json?address=' + address + '&key=' + googleKey);
const req = https.request(options, (res) => {
res.on('data', (chunk) => {
console.log(`BODY: ${chunk}`);
});
res.on('end', () => {
console.log('No more data in response.');
});
});
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
req.write();
req.end();
});
I get this error -
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be one of type string or Buffer. Received type undefined
at write_ (_http_outgoing.js:595:11)
at ClientRequest.write (_http_outgoing.js:567:10)
Any help would be greatly appreciated - I have tried about 4 variations on using "get" or "https"...
I found node-geocoder - and it worked great...
Basically I did this, it is 'generalized code', non-functional; but you'll get the idea.
A bunch of checks and compares went into it so I am not hitting API's when I do not need to.
var NodeGeocoder = require('node-geocoder');
var options = {
provider: process.env.GEOCODING_PROVIDER,
httpAdapter: 'https',
apiKey: process.env.GEOCODING_KEY,
formatter: null
};
var geocoder = NodeGeocoder(options);
collection.getExistingId( req.params.id, ( err, record ) => {
const existingAddress = addresstoString(record.address);
const newAddress = addresstoString(newRecord.address);
if ( !compareAddresses(existingAddress,newAddress) ) {
geocoder.geocode(newAddress, function(err, geocode) {
let coords = []; // mongoDB wants [Long,Lat]
coords[0] = geocode[0].longitude;
coords[1] = geocode[0].latitude;
// set existingAddress.COORDINATES = coords
// save the record
});
}
});
I am running into a sync/async issue with my program and am having a hard time implementing async into my middleware. I have tried a few times and broken it a bunch, what would be a fairly simple way to implement async await?
Here is the route
router.post("/testmedia", uploadFiles, testMedia);
Here is the middleware
const uploadFiles = (req,res,next)=> {
let b2fileIDs = [];
let savedPhotoLinks = {};
upload(req,res,function(err){
req.files.forEach(function(image){
var b2accountId;
var b2authToken;
var b2uploadURL;
//console.log(image);
b2.authorize()
.then(function(auth){
b2accountId = auth.accountId;
b2authToken = auth.authorizationToken;
}).then(function(ret){
b2.getUploadUrl(process.env.B2BUCKET_ID)
}).then(function(cb){
b2uploadURL = cb.uploadUrl;
b2authToken = cb.authorizationToken;
}).then(function(cb){
b2.uploadFile({
uploadUrl: b2uploadURL,
uploadAuthToken: b2authToken,
filename: 'test',
data: image.buffer
}).then(function(cb){
var uploadLink = {link: cb.fileId};
b2fileIDs.push(uploadLink);
}).then(function(){
successFn(response){
console.log(b2fileIDs);
//Save ID's to DB here
}
}).catch((error) => {
console.log("This is an ERROR " + error );
});
})
console.log("1");
console.log('2');
})
console.log('3');
console.log(b2fileIDs);
})
console.log('4');
next();
};
The problem is that when I upload multiple files the async nature moves on without capturing the b2fileIDs line.
As you can see below, in my server.js file I have a /POST Info request that gets called on a form submittal.
I started to get confused on reading about the different between app.post and express routes and if in anyway using routes would benefit my code here.
Within the /POST Info I have two axios requests to 2 different APIs and I think it would be wise to move the code elsewhere to make it cleaner.
Would knowing how routes work here benefit me anyway?And if you can explain the difference here that would be great.
app.post('/Info', function (req, res) {
var State = req.body.State;
var income = Number(req.body.income);
var zip = req.body.ZIP;
axios.post('https://taxee.io/api/v2/calculate/2017', {
//data sent to Taxee.io
"exemptions": 1
, "filing_status": "single"
, "pay_periods": 1
, "pay_rate": income || 100000
, "state": State || "NY"
}, {
headers: {
'Authorization': "Bearer <API_KEY>"
//headers
}
}).then(function (response) {
var obj = {
income: '$' + income
, fica: response.data.annual.fica.amount
, federal: response.data.annual.federal.amount
, residence: State + ", " + zip
, state: response.data.annual.state.amount
}
axios.get("https://www.quandl.com/api/v3/datasets/ZILL/Z" + zip + "_RMP.json?api_key=<API_KEY>").then(function (response) {
var monthRent = response.data.dataset.data[0][1]
obj.rent = monthRent
obj.yearlyRent = Number(monthRent) * 12;
}).then(function (response) {
res.send(obj);
});
}).catch(function (error) {
alert('error');
});
}
There are two ways to define routes in an Express application:
Use the Express application (app) object directly:
const express = require('express')
const app = express()
app.post(...)
app.get(...)
app.put(...)
// and so on
Or use the router object:
const express = require('express')
const app = express()
const router = express.Router()
router.post(...)
router.get(...)
router.put(...)
// and so on
app.use(router)
My guess is that you've been reading about the latter snippet of code with the router object. Using Express' Router object can indeed make code cleaner to read as there more of a separation of concerns.
There's nothing wrong with calling an external API from your own API. For example, in a project of mine, I call the Google Calendar API on this line. The only difference between mine is yours is that I used the Google APIs Node.js Client while you used standard HTTP requests. I could have certainly used HTTP requests as shown here.
Your code is fine, but can be improved. For example, instead of:
axios.post('...', {
exemptions: 1,
filing_status: 'single',
pay_periods: 1,
pay_rate: income || 100000,
state: State || 'NY'
})
You could call an helper function that prepares the options object:
function prepareOptions (state = 'NY', income = 100000) {
return {
exemptions: 1,
filing_status: 'single',
pay_periods: 1,
pay_rate: income,
state: State
}
}
Then call it like so:
axios.post('...', prepareOptions(State, income))
This makes for more readable code.
Finally, there is no reason to use axios on the server side. Simply use Node's built in HTTP module.
app.post('/Info', function (req, res) {
var uData ={
state: req.body.State,
income : Number(req.body.income),
zip: req.body.ZIP
};
taxee(uData).then(function(data){
return rent(data) ;
}).then(function(fullData){
res.send(fullData);
}).catch(function (error) {
res.render('error');
});
function taxee(data) {
return new Promise((resolve, reject) => {
var income = data.income;
var state = data.state;
var zip = data.zip;
axios.post('https://taxee.io/api/v2/calculate/2017', {
//data sent to Taxee.io
"exemptions": 1
, "filing_status": "single"
, "pay_periods": 1
, "pay_rate": income || 100000
, "state": state || "NY"
, }, header).then(function (response) {
var taxData = {
income: '$' + income
, fica: response.data.annual.fica.amount
, federal: response.data.annual.federal.amount
, stateTax: response.data.annual.state.amount
, state
, zip: zip
}
resolve(taxData);
}).catch(function (error) {
console.log('break');
resolve(error);
});
});
};
function rent(data) {
return new Promise((resolve, reject) => {
axios.get("https://www.quandl.com/api/v3/datasets/ZILL/Z" + data.zip + "_RMP.json?api_key=d7xQahcKCtWUC4CM1LVd").then(function (response) {
console.log(response.status, ' status');
var monthRent = response.data.dataset.data[0][1];
data.rent = monthRent
data.yearlyRent = Number(monthRent) * 12;
return data;
}).then(function (response) {
resolve( data);
}).catch(function (error) {
reject(error);
});
});
}
module.exports = {
taxee
, rent
};
Ended up putting the code above into clean promise methods. Really happy how it worked out!