I am having difficulty getting my LatLon look up to work - I have read
Get Google Maps Geocoding JSON from Express - but that just says use HTTP...and I have read the docs on http/https - but I'm still getting an error.
Here is my code - so calling myUrl/LatLon should give me the Google API response - or at least that is what I want...
const https = require('https');
router.get( '/LatLon', ( res ) => {console.log('Here getting https');
const googleKey = '---';
const address = '1600 Amphitheatre Parkway, Mountain View, CA';
const options = new URL('https://maps.googleapis.com/maps/api/geocode/json?address=' + address + '&key=' + googleKey);
const req = https.request(options, (res) => {
res.on('data', (chunk) => {
console.log(`BODY: ${chunk}`);
});
res.on('end', () => {
console.log('No more data in response.');
});
});
req.on('error', (e) => {
console.error(`problem with request: ${e.message}`);
});
req.write();
req.end();
});
I get this error -
TypeError [ERR_INVALID_ARG_TYPE]: The first argument must be one of type string or Buffer. Received type undefined
at write_ (_http_outgoing.js:595:11)
at ClientRequest.write (_http_outgoing.js:567:10)
Any help would be greatly appreciated - I have tried about 4 variations on using "get" or "https"...
I found node-geocoder - and it worked great...
Basically I did this, it is 'generalized code', non-functional; but you'll get the idea.
A bunch of checks and compares went into it so I am not hitting API's when I do not need to.
var NodeGeocoder = require('node-geocoder');
var options = {
provider: process.env.GEOCODING_PROVIDER,
httpAdapter: 'https',
apiKey: process.env.GEOCODING_KEY,
formatter: null
};
var geocoder = NodeGeocoder(options);
collection.getExistingId( req.params.id, ( err, record ) => {
const existingAddress = addresstoString(record.address);
const newAddress = addresstoString(newRecord.address);
if ( !compareAddresses(existingAddress,newAddress) ) {
geocoder.geocode(newAddress, function(err, geocode) {
let coords = []; // mongoDB wants [Long,Lat]
coords[0] = geocode[0].longitude;
coords[1] = geocode[0].latitude;
// set existingAddress.COORDINATES = coords
// save the record
});
}
});
Related
I am trying to make a site for crypto data using coin-gecko's API.
They have 2 different end points for what i require and as such require 2 different URLs.
I had no problem using into the globalUrl to get data such as the total Market cap, volume, etc. which i was able to render into my ejs.
My problem is now i cannot use the other URL for this, seeing as I cannot make another get request, what would be the best way to get data from the topCoinsUrl such as say the "id" of bitcoin from the 2nd url please
const https = require('https');
const app = express();
app.get("/", function(req, res) {
const globalUrl = "https://api.coingecko.com/api/v3/global";
const topCoinsUrl = "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&order=market_cap_desc&per_page=100&page=1&sparkline=false&price_change_percentage=1h"
https.get(globalUrl , function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const globalMarket = JSON.parse(data);
const totalCryptocurrencies = globalMarket.data.active_cryptocurrencies
let totalMarketCap = globalMarket.data.total_market_cap.usd
let totalMarketCapUsd = totalMarketCap.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let totalVolume = globalMarket.data.total_volume.usd
let total24hVolume = totalVolume.toLocaleString('en-US', {
style: 'currency',
currency: 'USD',
});
let markets = globalMarket.data.markets
let bitcoinMarketShare = Math.round(globalMarket.data.market_cap_percentage.btc);
res.render("home", {
totalCryptocurrencies: totalCryptocurrencies,
totalMarketCap: totalMarketCapUsd,
total24hVolume: total24hVolume,
markets: markets,
bitcoinMarketShare: bitcoinMarketShare
});
})
}).on("error", function(error) {
console.error(error)
});
});
// Ideally i would like to add this to get the ID of bitcoin, but I get an error when i try to use the 2 get requests:
https.get(topCoinsUrl, function(response) {
let data = "";
response.on("data", function(chunk) {
data += chunk
});
response.on("end", function() {
const topCoinsUrl = JSON.parse(data);
let bitcoinId = topCoinsUrl[0].symbol
res.render("home", {
bitcoinId: bitcoinId
})
})
// Error handler
}).on("error", function(error) {
console.error(error)
});
});
If you wish to make 2 simultaneous requests, you should use something like Promise.all() . Create two network requests and fire them at the same time using Promise.all & collect their result.
You can use Blurebird as well... http://bluebirdjs.com/docs/api/promise.all.html
So I am making a kind of API middleware for my company that will grab information from the NOAA API and then store in in my database. It does more then but that a separate part. I have set it up so that it works it will get the information and store it in my sql database perfectly The issue is the information I get is based off of zipcode. One request is the information for one zipcode. I need to be able to 'loop" through a list of zipcode one at a time and store the information in the database. I am not sure how to properly get it to work. I have tested a couple of ways but have not been able to get it to work so if someone can get me pointed in the right direction it would be appreciated.
Sorry in advance my code is not cleaned up.
Everything below apiRequest.end() has little function for the question. I keep it for context.
let mysql = require('mysql');
let config = require('./config.js');
var https = require("https");
var express = require("express");
var app = express();
const port = 3000;
var fs= require('fs');
var csv = require('fast-csv');
//last test
//array will replace this zip variable
let zip = '90012';
api(zip);
function api(zips){
//All of the parts for building the get requests url
app.get("/", function(req, response) {
var apiKey = "gPaEVizejLlbRVbXexyWtXYkfkWkoBhd";
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid="datasetid=GHCND";
let datatypeid="&datatypeid=TMAX";
let location="&locationid=ZIP:";
const zipcode = zips;
let startdate="&startdate=2019-01-01";
let enddate="&enddate=2020-01-01";
let units = "&units=standard";
let limit="&limit=1000";
let url = webapi + datasetid + datatypeid + location + zipcode + startdate + enddate + units + limit;
var options = {
port: 443,
method: "GET",
headers: {
"token": apiKey
}
};
let data = "";
//request to grab from NOAA api
let apiRequest = https.request(url, options, function(res) {
console.log("Connected");
//grabing all data
res.on("data", chunk => {
data += chunk;
});
res.on("end", () => {
console.log("data collected");
//Format JSON data
response.send(JSON.parse(data));
var getData = JSON.parse(data);
if(isEmpty(getData)){
emptyCorrect();
}
dataFormat(getData);
});
});
apiRequest.end();
});
//fix format for date Can add more formating if needed here
function dataFormat(formData){
for(x in formData.results){
let date = formData.results[x].date;
formData.results[x].date = date.slice(0,10);
}
jsonToSQL(formData.results);
}
//test function is going to be used for inserting the zip
function test(){
var content = "";
console.log("your test worked see ***************");
return "92507";
}
//function to add grabed JSON data into the SQL database
function jsonToSQL(datafin){
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let values = [];
for(let x in datafin){
values.push([zipcode,datafin[x].date,datafin[x].value]);
}
// execute the insert statment
connection.query(stmt, [values], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function emptyCorrect(){
console.log("Eror correction");
var zipcode = zips;
let connection = mysql.createConnection(config);
// insert statment
let stmt = `INSERT INTO test1(ZIPCODE,DATE, TEMP) VALUES ? `;
let valueE = [];
valueE.push([zipcode,"0","No Data"]);
// execute the insert statment
connection.query(stmt, [valueE], (err, results, fields) => {
if (err) {
return console.error("error");
}
// get inserted rows
console.log('Row inserted:' + results.affectedRows);
});
// close the database connection
connection.end();
}
function isEmpty(obj) {
for(var key in obj) {
if(obj.hasOwnProperty(key))
return false;
}
return true;
}
app.listen(port, () => console.log(`Example app listening on port ${port}!`))
}
As I understand your problem can roughly be summarized as "How to loop through asynchronous evaluations in Nodejs".
There are some options for you. I would recommend wrapping call to the NOAA API with a promise and then chain those promises. This can be done as follows:
app.get('/', async function(req, response) {
var apiKey = 'some value';
let webapi = 'https://www.ncdc.noaa.gov/cdo-web/api/v2/data?';
let datasetid = 'datasetid=GHCND';
let datatypeid = '&datatypeid=TMAX';
let location = '&locationid=ZIP:';
let startdate = '&startdate=2019-01-01';
let enddate = '&enddate=2020-01-01';
let units = '&units=standard';
let limit = '&limit=1000';
var options = {
port: 443,
method: 'GET',
headers: {
token: apiKey
}
};
const zipCodes = ['90012', '90013']; // Place a call to your function for fetching zip codes here
let datas = [];
prom = Promise.resolve();
zipCodes.forEach(zipcode => {
prom = prom.then(() =>
new Promise((resolve, reject) => {
let url =
webapi +
datasetid +
datatypeid +
location +
zipcode +
startdate +
enddate +
units +
limit;
let apiRequest = https.request(url, options, function(res) {
console.log('Connected');
let data = '';
res.on('data', chunk => {
data += chunk;
});
res.on('end', () => {
console.log('data collected for zip ' + zipcode);
datas.push(data);
resolve();
});
});
apiRequest.end();
})
);
});
prom.then(() => {
// All requests have now been handled sequentially
response.send(/* You'll need to figure out what to do here */);
});
});
An alternative is to use something like the async library for dealing with sequentially calling callbacks. The async library (https://github.com/caolan/async) describes itself as:
Async is a utility module which provides straight-forward, powerful functions for working with asynchronous JavaScript.
See e.g. Node.js: How do you handle callbacks in a loop? for a similar problem (not with regards to callign an API, but dealing with asynchronous function in a loop).
How do I get the data from a https request outside of its scope?
Update
I've seen Where is body in a nodejs http.get response?, but it doesn't answer this question. In fact, that question isn't answered accurately, either. In the accepted answer (posted by the asker), a third party library is used. Since the library returns an object different from that returned by http.get() it doesn't answer the question.
I tried to set a variable to the return value of http.get() using await, but that returns a http.clientRequest and doesn't give me access to the response data that I need.
I'm using Node v8.9.4 with Express and the https module to request data from Google's Custom Search.
I have two routes. One for a GET request and one for a POST request used when submitting a form on the front page. They both basically serve the same purpose... request the data from CSE and present the data as a simple JSON string. Rather than repeat myself, I want to put my code for the CSE request into a function and just call the function within the callback for either route.
I thought about returning all the way up from the innermost callback, but that won't work because it wouldn't get to the request's error event handler or the necessary .end() call.
Here's a subset of the actual code:
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
res.status(200)
res.set('Content-Type', 'application/json')
// This doesn't work
let searchResults = JSON.stringify(cseSearch(req))
res.end(searchResults)
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams)
}
let cseRequest = https.request(options, cseResponse => {
let jsonString = ''
let searchResults = []
cseResponse.on('data', data => {
jsonString += data
})
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
// This doesn't work... wrong scope, two callbacks deep
return searchResults
})
})
cseRequest.on('error', e => {
console.log(e)
})
cseRequest.end()
}
If you're curious, it's for a freeCodeCamp project: Image Search Abstraction Layer
using promise method solve this issue.
cseSearch(req).then(searchResults=>{
res.end(searchResults)
}).catch(err=>{
res.status(500).end(searchResults)
})
function cseSearch (request) {
return new Promise((resolve, reject)=>{
...your http request code
cseResponse.on('end', () => {
let cseResult = JSON.parse(jsonString)
let items = cseResult.items
items.map(item => {
let resultItem = {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
}
searchResults.push(resultItem)
})
resolve(searchResults);
})
})
}
Based on what I explained in the comments, to give you an idea how compact your code could be using the request-promise library, here's what you could use:
const rp = require('request-promise-native');
app.get('/api/imagesearch/:query', newQuery)
app.post('/', newQuery)
function newQuery (req, res) {
let query = req.body.query || req.params.query
console.log(`Search Query: ${query}`)
cseSearch(req).then(results => {
res.json(results);
}).catch(err => {
console.log("newQueryError ", err);
res.sendStatus(500);
});
}
function cseSearch (request) {
let cseParams = '' +
`?q=${request.params.query}` +
`&cx=${process.env.CSE_ID}` +
`&key=${process.env.API_KEY}` +
'&num=10' +
'&safe=high' +
'&searchType=image' +
`&start=${request.query.offset || 1}`
let options = {
hostname: 'www.googleapis.com',
path: '/customsearch/v1' + encodeURI(cseParams),
json: true
};
return rp(options).then(data => {
return data.items.map(item => {
return {
url: item.link,
snippet: item.title,
thumbnail: item.image.thumbnailLink,
context: item.image.contextLink
};
});
});
I'm trying to fetch an image from the web and encode it with base64.
What I have so far is this:
var request = require('request');
var BufferList = require('bufferlist').BufferList;
bl = new BufferList(),
request({uri:'http://tinypng.org/images/example-shrunk-8cadd4c7.png',responseBodyStream: bl}, function (error, response, body)
{
if (!error && response.statusCode == 200)
{
var type = response.headers["content-type"];
var prefix = "data:" + type + ";base64,";
var base64 = new Buffer(bl.toString(), 'binary').toString('base64');
var data = prefix + base64;
console.log(data);
}
});
This seems to be pretty close to the solution, but I can't quite get it to work. It recognizes the data type and gives out this output:
data:image/png;base64
However, the bufferlist 'bl' seems to be empty.
BufferList is obsolete, as its functionality is now in Node core. The only tricky part here is setting request not to use any encoding:
var request = require('request').defaults({ encoding: null });
request.get('http://tinypng.org/images/example-shrunk-8cadd4c7.png', function (error, response, body) {
if (!error && response.statusCode == 200) {
data = "data:" + response.headers["content-type"] + ";base64," + Buffer.from(body).toString('base64');
console.log(data);
}
});
If anyone encounter the same issue while using axios as the http client, the solution is to add the responseType property to the request options with the value of 'arraybuffer':
let image = await axios.get('http://aaa.bbb/image.png', {responseType: 'arraybuffer'});
let returnedB64 = Buffer.from(image.data).toString('base64');
Hope this helps
LATEST, AS OF 2017 ENDING
Well, after reading above answers and a bit research, I got to know a new way which doesn't require any package installation, http module(which is built-in) is enough!
NOTE: I have used it in node version 6.x, so I guess its also applicable to above versions.
var http = require('http');
http.get('http://tinypng.org/images/example-shrunk-8cadd4c7.png', (resp) => {
resp.setEncoding('base64');
body = "data:" + resp.headers["content-type"] + ";base64,";
resp.on('data', (data) => { body += data});
resp.on('end', () => {
console.log(body);
//return res.json({result: body, status: 'success'});
});
}).on('error', (e) => {
console.log(`Got error: ${e.message}`);
});
I hope it helps!
Also, check more about the http.get(...) here !
Another way of using node fetch, which breaks down the steps per variable:
const fetch = require('node-fetch');
const imageUrl = "Your URL here";
const imageUrlData = await fetch(imageUrl);
const buffer = await imageUrlData.arrayBuffer();
const stringifiedBuffer = Buffer.from(buffer).toString('base64');
const contentType = imageUrlData.headers.get('content-type');
const imageBas64 =
`data:image/${contentType};base64,${stringifiedBuffer}`;
If you know the image type, it's a one-liner with the node-fetch package. Might not suit everyone, but I already had node-fetch as a dependency, so in case others are in a similar boat:
await fetch(url).then(r => r.buffer()).then(buf => `data:image/${type};base64,`+buf.toString('base64'));
If you are using axios then you can follow below steps
var axios = require('axios');
const url ="put your url here";
const image = await axios.get(url, {responseType: 'arraybuffer'});
const raw = Buffer.from(image.data).toString('base64');
const base64Image = "data:" + image.headers["content-type"] + ";base64,"+raw;
you can check with decode base64.
You can use the base64-stream Node.js module, which is a streaming Base64 encoder / decoder. The benefit of this method is that you can convert the image without having to buffer the whole thing into memory, and without using the request module.
var http = require('http');
var base64encode = require('base64-stream').Encode;
http.get('http://tinypng.org/images/example-shrunk-8cadd4c7.png', function(res) {
if (res.statusCode === 200)
res.pipe(base64encode()).pipe(process.stdout);
});
I use for load and encode image into base64 string node-base64-image npm module.
Download and encode an image:
var base64 = require('node-base64-image');
var options = {string: true};
base64.base64encoder('www.someurl.com/image.jpg', options, function (err, image) {
if (err) {
console.log(err);
}
console.log(image);
});
Encode a local image:
var base64 = require('node-base64-image');
var path = __dirname + '/../test.jpg',
options = {localFile: true, string: true};
base64.base64encoder(path, options, function (err, image) {
if (err) { console.log(err); }
console.log(image);
});
Oneliner:
Buffer.from(
(
await axios.get(image, {
responseType: "arraybuffer",
})
).data,
"utf-8"
).toString("base64")
Old post but could help someone.
On the basis of Dmytro response that help me.
const base64FromUrl = async (url: string) => {
try {
return Buffer.from((await axios.get(url, { responseType: "arraybuffer", })).data, "utf-8").toString("base64")
} catch (error) {
return ""
}
}
I've just add error handling.
You can use image-to-base64 Node.js module
The benefit of using this module is that you convert your image hassle-free
const imageToBase64 = require('image-to-base64');
const imageLink = 'Your image link'
imageToBase64(imageLink);
.then((response) => {
const base64Image = `data:image/png;base64,${response}`
console.log(base64Image);
})
.catch((error) => {
console.log(error);
})
I'm trying to convert the following Node.js snippet to Dart. In my conversion the 'data returned...' message is printed as soon as there is a response unlike in the Node.js version which waits until the page completes the requested 2 second delay.
Node.js
var http = require('http')
function fetchPage() {
console.log('fetching page');
http.get({ host: 'trafficjamapp.herokuapp.com', path: '/?delay=2000' }, function(res) {
console.log('data returned from requesting page');
}).on('error', function(e) {
console.log("There was an error" + e);
});
}
Dart
import 'dart:io';
import 'dart:uri';
fetchPage() {
print('fetching page');
var client = new HttpClient();
var uri = new Uri.fromComponents(scheme:'http', domain: 'trafficjamapp.herokuapp.com', path: '?delay=2000');
var connection = client.getUrl(uri);
connection.onRequest = (HttpClientRequest req) {
req.outputStream.close();
};
connection.onResponse = (HttpClientResponse res){
print('data returned from requesting page');
};
connection.onError = (e) => print('There was an error' ' $e');
}
How do I achieve the same delayed print in Dart as in Node? Thanks in advance.
Your Dart code was almost right, but there was a bug. You should have used query named parameter instead of path. I don't really know which Url is call with your code but the response has a 400 status code. For more readability, you can also use the Uri.fromString constructor.
Moreover you can omit your onRequest setting because the same code is done when onRequest is not defined.
Here's the code :
import 'dart:io';
import 'dart:uri';
main() {
print('fetching page');
//var uri = new Uri.fromString('http://trafficjamapp.herokuapp.com/?delay=2000');
var uri = new Uri.fromComponents(scheme:'http',
domain: 'trafficjamapp.herokuapp.com', query: 'delay=2000');
var client = new HttpClient();
var connection = client.getUrl(uri);
connection.onResponse = (HttpClientResponse res){
print('data returned from requesting page with status ${res.statusCode}');
};
connection.onError = (e) => print('There was an error' ' $e');
}
You can find this in the documentation of the onResponse callback:
The callback is called when all headers of the response are received and data is ready to be received.
So it works like this:
connection.onResponse = (res) {
print('Headers received, can read the body now');
var in = res.inputStream;
in.onData = () {
// this can be called multiple times
print('Received data');
var data = in.read();
// do something with data, probably?
};
in.onClosed = () {
// this is probably the event you are interested in
print('No more data available');
};
in.onError = (e) {
print('Error reading data: $e');
};
};