I am fetching JSON data from instagram api, which return something like {"pagination": {}, "data": [{"id": ...... and I am using node.js to fetch it. What's wrong with my code? I cannot see the expected console log of 'success'!
var cheerio = require('cheerio'),
request = require('request'),
url = require('url');
var results = [];
var target = 'https://api.instagram.com/v1/users/self/media/recent/?access_token=';
request.get(target, function(error, response, body) {
var $ = cheerio.load(body);
$('data').each(function(i, element) {
console.log('success');
results.push(element);
});
console.log(results);
});
Try this:
request.get(target, function(error, response, body) {
console.log(body);
// you can get pagination, etc with yourObj.pagination, ...
var yourObj = JSON.parse(body);
});
Related
So I am using the Request NPM module at the moment.
request( URL, function (error, response, body) {
console.log(body);
var bod = JSON.parse(body);
console.log(bod.url);
});
I want to utilize the bod.url parameter inside an EJS file.
The console.log(bod.url) part works fine and it prints out exactly what I am looking for.
I am looking to do something like the following.
request( URL, function (error, response, body) {
console.log(body);
var bod = JSON.parse(body);
console.log(bod.url);
var bod2 = bod.url;
});
res.render("image",{url:bod2});
but it keeps saying that bod2 is undefined.
If I make bod2 a variable outside of the request function then my EJS tempalte renders it correctly.
I know I am missing something fundamental but I cannot see where I am going wrong here.
Update
If I do the following it works...but is this bad practice inside of a route?
request( URL, function (error, response, body) {
console.log(body);
var bod = JSON.parse(body);
console.log(bod.url);
var bod2 = bod.url;
res.render("image",{url:bod2});
});
It is because of Scope of a variable, that is the issue which occurs sometimes.
So either you can use the following code to handle such situations.
request( URL, function (error, response, body) {
console.log(body);
var bod = JSON.parse(body);
console.log(bod.url);
var bod2 = bod.url;
res.render("image",{url:bod2});
});
OR you can just declare bod2 above request block as
var bod2;
return new Promise(resolve => {
request( URL, function (error, response, body) {
console.log(body);
var bod = JSON.parse(body);
console.log(bod.url);
this.bod2 = bod.url;
}, function (error, response, body) {
if(!error)
resolve(body);
});
How to make a proper post request to this endpoint. When I use the POSTMAN I get the correct response but when I call using the below function I get 503 error. The call seems to be fine according to me. I appreciate your help!!
const request = require('request');
const express = require('express');
// Initialize request
var img64Data = "/9j/4AAQSkZJRgABAQAAAQABAAD/2w… "; // Include the entire base64 encoding. // Shown Below in the next page
var send = {"img64": img64Data};
var api_address = "https://8n78hbwks0.execute-api.us-west-2.amazonaws.com/dev/";
// Make Post Request
module.exports = app => {
app.post('/axe', (req, res, next) => {
console.log("inside the axe");
request.post({
url: api_address,
body: JSON.stringify(send),
headers: {"Content-Type":"application/json"}
}, function (error, response, body) {
console.log("hiii");
console.log(response.statusCode);
if (!error && response.statusCode == 200) {
// Successful call
var results = JSON.parse(body);
console.log(results) // View Results
}
});
});
};
You get a 503 Error https://en.wikipedia.org/wiki/List_of_HTTP_status_codes because your server doesn't reply any http code.
if (!error && response.statusCode == 200) {
// Successful call
var results = JSON.parse(body);
console.log(results) // View Results
res.sendStatus(200);
} else {
res.sendStatus(response.statusCode);
}
I want to get the body of web-page from a list of more than 1000 urls (my goal is to do scraping using cheerio then).
The problem is that I get a weird GUNZIP result and I can't get the content of the body tag. This is the code that I'm using (I cant use a simple "request" cause it misses some request)
var async = require('async');
var fetch = require('isomorphic-unfetch');
const cheerio = require('cheerio');
let urls= // reading a list of ~1000 URLs from JSON file
async.mapLimit(urls, 1, async function(url) {
const response = await fetch(url);
return response.body
}, (err, results) => {
if (err) throw err
console.log(results);
});
The problem is that I get a weird GUNZIP result
use zlib,
var zlib = require('zlib');
async.mapLimit(urls, 1, async function(url) {
const response = await fetch(url);
zlib.gunzip(response.body, function(err, dezipped) {
return (dezipped.toString());
});
}, (err, results) => {
if (err) throw err
console.log(results);
});
then purseed your parsing with cheerio :)
i hope this helps.
I'm using the latest node.js (and express) to make an API call to a site that returns... XML... Ugh >_>.
I've scowered the web and found a million different ways, but I don't know the latest, most up to date / best way to make a request and get a response in node/express.
I tried using https://github.com/request/request and did the following:
var sendJsonResponse = function(res, status, content) {
res.status(status);
res.json(content);
};
var token = request
.get('some-website.com/api/stuff')
.on('response', function(response) {
console.log(response.statusCode);
console.log(response.headers['content-type']);
});
sendJsonResponse(res, 200, token);
in the console.log statements I get 200 and then application/xml;charset=utf-8.
But on my page I don't get the xml I'm looking for. Any ideas? I've tried using https://github.com/Leonidas-from-XIV/node-xml2js to attempt to "parse" the response, in case node just can't handle the xml response, but to no avail.
var xml2js = require('xml2js');
parser.parseString(response, function(err, result) {
console.dir(result);
console.log('Done');
});
Any help on accessing an API using Node and actually using the XML response, please?
EDIT ANSWER
For the Node.js request and xml parsing of the returned xml content:
var request = require('request');
var xml2js = require('xml2js');
var sendJsonResponse = function(res, status, content) {
res.status(status);
res.json(content);
};
/* GET XML Content*/
module.exports.dsRequest = function(req, res) {
var parser = new xml2js.Parser();
request('url_for_xml_request', function(error, response, body) {
parser.parseString(body, function(err, result) {
sendJsonResponse(res, 200, result);
});
});
};
I think this will work, because request is async, you should write like below:
var sendJsonResponse = function(res, status, content) {
res.status(status);
res.json(content);
};
request.get('http://some-website.com/api/stuff', function (err,response, body) {
sendJsonResponse(res, 200, body);
});
I have the following code:
var request = require('request');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
if (err) throw err;
var urls = data.split('\n');
urls = urls.filter(function(n){return n});
for(var i in urls) {
request(urls[i], function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
$('item').each(function(){
console.log("----------");
console.log($(this).find('title').text());
console.log($(this).find('link').text());
console.log($(this).find('pubDate').text());
});
}).end();
}
});
and from the urls.txt file I only have the following url:
http://www.visir.is/section/?Template=rss&mime=xml
When I use wget on that url I get a response which looks like an rss feed but when I do it in the code above the body is empty. Can someone explain to me why and how can I fix this?
Update: Simply removing .end() from your original script works. end() terminates the script on callback. IMO, in 2016, I'd definitely choose Request over Needle.
Request is an odd bird, and why it's not working in your case it's giving no information in the response at all.
Try with Needle instead:
var needle = require('needle');
var cheerio = require('cheerio');
var URL = require('url')
var fs = require('fs')
fs.readFile("urls.txt", 'utf8', function(err, data) {
if (err) throw err;
var urls = data.split('\n');
urls = urls.filter(function(n){return n});
for(var i in urls) {
needle.get(urls[i], function(err, resp, body) {
if (err)
throw err;
$ = cheerio.load(body,{lowerCaseTags: true, xmlMode: true});
$('item').each(function(){
console.log("----------");
console.log($(this).find('title').text());
console.log($(this).find('link').text());
console.log($(this).find('pubDate').text());
});
});
}
});