var request = require('request'),
cheerio = require('cheerio');
var i
for(i=1; i<908; i++){
var options = {
url:k,
var address='http://gallog.dcinside.com/inc/_mylog.php?
gid=chermy018&oneview=Y&cid=59&page=';
var k = address+'i';
request(options, function (err, response, body) {
console.log(body);
});
}
error
i selected parsing method by using cookie. if i run this code for one page, it work well, but for multiple pages it result sequence of weird symbols or letters like above. what cause this and how can i correct error? and because i only searched and modified nodejs codes for parsing, i don't know much well about cheerio. i want to parse texts but i don't know how designate under 'id' part.
Related
i try to get google suggestion from this url http://suggestqueries.google.com/complete/search?q=bob&client=firefox
when i run the url i get this result : ["bob",["bobby shmurda","bob marley","bobbi kristina","bobbi brown","bobbi kristina brown","bob dylan","bob evans","bobby hurley","bob\u0027s burgers","bob seger"]]
in node.js i used request , heres my code :
var request = require('request');
var url = 'http://suggestqueries.google.com/complete/search?q=bob&client=firefox';
request(url,function(error, response, result){
if(!error){
console.log(result);
}
});
until now everythings work fine
as you can see my output is an array with two values, in above code when i try to get result[1]instead of show array its just show a ".
i dont know why this happen.
probably because you get a String and not a JSON.
try JSON.parse
result = JSON.parse(result)
try to parse it first
var json_data = JSON.parse(result);
I would like to scrape the http://www.euromillones.com.es/ website to get the last winning 5 numbers and two stars. It can be seen on the left column of the website. I've been reading tutorials, but I am not capable of achieving this.
This is the code I wrote so far:
app.get('/winnernumbers', function(req, res){
//Tell the request that we want to fetch youtube.com, send the results to a callback function
request({uri: 'http://www.euromillones.com.es/ '}, function(err, response, body){
var self = this;
self.items = new Array();//I feel like I want to save my results in an array
//Just a basic error check
if(err && response.statusCode !== 200){console.log('Request error.');}
//Send the body param as the HTML code we will parse in jsdom
//also tell jsdom to attach jQuery in the scripts and loaded from jQuery.com
jsdom.env({
html: body,
scripts: ['http://code.jquery.com/jquery-1.6.min.js ']
}, function(err, window){
//Use jQuery just as in a regular HTML page
var $ = window.jQuery;
res.send($('title').text());
});
});
});
I am getting the following error:
Must pass a "created", "loaded", "done" option or a callback to jsdom.env.
It looks to me that you've just used a combination of arguments that jsdom does not know how to handle. The documentation shows this signature:
jsdom.env(string, [scripts], [config], callback);
The two middle arguments are optional but you'll note that all possible combinations here start with a string and end with a callback. The documentation mentions one more way to call jsdom.env, and that's by passing a single config argument. What you are doing amounts to:
jsdom.env(config, callback);
which does not correspond to any of the documented methods. I would suggest changing your code to pass a single config argument. You can move your current callback to the done field of that config object. Something like this:
jsdom.env({
html: body,
scripts: ['http://code.jquery.com/jquery-1.6.min.js'],
done: function (err, window) {
//Use jQuery just as in a regular HTML page
var $ = window.jQuery;
res.send($('title').text());
}
});
I am trying to take a variable from the original url, insert it into a second url, visit that url and then access variable from it. I have two problems:
Problem 1: 'myurl' variable returns value of
"http://api.trove.nla.gov.au/work/undefined?key=6k6oagt6ott4ohno&reclevel=full"
That is, it is not taking the 'myid' variable.
Problem 2: How do I then follow the 'myurl' url as I want to access the DOM? Do I make another 'request' for 'myurl'?
Here is my code so far:
var request = require('request'),
cheerio = require('cheerio');
request('http://api.trove.nla.gov.au/result?key=6k6oagt6ott4ohno&zone=book&l-advformat=Thesis&sortby=dateDesc&q=+date%3A[2000+TO+2014]&l-availability=y&l-australian=y&n=0&s=0', function(error, response, html) {
if (!error && response.statusCode == 200) {
var $ = cheerio.load(html, {
xmlMode: true
});
var myid = ($('work').attr('id'))
var myurl = "http://api.trove.nla.gov.au/work/" +(myid)+ "?key=6k6oagt6ott4ohno&reclevel=full"
console.log(myurl)
}
});
Your selector is probably wrong, did you mean '#work' or '.work' perhaps?
Yes, you'll have to make another request to myurl using request() (since that's what you're using)
How to check if youtube video exists on node.js app server side:
var youtubeId = "adase268_";
// pseudo code
youtubeVideoExist = function (youtubeId){
return true; // if youtube video exists
}
You don't need to use the youtube API per-se, you can look for the thumbnail image:
Valid video = 200 - OK:
http://img.youtube.com/vi/gC4j-V585Ug/0.jpg
Invalid video = 404 - Not found:
http://img.youtube.com/vi/gC4j-V58xxx/0.jpg
I thought I could make this work from the browser since you can load images from a third-party site without security problems. But testing it, it's failing to report the 404 as an error, probably because the content body is still a valid image. Since you're using node, you should be able to look at the HTTP response code directly.
I can't think of an approach that doesn't involve making a separate HTTP request to the video link to see if it exists or not unless you know beforehand of a set of video IDs that are inactive,dead, or wrong.
Here's an example of something that might work for you. I can't readily tell if you're using this as a standalone script or as part of a web server. The example below assumes the latter, assuming you call a web server on /video?123videoId and have it respond or do something depending on whether or not the video with that ID exists. It uses Node's request library, which you can install with npm install request:
var request = require('request');
// Your route here. Example on what route may look like if called on /video?id=123videoId
app.get('/video', function(req, response, callback){
var videoId = 'adase268_'; // Could change to something like request.params['id']
request.get('https://www.youtube.com/watch?v='+videoId, function(error, response, body){
if(response.statusCode === 404){
// Video doesn't exist. Do what you need to do here.
}
else{
// Video exists.
// Can handle other HTTP response codes here if you like.
}
});
});
// You could refactor the above to take out the 'request.get()', wrap it in a function
// that takes a callback and re-use in multiple routes, depending on your problem.
#rodrigomartell is on the right track, in that your check function will need to make an HTTP call; however, just checking the youtube.com URL won't work in most cases. You'll get back a 404 if the videoID is a malformed ID (i.e. less than 11 characters or using characters not valid in their scheme), but if it's a properly formed videoID that just happens to not correspond to a video, you'll still get back a 200. It would be better to use an API request, like this (note that it might be easier to use the request-json library instead of just the request library):
request = require('request-json');
var client = request.newClient('https://www.googleapis.com/youtube/v3/');
youtubeVideoExist = function (youtubeId){
var apikey ='YOUR_API_KEY'; // register for a javascript API key at the Google Developer's Console ... https://console.developers.google.com/
client.get('videos/?part=id&id='+youtubeId+'&key='+apikey, function(err, res, body) {
if (body.items.length) {
return true; // if youtube video exists
}
else {
return false;
}
});
};
Using youtube-feeds module. Works fast (~200ms) and no need API_KEY
youtube = require("youtube-feeds");
existsFunc = function(youtubeId, callback) {
youtube.video(youtubeId, function(err, result) {
var exists;
exists = result.id === youtubeId;
console.log("youtubeId");
console.log(youtubeId);
console.log("exists");
console.log(exists);
callback (exists);
});
};
var notExistentYoutubeId = "y0srjasdkfjcKC4eY"
existsFunc (notExistentYoutubeId, console.log)
var existentYoutubeId = "y0srjcKC4eY"
existsFunc (existentYoutubeId, console.log)
output:
❯ node /pathToFileWithCodeAbove/FileWithCodeAbove.js
youtubeId
y0srjcKC4eY
exists
true
true
youtubeId
y0srjasdkfjcKC4eY
exists
false
false
All you need is to look for the thumbnail image. In NodeJS it would be something like
var http = require('http');
function isValidYoutubeID(youtubeID) {
var options = {
method: 'HEAD',
host: 'img.youtube.com',
path: '/vi/' + youtubeID + '/0.jpg'
};
var req = http.request(options, function(res) {
if (res.statusCode == 200){
console.log("Valid Youtube ID");
} else {
console.log("Invalid Youtube ID");
}
});
req.end();
}
API_KEY is not needed. It is quite fast because there is only header check for statusCode 200/404 and image is not loaded.
I'm trying to use the Node.js packages request and jsdom to scrape web pages, and I want to know how I can submit forms and get their responses. I'm not sure if this is possible with jsdom or another module, but I do know that request supports cookies.
The following code demonstrates how I'm using jsdom (along with request and jQuery) to retrieve and parse a web page (in this case, the Wikipedia home page). (Note that this code is adapted from the jquery-request.js code from this tutorial http://blog.nodejitsu.com/jsdom-jquery-in-5-lines-on-nodejs)
var request = require('request'),
jsdom = require('jsdom'),
url = 'http://www.wikipedia.org';
request({ uri:url }, function (error, response, body) {
if (error && response.statusCode !== 200) {
console.log('Error when contacting '+url);
}
jsdom.env({
html: body,
scripts: [
'http://code.jquery.com/jquery-1.5.min.js'
]
}, function (err, window) {
var $ = window.jQuery,
// jQuery is now loaded on the jsdom window created from 'agent.body'
$searchform = $('#searchform'); //search form jQuery object
$('#searchInput').val('Wood');
console.log('form HTML is ' + $searchform.html(),
'search value is ' + $('#searchInput').val()
//how I'd like to submit the search form
$('#searchform .searchButton').click();
);
});
});
The above code prints the HTML from Wikipedia's search form, then "Wood", the value I set the searchInput field to contain. Of course, here the click() method doesn't really do anything, because jQuery isn't operating in a browser; I don't even know if jsdom supports any kind of event handling.
Is there any module that can help me to interact with web pages in this way, or in a similar non-jQuery way? Can this be done in jsdom?
Thanks in advance!
If you don't want to handle the POST request yourself like in the other answer, you can use an alternative to jsdom that does support more things in a browser.
http://www.phantomjs.org/
I'm not familiar with a nodejs library that will let you get a fully interactive client-side view of a web-page, but you can get the results of a form submission without too much worry.
HTML forms are essentially just a way of sending HTTP requests to a specific URL (which can be found as the action attribute of the form tag). With access to the DOM, you can just pull out these values and create your own request for the specified URL.
Something like this as the callback from requesting the wikipedia home page will get you the result of doing a search for "keyboard cat" in english:
var $ = window.jQuery;
var search_term = "keyboard cat";
var search_term_safe = encodeURIComponent(search_term).replace("%20", "+");
var lang = "en";
var lang_safe = encodeURIComponent(lang).replace("%20", "+");
var search_submit_url = $("#searchform").attr("action");
var search_input_name = $("#searchInput").attr("name");
var search_language_name = $("#language").attr("name");
var search_string = search_input_name + "=" + search_term_safe + "&" + search_language_name + "=" + lang_safe;
// Note the wikipedia specific hack by prepending "http:".
var full_search_uri = "http:" + search_submit_url + "?" + search_string;
request({ uri: full_search_uri }, function(error, response) {
if (error && response.statusCode != 200) {
console.log("Got an error from the search page: " + error);
} else {
// Do some stuff with the response page here.
}
});
Basically the important stuff is:
"Submitting a search" really just means sending either a HTTP GET or POST request to the URL specified at the action attribute of the form tag.
Create the string to use for form submission using the name attributes of each of the form's input tags, combined with the value that they are actually submitting, in this format: name1=value1&name2=value2
For GET requests, just append that string to the URL as a query string (URL?query-string)
For POST requests, post that string as the body of the request.
Note that the string used for form submission must be escaped and have spaces represented as +.