var request = require('request'),
cheerio = require('cheerio');
var i
for(i=1; i<908; i++){
var options = {
url:k,
var address='http://gallog.dcinside.com/inc/_mylog.php?
gid=chermy018&oneview=Y&cid=59&page=';
var k = address+'i';
request(options, function (err, response, body) {
console.log(body);
});
}
error
i selected parsing method by using cookie. if i run this code for one page, it work well, but for multiple pages it result sequence of weird symbols or letters like above. what cause this and how can i correct error? and because i only searched and modified nodejs codes for parsing, i don't know much well about cheerio. i want to parse texts but i don't know how designate under 'id' part.
I have a nodejs piece of code where am trying to do a db.get to cloudant trying to build the URL.
var contactsFile = result.docs[0]._id;
var finalUrl = contactsFile+'/'+contactsFile;
testDb.get(urlencode.decode(finalUrl), function(err, data) {
if (!err) {
abc = data;
console.log("file" , abc);
}
else{
console.log("this is the error " , err);
}
});
When i try to print the URL that is passed the get request, the URL is passed as
https://cloudant/db/contactsFile%2FcontactsFile.
But I want to pass it as https://cloudant/db/contactsFile/contactsFile.
I have tried uriencode and decode and tried what i understood, but when i print with urlencode.decode it shows / and in the get request no matter wat i do it gets converted to %2F.
This might be a simple question but as I am new to node I am still learning.
If I understood correctly you are now passing "https://cloudant/db/contactsFile%2FcontactsFile" to your testDb.get, but you'd need to pass "https://cloudant/db/contactsFile/contactsFile".
If that's so you could try this:
testDb.get(decodeURIComponent(finalUrl), ...);
For further reference: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent
When I tried to get the response from the requested URL, I can get only printed statements of 'out' its not going in the requestify and how to get the response from requestify. Can any one tell me what is the error behind this.
console.log('out');
requestify.get('http://www.google.com').then(function(response) {
// Get the response body
console.log('in');
response.getBody();
console.log(response.body);
});
console.log('out');
Did you add
var requestify = require('requestify');
above your code?
Also, due to the async behavior of NodeJS it will probably first display 'out' twice and after you recieved a response from Google it will display the google homepage HTML.
I want to create a koa route that acts like a proxy for another url, which delivers a file that is usually a few dozens of Megabytes.
Therefore I would like not to block when making the response. I am using this.body = yield request.get(url); currently, where request is the [co-request]1 module.
How do I stream the response back to the client ?
Edit :
I am now doing the following :
var req = require('request');
//...
this.body = req(url).pipe(fs.createWriteStream(this.params.what));
If I paste the url in my browser, I get a file just fine.
However if I get a Error: Cannot pipe. Not readable. in my route.
Turns out the solution was simply :
var req = require('request');
//...
this.body = req(url);
This is because this.body has to be a readable stream, which req(url) returns. Thanks to #danneu for the explanation.
How to check if youtube video exists on node.js app server side:
var youtubeId = "adase268_";
// pseudo code
youtubeVideoExist = function (youtubeId){
return true; // if youtube video exists
}
You don't need to use the youtube API per-se, you can look for the thumbnail image:
Valid video = 200 - OK:
http://img.youtube.com/vi/gC4j-V585Ug/0.jpg
Invalid video = 404 - Not found:
http://img.youtube.com/vi/gC4j-V58xxx/0.jpg
I thought I could make this work from the browser since you can load images from a third-party site without security problems. But testing it, it's failing to report the 404 as an error, probably because the content body is still a valid image. Since you're using node, you should be able to look at the HTTP response code directly.
I can't think of an approach that doesn't involve making a separate HTTP request to the video link to see if it exists or not unless you know beforehand of a set of video IDs that are inactive,dead, or wrong.
Here's an example of something that might work for you. I can't readily tell if you're using this as a standalone script or as part of a web server. The example below assumes the latter, assuming you call a web server on /video?123videoId and have it respond or do something depending on whether or not the video with that ID exists. It uses Node's request library, which you can install with npm install request:
var request = require('request');
// Your route here. Example on what route may look like if called on /video?id=123videoId
app.get('/video', function(req, response, callback){
var videoId = 'adase268_'; // Could change to something like request.params['id']
request.get('https://www.youtube.com/watch?v='+videoId, function(error, response, body){
if(response.statusCode === 404){
// Video doesn't exist. Do what you need to do here.
}
else{
// Video exists.
// Can handle other HTTP response codes here if you like.
}
});
});
// You could refactor the above to take out the 'request.get()', wrap it in a function
// that takes a callback and re-use in multiple routes, depending on your problem.
#rodrigomartell is on the right track, in that your check function will need to make an HTTP call; however, just checking the youtube.com URL won't work in most cases. You'll get back a 404 if the videoID is a malformed ID (i.e. less than 11 characters or using characters not valid in their scheme), but if it's a properly formed videoID that just happens to not correspond to a video, you'll still get back a 200. It would be better to use an API request, like this (note that it might be easier to use the request-json library instead of just the request library):
request = require('request-json');
var client = request.newClient('https://www.googleapis.com/youtube/v3/');
youtubeVideoExist = function (youtubeId){
var apikey ='YOUR_API_KEY'; // register for a javascript API key at the Google Developer's Console ... https://console.developers.google.com/
client.get('videos/?part=id&id='+youtubeId+'&key='+apikey, function(err, res, body) {
if (body.items.length) {
return true; // if youtube video exists
}
else {
return false;
}
});
};
Using youtube-feeds module. Works fast (~200ms) and no need API_KEY
youtube = require("youtube-feeds");
existsFunc = function(youtubeId, callback) {
youtube.video(youtubeId, function(err, result) {
var exists;
exists = result.id === youtubeId;
console.log("youtubeId");
console.log(youtubeId);
console.log("exists");
console.log(exists);
callback (exists);
});
};
var notExistentYoutubeId = "y0srjasdkfjcKC4eY"
existsFunc (notExistentYoutubeId, console.log)
var existentYoutubeId = "y0srjcKC4eY"
existsFunc (existentYoutubeId, console.log)
output:
❯ node /pathToFileWithCodeAbove/FileWithCodeAbove.js
youtubeId
y0srjcKC4eY
exists
true
true
youtubeId
y0srjasdkfjcKC4eY
exists
false
false
All you need is to look for the thumbnail image. In NodeJS it would be something like
var http = require('http');
function isValidYoutubeID(youtubeID) {
var options = {
method: 'HEAD',
host: 'img.youtube.com',
path: '/vi/' + youtubeID + '/0.jpg'
};
var req = http.request(options, function(res) {
if (res.statusCode == 200){
console.log("Valid Youtube ID");
} else {
console.log("Invalid Youtube ID");
}
});
req.end();
}
API_KEY is not needed. It is quite fast because there is only header check for statusCode 200/404 and image is not loaded.