I'm trying to download a favicon from a website using Node.js, but am having some trouble.
My code is as follows:
//Imports ...
var theurl = http.createClient(80, 'a1.twimg.com');
var requestUrl = 'http://a1.twimg.com/a/1284159889/images/favicon.ico';
var request = theurl.request('GET', requestUrl, {"host": "a1.twimg.com"});
request.end();
request.addListener('response', function (response)
{
var body = '';
response.addListener('data', function (chunk) {
body += chunk;
});
response.addListener("end", function() {
fs.writeFileSync('favicon.ico', body.toString('binary'), 'binary');
});
});
The resulting icon is just garbage, however, and I suspect it has something to do with the encoding of the favicon when I grab it this way. What's the correct way to do something like this?
Try as a first line in the response callback response.setEncoding('binary'), or (since that's not a preferred (by node) encoding to set) response.setEncoding(null), which will make it a Buffer. And then just write body directly, without performing anything on it.
fs.writeFileSync('favicon.ico', body, 'binary');
I had to do response.setEncoding("binary") and provide the third argument to writeFileSync:
fs.writeFileSync('favicon.ico', body, 'binary')
This combination worked for me. Thanks.
Related
I want to get the html of this page for parsing(click the link to understand what content i want to get).
750-bond list
Here's my code to request this page content
var https = require("https");
var fs = require("fs");
var options = {
hostname: "www.prizebond.net",
port: 443,
path: "/dlist.php?num=455",
method: "GET"
};
var response = "";
var req = https.request(options, function (res) {
res.setEncoding("UTF-8");
console.log(res.statusCode);
res.on("data", function (chunk) {
response += chunk;
});
res.on("end", function () {
fs.writeFile("750-bond.html", response, function (err) {
if (err) {
console.log(err.message);
}
console.log("File downloaded");
});
console.log("end");
});
});
req.end();
Now the problem is that in my 750-bont.html file, I am getting the weird the
result of "Checking your browser before accessing the prizebond.net" not the
original content. Here's the screenshot what I got when I open the 750-
bond.html file in browser.
What I am doing wrong? And how can I get the original content of this webpage?
You can't, unless you write something more sophisticated, but you probably shouldn't.
The purpose of Cloudflare-protection is to prevent what you are trying to realize unfortunately.
You could look into a possibility to access whatever you want to access by a public API or something that prizebond.net provides for example.
I'm using Node.js and connect to create a simple web server. I have something similar to the following code and I can't figure out how to access the actual request message body from the request object. I'm new to this so bear with me. I'm also taking out some of the stuff that's not necessary for the example.
function startServer(dir) {
var port = 8888,
svr = connect().use(connect.static(dir, {"maxAge" : 86400000}))
.use(connect.directory(dir))
/*
* Here, I call a custom function for when
* connect.static can't find the file.
*/
.use(custom);
http.createServer(svr).listen(port);
}
function custom(req, res) {
var message = /* the message body in the req object */;
// Do some stuff with message...
}
startServer('dirName');
Make sense? I've tried logging that object to the console and it is full of TONS of stuff. I can easily see headers in there plus the request URL and method. I just can't seem to isolate the actual message body.
You should include the connect.bodyParser middleware as well:
svr = connect().use(connect.static(dir, {"maxAge" : 86400000}))
.use(connect.directory(dir))
.use(connect.bodyParser())
.use(custom);
That will provide the parsed message body as req.body to your handler.
If you want the raw message body, you shouldn't use it but instead read the req stream yourself:
function custom(req, res) {
var chunks = [];
req.on('data', function(chunk) {
chunks.push(chunk);
});
req.on('end', function() {
var rawbody = Buffer.concat(chunks);
...do stuff...
// end the request properly
res.end();
});
}
if(req.method == "POST"){
var body = '';
req.on('data', function(data){
body += data;
});
}
Then body should contain your message if you posted correctly.
A better idea would be to use Express, then use the bodyparser middleware - which will give you this functionality out of the box without worrying about somebody hammering your server. The code above has NO functionality to worry about attacks - but it will get you started.
I am still a beginner in Node.js and I am trying to explore as much as I can.
I know that Express.js is a framework used by many people for creating websites in Node.js.
But without using Express.js, I know that it is it possible to read .html files using 'fs.readFile', and then "display" this .html file in the browser.
Is there a way to get user input (say a button click, or fill in a box) from this web page into Node.js? So far, I have not found any examples of this.
Yes, this is possible. Study how the connect bodyParser's urlencoded function works.
When a request comes in from the browser, node is going to represent this as a readable data stream. For web forms, the pattern will be:
Use the request's data and end events to buffer the chunks of data from the stream into a single string.
Parse that string appropriately given its data format. In the case of a web form, this will normally urlencoded (application/x-www-form-urlencoded) MIME type
.
var qs = require('qs'); //https://github.com/visionmedia/node-querystring
function handle(req, res) {
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){
//assemble the request from distinct chunks into a single string
buf += chunk
});
req.on('end', function(){
//OK, you have a usable string request body, parse it and handle it
try {
var formData = qs.parse(buf);
//Yay, it parsed. Now you have your form data
//depending on your form's html, you might have formData.email, for example
} catch (err){
//oops, respond with an error
}
});
}
Tutorial
Long story short:
http.createServer(function (req, res) {
var data = '';
req.on('data', function(chunk) {
console.log("Received body data:");
console.log(chunk);
data += chunk.toString();
});
req.on('end', function() {
console.log('Received Data: ', data);
res.end();
});
}
I would like to download a file, write it to a temporary file, read it and give the readFileSync Buffer to a function. I tried this:
var file = fs.createWriteStream("temp.pdf")
var request = http.get(linkArray[1], function(response) {
response.on('data', function(data){
file.write(data)
}).on('end', function(){
postData(fs.readFileSync('temp.pdf'))
})
});
Sometimes it works, but sometimes it doesn't - my guess is that the file isn't written completely, when it is read. (But than the 'end' event shouldn't be fired ?!
As you can see, I would like to download a bunch of files and do this. Do you have any advise how to solve this? Maybe this isn't the best way to solve this...
You shouldn't link streams with on('data' you should use pipe. Pipe will link the streams data events to writes and end events to ends.
var file = fs.createWriteStream("temp.pdf");
var request = http.get(linkArray[1], function(response) {
response.pipe(file).on('close', function(){
postData(fs.readFileSync('temp.pdf'));
});
});
also you should use https://github.com/mikeal/request
var request = require('request');
request.get(linkArray[i], function (err, response, body) {
postData(body);
});
or
var request = require('request');
var file = fs.createWriteStream("temp.pdf");
request.get(linkArray[i]).pipe(file).on('close', function () {
postData(fs.readFileSync('temp.pdf'));
});
You need to call file.end(); at the top of your .on('end', ...) handler. The end() method itself is asynchronous, though, so you'll want to read the file once that's complete. E.g.,
var file = fs.createWriteStream("temp.pdf")
var request = http.get(linkArray[1], function(response) {
response.on('data', function(data){
file.write(data)
}).on('end', function(){
file.end(function() {
postData(fs.readFileSync('temp.pdf'))
});
})
});
I'm currently messing around with NodeJS and try to use the Twitter-Search API. With curl it works fine - so there's nothing wrong with my firewall or whatever. Yet, I never get a response within NodeJS.
var sys = require("sys"),
http = require("http"),
events = require("events");
sys.puts("Hi there… ");
var client = http.createClient(80, "search.twitter.com"),
body = "",
query = "foobar";
function getResults() {
sys.puts("fetching for "+query);
var request = client.request("GET", "/search.json?q="+query);
request.on("response", function(data){
/* this somehow never gets fired :( */
sys.puts("BODY:"+ data);
});
}
var interval = setInterval(getResults, 5000);
And the URL is also working.
Any hints or solutions are welcome!
Thanks in advance.
You never send the request.
You need to use request.end()
NOTE: the request is not complete. This method only sends the header of the request. One needs to call request.end() to finalize the request and retrieve the response. (This sounds convoluted but it provides a chance for the user to stream a body to the server with request.write().)
Also the response event'S parameter is the response object NOT the body. You need to set the data event on the response object and then listen for the end event to make sure you got all the data.
request.on('response', function (response) {
var body = '';
response.on('data', function (chunk) {
body += chunk;
});
response.on('end', function () {
console.log('BODY: ' + body);
});
});
request.end(); // start the request
See: http://nodejs.org/api/http.html#http_class_http_clientrequest
A few more tips
You might want to use querystring.escape to urlencode your search parameter
You also need to set the Host header, otherwise Twitter will return a 404
Fixed code:
var querystring = require('querystring');
...
var request = client.request("GET", "/search.json?q=" + querystring.escape(query), {'Host': 'search.twitter.com'});