When I do a simple http.get for a URL that goes to a SquareSpace (SS) site I'm getting a 403 message. I know the site is working and that the server can reach it. Here's a simple example against a SS site (not mine, but produces the same issue):
Show that server can access site:
curl http://www.letsmoveschools.org
This returns all the HTML from the site...
Node app
var http = require('http');
var url;
url = 'http://www.letsmoveschools.org/';
var req = http.get(url, function(res) {
res.on('data', function(chunk) {
//Handle chunk data
});
res.on('end', function() {
// parse xml
console.log(res.statusCode);
});
// or you can pipe the data to a parser
//res.pipe(dest);
});
req.on('error', function(err) {
// debug error
console.log('error');
});
When I run the node app now node app.js it outputs the 403 status code.
I have tried this code with other sites and it works fine, just not against squarespace sites. Any idea of either configuration on SS or something else I need to do in Node?
The problem is that the remote server is expecting/requiring a User-Agent header and node does not send such headers automatically. Add that and you should get back a 200 response:
// ...
url = 'http://www.letsmoveschools.org/';
var opts = require('url').parse(url);
opts.headers = {
'User-Agent': 'javascript'
};
var req = http.get(opts, function(res) {
// ...
Related
I have found very simple code that creates a server and displays a string, using the "http" module.
eg. from https://garywoodfine.com/simple-lightweight-nodejs-webserver/:
var http = require('http');
var server = http.createServer(function (req, res) {
var body = 'Amazing lightweight webserver using node.js\n';
var content_length = body.length;
res.writeHead(200, {
'Content-Length': content_length,
'Content-Type': 'text/plain' });
res.end(body);
});
server.listen(3939);
console.log('Server is running on port 3939');
I have found very simple code that gets data over HTTP, using the "got" module.
eg. from https://nodesource.com/blog/express-going-into-maintenance-mode:
const got = require('got');
(async () => {
try {
const response = await got('https://www.nodesource.com/');
console.log(response.body);
//=> '<!doctype html> ...'
} catch (error) {
console.log(error.response.body);
//=> 'Internal server error ...'
}
})();
However I am failing to integrate the two to create a server that, when visited, makes the HTTP call and returns the result. I essentially just want to replace the var body = 'Amazing lightweight webserver using node.js\n'; line from the Gary Woodfine example with the output of the Nodesource example.
I'm not particularly looking for comments or questions as to why I would want to make something that does this, I'm trying to understand fundamentally why I can't just do what feels like a very simple and natural thing to do: return content based on a server side request to another web service. I get the impression that the issue is to do with the asynchronous paradigm and obviously I understand the performance improvements it offers, I'm failing to understand how you structure something that works for this simple usecase.
With thanks to Brad for his comment, I now have code that integrates the two samples:
var http = require('http');
const got = require('got');
var server = http.createServer(function (req, res) {
var body = (async () => {
try {
const response = await got('https://www.nodesource.com/');
var body = response.body;
var content_length = body.length;
res.writeHead(200, {
'Content-Length': content_length,
'Content-Type': 'text/plain' });
res.end(body);
//=> '<!doctype html> ...'
} catch (error) {
console.log(error.response.body);
//=> 'Internal server error ...'
}
})();
});
server.listen(3939);
console.log('Server is running on port 3939');
This code can be stripped down further obviously, to the sort of level of simplicity I had in mind.
Where I was going wrong was by trying to handle all the http response code after the async block, when I needed to do it inside it.
I am working on a project that has a mobile appplication, and two web servers(let it be named tub and cloud). THe web servers are developed in node.js environment.
The mobile app will request for some information from the web server named 'tub' ;
this 'tub' will in turn act as a http client and request some information from the web server named 'cloud'.
When the 'tub' requests the 'cloud', the 'cloud' has to process the incoming request and then has to do header redirection to another url(possibly in the 'cloud' or some other server) and return the response back to the 'tub'.
I am facing certain difficulty in figuring out header redirection in the 'cloud' server. Below I have mentioned the coding for putting a rest api call from 'tub' to 'cloud' :
var options =
{
hostname: '10.0.0.1',
port: 1048,
path: '/api/v1/vendor?DevType=mobile,
method: 'GET',
headers:
{
'Content-Type': 'application/x-www-form-urlencoded'
}
};
var request = http.request(options, function(cloudres)
{
cloudres.setEncoding('utf8');
cloudres.on('data', function (chunk)
{
console.log("chunk");
console.log(chunk);
});
});
request.on('error', function(e)
{
console.log('Cloud Package http Request Error : ' + e.message);
});
request.end();
Below mentioned code is in the 'cloud' server for header redirection :
var express = require('express');
var http = require('http');
var app = express(); // Create an Express instance.
var httpserver = http.createServer(app); // Create a http server and hook the Express instance to it.
httpserver.listen(1048,function() // Make the http server listen to a port.
{
console.log("Http Server Listening On Port : 1048");
});
app.get('/api/v1/vendor',function(req,res)
{
res.statusCode = 302;
res.setHeader('Content-Disposition', 'attachment');
res.setHeader('Location', 'http://10.0.0.1:80/mobilelib.zip');
res.end();
});
I am not getting neither any response nor any error in the 'tub' . Am I going wrong anywhere ? How can we track whether header redirection is taking place or not.
Express can handle the redirection response automatically, using res.redirect('/foo/bar');
app.get('/api/v1/vendor',function(req,res)
{
res.redirect('http://10.0.0.1:80/mobilelib.zip');
});
It's inadvisable to set both the Location and Content-Disposition headers together: Content-Disposition with 302 redirect
Http request does not seem to work as it should with iisnode ...
When I run this without running it under IIS work it just fine ...
Anyone know if I need to give any permission or something, or someone who has experience in general with http request to other domain using IIS Node.
I can unfortunately not show debug, since I have not gotten it to work real under IISNode.
var http = require('http');
var cached_defs = [];
function loadArticleDef() {
var options = {
host: 'http://someurl.com',
path: '/data/DataObjectDef/GetDef?ArticeID=mypage'
};
callback = function(response) {
response.on('data', function (chunk) {
cached_defs.push(JSON.parse(chunk));
});
response.on('end', function () {
console.log("end");
});
}
http.request(options, callback).end();
}
I am trying to access the body of a json page that has a callback url.
In client side it is easy by using jquery and using jsonp option and so on...
How do i do it on the nodejs server side to get the title for example??
the url prints out: data({"log":{"title":"example","description" ...
var http = require("http");
function getPost(){
var reqest = http.get("http://news.example.com/api/read/json?callback=data", function(response){
response.setEncoding('utf8');
response.on('data', function(posts){
var x = JSON.parse(posts);
var callback = function(x){
console.log(callback);
}
});
});
}
getPost();
I have a case where i have to read the data from the request body and create a file and write the data into it. If the operation is successful I set the response header to 201 and add the location of file in Location header. The file creation is done using Java methods and node.js code is below.
var server = http.createServer(function(req, res)
{
var body = "";
req.on("data", function(chunk)
{
body += chunk.toString();
});
req.on("end", function() {
var rtn = obj.AddonPostMethod(filepath,body);
if(rtn.length < 13)
{
res.writeHead(201, {"Location" : rtn});
res.end();
}
else
{
res.writeHead(400, {"Content-Type" : application/json"});
res.write(''+rtn);
res.end();
}
});
}});
The problem is that the response headers are not getting updated and are always set to the default headers 200 Ok. In addition to this the server is always busy even after the response is received.
I don't think you're actually listening on a port with the code you reference.
var http = require('http');
http.createServer(function(req,res){
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
}).listen(8000);
console.log('Server running at http://127.0.0.1:8000/');
You never declare the http object as actually listening on a port/ip with the .listen() function.
Also, you don't need to wait for the req object to emit anything to respond. The function is called when the request is complete. You can listen for specific requests and route them appopriately by storing the http.Server object to a variable.
var server = http.createServer();
server.listen(8000);
server.on('request', function(req,res){ /* do something with the request */ });
More documentation on the http object can be found on the node.js documents for http