I am trying to make a webserver in node.js that downloads an image from Wikipedia and servers it on a page. I cant get it to work. I pasted my code in an online sandbox: http://runnable.com/UXWTyD3pTQ1RAADe.
Heres my code:
var http = require('http');
var fs = require('fs');
var fd = fs.open('name.jpeg', 'r+');
var options = {
host:'upload.wikimedia.org',
port:80,
path:'/wikipedia/commons/1/15/Jagdschloss_Granitz_4.jpg'
};
var server = http.createServer(function(req, res){
res.writeHead(200, ['Content-Type', 'text/html']);
http.get(options,function(res) {
res.on('data', function (chunk) {
fs.write(fd, chunk, 0, chunk.length, 0, null);
});
res.on('end',function(){
fd.end();
res.send("<img src='name.jpeg'></img>");
res.end();
});
});
});
server.listen(process.env.OPENSHIFT_NODEJS_PORT, process.env.OPENSHIFT_NODEJS_IP);
I keep running into:
node server.js
Running...
fs.js:415
binding.write(fd, buffer, offset, length, position, wrapper);
^
TypeError: Bad argument
at Object.fs.write (fs.js:415:11)
at IncomingMessage.<anonymous> (server.js:18:12)
at IncomingMessage.EventEmitter.emit (events.js:96:17)
at IncomingMessage._emitData (http.js:359:10)
at HTTPParser.parserOnBody [as onBody] (http.js:123:21)
at Socket.socketOnData [as ondata] (http.js:1485:20)
at TCP.onread (net.js:404:27)
Working code - saving image file:
/**Try to get an image from Wikipedia and return it**/
var http = require('http');
var fs = require('fs');
var options = {
host:'upload.wikimedia.org',
port:80,
path:'/wikipedia/commons/1/15/Jagdschloss_Granitz_4.jpg'
};
var server = http.createServer(function(req, res){
res.writeHead(200, ['Content-Type', 'text/html']);
http.get(options,function(imgRes) {
imgRes.pipe(fs.createWriteStream('name.jpeg'));
res.end("<html><img src='name.jpeg'></img></html>");
});
});
server.listen(process.env.OPENSHIFT_NODEJS_PORT, process.env.OPENSHIFT_NODEJS_IP);
You would also need node-static (http://www.sitepoint.com/serving-static-files-with-node-js/) for serving static file name.jpeg.
But the other way is to do it manually:
var http = require('http');
var fs = require('fs');
var options = {
host:'upload.wikimedia.org',
port:80,
path:'/wikipedia/commons/1/15/Jagdschloss_Granitz_4.jpg'
};
var server = http.createServer(function(req, res){
if(req.url == '/name.jpeg') {
res.writeHead(200, ['Content-Type', 'image/jpg']);
try {
var imgData = fs.readFileSync('name.jpeg');
res.end(fs.readFileSync('name.jpeg'));
} catch(err) {
res.end();
}
}
else {
res.writeHead(200, ['Content-Type', 'text/html']);
http.get(options,function(imgRes) {
imgRes.pipe(fs.createWriteStream('name.jpeg'));
res.end("<html><img src='name.jpeg'></img></html>");
});
}
});
server.listen(process.env.OPENSHIFT_NODEJS_PORT, process.env.OPENSHIFT_NODEJS_IP);
Related
I have a simple nodeJS server that fetches data from another server and store them in a JSON files, i need to write a status about each file fetched and generated, but that doesn't work, because i have to execute response.end(), which implies that i can't write to the stream again, without ending the stream
here's my code:
var http = require('http');
var module = require('fs');
var APIs = [ '/servlet/en', '/servlet/fr' ];
var langs =[ 'en', 'fr' ];
var finish = false;
var host = 'http://www.localtest';
const port = process.argv[2] || 9000;
var responses = [];
http.createServer(function (req, response) {
for (x in APIs){
console.log(x);
var options = {
host: 'localtest',
port: 8888,
path: APIs[x],
lang: langs[x]
};
http.get(options, function(res) {
res.setEncoding('utf8');
var body='';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(chunk){
responses.push(body);
if (responses.length == 2){
var d = JSON.parse(responses[1]);
var d2 = JSON.parse(responses[0]);
module.writeFileSync("options.lang1"+".json",JSON.stringify(d) , 'utf-8');
module.writeFileSync("options.lang2"+".json",JSON.stringify(d2) , 'utf-8');
}
});
});
}
}).listen(parseInt(port));
console.log(`Server listening on port ${port}`);
An example, i tried to write a message to the user after the line :
responses.push(body);
using response.write(), but this method needs an response.end() in order to be executed and displayed on the browser, If i do that i can't write to the stream anymore!
Couple issues with your code here. First off, you shouldn't use module as a variable, as that is a word that's already used in node's moduling system, e.g. in module.exports
Second, You really want to have some control flow in there. here's a complete example using the async library, though others prefer Promises.
var http = require('http');
var fs = require('fs');
var APIs = [ '/servlet/en', '/servlet/fr' ];
var langs =[ 'en', 'fr' ];
var host = 'http://www.localtest';
const port = process.argv[2] || 9000;
const async = require('async');
let responses = [];
function fetchAndWriteFile(lang, callback){
var options = {
host: 'localtest',
port: 8888,
path: '/servlet/'+lang,
lang: lang
};
http.get(options, function(res) {
res.setEncoding('utf8');
const filename = 'options.'+lang+'.json';
const fileStream = fs.createWriteStream(filename, {defaultEncoding: 'utf-8'});
fileStream.on('end', (e)=> {
if(e) return callback(e);
return callback(null, filename);
});
res.pipe(fileStream)
});
}
http.createServer(function (req, response) {
// this will run the fetchAndWriteFile once for each lang in langs
async.map(langs, fetchAndWriteFile, (e, files) => {
response.end(files); // files will be an array of filenames saved
});
}).listen(parseInt(port));
console.log(`Server listening on port ${port}`);
This is my code:
var express = require('express');
var http = require('http');
var redis = require('redis');
var url = require('url');
var client = redis.createClient().setMaxListeners(0);
var app = express();
app.set('port', 3000);
app.get('/*', function(req, res) {
var key = url.parse(req.url).pathname;
client.on('connect', function() {
console.log('connected to redis!');
});
client.get(key, function(err, reply) {
if( reply == null) {
client.set(key, 1);
client.expire(key, 300);
res.send('1');
}
else {
client.incr(key, function(err, reply) {
console.log('increment value: ' + reply);
res.sendStatus(reply);
});
}
});
});
http.createServer(app).listen(app.get('port'), function() {
console.log('listening');
});
This is my output when I run the file ($ node test.js):
I tried this on my ubuntu machine and it perfectly works. This is what I get on my mac. Could someone explain me why this is happening. Any help would be appreciated.
listening
increment value: 2
_http_server.js:192
throw new RangeError(`Invalid status code: ${statusCode}`);
^
RangeError: Invalid status code: 2
at ServerResponse.writeHead (_http_server.js:192:11)
at ServerResponse._implicitHeader (_http_server.js:157:8)
at ServerResponse.OutgoingMessage.end (_http_outgoing.js:559:10)
at ServerResponse.send (/Users/sharath/webapps/docker/node_modules/express/lib/response.js:209:10)
at ServerResponse.sendStatus (/Users/sharath/webapps/docker/node_modules/express/lib/response.js:346:15)
at Command.callback (/Users/sharath/webapps/docker/test.js:24:13)
at normal_reply (/Users/sharath/webapps/docker/node_modules/redis/index.js:714:21)
at RedisClient.return_reply (/Users/sharath/webapps/docker/node_modules/redis/index.js:816:9)
at JavascriptRedisParser.returnReply (/Users/sharath/webapps/docker/node_modules/redis/index.js:188:18)
at JavascriptRedisParser.execute (/Users/sharath/webapps/docker/node_modules/redis-parser/lib/parser.js:415:12)
Http response statuses should be integers. It cannot be strings, objects, array or like that and should begin from 100.
From your code i see that you try to do
res.sendStatus(reply);
Check reply variable. From redis incr response im thinking it's string "OK".
Which is bad.. So to fix it just use
res.sendStatus(reply ? 200 : 500);
Also check this.
http://expressjs.com/en/4x/api.html#res.sendStatus
And this
https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
EDIT
If you need to send some JSON or data into front-end just do like this
res.json({thisIsMyNumber: reply});
or
res.send({thisIsMyNumber: reply});
Hope this helps.
I am getting following error in my server console when user click signout button.
Error: socket hang up
at createHangUpError (http.js:1472:15)
at Socket.socketCloseListener (http.js:1522:23)
at Socket.EventEmitter.emit (events.js:95:17)
at TCP.close (net.js:466:12)
Here is my proxy_server:
var fs=require('fs');
var options = {
key: fs.readFileSync('key/xxxxxxxxx.pem'),
cert: fs.readFileSync('key/xxxxx.pem'),
ca: fs.readFileSync('key/gd_bundle-g2-g1.crt')
};
var express=require('express'),
https=require('https'),
httpProxy = require('http-proxy'),
http=require('http'),
app=express(),
app1=express(),
server=https.createServer(options,app),
serverhttp=http.createServer(app1);
var proxy = httpProxy.createProxy({ target: 'http://localhost:9898',secureOptions:'constants.SSL_OP_NO_TLSv1_2'});
var proxySig = httpProxy.createProxy({target:'http://localhost:8881',secureOptions:'constants.SSL_OP_NO_TLSv1_2'});
var callSig = httpProxy.createProxy({target:'http://localhost:6666',secureOptions:'constants.SSL_OP_NO_TLSv1_2'});
var proxyCdn = httpProxy.createProxy({target:'http://localhost:3030',secureOptions:'constants.SSL_OP_NO_TLSv1_2'});
// var proxyhttps= new httpProxy.createProxy({ target: 'https://localhost:443',secure:false});
var errorhandler = require('errorhandler');
app.all('*', function(req,res){
if(req.hostname=='xxxxxxxx.in')
{
proxy.web(req, res);
}
else if(req.hostname=='xxxx.in')
{
proxyCdn.web(req, res);
}
else if(req.hostname=='xxxxxx.in')
{
proxySig.web(req, res);
}
else if(req.hostname=='xxxxx.in')
{
callSig.web(req, res);
}
});
app1.all('*', function(req,res){
res.redirect('https://'+req.hostname);;
});
serverhttp.listen(80);
server.listen(443);
You need to handle errors on each of your httpProxy objects. For example:
proxy.on('error', function (error, req, res) {
var json;
console.log('proxy error', error);
if (!res.headersSent) {
res.writeHead(500, { 'content-type': 'application/json' });
}
json = { error: 'proxy_error', reason: error.message };
res.end(JSON.stringify(json));
});
This thread was useful to me: https://github.com/nodejitsu/node-http-proxy/issues/527
Add changeOrigin: true, to the options when you set up the proxy. It changes the origin of the host header. When the proxy is "name based" /not an ip/, its a requirement.
Short summary:
Technical prerequisite needed for name-based virtual hosts is a web browser with HTTP/1.1 support (commonplace today) to include the target hostname in the request
I'm trying to create an HTTP/S MitM forwarding proxy using Node.js.
The way I'm tackling this project is by reusing the solution found in ./lib/proxy.js file of the NPM Proxy Cache project created by #runk after he raised the issue on the Node HTTP Proxy project issue tracker.
My Proxy() class looks like this:
var request = require('request')
, https = require('https')
, http = require('http')
, net = require('net')
, url = require('url')
, os = require('os')
, fs = require('fs');
var SOCKET_PATH = os.tmpdir() + 'mitm.sock';
console.log('[SOCKET PATH] ' + SOCKET_PATH);
function Proxy (config) {
config = config || {};
if(fs.existsSync(SOCKET_PATH)) {
fs.unlinkSync(SOCKET_PATH);
}
var options = {
key: fs.readFileSync('./certs/dummy.key', 'utf8'),
cert: fs.readFileSync('./certs/dummy.crt', 'utf8')
};
// HTTPS Server
https.createServer(options, this.handler).listen(config.port + 1, this.hostname, function (e) {
if(e) {
console.log('[HTTPS] Server listen() error !');
throw e;
}
});
// HTTP Server
var server = http.createServer(this.handler);
server.listen(config.port, this.hostname, function (e) {
if(e) {
console.log('[HTTP] Server listen() error !');
throw e;
}
});
// Intercept CONNECT requests for HTTPS handshake
server.addListener('connect', this.httpsHandler);
}
Proxy.prototype.handler = function (req, res) {
var schema = !!req.client.pair ? 'https' : 'http'
, path = url.parse(req.url).path;
var dest = schema + '://' + req.headers['host'] + path;
console.log('(1) - [' + schema.toUpperCase() + '] ' + req.method + ' ' + req.url);
var params = {
rejectUnauthorized: false,
url: dest
};
if(req.method.toUpperCase() !== 'GET') {
return console.log('[HTTP] Request is not HTTP GET.');
}
var onResponse = function (e, response) {
if(e == null && response.statusCode === 200) {
return r.pipe(res);
}
var body = 'Status ' + response.statusCode + ' returned';
if(e) {
body = e.toString();
}
res.end(body);
};
var r = request(params);
r.on('response', onResponse.bind(null, null));
r.on('error', onResponse.bind(null));
};
Proxy.prototype.httpsHandler = function (request, socketRequest, bodyHead) {
var httpVersion = request['httpVersion']
, url = request['url'];
console.log('(2) - [HTTPS] ' + request['method'] + ' ' + request['url']);
var proxySocket = new net.Socket();
// ProxySocket event handlers
proxySocket.connect(SOCKET_PATH, function () {
proxySocket.write(bodyHead);
proxySocket.write('HTTP/' + httpVersion + ' 200 Connection established\r\n\r\n');
});
proxySocket.on('data', function (chunk) {
console.log('ProxySocket - "data"');
socketRequest.write(chunk);
});
proxySocket.on('end', function () {
console.log('ProxySocket - "end"');
socketRequest.end();
});
proxySocket.on('error', function (e) {
console.log('ProxySocket - "error"');
console.log(e);
console.log(e.stack);
socketRequest.write('HTTP/' + httpVersion + ' 500 Connection error\r\n\r\n');
socketRequest.end();
});
// SocketRequest event handlers
socketRequest.on('data', function (chunk) {
console.log('SocketRequest - "data"');
proxySocket.write(chunk);
});
socketRequest.on('end', function () {
console.log('SocketRequest - "end"');
proxySocket.end();
});
socketRequest.on('error', function (e) {
console.log('socketRequest - "error"');
console.log(e);
console.log(e.stack);
proxySocket.end();
});
};
module.exports = Proxy;
And my Index.js file that start my program looks like this:
var Proxy = require('./lib/proxy');
var proxy = new Proxy({
hostname: '127.0.0.1',
port: 8000
});
Here's my directory / file structure this:
/my_project
/certs
dummy.crt // Copied from the NPM Proxy Cache project
dummy.csr // Copied from the NPM Proxy Cache project
dummy.key // Copied from the NPM Proxy Cache project
/lib
proxy.js
index.js
I'm testing my program by setting (in Mac OSX Maverick) an HTTP and HTTPS proxy as IP address 127.0.0.1 and port 8000.
When browsing an HTTP only website everything works fine, but if I browse an HTTPS website I get the following error:
{[Error: connect ENOENT] code: 'ENOENT', errno: 'ENOENT', syscall: 'connect'}
Error: connect ENOENT
at errnoException (net.js:904:11)
at Object.afterConnect [as oncomplete] (net.js:895:19)
Any ideas from where this issue could come from and how to fix this ?
Thank you very much in advance !
(If you want to test my code, the NPM module request is the only dependency needed to run the code.)
EDIT: The certs can be downloaded from here : https://github.com/runk/npm-proxy-cache/tree/master/cert.
I'm an author of npm-proxy-cache. In fact I've created another project called thin https://www.npmjs.org/package/thin and I hope in future the npm proxy cache thing will utilize it. Despite the fact that it's still very rough it's usable and it does what you need.
E.g.
proxy code
var Thin = require('thin')
var proxy = new Thin;
// `req` and `res` params are `http.ClientRequest` and `http.ServerResponse` accordingly
// be sure to check http://nodejs.org/api/http.html for more details
proxy.use(function(req, res, next) {
console.log('Proxying:', req.url);
next();
});
// you can add different layers of "middleware" similar to "connect",
// but with few exclusions
proxy.use(function(req, res, next) {
if (req.url === '/foobar')
return res.end('intercepted');
next();
});
proxy.listen(8081, 'localhost', function(err) {
// .. error handling code ..
});
server code
var express = require('express'); // v3.4
var app = express();
app.use(express.urlencoded({limit: '10mb'}));
app.get('/test', function(req, res){
console.log(req.protocol, 'get req.query', req.query);
res.end('get: hello world');
});
app.post('/test', function(req, res) {
console.log(req.protocol, 'post req.query', req.query);
console.log(req.protocol, 'post req.body', req.body);
res.end('post: hello world');
});
app.listen(3000);
var fs = require('fs');
var https = require('https');
https.createServer({
key: fs.readFileSync('./cert/dummy.key'), // your mitm server keys
cert: fs.readFileSync('./cert/dummy.crt')
}, app).listen(3001);
You need to start proxy and server in two terminal sessions, then
curl -d "foo=baz" -k -x https://localhost:8081 https://localhost:3001/test?foo=bar
curl -d "foo=baz" -x http://localhost:8081 http://localhost:3000/test?foo=bar
After that you should be able to see following output from the server
https post req.query { foo: 'bar' }
https post req.body { foo: 'baz' }
http post req.query { foo: 'bar' }
http post req.body { foo: 'baz' }
Small example for interceptor
curl -d "foo=baz" -k -x https://localhost:8081 https://localhost:3001/foobar
It should return intercepted
Hope that helps :)
I am using express and nodejs and am having problems saving facebook profile pictures to my server.
Location of picture: http://profile.ak.fbcdn.net/hprofile-ak-ash2/275619_223605264_963427746_n.jpg
Script Being Used:
var http = require('http')
var fs = require('fs')
var options = {
host: 'http://profile.ak.fbcdn.net',
port: 80,
path: '/hprofile-ak-ash2/275619_223605264_963427746_n.jpg'
}
var request = http.get(options, function(res){
res.setEncoding('binary')
var imagedata = ''
res.on('data', function (chunk) {imagedata += chunk})
res.on('end', function(){
fs.writeFile('logo.jpg', imagedata, 'binary', function (err) {
if(err){throw err}
console.log('It\'s saved!');
})
})
})
The image saves but is empty. Console logging the image data is blank too. I followed this example origionally which does work for me. Just changing the location of the image to the facebook pic breaks the script.
I ended up coming up with a function that worked:
var http = require('http');
var fs = require('fs');
var url = require('url');
var getImg = function(o, cb){
var port = o.port || 80,
url = url.parse(o.url);
var options = {
host: url.hostname,
port: port,
path: url.pathname
};
http.get(options, function(res) {
console.log("Got response: " + res.statusCode);
res.setEncoding('binary')
var imagedata = ''
res.on('data', function(chunk){
imagedata+= chunk;
});
res.on('end', function(){
fs.writeFile(o.dest, imagedata, 'binary', cb);
});
}).on('error', function(e) {
console.log("Got error: " + e.message);
});
}
USAGE:
getImg({
url: "http://UrlToImage.com",
dest: __dirname + '/your/path/to/save/imageName.jpg'
},function(err){
console.log('image saved!')
})
I know my answer is a little late, but I hope it'll help others we get to this question, so here it is:
Saving the file to the root directory of your Node server can be done this way:
var request = require("request");
var fs = require("fs");
var fbUserId = 4;
var imageLink = "https://graph.facebook.com/"+ fbUserId +"/picture?width=500&height=500";
request(imageLink).pipe(fs.createWriteStream("resultIMG.png"))
.on('close', function(){
console.log("saving process is done!");
});
Of course, you can add any path you want for the image prior the the file name string.
If you still are seeing empty images, set the encoding of the request module to null , like this:
var request = require("request").defaults({ encoding: null });
That should do it.