I'm trying to read the content from a URL with Node.js but all I seem to get are a bunch of bytes. I'm obviously doing something wrong but I'm not sure what. This is the code I currently have:
var http = require('http');
var client = http.createClient(80, "google.com");
request = client.request();
request.on('response', function( res ) {
res.on('data', function( data ) {
console.log( data );
} );
} );
request.end();
Any insight would be greatly appreciated.
try using the on error event of the client to find the issue.
var http = require('http');
var options = {
host: 'google.com',
path: '/'
}
var request = http.request(options, function (res) {
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
console.log(data);
});
});
request.on('error', function (e) {
console.log(e.message);
});
request.end();
HTTP and HTTPS:
const getScript = (url) => {
return new Promise((resolve, reject) => {
const http = require('http'),
https = require('https');
let client = http;
if (url.toString().indexOf("https") === 0) {
client = https;
}
client.get(url, (resp) => {
let data = '';
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
resolve(data);
});
}).on("error", (err) => {
reject(err);
});
});
};
(async (url) => {
console.log(await getScript(url));
})('https://sidanmor.com/');
the data object is a buffer of bytes. Simply call .toString() to get human-readable code:
console.log( data.toString() );
reference: Node.js buffers
A slightly modified version of #sidanmor 's code. The main point is, not every webpage is purely ASCII, user should be able to handle the decoding manually (even encode into base64)
function httpGet(url) {
return new Promise((resolve, reject) => {
const http = require('http'),
https = require('https');
let client = http;
if (url.toString().indexOf("https") === 0) {
client = https;
}
client.get(url, (resp) => {
let chunks = [];
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
chunks.push(chunk);
});
// The whole response has been received. Print out the result.
resp.on('end', () => {
resolve(Buffer.concat(chunks));
});
}).on("error", (err) => {
reject(err);
});
});
}
(async(url) => {
var buf = await httpGet(url);
console.log(buf.toString('utf-8'));
})('https://httpbin.org/headers');
Related
Well I had a bit problems
function getId(username){
const https = require("https")
let id = ""
let data = ``
https.get(`https://api.roblox.com/users/get-by-username?username=${username}`, (response) =>{
response.on('data', (chunk) => {
data += chunk;
})
response.on('end', () =>{
if(data){
id = JSON.parse(data).Id
}
})
})
.on('error', (error) => {
console.log(error)
})
return id
}
So my goal here to use getId("IHZAQSTORM33") and expected result, it will return user id (1684676332). but instead, it give me (""). It give me a colons
yes I'm trying to connect to roblox api.
Use promise to return the response object as shown in the code.
You can also refer to link for the comment by nkron:
Where is body in a nodejs http.get response?
function getId(username) {
const https = require('https');
let id = '';
let data = '';
const url = `https://api.roblox.com/users/get-by-username?username=${username}`;
return new Promise((resolve, reject) => {
https
.get(url, (response) => {
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', () => {
resolve(data);
});
})
.on('error', reject);
});
//return id
};
(async () => {
const responseObject = await getId('IHZAQSTORM33');
})();
For more on async and await, refer this link:
await is only valid in async function
I am trying to send a http request when an event is entering, but the console outputs in the higher-order function are not working.
exports.handler = async (event) => {
const req = https.get('https://test.com', (res) => {
console.log('statusCode:', res);
console.log('headers:', res.headers);
let data = "";
res.on('data', (d) => {
data += d;
});
res.on('end', () => {
let url = JSON.parse(data);
console.log(url);
});
}).on('error', (e) => {
console.error(e);
});
return 0;
};
Happy for every help i can get!
I updated the code snippet you provided,
const https = require('https');
const func = async () => {
const req = https.get('https://test.com', (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
let data = "";
res.on('data', (d) => {
data += d;
});
res.on('end', () => {
console.log({data})
let url = JSON.parse(data);
console.log(url);
});
}).on('error', (e) => {
console.error(e);
});
return 0;
};
func();
console.log('statusCode:', res.statusCode);
is the correct way to retrieve statusCode from res.
let url = JSON.parse(data);
data is not json, it cannot be parsed
res is an instanceOf IncomingMessage
https://nodejs.org/dist/latest-v14.x/docs/api/all.html#http_class_http_incomingmessage
Have a look at the documentation to understand what is passed on the events emitted
Run the updated code so that you can see what is logged to stdOut
How can I send Id and password means basic authentication to below code. I mean where I need to put id/pass in below code? My API needs basic authentication
const https = require('https');
https.get('https://rws322s213.infosys.com/AIAMFG/rest/v1/describe', (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end', () => {
console.log(JSON.parse(data).explanation);
});
}).on("error", (err) => {
console.log("Error: " + err.message);
});
To request Basic authentication, a client passes a http Authorization header to the server. That header takes the form
Authorization: Basic Base64EncodedCredentials
Therefore, your question is "how to pass a header with a https.get() request?"
It goes in options.headers{} and you can put it there like this:
const encodedCredentials = /* whatever your API requires */
const options = {
headers: {
'Authorization' : 'Basic ' + encodedCredentials
}
}
const getUrl = https://rws322s213.infosys.com/AIAMFG/rest/v1/describe'
https.get (getUrl, options, (resp) => {
/* handle the response here. */
})
const https = require('https');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
// Allow SELF_SIGNED_CERT, aka set rejectUnauthorized: false
let options = {
agent: httpsAgent
}
let address = "10.10.10.1";
let path = "/api/v1/foo";
let url = new URL(`https://${address}${path}`);
url.username = "joe";
url.password = "password123";
let apiCall = new Promise(function (resolve, reject) {
var data = '';
https.get(url, options, res => {
res.on('data', function (chunk){ data += chunk })
res.on('end', function () {
resolve(data);
})
}).on('error', function (e) {
reject(e);
});
});
try {
let result = await apiCall;
} catch (e) {
console.error(e);
} finally {
console.log('We do cleanup here');
}
want to make few async requests from client to server.
i setting up local server with http module , and export this function to main app file. at the client file i write function that makes http request and i call this function number of times.
//server
const http = require('http');
const ms = 2000;
const init = () => {
http.createServer((req,res) => {
sleep(ms);
console.log(req.method);
console.log("After sleeping 2 seconds,hello from server");
res.end();
}).listen(5000, () => {
console.log("server running");
});
}
function sleep(ms) {
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms);
console.log("Sleep 2 seconds.");
}
module.exports.init = init;
//client
const url = "http://127.0.0.1:5000";
const http = require('http');
const getData = async url => {
await http.get(url, res => {
res.on('data', chunk => {
console.log("chunk : "+chunk);
});
res.on('end', () => {
console.log("response ended.");
});
}).on("error", (error) => {
console.log("Error: " + error.message);
});
};
const makeRequests = () => {
for (let i = 0; i < 3; i++) {
getData(url);
}
}
module.exports.makeRequests = makeRequests;
//app
const server1 = require('./server1');
const client = require('./client');
server1.init();
client.makeRequests();
how do i use the async await proprely ? and why its now printing the "chunk" ?
want to make few async requests from client to server.
Well, your code is actually async.
how do i use the async await proprely ?
How to use async/await correctly. There are examples how to use.
and why its now printing the "chunk" ?
http.get(url, res => {
res.on('data', chunk => {
console.log("chunk : "+chunk);
});
res.on('end', () => {
console.log("response ended.");
});
http.get(url, callback) ... response.on("data") gets fired if a new chunk is received. So it will read until the response stream gets an EOF ( End Of File ). If you wanna save & read the whole data at once you can write your chunks into a variable by append and read at "end".
As part of a program I am writing, I'd like to have a helper function that shortens some long URLs that are created. I found this package:
and am attempting to change the code so that it simply stores the string in a variable and returns it. I am having some issues with scope. the variable resData is never actually updated and return is always an empty string. How can I get this returned string into the global variable and return it? Thanks
var http = require("http")
module.exports = {
shorten: function(url) {
var resData = ''
http.get('[tinyurl api endpoint]' + encodeURIComponent(url), (res) => {
res.setEncoding('utf8')
res.on('data', (chunk) => {resData += chunk})
res.on('end', () => {
resData = resData.toString()
//this contains the link, and can be console.logged
})
}).on('error', (e) => {
console.log(e)
})
return resData //returns empty string
}
};
do this
var http = require("http")
module.exports = {
shorten: function(url,cb) {
var resData = ''
http.get('[tinyurl api endpoint]' + encodeURIComponent(url), (res) => {
res.setEncoding('utf8')
res.on('data', (chunk) => {resData += chunk})
res.on('end', () => {
resData = resData.toString()
//this contains the link, and can be console.logged
cb(null,resData) //<----- use callback (thanks robertklep)
})
}).on('error', (e) => {
console.log(e)
})
//--> return resData //returns empty string as node is non-blocking io, this line will be executed before http response is received
}
};
//index.js
var http = require('./path/to/that/module')
http.shorten(url,function(error,result){ console.log(result) })
Try to use with callback function
shorten: function(url,callback) {
var resData = ''
http.get('[tinyurl api endpoint]' + encodeURIComponent(url), (res) => {
res.setEncoding('utf8')
res.on('data', (chunk) => {resData += chunk})
res.on('end', () => {
resData = resData.toString()
//this contains the link, and can be console.logged
callback(null,resData); //<----here
})
}).on('error', (e) => {
console.error(e);
callback(e,null); //<----- and here
})
}