I have created a node server that scrapes through a certain website and then returns a nicely formatted json data. Everything works perfectly fine on my localhost. But now I want to upload it to a server and get a link that returns the json data (just like an api).
How do I do that?
Below is the code outline:
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/', gotdata);
function gotdata(req, res){
url = 'myurlhere';
request(url, inside_request);
function inside_request(error, response, html){
if(!error){
var $ = cheerio.load(html);
var title, date, permalink;
var obj = { title : "", date : "", permalink : ""};
// ALL MY CODES HERE
res.send(obj);
}
}
}
app.listen('8081');
console.log('Visit http://localhost:8081/');
exports = module.exports = app;
OUTPUT from my localhost:8081/
You can use an nginx server instead if you have a public IP on the server where you want to deploy your API.
Define a virtual host to your API from nginx configuration.
Please follow the tutorial in the following link for virtual host creation
https://gist.github.com/soheilhy/8b94347ff8336d971ad0
Related
It's the first time I use Node.js and Express.
I would create a web scraping.
This is my project structure:
WebScrape:
|_ bin
|_ node_modules
|_ public
|_ routes
|_ view
|_ app.js
|_ package.js
|_ package-lock.json
I've created a scrape.js file inside routes directory:
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
app.get('/scrape', function(req, res) {
// the URL we will scrape from - in our example Anchorman 2
url = 'http://www.imdb.com/title/tt1229340/';
/**
* The structure of our request call.
* The first parameter is our URL.
* The callback function takes 3 parameters: an error, a response status code and the html.
*/
request(url, function(error, response, html) {
// check to make sure no errors occurred when making the request
if(!error) {
// utilize the cheerio library on the returned html which will essentially give us jQuery functionality
var $ = cheerio.load(html);
// finally, we'll define the variables we're going to capture
var title, release, rating;
var json = { title : "", release : "", rating : ""};
}
}) // end request
}) // end get
app.listen('8081')
console.log('Magic happens on port 8081');
exports = module.exports = app;
How can I test it? Is this the right place to put it?
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var router = express.Router();
router.get('/scrape', function(req, res) {
// the URL we will scrape from - in our example Anchorman 2
url = 'http://www.imdb.com/title/tt1229340/';
/**
* The structure of our request call.
* The first parameter is our URL.
* The callback function takes 3 parameters: an error, a response status code and the html.
*/
request(url, function(error, response, html) {
// check to make sure no errors occurred when making the request
if(!error) {
// utilize the cheerio library on the returned html which will essentially give us jQuery functionality
var $ = cheerio.load(html);
// finally, we'll define the variables we're going to capture
var title, release, rating;
var json = { title : "", release : "", rating : ""};
}
}) // end request
}) // end get
exports = module.exports = router;
Generally, app.js listens on ports for requests. You use express.Router to further extend and add routes in seperate router files.
In app.js you have to do this to actually add the routes:
const routes = require('./routes/scraper.js');
// app is the express() app
app.use(routes);
Made a very simple Timestamp Microservice app with node that I want to be able to run on a webpage on my website. How would I go about doing this? It currently works fine on my local server.
I feel like this would be very simple but from searching can only find how to deploy to Heroku/AWS.
const express = require('express');
const bodyParser = require('body-parser');
const cors = require('cors');
//Create an instance of Express for the app and instantiate bodyParser and cors
const app = module.exports = express();
app.use(bodyParser.json());
app.use(cors());
app.get(`/dateValues/:dateVal`, (req,res,next) => {
//gets date from request
var dateVal = req.params.dateVal;
//Options for formatting date in natural state
var options = { year: 'numeric', month: 'long', day: 'numeric' };
if(isNaN(dateVal)) {
var naturalDate = new Date(dateVal);
naturalDate= naturalDate.toLocaleDateString('en-US', options);
var unixDate = new Date(dateVal).getTime()/1000-21600;
} else {
var unixDate = dateVal;
var naturalDate = new Date((parseInt(dateVal)+21600)*1000);
naturalDate= naturalDate.toLocaleDateString('en-US', options);
}
res.json({unix: unixDate, natural: naturalDate});
});
app.listen(3000, () => {
console.log('App is running');
});
is you want to push this online on your own server, it will be the same as you did in local.
Install your server, install npm/node, push your project on it and run npm start. This will work.
If you want something a bit better for production, you can use a proxy webserver, like apache or nginx and run your nodejs project with pm2
https://www.phusionpassenger.com/library/walkthroughs/deploy/nodejs/ownserver/nginx/oss/trusty/deploy_app.html
Heroku is the easiest deployment platfrom when it comes to node.js application. You can host it for free too. Checkout the url below.
https://devcenter.heroku.com/articles/getting-started-with-nodejs#introduction
I am new in Koa and I am creating a demo app. I want to create an API to handle POST request, But when I console.log(ctx); then there is nothing in ctx in indexRouter.js, the console only printing empty object like {}.
I don't know why this is happening. Please anyone suggest me where I am doing wrong?
And please help me to get the request.body by POST Method.
serverKoa.js:
var koa = require('koa');
var router = require('koa-router');
var app = new koa();
var route = router(); //Instantiate the router
app.use(route.routes()); //Use the routes defined using the router
const index = require('./router/indexRouter')(route);
app.listen(3005 ,function(){
console.log("\n\nKoa server is running on port: 3005");
});
indexRouter.js:
var indexController=require('../controller/indexController');
module.exports = function (route) {
route.post('/postnew',async ( ctx, next) => {
console.log(ctx); // here printing {}
});
}
and my request object ( by postman ) is:
Method: POST,
url:http://localhost:3005/postnew
body:{
"firstName":"viki",
"lastName":"Kumar",
"userName":"vk12kq14",
"password":"098765432"
}
content-type:application/json
It looks like your code has two issues:
you're not using the "next" version of koa-router, which is required if you want to use async/await. You can install it like this:
npm i koa-router#next --save
you're not using koa-bodyparser to parse the request data:
npm i koa-bodyparser --save
To use:
var koa = require('koa');
var router = require('koa-router');
var bodyParser = require('koa-bodyparser');
var app = new koa();
app.use(bodyParser());
...
The parsed body data will be available in your route handlers as ctx.request.body.
I am trying to run a simple node application using nide modules and testing it using the Advance Rest Client.
The console is not showing any error.
But I am not getting anything in the output.
While running this on ARC, I am getting : Cannot /GET data
Text version of the code:
MainFile:
var express = require('express');
//var morgan = require('morgan');
var bodyparser = require('body-parser');
var hostname = 'localhost';
var port = '3000';
var app = express();
//app.use(morgan('dev'));
var dishRouter = express.Router();
dishRouter.use(bodyparser.json());
var allDishes = require('./dishRouter');
//For all dishes
dishRouter.route('/dishes')
.get(allDishes.dishesGet)
.delete(allDishes.dishesDelete)
.post(allDishes.dishesPost)
;
//For specific dishesDelete
dishRouter.route('/dishes/:dishid')
.get(allDishes.dishSpecificGet)
.delete(allDishes.dishSpecificDelete)
.put(allDishes.dishSpecificPut)
;
app.listen(port,hostname,function(){
console.log('server runing properly');
});
dishRouter file:
console.log('in dishrouter file');
module.exports.dishesGet = function(req,res,next){
console.log('inside GET');
res.end('Will be displaying all the dishes');
};
module.exports.dishesDelete = function(req,res,next){
res.end('Will delete all the dishes');
};
module.exports.dishesPost = function(req,res,next){
res.end('will add the new dishes');
};
module.exports.dishSpecificGet = function(req,res,next){
res.end('displaying the specific dish :'+req.params.dishid);
};
module.exports.dishSpecificDelete = function(req,res,next){
res.end('Will delete the specific dish with id : '+req.params.dishid);
};
module.exports.dishSpecificPut = function(req,res,next){
res.write('will update the specific dish :'+req.params.dishid);
res.end('Updating the dish with values as name : '+req.body.name);
};
According to body-parser docs
Looks like your router is a bit broken here:
dishRouter.use(bodyParser.json())
Try switching this to:
app.use(bodyParser.json())
And I can recommend creating router in the file, where you write handlers and just export router.
UPDATE:
Here is what you forgot:
app.use(dishRouter)
When calling express.Router() you're just creating an instance of the router, but you have to connect it to the express application instance.
I have looked through stackoverflow and read about require. However I cannot understand why my require function does not run.
app.js code:
var http = require('http');
var express = require("express");
var app = express();
//Twitter Search -------------------
app.get("/tweet, function(req,res){
var twiter = require('twiter.js');
});
app.listen(3000);
twitter.js code:
console.log("twitter.js ran");
Make sure both app.js and twitter.js in same directory
And add ./ before it. Just use following
var twitter = require('./twitter'); // .js extension is not necessary
Also as alexey mentioned. twiter is not same as twitter :)
Take care of your typos. (I think I'm too lazy to read it carefully)
app.js
var http = require('http');
var express = require("express");
var app = express();
//Twitter Search -------------------
app.get("/tweet", function (req, res) {
var twitter = require('./twitter');
twitter.log();
});
app.listen(3000);
twitter.js should be exposed using module.exports
var twitter = {
log: function () {
console.log('twitter is loaded');
}
};
module.exports = twitter;
This should now print "twitter is loaded" in your console, when you visit localhost:3000/tweet