this is 1.js
let's run it
node 1.js
and then on the web
execute: domain.tld/-2
it all works perfect.
i see it loaded 2.js and then
used a function to do the mysql query
and displays all the results. perfect.
now on the web let's try to
go to : domain.tld/-3
1.js tries to load 3.js
because the urlpath name is '/-3'
but there is no '3.js'
i am expecting it to say
"page not found'
node 1.js is not crashing.
and web browser is still.. 'loading.."
here is the 1.js :
var http = require('http');
var url = require('url');
var mysql = require('mysql');
var Memcached = require('memcached');
var memcached = new Memcached('localhost:11211');
var connection = mysql.createConnection({
host : '--------------',
user : '2',
password : '-------------',
database : '1'
});
connection.connect();
var server=http.createServer(function(req,res){
res.writeHead(200,{'Content-Type': 'text/html; charset=utf-8'});
var pathname=url.parse(req.url).pathname;
if (pathname.match(/\/-\d{1,2}/)) {
pathname = pathname.replace('-', '')
try{
var o = require('/' + pathname + '.js')
} catch (err){
var o = '0'
}
if (o == '0'){
o = 'page not found'
}else{
o.makeQuery(connection, function(err, result){
if(err) return res.end(err);
o = result;
res.end(o)
});
}
} else if (pathname.match(/^\/$/)) {
res.end('welcome to index page');
} else {
pathname = pathname.replace('/', 'o');
res.end('welcome to user profile page');
}
}).listen(80);
although it may not matter..
here is the 2.js
exports.makeQuery = function(connection, callback) {
var queryString = 'SELECT * FROM 1_accounts order by ac_nu asc limit 5';
connection.query(queryString, function(err,res,fields){
if (err) {return callback(err)};
bb = JSON.stringify(res);
callback(null, bb);
});
}
before i added the following segment to
1.js it was working fine via web.
if (pathname.match(/\/-\d{1,2}/)) {
} else if (pathname.match(/^\/$/)) {
} else {
}
in other words it must have something to do
with those perhaps.
before i added those lines..
it would smoothly display:
"page not found"
You have set o='page not found' but there is no response.write(o) or res.end(o) for it as it is in the else part hence it will never be displayed
Related
I am trying to build an online compiler IDE and I always get an error that I can't understand or solve. the project/application consists of (index.html - package.json - app.js) the "app.js" refers to the server side which surely causes this error ("File not found: Firefox can’t find the file at /C:/Users/Mr_Tech/Desktop/online_judge_system/compilecode"). this is the code:
// including all packages and modules that will be used
var express = require('express');
var path = require('path');
var bodyParser = require('body-parser');
var compiler = require('compilex');
// create a app variable for using express packages and body-parser
var app = express();
//app.use(bodyParser());
app.use(express.urlencoded({ extended: true }))
//Then initialize comiler options from compilex package and link "/" root page with the "index.html".
var option = {stats : true};
compiler.init(option);
app.get('/' , function (req , res ) {
res.sendfile(__dirname +"/index.html");
});
// define the post funtion for the language you want your compiler to have and with input or without input options
app.post('\compilecode' , function (req , res ) {
var code = req.body.code;
var input = req.body.input;
var inputRadio = req.body.inputRadio;
var lang = req.body.lang;
if((lang === "C") || (lang === "C++"))
{
if(inputRadio === "true")
{
var envData = { OS : "windows" , cmd : "g++"};
compiler.compileCPPWithInput(envData , code ,input , function (data) {
if(data.error)
{
res.send(data.error);
}
else
{
res.send(data.output);
}
});
}
else
{
var envData = { OS : "windows" , cmd : "g++"};
compiler.compileCPP(envData , code , function (data) {
if(data.error)
{
res.send(data.error);
}
else
{
res.send(data.output);
}
});
}
}
if(lang === "Java")
{
if(inputRadio === "true")
{
var envData = { OS : "windows" };
console.log(code);
compiler.compileJavaWithInput( envData , code , function(data){
res.send(data);
});
}
else
{
var envData = { OS : "windows" };
console.log(code);
compiler.compileJavaWithInput( envData , code , input , function(data){
res.send(data);
});
}
}
if( lang === "Python")
{
if(inputRadio === "true")
{
var envData = { OS : "windows"};
compiler.compilePythonWithInput(envData , code , input , function(data){
res.send(data);
});
}
else
{
var envData = { OS : "windows"};
compiler.compilePython(envData , code , function(data){
res.send(data);
});
}
}
}); // end POST function
// set the port for the server to listen and and define the get function to send the output data generated after the executed code to the server.
app.get('/fullStat' , function(req , res ){
compiler.fullStat(function(data){
res.send(data);
});
});
app.listen(8000);
// function to delete all the temporary files created during the process
compiler.flush(function(){
console.log('All temporary files flushed !');
});
Nope. The problem is you are using a file:// path when this needs to be run as a HTTP server and you need to go to localhost:8000. You also used a backslash as a web path which is invalid.
I am using firebase for my android application and am performing full text search using Algolia which is suggested by all the blogs. I have successfully developed the script and its functioning properly. Now I want to host the script to run 24* 7. As I have an azure account , how do I go about uploading the script ? I have tried uploading the following as a function , web app but have been unsuccessful .
PS:- I have tried Heroku but wasn't satisfied.
The Script.
var http = require('http');
var port = process.env.port || 1337;
http.createServer(function (req, res) {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Hello World\n');
}).listen(port);
var dotenv = require('dotenv');
var firebaseAdmin = require("firebase-admin");
var algoliasearch = require('algoliasearch');
var algoliasearchHelper = require('algoliasearch-helper');
// load values from the .env file in this directory into process.env
dotenv.load();
// configure firebase
var serviceAccount = require("./serviceAccountKey.json");
firebaseAdmin.initializeApp({
credential: firebaseAdmin.credential.cert(serviceAccount),
databaseURL: process.env.FIREBASE_DATABASE_URL
});
var database = firebaseAdmin.database();
// configure algolia
var algolia = algoliasearch(process.env.ALGOLIA_APP_ID, process.env.ALGOLIA_API_KEY);
var index = algolia.initIndex('books');
var contactsRef = database.ref("/BookFair");
contactsRef.on('child_added', addOrUpdateIndexRecord);
contactsRef.on('child_changed', addOrUpdateIndexRecord);
contactsRef.on('child_removed', deleteIndexRecord);
function addOrUpdateIndexRecord(dataSnapshot) {
// Get Firebase object
var firebaseObject = dataSnapshot.val();
// Specify Algolia's objectID using the Firebase object key
firebaseObject.objectID = dataSnapshot.key;
// Add or update object
index.saveObject(firebaseObject, function(err, content) {
if (err) {
throw err;
}
console.log('Firebase object indexed in Algolia', firebaseObject.objectID);
});
}
function deleteIndexRecord(dataSnapshot) {
// Get Algolia's objectID from the Firebase object key
var objectID = dataSnapshot.key;
// Remove the object from Algolia
index.deleteObject(objectID, function(err, content) {
if (err) {
throw err;
}
console.log('Firebase object deleted from Algolia', objectID);
});
}
var queries = database.ref("/queries");
queries.on('child_added', addOrUpdateIndexRecordN);
function addOrUpdateIndexRecordN(dataSnapshot) {
// Get Firebase object
var firebaseObject = dataSnapshot.val();
// Specify Algolia's objectID using the Firebase object key
firebaseObject.objectID = dataSnapshot.key;
// Add or update object
var collegeName = "";
var query_ID_LOLWA= "";
var year="";
var query = "";
var counter = 0;
for(var i in firebaseObject){
var c = firebaseObject.charAt(i);
if(c=='/'){
counter = counter + 1;
continue;
}
else{
if(counter==2)
collegeName = collegeName + c;
else if(counter == 3)
year = year+c;
else if(counter == 1)
query_ID_LOLWA = query_ID_LOLWA + c;
else
query = query +c;
}
}
console.log(collegeName);
console.log(year);
console.log(query_ID_LOLWA);
console.log(query);
const query_final = query_ID_LOLWA;
var helper = algoliasearchHelper(algoliasearch("****", "****"), 'books', {
facets: ['collegeName', 'year','priority']});
helper.on('result', function(data,query_ID_LOLWA){
data.getFacetValues('priority',{sortBy: ['count:desc']});
console.log(data.hits);
var path_query = "/queries_answers/"+query_final;
path_query = path_query.toString();
console.log(path_query);
if(data.hits.length==0){
console.log("No results");
database.ref(path_query).push(-1);
}
else if(data.hits.length>1){
var ID = 1;
var counter = -1;
var length = data.hits.length-1;
for(var h in data.hits){
counter = counter + 1;
if( (counter%5 == 0) && (counter != 0)){
ID = ID + 1;
}
database.ref(path_query+"/"+ID).push(data.hits[h].uuid);
}
database.ref(path_query+"/totalResults").push(data.hits.length);
}
else{
database.ref(path_query+"/totalResults").push(data.hits.length);
for(var h in data.hits)
database.ref(path_query+"/1").push(data.hits[h].uuid);
}
});
helper.addFacetRefinement('collegeName', collegeName);
helper.addFacetRefinement('year',year);
helper.setQuery(query);
helper.search();
/*index.search(firebaseObject, function(err, content) {
if (err) {
console.error(err);
return;
}
console.log(content.hits);
for (var h in content.hits) {
console.log('Hit(' + content.hits[h].objectID + '): ' + content.hits[h].uuid);
}
database.ref("/query_result").push(content.hits);
});*/
}
Without more details than but have been unsuccessful, the only advice one could give you is to follow the usual steps to get a time-based Azure function deployed.
The simplest way is to use the Azure Portal:
Login to your Microsoft Azure account
Create a Function App to host your function
Add a Timer-triggered Function
Select the TimerTrigger-Javascript template to get started
At this point, you'll have a function that runs every minute. You can check the logs to confirm it is working.
You now want to configure its frequency:
Update the function's Timer Schedule (in the Integrate tab) to set how frequently the function should run
Finally, replace the template's code with your own.
You can find a detailed tutorial here with explanations on how to achieve each of these steps.
I developed a light node web server with Koa. I contain three services that retrieve data from a mongo database, then return them to the client.
The object returned is an array of JSON objects. It can contain until about 750.000 objects.
One object looks like the following :
{"_id":"58a4779b783dbfa853a93e09","attributes":[{"key":"nb_mots","value":"16"},{"key":"fonction","value":"président"},{"key":"groupe_acronyme","value":"UMP"}],"datatype":"deputes","date":"2007-10-23T00:00:00.000Z","eventtype":"loi","actor":"Bernard Accoyer"}
My server runs under Linux (Debian, I guess...).
My problem is: when I want to retrieve the whole dataset (so the 750.000 objects - using the "allevents" service as given below), my server fails. The only message I get is "Killed". I survey the state of the server by using command "top". Of course, the %CPU goes high (between 90 & 95), so do for the %MEM, but nothing critical (moreover that does the same if I try to retrieve less data).
Any idea to fix that?
Code of my server :
var kr = require('koa-route');
var koa = require('koa');
var app = koa();
var _ = require('lodash');
var MongoClient = require("mongodb").MongoClient;
var someeventsPromise = function(datatype, number){
return new Promise(function(resolve, reject){
MongoClient.connect("mongodb://localhost/eventdata", function(error, db) {
if (error){
return reject(error);
}
console.log("Connecté à la base de données");
db.collection('events').find({'datatype' : datatype},{"limit": number ? number : 1000}).toArray(function(err, array)
{
console.log('returning %d objects', array.length);
db.close();
return resolve({eventsArray : array});
});
});
});
}
var someevents = function * (datatype, number)
{
var event = yield someeventsPromise(datatype, parseInt(number));
this.body = event;
}
var alleventsPromise = function(datatype){
return new Promise(function(resolve, reject){
MongoClient.connect("mongodb://localhost/eventdata", function(error, db) {
if (error){
return reject(error);
}
console.log("Connecté à la base de données");
db.collection('events').find({'datatype' : datatype}).toArray(function(err, array)
{
console.log('returning %d objects', array.length);
db.close();
return resolve({eventsArray : array});
});
});
});
}
var allevents = function * (datatype)
{
var event = yield alleventsPromise(datatype);
this.body = event;
}
var minmaxdatesPromise = function (datatype)
{
return new Promise(function(resolve, reject){
MongoClient.connect("mongodb://localhost/eventdata", function(error, db) {
if (error){
return reject(error);
}
console.log("Connecté à la base de données");
db.collection('events').find({'datatype' : datatype}, {date : 1}).toArray(function(err, array)
{
console.log('returning %d objects', array.length);
db.close();
let g = _.map(_.uniqBy(array, function(d) {return d.date}), function(d) { return new Date(d.date);});
console.log('mapping to %d objects', g.length);
g.sort(function(a,b) { return b - a ; });
let minmax = { maxdate : g[0], mindate : g[g.length - 1]};
return resolve(minmax);
});
});
});
}
var minmaxdates = function * (datatype)
{
var dates = yield minmaxdatesPromise(datatype)
this.body = dates;
}
app.use(kr.get('/someevents/:datatype/:number', someevents));
app.use(kr.get('/allevents/:datatype', allevents));
app.use(kr.get('/minmaxdates/:datatype', minmaxdates));
app.listen(3010);
As I'm pretty sure that my issue is related to the fact that I have to many data to retrieve, I tried to send them through a stream (instead of storing them in a big array).
No luck, I have another issue then :
Error: write EPIPE
at exports._errnoException (util.js:1023:11)
at WriteWrap.afterWrite [as oncomplete] (net.js:804:14)
My code is the following :
var kr = require('koa-route');
var koa = require('koa');
var app = koa();
var _ = require('lodash');
var MongoClient = require('mongodb').MongoClient;
var allevents = function * (datatype)
{
var ctx = this;
var db = yield MongoClient.connect('mongodb://localhost/eventdata');
console.log("Querying all data for " + datatype);
// Get the collection
var col = db.collection('events');
ctx.body = col.find({'datatype' : datatype}).stream(
{
transform: function(doc) {
return JSON.stringify(doc);
}
});
}
app.use(kr.get('/allevents/:datatype', allevents));
app.listen(3010);
Last things : all results are used in a C# Unity script (just in case).
Thanks in advance.
BTW: I can't retrieve less data, I need all of them.
Heyllo, everyone!
I am making a scraper which uses node-simplecrawler. Everything runs fine, but what I can't figure out is how to stop one instance when creating a new one (I want to have only one running at a time). I am using express for this and all the scraping logic is in one route. In order to cancel the crawling right now, I need to stop the node process and run the app again.
Here is the part of the code that concerns running the crawler (note: I've simplified the code a little bit, so it's shorter):
module.exports = function(socket) {
var express = require('express');
var router = express.Router();
[... requires continue...]
/* GET scaning page. */
router.post('/', function(req, res, next) {
res.render('scanning'); // Load the socket.io host page
var render = {};
var pages = [];
var timer = new Date();
// Helper func to log the requests.
function log(message) {
var now = new Date();
console.log(now - timer + 'ms', message);
timer = now;
}
// Ensure URL format, parse URL
// Check if URL exist
request(url.href, function (error, response, body) {
if (!error && response.statusCode == 200) {
// URL exists, so let's scan it
// Exclude links to the following extensions:
var exclude = ['gif', 'jpg', 'jpeg', 'png', 'ico', 'bmp', 'ogg', 'webp',
'mp4', 'webm', 'mp3', 'ttf', 'woff', 'json', 'rss', 'atom', 'gz', 'zip',
'rar', '7z', 'css', 'js', 'gzip', 'exe', 'xml', 'svg'];
var exts = exclude.join('|');
var regexReject = new RegExp('\.(' + exts + ')', 'i');
var rootURL = url.protocol + '//' + url.host + '/';
// Crawler configuration
var crawler = new Crawler(url.host);
crawler.initialPort = 80;
crawler.initialPath = url.pathname;
crawler.maxConcurrency = 1;
crawler.ignoreWWWDomain = false; // This is a little suspicious...
crawler.filterByDomain = true; // Only URLs from the current domain
crawler.scanSubdomains = true;
crawler.downloadUnsupported = false;
crawler.parseHTMLComments = false;
crawler.parseScriptTags = false;
crawler.acceptCookies = false;
// crawler.maxDepth = 1 // Debug only!
/*
* Fetch Conditions
*/
// Get only URLs, ignore feeds, only from this host
crawler.addFetchCondition(function (parsedURL) {
return (
!parsedURL.path.match(regexReject) && // Only links
(parsedURL.path.search('/feed') === -1) && // Igrnore feeds
(parsedURL.host === url.host) // Page is from this domain
);
});
// Should we only include subpages?
if(onlySubpages) {
crawler.addFetchCondition(function(parsedURL) {
// return parsedURL.path.search(url.pathname) > -1;
return parsedURL.path.startsWith(url.pathname);
// console.log(url, parsedURL);
});
}
// Exclude urls with fragments?
if(excludeUrls.length >= 1 ) {
crawler.addFetchCondition(function(parsedURL) {
var urlFragmentsOk = true;
excludeUrlFragments.forEach(function(fragment) {
if(parsedURL.path.search('/'+fragment) > -1) {
urlFragmentsOk = false;
}
});
return urlFragmentsOk;
});
}
// Include only URLs with fragments
if(includeOnlyUrls.length >= 1) {
crawler.addFetchCondition(function(parsedURL) {
var urlFragmentsOk = false;
var includeUrlFragments = includeOnlyUrls.replace(/\s/, '').split(',');
includeUrlFragments.forEach(function(fragment) {
if(parsedURL.path.search('/'+fragment) !== -1) {
urlFragmentsOk = true;
}
});
return urlFragmentsOk;
});
}
// Run the crawler
crawler.start();
// Execute for each URL, on fetchcomplete
crawler.on('fetchcomplete', function(item, responseBuffer, response) {
[Do stuff with the scraped page]
});
// Completed crawling. Now let's get to work!
crawler.on('complete', function() {
[Get all scraped pages and do something with them]
});
// Error handling
crawler.on('queueerror', function(errorData, URLData) {
console.log('Queue error:', errorData, URLData);
});
crawler.on('fetchdataerror', function(queueitem, response) {
console.log('Fetch error:', queueitem, response);
});
crawler.on('fetchtimeout', function(queueItem, crawlerTimeoutValue) {
console.log('Fetch timeout:', queueItem, crawlerTimeoutValue);
});
crawler.on('fetchclienterror', function(queueItem, errorData) {
console.log('Fetch local error:', queueItem, errorData);
});
crawler.on('fetchtimeout', function(queueItem, crawlerTimeoutValue) {
console.log('Crawler timeout:', queueItem, crawlerTimeoutValue);
});
} else if(error) {
console.log(error);
}
});
});
return router;
}
Every simplecrawler instance has a stop method that can be called to prevent the crawler from making any further requests (requests won't be stopped in flight, however).
I would probably store the crawler instance in a scope outside of the route handler, check if its defined first thing in the route handler, in that case call the stop method and then construct a new scraper.
I stripped out a lot of the meat of your code, but something like this is what I had in mind:
module.exports = function(socket) {
var express = require('express');
var router = express.Router();
var Crawler = requrie('simplecrawler');
var crawler;
router.post('/', function(req, res, next) {
// Check if URL exist
request(url.href, function (error, response, body) {
if (!error && response.statusCode == 200) {
// Stop any crawler that's already running
if (crawler instanceof Crawler) {
crawler.stop();
}
// Crawler configuration
crawler = new Crawler(url.host);
crawler.initialPort = 80;
crawler.initialPath = url.pathname;
// Run the crawler
crawler.start();
// Execute for each URL, on fetchcomplete
crawler.on('fetchcomplete', function(item, responseBuffer, response) {
// [Do stuff with the scraped page]
});
// Completed crawling. Now let's get to work!
crawler.on('complete', function() {
// [Get all scraped pages and do something with them]
});
} else if(error) {
console.log(error);
}
});
});
return router;
}
Is there a gist which I can follow to use to connect to cloudant using nano / nodejs. Or, is there any other library which I can use to connect from nodejs to cloudant. Here is the error I encounter, when i try to connect.
description: 'one of _writer, _creator is required for this request',
Here is how my code looks
// require nano, point it at cloudant's root
var config = require('config');
var nano = require('nano')({url: config.cloudant.url})
, username = config.cloudant.username
, userpass = config.cloudant.password
, cookies = {}
, callback = console.log // this would normally be some callback
;
nano.auth(username, userpass, function (err, body, headers) {
if (err) {
return callback(err);
}
if (headers && headers['set-cookie']) {
cookies['user'] = headers['set-cookie'];
}
callback(null, "it worked");
});
var db = require('nano')({
'url': config.cloudant.url + '/' + config.cloudant.database,
'cookie': cookies['user']
});
After this, when i try to connect - i get the error
var db = require('nano')({
'url': config.cloudant.url + '/' + config.cloudant.database,
'cookie': cookies['user']
});
var doc = {"hello": "world"};
db.insert(doc, function (err, body, headers) {
if (err) {
return callback(err);
}
// change the cookie if couchdb tells us to
if (headers && headers['set-cookie']) {
cookies['user'] = headers['set-cookie'];
}
callback(null, "it worked");
});
You can try using this https://github.com/cloudant/nodejs-cloudant . It is a wrapper over nano to directly connect with cloudant.