node phantomjs seo running via node write the file - node.js

I'm wondering if this is the write way:
node.js
var fs = require('fs');
var path = require('path');
var childProcess = require('child_process');
var phantomjs = require('phantomjs');
var binPath = phantomjs.path;
var childArgs = [
path.join(__dirname, 'phantomjs-runner.js'),
'http://localhost:3000'
]
childProcess.execFile(binPath, childArgs, function(err, stdout, stderr) {
if(err){
}
if(stderr){
}
fs.writeFile(__dirname+'/public/snapshots/index.html', stdout, function(err) {
if(err) {
console.log(err);
}
else {
console.log("The file was saved!");
}
});
});
phantomjs-runner.js
var system = require('system');
var url = system.args[1] || '';
if(url.length > 0) {
var page = require('webpage').create();
page.open(url, function (status) {
if (status == 'success') {
var delay, checker = (function() {
var html = page.evaluate(function () {
var body = document.getElementsByTagName('body')[0];
if(body.getAttribute('data-status') == 'ready') {
return document.getElementsByTagName('html')[0].outerHTML;
}
});
if(html) {
clearTimeout(delay);
console.log(html);
phantom.exit();
}
});
delay = setInterval(checker, 100);
}
});
}
may be could be a better way like
clearTimeout(delay);
fs.writeFile
phantom.exit();
How can I manage ie
different urls and different files
I mean
http://localhost:3000 index.html
http://localhost:3000/blog blog.html
http://localhost:3000/blog/postid postid.html
ENDED UP
'use strict';
var fs = require('fs'),
path = require('path'),
childProcess = require('child_process'),
phantomjs = require('phantomjs'),
binPath = phantomjs.path,
config = require('../config/config');
var env = (process.env.NODE_ENV === 'production') ? 'production' : null,
currentPath = (env) ? config.proot + '/build/default/snapshots' : config.proot + '/default/snapshots';
function normalizeUrl(url){
if( (typeof url === 'undefined') || !url){
return '';
}
if ( url.charAt( 0 ) === '/' ){
url = url.substring(1);
}
return '/'+url.replace(/\/$/, "");
}
function normalizePage(route){
if(!route){
return 'index';
}
var chunks = route.substring(1).split('/');
var len = chunks.length;
if(len===1){
return chunks[0];
}
chunks.shift();
//I get the id
return chunks[0];
}
module.exports = function (url) {
var route = normalizeUrl(url);
var page = normalizePage(route);
var childArgs = [
path.join(__dirname, './phantomjs-runner.js'),
config.url+route
];
childProcess.execFile(binPath, childArgs, function(err, stdout, stderr) {
if(err){
}
if(stderr){
}
fs.writeFile(currentPath + '/' + page + '.html', stdout, function(err) {
if(err) {
console.log(err);
}
else {
console.log("The file was saved!");
}
});
});
};
So I worked out for the parameters I'm still
waiting for way to get the output ^^

Related

change data of a file which contain an array using node

I have created a file called config.js which looks like below:
const config = {
staticFiles:{
name:[
'./',
'./index.html',
'./script.js',
'./icon.jpg'
]
},
outputFolderName: "D:\\DemoApp",
sourceApplicationParentPath: "D:\\DemoApp\\"
};
Now I am reading list of files from sourceApplicationParentPath folder using node and have to update staticFiles array of above file. I am not sure how should I do it. Can someone please help.
Thanks in advance.
config.js
const config = {
staticFiles: {
name: ['./',
'./index.html',
'./script.js',
'./icon.jpg',
]
},
outputFolderName: 'D:\\DemoApp',
sourceApplicationParentPath: 'D:\\DemoApp'
};
module.exports = config;
index.js
var fs = require('fs'),
config = require('./config'),
util = require('util');
fs.readdir(config.sourceApplicationParentPath, function(err, files) {
if (err) console.log(err.message);
for (var i = 0; i < files.length; i++) {
if (config.staticFiles.name.indexOf(`./${files[i]}`) == -1) {
config.staticFiles.name.push('./' + files[i]);
}
if (i == (files.length - 1)) {
var buffer = `const config = \n ${util.inspect(config, false, 2, false)}; \n module.exports = config;`;
fs.writeFile('./config.js', buffer, function(err) {
err || console.log('Data replaced \n');
})
}
}
});
The Above code is tested and working fine.
You can add or change the object or an array or value in config.js without duplicate entry.
config.js
const config = {
staticFiles:{
name:[
'./',
'./index.html',
'./script.js',
'./icon.jpg'
]
},
outputFolderName: "D:\\DemoApp",
sourceApplicationParentPath: "D:\\DemoApp\\"
};
exports.config = config;
code for the file from where you want to change the data
var fs = require('fs');
var bodyparser = require('body-parser');
var config = require('./config.js')
//path of directory
var directoryPath = "D:\\DemoApp\\"
var data = config.config;
//passsing directoryPath and callback function
fs.readdir(directoryPath, function (err, files) {
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
var dataToUpdate = data.staticFiles.name;
//listing all files using forEach
files.forEach(function (file) {
// Do whatever you want to do with the file
console.log(file)
dataToUpdate.push(file)
});
data.staticFiles.name = dataToUpdate;
var value = 'const config = ' + JSON.stringify(data) + ';' + '\n' + 'exports.config = config';
fs.writeFile('./config.js',value, er => {
if(er){
throw er;
}
else{console.log('success')}
});
});

Async Recursive File System Traversal

I am trying to compile an array out of data present in some specific files, using a recursive approach to read directories and the file system methods are asynchronous. I am unable to figure out the apt place for the callback invocation.
const fs = require('fs');
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
function doThisOnceDone() {
console.log(nodes);
}
function readFile(path) {
fs.readFile(path + '/' + FILE_NAME,{
encoding:"UTF-8"
}, function(err, data) {
if(err) {
return;
}
nodes.push(data);
});
}
function compileArray(path, callback) {
fs.readdir(path, {
encoding:"UTF-8"
}, function(err, files) {
if(err) {
console.error(err);
return;
}
files.forEach(function(file) {
var nextPath = path + '/' + file;
fs.stat(nextPath, function(err, stats) {
if(err) {
return;
}
if(stats.isDirectory()) {
if(file === 'specific') {
readFile(nextPath);
}
else {
compileArray(nextPath);
}
}
});
});
});
}
compileArray(ENTRY, doThisOnceDone);
When do I know that the recursion tree has been done with , and I can access the nodes array ?
Try this
const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach((file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}
const util = require('util');
const fs = require('fs');
const stat = util.promisify(fs.stat);
const readFile = util.promisify(fs.readFile);
const readdir = util.promisify(fs.readdir);
const ENTRY = "../a/b";
const FILE_NAME = 'index.json';
var nodes = [];
const compileArray = async (path) => {
try {
const files = await readdir(path);
files.forEach(async (file) => {
try {
var nextPath = path + '/' + file;
const stats = await stat(nextPath); //check other conditions
if (file === 'specific') {
const data = await readFile(path + '/' + FILE_NAME, {
encoding: "UTF-8"
});
nodes.push(data);
}
} catch (error) {
console.log(error);
}
});
} catch (error) {
console.log(error);
}
}
compileArray(ENTRY)

Initialize modules asynchronously

I have created a function to require my modules (middlewares and controllers) but I want to initialize middlewares first.
My code bellow initialize both modules in the same time :
function loadModule(app, modulePath) {
var folder = path.join(__dirname, modulePath);
fs.readdir(folder, function(err, files) {
_.forEach(files, function(file) {
var controllerPath = path.join(folder, file);
fs.stat(controllerPath, function(err, stats) {
if (stats.isFile() && file !== 'Controller.js') {
var loadingModulePath = './' + modulePath + '/' + file;
console.log('Loading module : ' + loadingModulePath);
var Module = require(loadingModulePath)(app);
}
});
});
});
}
loadModule(app, 'middlewares');
loadModule(app, 'controllers');
Issue : sometimes controllers are initialized in first, sometimes that are middlewares...
Edit #1 :
const express = require('express'),
app = express(),
async = require('async');
function loadModule(module, app) {
var folder = path.join(__dirname, module);
fs.readdir(folder, function(err, files) {
_.forEach(files, function(file) {
var controllerPath = path.join(folder, file);
fs.stat(controllerPath, function(err, stats) {
if (stats.isFile() && file !== 'Controller.js') {
var loadingModulePath = './' + module + '/' + file;
console.log('Loading module : ' + loadingModulePath);
var Module = require(loadingModulePath)(app);
}
});
});
});
}
async.series([
function(callback, app) {
loadModule('middlewares', app);
callback(null);
},
function(callback, app) {
loadModule('controllers', app);
callback(null);
}
], function(err, results) {
console.log(err, results);
});
Issue edit #1 : app is undefined...

WebSocket connection to 'ws://localhost:3434/' failed: Connection closed before receiving a handshake response

When using the cmd to run the node server.js and launch the localhost:3434 in the web browser I am getting the error:
in Chrome:
Connection closed before receiving a handshake response.
in firefox:
Firefox can't establish a connection to the server at ws://localhost:3434/
Could you pls help
server.js
var http = require('http'),
fs = require('fs');
fs.readFile('../index.html', function (err, html) {
if (err) {
throw err;
}
http.createServer(function (request, response) {
response.writeHeader(200, { "Content-Type": "text/html" });
response.write(html);
response.end();
}).listen(3434);
});
index.html
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<!--<script type="text/javascript" src="../../js/adapter.js"></script>-->
<!--<script type="text/javascript" src="../../js/webrtc.js"></script>-->
<title>Simple WebRTC video chat</title>
</head>
<body>
<header>
<video id="localVideo" autoplay="autoplay" muted="muted" style="width:40%;"></video>
<video id="remoteVideo" autoplay="autoplay" style="width:40%;"></video>
<input type="button" id="start" onclick="start(true)" value="Start Video"; />
</header>
<script>
var RTCPeerConnection = null;
var getUserMedia = null;
var attachMediaStream = null;
var reattachMediaStream = null;
var webrtcDetectedBrowser = null;
var webrtcDetectedVersion = null;
function trace(text) {
// This function is used for logging.
if (text[text.length - 1] == '\n') {
text = text.substring(0, text.length - 1);
}
console.log((performance.now() / 1000).toFixed(3) + ": " + text);
}
if (navigator.mozGetUserMedia) {
console.log("This appears to be Firefox");
webrtcDetectedBrowser = "firefox";
webrtcDetectedVersion =
parseInt(navigator.userAgent.match(/Firefox\/([0- 9]+)\./)[1]);
// The RTCPeerConnection object.
RTCPeerConnection = mozRTCPeerConnection;
// The RTCSessionDescription object.
RTCSessionDescription = mozRTCSessionDescription;
// The RTCIceCandidate object.
RTCIceCandidate = mozRTCIceCandidate;
// Get UserMedia (only difference is the prefix).
getUserMedia = navigator.mozGetUserMedia.bind(navigator);
// Creates iceServer from the url for FF.
createIceServer = function (url, username, password) {
var iceServer = null;
var url_parts = url.split(':');
if (url_parts[0].indexOf('stun') === 0) {
// Create iceServer with stun url.
iceServer = { 'url': url };
} else if (url_parts[0].indexOf('turn') === 0 &&
(url.indexOf('transport=udp') !== -1 ||
url.indexOf('?transport') === -1)) {
// Create iceServer with turn url.
// Ignore the transport parameter from TURN url.
var turn_url_parts = url.split("?");
iceServer = {
'url': turn_url_parts[0],
'credential': password,
'username': username
};
}
return iceServer;
};
// Attach a media stream to an element.
attachMediaStream = function (element, stream) {
console.log("Attaching media stream");
element.mozSrcObject = stream;
element.play();
};
reattachMediaStream = function (to, from) {
console.log("Reattaching media stream");
to.mozSrcObject = from.mozSrcObject;
to.play();
};
MediaStream.prototype.getVideoTracks = function () {
return [];
};
MediaStream.prototype.getAudioTracks = function () {
return [];
};
} else if (navigator.webkitGetUserMedia) {
console.log("This appears to be Chrome");
webrtcDetectedBrowser = "chrome";
webrtcDetectedVersion =
parseInt(navigator.userAgent.match(/Chrom(e|ium)\/([0-9]+)\./)[2]);
// Creates iceServer from the url for Chrome.
createIceServer = function (url, username, password) {
var iceServer = null;
var url_parts = url.split(':');
if (url_parts[0].indexOf('stun') === 0) {
// Create iceServer with stun url.
iceServer = { 'url': url };
} else if (url_parts[0].indexOf('turn') === 0) {
if (webrtcDetectedVersion < 28) {
var url_turn_parts = url.split("turn:");
iceServer = {
'url': 'turn:' + username + '#' + url_turn_parts[1],
'credential': password
};
} else {
iceServer = {
'url': url,
'credential': password,
'username': username
};
}
}
return iceServer;
};
// The RTCPeerConnection object.
RTCPeerConnection = webkitRTCPeerConnection;
// Get UserMedia (only difference is the prefix).
// Code from Adam Barth.
getUserMedia = navigator.webkitGetUserMedia.bind(navigator);
// Attach a media stream to an element.
attachMediaStream = function (element, stream) {
if (typeof element.srcObject !== 'undefined') {
element.srcObject = stream;
} else if (typeof element.mozSrcObject !== 'undefined') {
element.mozSrcObject = stream;
} else if (typeof element.src !== 'undefined') {
element.src = URL.createObjectURL(stream);
} else {
console.log('Error attaching stream to element.');
}
};
reattachMediaStream = function (to, from) {
to.src = from.src;
};
// The representation of tracks in a stream is changed in M26.
// Unify them for earlier Chrome versions in the coexisting period.
if (!webkitMediaStream.prototype.getVideoTracks) {
webkitMediaStream.prototype.getVideoTracks = function () {
return this.videoTracks;
};
webkitMediaStream.prototype.getAudioTracks = function () {
return this.audioTracks;
};
}
if (!webkitRTCPeerConnection.prototype.getLocalStreams) {
webkitRTCPeerConnection.prototype.getLocalStreams = function () {
return this.localStreams;
};
webkitRTCPeerConnection.prototype.getRemoteStreams = function () {
return this.remoteStreams;
};
}
} else {
console.log("Browser does not appear to be WebRTC-capable");
}
var localVideo;
var remoteVideo;
var peerConnection;
var localStream;
var optional = { optional: [{ DtlsSrtpKeyAgreement: true }] }
var peerConnectionConfig = { 'iceServers': [{ 'url': 'stun:stun.services.mozilla.com' }, { 'url': 'stun:stun.l.google.com:19302' }] };
navigator.getUserMedia = navigator.getUserMedia || navigator.mozGetUserMedia || navigator.webkitGetUserMedia;
window.RTCPeerConnection = window.RTCPeerConnection || window.mozRTCPeerConnection || window.webkitRTCPeerConnection;
window.RTCIceCandidate = window.RTCIceCandidate || window.mozRTCIceCandidate || window.webkitRTCIceCandidate;
window.RTCSessionDescription = window.RTCSessionDescription || window.mozRTCSessionDescription || window.webkitRTCSessionDescription;
localVideo = document.getElementById("localVideo");
remoteVideo = document.getElementById("remoteVideo");
serverConnection = new WebSocket('ws://localhost:3434');
serverConnection.onmessage = gotMessageFromServer;
var constraints = {
video: true,
audio: true,
};
if (navigator.getUserMedia) {
navigator.getUserMedia(constraints, getUserMediaSuccess, getUserMediaError);
} else {
alert('Your browser does not support getUserMedia API');
}
function getUserMediaSuccess(stream) {
localStream = stream;
localVideo.src = window.URL.createObjectURL(stream);
}
function start(isCaller) {
peerConnection = new RTCPeerConnection(peerConnectionConfig, optional);
peerConnection.onicecandidate = gotIceCandidate;
peerConnection.onaddstream = gotRemoteStream;
peerConnection.addStream(localStream);
if (isCaller) {
peerConnection.createOffer(gotDescription, createOfferError);
}
}
function gotMessageFromServer(message) {
if (!peerConnection) start(false);
var signal = JSON.parse(message.data);
if (signal.sdp) {
peerConnection.setRemoteDescription(new RTCSessionDescription(signal.sdp), function () {
peerConnection.createAnswer(gotDescription, createAnswerError);
});
} else if (signal.ice) {
peerConnection.addIceCandidate(new RTCIceCandidate(signal.ice));
}
}
function gotIceCandidate(event) {
if (event.candidate != null) {
serverConnection.send(JSON.stringify({ 'ice': event.candidate }));
}
}
function gotDescription(description) {
console.log('got description');
peerConnection.setLocalDescription(description, function () {
serverConnection.send(JSON.stringify({ 'sdp': description }));
}, function () { console.log('set description error') });
}
function gotRemoteStream(event) {
console.log("got remote stream");
remoteVideo.src = window.URL.createObjectURL(event.stream);
}
// Error functions....
function getUserMediaError(error) {
console.log(error);
}
function createOfferError(error) {
console.log(error);
}
function createAnswerError(error) {
console.log(error);
}
</script>
</body>
</html>
In order to handle websocket connections your http-server should handle 'upgrade'-event (see here - https://nodejs.org/api/http.html#http_event_upgrade_1)
For example, here is the working example (it uses ws-module):
var WebSocketServer = require('ws').Server;
var wss = new WebSocketServer({noServer: true});
var http = require('http'),
fs = require('fs');
fs.readFile('../index.html', function (err, html) {
if (err) {
throw err;
}
var server = http.createServer(function (request, response) {
response.writeHeader(200, { "Content-Type": "text/html" });
response.write(html);
response.end();
})
server.listen(3434);
server.on('upgrade', wss.handleUpgrade);
wss.on('connection', function connection(ws) {
ws.on('message', function incoming(message) {
console.log('received: %s', message);
});
ws.send('something');
});
});
Here are docs for ws - https://github.com/websockets/ws

How to know non blocking Recursive job is complete in nodejs

I have written this non-blocking nodejs sample recursive file search code, the problem is I am unable to figure out when the task is complete. Like to calculate the time taken for the task.
fs = require('fs');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var scan = function(dir, done) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath, done);;
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
var onComplte = function () {
console.log('Task is completed');
}
scan(dirToScan,onComplte);
}
searchApp();
Above code do the search perfectly, but I am unable to figure out when the recursion will end.
Its not that straight forward, i guess you have to rely on timer and promise.
fs = require('fs');
var Q = require('q');
searchApp = function() {
var dirToScan = 'D:/';
var stringToSearch = 'test';
var promises = [ ];
var traverseWait = 0;
var onTraverseComplete = function() {
Q.allSettled(promises).then(function(){
console.log('Task is completed');
});
}
var waitForTraverse = function(){
if(traverseWait){
clearTimeout(traverseWait);
}
traverseWait = setTimeout(onTraverseComplete, 5000);
}
var scan = function(dir) {
fs.readdir(dir, function(err, files) {
files.forEach(function (file) {
var abPath = dir + '/' + file;
var future = Q.defer();
try {
fs.lstat(abPath, function(err, stat) {
if(!err && stat.isDirectory()) {
scan(abPath);
}
});
}
catch (e) {
console.log(abPath);
console.log(e);
}
matchString(file,abPath);
future.resolve(abPath);
promises.push(future);
waitForTraverse();
});
});
}
var matchString = function (fileName, fullPath) {
if(fileName.indexOf(stringToSearch) != -1) {
console.log(fullPath);
}
}
scan(dirToScan);
}
searchApp();

Resources