SFTP Server for reading directory using node js - node.js

I have created a node based SSH2 SFTP Server and Client. My objective is to read directory structure of SFTP Server. Suppose I have an SFTP Server containing folder temp, I want to read files inside the temp directory. I am using ssh2 npm module for creating SFTP Server and Client. It is making connection to SFTP Server but not listing the directory
Below is the code
CLIENT SIDE SFTP
var Client = require('ssh2').Client;
var connSettings = {
host: 'localhost',
port: 22,
username:'ankit',
password:'shruti',
method:'password'
// You can use a key file too, read the ssh2 documentation
};
var conn = new Client();
conn.on('ready', function() {
console.log("connected to sftp server")
conn.sftp(function(err, sftp) {
if (err)
throw err;
sftp.readdir('',function(err,list)
{
console.log('Inside read')
if(err)
{
console.log(err);
throw err;
}
console.log('showing directory listing')
console.log(list);
conn.end();
})
/**
var moveFrom = "/remote/file/path/file.txt";
var moveTo = __dirname+'file1.txt';
sftp.fastGet(moveFrom, moveTo , {}, function(downloadError){
if(downloadError) throw downloadError;
console.log("Succesfully uploaded");
});
*/
})
}).connect(connSettings);
SERVER SIDE SFTP :
var constants = require('constants');
var fs = require('fs');
var ssh2 = require('ssh2');
var OPEN_MODE = ssh2.SFTP_OPEN_MODE;
var STATUS_CODE = ssh2.SFTP_STATUS_CODE;
var srv = new ssh2.Server({
hostKeys: [fs.readFileSync(__dirname+'/key/id_rsa')],debug:console.log
}, function(client) {
console.log('Client connected!');
client.on('authentication', function(ctx) {
if (ctx.method === 'password'
// Note: Don't do this in production code, see
// https://www.brendanlong.com/timing-attacks-and-usernames.html
// In node v6.0.0+, you can use `crypto.timingSafeEqual()` to safely
// compare two values.
&& ctx.username === 'ankit'
&& ctx.password === 'shruti')
ctx.accept();
else
ctx.reject();
}).on('ready', function() {
console.log('Client authenticated!');
});
client.on('session',function(accept,reject)
{
console.log("Client asking for session");
var session = accept();
var handleCount = 0;
var names=[]
session.on('sftp',function(accept,reject)
{
console.log('Client sftp connection')
var sftpStream = accept();
sftpStream.on('OPENDIR',function(reqID,path)
{
var handle = new Buffer(4);
handle.writeUInt32BE(handleCount++, 0, true);
sftpStream.handle(reqID,handle);
console.log(handle);
console.log('Opening Read Directory for --'+path);
console.log(reqID);
console.log(path);
}).on('READDIR',function(reqID,handle)
{
console.log('Read request')
console.log(reqID);
if(handle.length!==4)
return sftpStream.status(reqID, STATUS_CODE.FAILURE,'There was failure')
sftpStream.status(reqID, STATUS_CODE.OK,'Everything ok')
sftpStream.name(reqID,names);
})
})
})
}).listen(0, '127.0.0.1', function() {
console.log('Listening on port ' + this.address().port);
});
srv.listen(22)

The client will send multiple READDIR requests. On the first request, you should send the file listing, on the second you should send an EOF status, to tell the client the listing has finished.
An example would be:
let listSent = false;
let names = [];
sftpStream.on('OPENDIR', function (reqID, path) {
listSent = false;
sftpStream.handle(new Buffer());
});
sftpStream.on('READDIR', function (reqID, handle) {
if (listSent) {
sftpStream.status(reqID, STATUS_CODE.EOF);
return;
}
sftpStream.name(reqID, names);
listSent = true;
});

Related

Clear the client console in SSH2 for NodeJS?

I am building a command-line game that players will connect to over SSH. I have an SSH2 server script that responds to commands, but I would like to know if it is possible to clear the client console from the server?
I would like to have a cls/clear command that clears the console at a user's request, like the Unix command 'clear'.
Here is my code:
var fs = require('fs');
var crypto = require('crypto');
var inspect = require('util').inspect;
var ssh2 = require('ssh2');
var utils = ssh2.utils;
var allowedUser = Buffer.from('foo');
var allowedPassword = Buffer.from('bar');
var allowedPubKey = utils.parseKey(fs.readFileSync('public.key')); ;
new ssh2.Server({
hostKeys: [{
key: fs.readFileSync('private.key'),
passphrase: '<private key passphrase>'
}
]
}, function (client) {
console.log('Client connected!');
client.on('authentication', function (ctx) {
var user = Buffer.from(ctx.username);
if (user.length !== allowedUser.length
|| !crypto.timingSafeEqual(user, allowedUser)) {
return ctx.reject();
}
switch (ctx.method) {
case 'password':
var password = Buffer.from(ctx.password);
if (password.length !== allowedPassword.length
|| !crypto.timingSafeEqual(password, allowedPassword)) {
return ctx.reject(['password'],false);
}
break;
default:
return ctx.reject(['password'],false);
}
ctx.accept();
}).on('ready', function () {
console.log('Client authenticated!');
client.once('session', function (accept, reject) {
var session = accept();
session.on('shell', (accept, reject) => {
var stream = accept();
stream.write('Welcome to EnLore!\n\r');
stream.on('data', data => {
var args = data.toString().replace("\n","").split(" ");
if (args[0] == '\r') return;
switch(args[0]) {
case 'exit':
stream.exit(0);
stream.end();
stream = undefined;
break;
case 'echo':
stream.write(args[1]);
break;
case 'clear':
// -------------- CLEAR CONSOLE HERE --------------
break;
default:
stream.write('Unknown command');
break;
}
stream.write('\n\r');
});
});
});
}).on('end', function () {
console.log('Client disconnected');
});
}).listen(8080, '0.0.0.0', function () {
console.log('Listening on port ' + this.address().port);
});
EDIT:
I have found that writing \f to the stream will clear the current window, however, it only appends to the console, rather than clearing it

New connection cause the current connections to stop working

I am making the chat application using socket (which I'm new at) with multiple tenants structure and using namespaces. Here's my code:
Socket server:
index.js
class Server {
constructor() {
this.port = process.env.PORT || 3000;
this.host = process.env.HOST || `localhost`;
this.app = express();
this.http = http.Server(this.app);
this.rootSocket = socketio(this.http);
}
run() {
new socketEvents(this.rootSocket).socketConfig();
this.app.use(express.static(__dirname + '/uploads'));
this.http.listen(this.port, this.host, () => {
console.log(`Listening on ${this.host}:${this.port}`);
});
}
}
const app = new Server();
app.run();
socket.js
var redis = require('redis');
var redisConnection = {
host: process.env.REDIS_HOST,
password: process.env.REDIS_PASS
};
var sub = redis.createClient(redisConnection);
var pub = redis.createClient(redisConnection);
class Socket {
constructor(rootSocket) {
this.rootIo = rootSocket;
}
socketEvents() {
/**
* Subscribe redis channel
*/
sub.subscribe('visitorBehaviorApiResponse');
//TODO: subscribe channel..
// Listen to redis channel that published from api
sub.on('message', (channel, data) => {
data = JSON.parse(data);
console.log(data);
const io = this.rootIo.of(data.namespace);
if (channel === 'visitorBehaviorApiResponse') {
io.to(data.thread_id).emit('receiveBehavior', data);
io.to('staff_room').emit('incomingBehavior', data);
}
})
sub.on('error', function (error) {
console.log('ERROR ' + error)
})
var clients = 0;
this.rootIo.on('connection', (rootSocket) => {
clients++;
console.log('root:' + rootSocket.id + ' connected (total ' + clients + ' clients connected)');
const ns = rootSocket.handshake['query'].namespace;
// Dynamic namespace for multiple tenants
if (typeof (ns) === 'string') {
const splitedUrl = ns.split("/");
const namespace = splitedUrl[splitedUrl.length - 1];
const nsio = this.rootIo.of(namespace);
this.io = nsio;
this.io.once('connection', (socket) => {
var visitors = [];
console.log('new ' + socket.id + ' connected');
// once a client has connected, we expect to get a ping from them saying what room they want to join
socket.on('createChatRoom', function (data) {
socket.join(data.thread_id);
if (typeof data.is_staff !== 'undefined' && data.is_staff == 1) {
socket.join('staff_room');
} else {
if (visitors.some(e => e.visitor_id === data.visitor_id)) {
visitors.forEach(function (visitor) {
if (visitor.visitor_id === data.visitor_id) {
visitor.socket_ids.push(socket.id);
}
})
} else {
data.socket_ids = [];
data.socket_ids.push(socket.id);
visitors.push(data);
}
socket.join('visitor_room');
}
//TODO: push to redis to check conversation type
});
socket.on('sendMessage', function (data) {
console.log(data);
pub.publish('chatMessage', JSON.stringify(data));
this.io.in(data.thread_id).emit('receiveMessage', data);
this.io.in('staff_room').emit('incomingMessage', data);
// Notify new message in room
data.notify_type = 'default';
socket.to(data.thread_id).emit('receiveNotify', data);
}.bind(this))
socket.on('disconnect', (reason) => {
sub.quit();
console.log('client ' + socket.id + ' left, ' + reason);
});
socket.on('error', (error) => {
console.log(error);
});
});
}
// Root disconnect
rootSocket.on('disconnect', function () {
clients--;
console.log('root:' + rootSocket.id + ' disconnected (total ' + clients + ' clients connected)');
});
});
}
socketConfig() {
this.socketEvents();
}
}
module.exports = Socket;
Client:
const server = 'https://socket-server'
const connect = function (namespace) {
return io.connect(namespace, {
query: 'namespace=' + namespace,
resource: 'socket.io',
transports: ['websocket'],
upgrade: false
})
}
const url_string = window.location.href
const url = new URL(url_string)
const parameters = Object.fromEntries(url.searchParams)
const socket = connect(`${server}/${parameters.shopify_domain}`)
var handleErrors = (err) => {
console.error(err);
}
socket.on('connect_error', err => handleErrors(err))
socket.on('connect_failed', err => handleErrors(err))
socket.on('disconnect', err => handleErrors(err))
The problem that I met is when socket server got a new connection, the existing connections will be stopped working util they make a page refreshing to reconnect a new socket.id.
And when a namespace's client emit data, it sends to other namespaces, seem my code is not work correctly in a namespace.
Could you take a look at my code and point me where I'm wrong?
Thanks
1) Get UserId or accessToken while handshaking(in case of accessToken decrypt it).
and store userID: socketId(in Redis or in local hashmap) depends upon the requirement .
2) When u are going to emit to particular user fetch the socketid to that userid from redis or local hashmap
and emit to it.
**io.to(socketId).emit('hey', 'I just met you');**
3) If you are using multiple servers use sticky sessions
4) Hope this will help you

Dynamically Create Kafka Consumers That Send Data Through a Websocket

I am using kafka-node to read stream data and pass it to my web app with Web Sockets using NodeJS. This works fine if I able to define the kafka producer server and the topic I am interested in, however for my use case the end users will input the kafka producer server and the topic and my NodeJS backend will be responsible to receive that request and to create the appropriate kafka/websocket connections.
My idea was the following:
Create a rest API to which the web app could send requests to in order to create a new kafka consumer/web socket connection (/registerTopic)
Save the new kafka consumers in a global array when I create a new kafka consumer so that I can later pause or resume the stream with another rest API call (/pauseTopic and /resumeTopic)
I ran into problems trying to move the WebSocket code into /registerTopic...Whenever I do this everything acts very strangely and I suddenly get 1000x messages at once and then 40-50x messages every second even though the kafka producer is only sending 1 message per second. Any ideas on how I can get this working?
const express = require("express");
const app = express();
const cors = require('cors');
const bodyParser = require('body-parser');
const fs = require('fs');
app.use(cors());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
var WebSocketServer = require('websocket').server;
var http = require('http');
var https = require('https');
var kafka = require('kafka-node');
var topics = [];
var privateKey = fs.readFileSync('PATH', 'utf8');
var certificate = fs.readFileSync('PATH', 'utf8');
var credentials = { key: privateKey, cert: certificate };
var consumers = new Set();
// This was working without any issues before I tried to make this dynamic!
/* var Consumer = kafka.Consumer,
client = new kafka.KafkaClient('localhost:9092'),
consumer = new Consumer(
client, [{ topic: 'numtest', partition: 0 }], { autoCommit: false }); */
var server = http.createServer(function (request, response) {
console.log(' Request recieved : ' + request.url);
response.writeHead(404);
response.end();
});
server.listen(8080, function () {
console.log('Listening on port : 8080');
});
webSocketServer = new WebSocketServer({
httpServer: server,
autoAcceptConnections: false
});
function iSOriginAllowed(origin) {
return true;
}
// This was working without any issues before I tried to make this dynamic!
/* webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
}); */
var httpsServer = http.createServer(credentials, app);
httpsServer.listen(3000, () => {
console.log("Server running on port 3000");
});
app.get("/getTopics", (req, res, next) => {
res.json(topics);
});
app.post("/registerTopic", (req, res) => {
try {
var client = new kafka.KafkaClient(req.body.host);
var Consumer = kafka.Consumer;
consumer = new Consumer(
client, [{ topic: req.body.topic, partition: 0 }], { autoCommit: false });
let consumerExists = false;
for (let c = 0; c < [...consumers].length; c++) {
if ([...consumers][c].topic == req.body.topic && [...consumers][c].sessionId == req.body.sessionId) {
consumerExists = true;
}
}
if (!consumerExists) {
consumers.add({ 'topic': req.body.topic, 'sessionId': req.body.sessionId, 'consumer': consumer });
}
client.loadMetadataForTopics([], function (error, results) {
Object.keys(results[1].metadata).forEach(function (key) {
var value = results[1].metadata[key];
if (!value['0'].topic.includes('__') && !value['0'].topic.includes('offset')) {
topics.push({ 'producer': req.body.host, 'topic': value['0'].topic });
}
});
});
webSocketServer.on('request', function (request) {
if (!iSOriginAllowed(request.origin)) {
request.reject();
console.log(' Connection from : ' + request.origin + ' rejected.');
return;
}
let connection = request.accept('echo-protocol', request.origin);
console.log(' Connection accepted : ' + request.origin);
connection.on('message', function (message) {
if (message.type === 'utf8') {
console.log('Received Message: ' + message.utf8Data);
}
});
consumer.on('message', function (message) {
console.log('msg');
connection.sendUTF(message.value);
});
connection.on('close', function (reasonCode, description) {
console.log('Connection ' + connection.remoteAddress + ' disconnected.');
});
});
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/pauseTopic", (req, res) => {
try {
console.log(req.body);
let filteredConsumer = [...consumers].filter(function (item) {
console.log(req.body.topic, item.sessionId);
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return c;
}
});
console.log(filteredConsumer);
//filteredConsumer[0].consumer.pause();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});
app.post("/resumeTopic", (req, res) => {
try {
let filteredConsumer = [...consumers].filter(function (item) {
if (item.topic == req.body.topic && item.sessionId == req.body.sessionId) {
return item;
}
});
filteredConsumer[0].consumer.resume();
res.json("Working");
} catch (error) {
console.error(error);
res.status(400).send('Unable to register new topic')
}
});

node.js (or nwjs) ftp download

Need to download new files periodicaly from public ftp:
ftp://ftp.cmegroup.com/bulletin/
But I cannot connect ftp module to sever. My code is:
var url = "ftp://ftp.cmegroup.com/bulletin/";
var path = require('path');
var fs = require('fs');
//var Promise = require('bluebird');
var Client = require('ftp');
var c = new Client();
var connectionProperties = {
host: "ftp.cmegroup.com",
};
c.on('ready', function () {
console.log('ready');
c.list(function (err, list) {
if (err) throw err;
list.forEach(function (element, index, array) {
//Ignore directories
if (element.type === 'd') {
console.log('ignoring directory ' + element.name);
return;
}
//Ignore non zips
if (path.extname(element.name) !== '.zip') {
console.log('ignoring file ' + element.name);
return;
}
//Download files
c.get(element.name, function (err, stream) {
if (err) throw err;
stream.once('close', function () {
c.end();
});
stream.pipe(fs.createWriteStream(element.name));
});
});
});
});
c.connect(connectionProperties);
Error:
Uncaught Error: connect ECONNREFUSED 127.0.0.1:21
Can't understand why it connects to localhost despite I pointed connection params.
lack of one line
c.connect(connectionProperties);

nodejs tcp file upload missing parts

I am trying to make a server and client program just on localhost to be able to upload and list files from the server. I have the code to upload files to the server but for some reason after the server writes the file from the client it has less bytes than the original file.
Here is the client code:
var net = require('net');
var fs = require('fs');
var port='3000';
var host='localhost';
var path=__dirname;
var fileUploadDir = __dirname+'/uploadfolder/';
process.stdin.resume();
var socket = net.connect(port,host);
var stdin = process.stdin;
var stdout = process.stdout;
stdout.on('data', function(d){
console.log("d: "+d);
});
socket.on('connect',function(){
console.log("connected to server");
//stdin.pipe(stdout);
stdin.pipe(socket, {end: false});
});
socket.on('data', function(data){
var b = new Buffer(data, "utf8");
var bStr = b.toString();
if(bStr=="okupload"){
console.log("uploading file");
folderFilesUpload(socket);
}else{
console.log(bStr);
}
});
socket.on('end', function(){
socket = net.connect(port,host);
console.log('socket ended');
});
function folderFilesUpload(socket){
fs.readdir(fileUploadDir, function(err, results){
if(err)
console.log("upload to upload files");
else{
if(results.length > 0){
for(var r in results){
var fullPath = fileUploadDir+results[r];
var fileStream = fs.createReadStream(fullPath);
fileStream.on('open', function(){
fileStream.pipe(socket);
});
}
}else{
console.log("no files to upload");
}
}
});
}
Server Code:
var net=require('net');
var fs=require('fs');
var host ='localhost';
var port='3000';
var outputDir = __dirname+'/outputdir/';
var uploadState = false;
var server = net.createServer(function(conn){
console.log('server connected');
conn.on('data', function(data) {
if(data == "l\n" && !uploadState){
listFiles(conn);
}
if(data == "u\n"){
console.log("switching to upload file state");
uploadState=true;
conn.write("okupload");
}
if(uploadState){
console.log("data");
//console.log("data: "+data);
var file = fs.createWriteStream(outputDir+'temp');
conn.pipe(file);
}
});
conn.on('end', function(){
uploadState = false;
console.log('data has ended');
});
});
server.listen(port,host, function(){
server.on('connection', function(){
console.log('connection made');
});
});
function listFiles(conn){
console.log("listing files");
fs.readdir(outputDir, function(err, results){
if(err){
var b = new Buffer("error listing files", "utf8");
conn.write(b);
}else{
var output = "files: \n";
if(results !== null && results.length >0){
for(var r in results){
output += results[r]+"\n";
}
}else{
output += "no files.\n";
}
var b = new Buffer(output, "utf8");
conn.write(b);
}
});
}

Resources