I am working on a small project which will be used for crawling. I am having problem while working with sockets.
I am emitting some events from the server in a loop.
for(i=0;i<blogs.length;i++){
socket.emit('crawling',blogs[i]);
const startTime = Date.now();
const post = await crawlPost(blogs[i]);
const endTime = Date.now();
socket.emit('crawled',blogs[i]);
socket.emit('timeTaken',{
time: endTime-startTime,
blog : blogs[i],
});
}
I want to listen for these events one by one in react component, but I am unable to do so, code I have written for listening to the events.
const App = () => {
const [crawlingBlog, setCrawlingBlog] = useState([]);
const ENDPOINT = 'localhost:4000';
socket = io(ENDPOINT);
useEffect(() => {
socket.on('crawling' , (blog) => {
setCrawlingBlog(crawlingBlog.concat(blog))
})
},[])
}
did you wrap your socket at the server like that?
module.exports = connectSockets
function connectSockets(io) {
io.on('connection', socket => {
for(i=0;i<blogs.length;i++){
socket.emit('crawling',blogs[i]);
const startTime = Date.now();
const post = await crawlPost(blogs[i]);
const endTime = Date.now();
socket.emit('crawled',blogs[i]);
socket.emit('timeTaken',{
time: endTime-startTime,
blog : blogs[i],
});
}
})
}
Related
I am trying to create an RTMP-server with the npm package: http://github.com/illuspas/Node-Media-Server. So the server works fine but I need to implement authentication in it. I am trying to check the authentication on "prePublish" event. I am querying the database and retrieving the user if the user was found then I want to let the user stream otherwise rejected. But the problem is, it doesn't leave it instead disconnects and then the stream automatically reconnected to it then it disconnects again and the loop goes on. How do I fix this problem?
Here is the code for the event:
const NodeMediaServer = require('node-media-server');
const config = require('./config').rtmp_server;
const db = require('./db');
const nms = new NodeMediaServer(config);
const getStreamKeyFromStreamPath = (path) => {
const parts = path.split('/');
return parts[parts.length - 1];
};
nms.on('prePublish', async (id, StreamPath, args) => {
const session = nms.getSession(id);
try {
const streamKey = getStreamKeyFromStreamPath(StreamPath);
const validStream = (
await db.query('SELECT * FROM public."People" WHERE stream_key = $1', [streamKey])
).rows[0];
console.log(validStream);
if (validStream) {
// do stuff
} else {
session.reject((reason) => {
console.log(reason);
});
}
console.log(
'[NodeEvent on prePublish]',
`id=${id} StreamPath=${StreamPath} args=${JSON.stringify(args)}`
);
} catch (err) {
session.reject();
}
});
module.exports = nms;
Here is the code of the entry point of the server:
require("dotenv").config();
const db = require("./db");
const nms = require("./nms");
// database connection
db.connect()
.then(() => {
console.log("Connected to database");
// start the rtmp server
nms.run();
})
.catch((err) => console.log(err.message));
Here is the db file:
const { Pool } = require('pg');
const connectionString = process.env.PG_CONNECTION_STRING;
const poolOptions = {
host: process.env.PG_HOST,
user: process.env.PG_USER,
port: process.env.PG_PORT,
password: process.env.PG_PASSWORD,
database: process.env.PG_DATABASE,
};
const pool = new Pool(process.env.NODE_ENV === 'production' ? connectionString : poolOptions);
module.exports = pool;
My procedures to solve that problem:
Instead of the async function, I tried to handle the database query using a callback but it didn't work.
Before I was calling session.reject() now I am passing a callback there but the behavior is still the same
If you have any solution for that, please let me know.
Thanks in advance
I have a NFT deployed on the mainnet, and I want to listen any mint event by watching the Transfer event via web3js. But unfortunately I am not able to retrieve any Transfer event happening. But the thing is that, when I try to getPastEvents, I successfully retrieve the correct data so it is most likely not due to another part of the code.
Here is my relevant piece of code:
const Web3 = require('web3')
const nodeClient = require('node-rest-client-promise').Client()
const dotenv = require('dotenv')
dotenv.config()
const CONTRACT_ADDRESS = '0x8d4B648F7fAB1c72d1690b42693fb7525ce3025e'
const projectId = process.env.INFURA_KEY
const etherscanKey = process.env.ETHERSCAN_KEY
const etherscan_url = `http://api.etherscan.io/api?module=contract&action=getabi&address=${CONTRACT_ADDRESS}&apikey=${etherscanKey}`
async function getContractAbi() {
const etherscan_response = await nodeClient.getPromise(etherscan_url)
const CONTRACT_ABI = JSON.parse(etherscan_response.data.result);
return CONTRACT_ABI;
}
async function handleTransferEvent(event) {
try{
const fromAddress = event.returnValues.from
const tokenId = event.returnValues.tokenId
console.log("SOMEONE TRANSFERED NFT!")
if(fromAddress == '0x0000000000000000000000000000000000000000') {
console.log("Minted:\n", event.returnValues)
/* Do stuff */
}
}
catch(err) {
console.log(err)
console.log("ERROR WHILE HANDLING TRANSFER EVENT")
}
}
const init = async () => {
var web3 = new Web3('wss://mainnet.infura.io/ws/v3/' + projectId)
console.log("Connected to mainnet")
const CONTRACT_ABI = await getContractAbi()
console.log("Retrieved contract abi")
const contract = new web3.eth.Contract(
CONTRACT_ABI,
CONTRACT_ADDRESS
)
contract.events.Transfer({})
.on('data', handleTransferEvent)
.on('error', console.error)
console.log('Started listening minting events...')
}
init()
You can check the smart contract from https://etherscan.io/address/0x8d4b648f7fab1c72d1690b42693fb7525ce3025e#code
EDIT: I think problem might be related to calling listen event inside a function.
Tags:
node-cron, ExpressJs, NodeJs, Replit, Uptimerobot
Situation:
Hey all!
I am trying to get my discord bot to send multiple messages every day on specific times.
I deployed my bot on Replit and use Uptimerobot to ping my app every 10 min to keep the bot live.
In my code I used node-cron shedules for each spicific time it should send a message:
imports
const express = require("express");
const router = express.Router();
const { Client, Intents, Guild } = require("discord.js");
const cron = require("node-cron");
const token = process.env['BOT_TOKEN']
const { promotions } = require("./promotions");
const { testServers } = require("./test-servers");
const { buildMessage } = require("./generateMessage");
Message generator
router.get("/", function(req, res, next) {
const client = new Client({
intents: [Intents.FLAGS.GUILDS],
allowedMentions: { parse: ["users", "roles"] }
});
const composeMessage = guilds => {
let thisGuild;
let discordChannel;
let role;
let holidays;
let start;
let end;
guilds.map((guild, key) => {
guild.channels.cache.map(channel => {
testServers.forEach((promo, index) => {
thisGuild = promo.guild_id;
discordChannel = promo.channel_id;
role = promo.role_id;
holidays = promo.holidays;
start = promo.start;
end = promo.end;
// All relevant promotions
if (discordChannel === channel.id.toString()) {
const notAHoliday = [];
const currentDate = new Date();
holidays.forEach(holiday => {
if (
currentDate >= holiday.start &&
currentDate.setUTCHours(23, 59, 59) <= holiday.end
) {
notAHoliday.push(false);
}
});
if (
notAHoliday.length === 0 &&
(currentDate >= promo.start &&
currentDate.setUTCHours(23, 59, 59) <= promo.end)
) {
const unfilteredMessage = buildMessage(role);
channel.send(unfilteredMessage);
}
}
});
});
});
};
When running the Bot
client.once("ready", () => {
console.log("READY!");
const guilds = client.guilds.cache.map(guild => guild);
cron.schedule("0 55 7 * * Mon,Tue,Wed,Thu,Fri", () => {
console.log("morning");
composeMessage(guilds);
});
cron.schedule("0 31 11 * * Mon,Tue,Wed,Thu,Fri", () => {
console.log("start lunch");
composeMessage(guilds);
});
cron.schedule("0 25 12 * * Mon,Tue,Wed,Thu,Fri", () => {
console.log("end lunch");
composeMessage(guilds);
});
cron.schedule("0 0 16 * * Mon,Tue,Wed,Thu,Fri", () => {
console.log("evening");
composeMessage(guilds);
});
});
client.login(token);
botStatus = "Active";
res.render('index', { status: botStatus, version: "1.0.0" })
});
module.exports = router;
Issue:
The timers work but every time it runs a schedule, I get back a bunch of messages (the longer between schedules, the more messages my bot sends)
I suspect that it has to do with the pinging and the schedules stocking those runs until the shedule runs active and releases it all..
But how should I fix this?
Thanks in advance!
I'm doing some simple experiments with RxJS in a simple NodeJS Express server where I am comparing different approaches to handling and processing requests (based on this post https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/). This is the basic setup:
const express = require('express')
const crypto = require('crypto')
const { asyncScheduler, asapScheduler, range } = require('rxjs')
const { promisify } = require('util')
const setImmediatePromise = promisify(setImmediate)
const PID = process.pid
function log(msg) {
console.log(`[${PID}]`, new Date(), msg)
}
const app = express()
function randomString() {
return crypto.randomBytes(100).toString('hex')
}
app.get('/compute-sync', function computeSync(req, res) {
log('computing sync!')
const hash = crypto.createHash('sha256')
for (let i = 0; i < 1e6; i++) {
hash.update(randomString())
}
res.send(hash.digest('hex') + '\n')
})
app.get('/compute-immediate', function computeImmediate(req, res) {
log('computing immediate!')
const hash = crypto.createHash('sha256')
for (let i = 0; i < 1e6; i++) {
await setImmediatePromise(hash.update, randomString())
}
res.send(hash.digest('hex') + '\n')
})
app.get('/compute-rxjs', async function computeRxjs(req, res) {
log('computing Rxjs!')
const hash = crypto.createHash('sha256')
range(0, 1e6, asapScheduler).subscribe({
next() {
hash.update(randomString())
},
complete() {
res.send(hash.digest('hex') + '\n')
},
})
})
app.get('/healthcheck', function healthcheck(req, res) {
log('they check my health')
res.send('all good!\n')
})
const PORT = process.env.PORT || 1337
let server = app.listen(PORT, () => log('server listening on :' + PORT))
It was my understanding that the asapScheduler would use setImmediate under the hood, so why does the /compute-immediate endpoint NOT block the event loop (keeps the server responsive to new requests) but the /compute-rxjs does block and leads to server timeouts on the health endpoint?
I have also tried the asyncScheduler - this does not block, but it does take perhaps an order of magnitude longer to complete then the /compute-immediate endpoint.
I would really like to use RxJS for more complex processing of incoming requests, but feel this issue makes that choice undesirable. Is there something I am missing? Is there a way to get the RxJS solution to work in the same way as the setImmediate solution?
Thanks for your responses!
I now have a solution that I based on this gist: https://gist.github.com/neilk/5380684
The /compute-rxjs now looks like this:
app.get('/compute-rxjs', function computeRxjs(req, res) {
log('computing Rxjs!')
const hash = crypto.createHash('sha256')
new Observable(subscriber => {
;(iter = (i = 0, max = 1e6) => {
if (i === max) return subscriber.complete()
subscriber.next(i)
return setImmediate(iter, i + 1, max)
})()
}).subscribe({
next() {
hash.update(randomString())
},
complete() {
res.send(hash.digest('hex') + '\n')
},
})
})
It seems to behave exactly as I wanted (recursion - who knew?) - it does not block the event loop, and takes the same amount of time to run as the /compute-immediate endpoint, but gives me the flexibility to use the RxJS pipeline capabilities.
I'm following this tutorial to create Reactive TCP server in Nodejs
here's code that i've been working on
const Rx = require('rxjs')
const net = require('net')
const uuid = require('uuid');
module.exports = () => {
const sockets = new Map();
const ids = new Map();
const GetSocket = _id => sockets.get(_id);
const GetId = _socket => ids.get(_socket);
const SetSocket = _socket =>{
_socket.setEncoding('utf8');
const _id = uuid();
sockets.set(_id, _socket);
ids.set(_socket,_id);
return _id;
};
const server = net.createServer({ allowHalfOpen: true });
const socketStream = Rx.Observable.fromEvent(server, 'connection');
const RemoveSocket = socket = () => {
console.log("connection closed && removing socket from Map");
const id = ids.get(socket);
sockets.delete(id);
ids.delete(socket)
};
const socketObservable = socket => SetSocket(socket) &&
Rx.Observable
.of({
action: 'CONNECTION',
socket: GetId(socket)
})
.merge(
Rx.Observable
.fromEvent(socket,'data')
.map(d=>{
try {return JSON.parse(d);}
catch (e) {
console.log(e);
return d;
}
})
.map(msg=>{
return Object.assign({action:msg,socket:GetId(socket)})
})
)
.takeUntil(Rx.Observable.fromEvent(socket, 'close').map(d=>{
console.log("!!!! Should remove !!!");
RemoveSocket(socket);
}));
const Print = ()=>{
//ids.forEach(id=> console.log(GetSocket(id)));
console.log("total connected socket : " + ids.size);
};
const startServer = port => server.listen(port) &&
socketStream
.flatMap(socketObservable);
return {startServer, Print , stop: () => server.close()};
};
and here's my test result(just sending test msg and connect/reconnect to server )
{ action: 'CONNECTION',
socket: '8989b581-dc54-479b-a8c0-870cc8103c5b' }
total connected socket : 1
{ action: { test: 1 },
socket: '8989b581-dc54-479b-a8c0-870cc8103c5b' }
total connected socket : 1
{ action: { test: 2 },
socket: '8989b581-dc54-479b-a8c0-870cc8103c5b' }
total connected socket : 1
{ action: 'CONNECTION',
socket: 'b868104b-d1cf-41c9-950f-472f63bac27a' }
total connected socket : 2
{ action: { test: 1 },
socket: 'b868104b-d1cf-41c9-950f-472f63bac27a' }
total connected socket : 2
{ action: 'CONNECTION',
socket: 'b9a579fe-3715-4952-aaf7-d7f64a0bea99' }
total connected socket : 3
Everything working fine till detecting socket close event by TakeUntil()
I tried using takewhile() by simply adding counter like this TakeWhile(cnt < 5)
and socket stream completed as expected.
this is my first attempt to make something with Node.js and feel like i'm missing something.
can anyone help me to understand why takeUntil() is not working here?
Thank you :)
So my confusion was from understanding 'close' and 'end' events
'end' event gets triggered when the client disconnected or server calls socket.end(..) when server receives FIN packet
and 'close' event gets called after socket.destroy()
if anyone wants to see all socket events in action, I recommend watching this video
#Brannon, Thank you for pointing out the right event usage and thank you, everyone, for helping me out with this!!
also just in case, anyone wants working TCP server code.
dependency : rxjs 5.5.0
const Rx = require('rxjs');
const net = require('net');
const uuid = require('uuid');
module.exports = () => {
const sockets = new Map();
const ids = new Map();
const GetSocket = _id => sockets.get(_id);
const GetId = _socket => ids.get(_socket);
const SetSocket = _socket =>{
_socket.setEncoding('utf8');
const _id = uuid();
sockets.set(_id, _socket);
ids.set(_socket,_id);
return _id;
};
const server = net.createServer({ allowHalfOpen: true });
const socketStream = Rx.Observable.fromEvent(server, 'connection');
const RemoveSocket = socket => {
const id = ids.get(socket);
sockets.delete(id);
ids.delete(socket)
console.log("[server.js] socket closed..");
};
const socketObservable = socket => SetSocket(socket) &&
Rx.Observable
.of({
action: 'CONNECTION',
socket: GetId(socket)
})
.merge(
Rx.Observable
.fromEvent(socket,'data')
.map(d=>{
try {return JSON.parse(d);}
catch (e) {
console.log(e);
return d;
}
})
.map(msg=>{
return Object.assign({action:msg,socket:GetId(socket)})
})
)
.takeUntil(Rx.Observable.fromEvent(socket, 'end')
.map(()=>RemoveSocket(socket)));
const Print = ()=>{
//ids.forEach(id=> console.log(GetSocket(id)));
//ids.clear();
console.log("total connected socket : " + ids.size);
};
const startServer = port => server.listen(port) &&
socketStream
.flatMap(socketObservable);
console.log("[server.js] Starts Started" );
return {startServer, Print , stop: () => server.close()};
};