Unexpected behavior in sockets in nodejs (double connections, - node.js

PROBLEM:
So I have a reactjs application that I created with npx create-react-app.
I also have a server running on my machine on port 8174(no significance just random). On this server I have some socket.io action going on.
io.on('connection', (socket) => {
console.log('client connected')
}
Something weird which isn't really the problem, but may be related is that when I connect to the server it always runs twice. It will give me the "client connected" output twice in the server console.
The real problem is that I cannot figure out how to get this data into an array on the actual react application.
Here is the code in question:
import React, {useState, useEffect} from 'react'
import socketIOClient from 'socket.io-client'
import TweetCard from './TweetCard'
export default function TweetList() {
const [tweetItems, setTweetItems] = useState([])
const [socket] = useState(() => socketIOClient('http://localhost:8174/', {reconnection: true, forceNew: false}))
var items = []
const sleep = (milliseconds) => {
return new Promise(resolve => setTimeout(resolve, milliseconds))
}
const streamTweets = async () => {
socket.on('tweets', async data => {
await sleep(1000)
items = [...tweetItems, data]
items = items.reverse()
// items = items.filter(d => {return d.user != data.user})
var count = 15
items = items.filter(function(d) {
if(count != 0){
count--
return true
}
return false
})
console.log(items)
setTweetItems(items)
})
}
useEffect(() => {
socket.once('connect', () => {
console.log("Socket has successfully connected to server")
})
streamTweets()
return () => {
socket.on('disconnect', () => {
socket.off("tweets");
socket.removeAllListeners("tweets")
console.log("Socket Disconnected")
})
}
}, [])
return (
<div style={{ height: "300rem", scrollY: "auto", overflowY: "auto"}} >
{tweetItems.map(tweet => {
return <TweetCard tweet={tweet}/>
})}
</div>
)
}
THINGS I'VE TRIED:
On the server level I did a io.once instead of io.on. This fixed the issue of the client connecting twice or a crazy number of times like 5 in one second. It predictively only connected once. The problem with this however is if I refresh the page on the app it disconnects forever and I couldn't figure out how to make it reconnect even though it was saying it was reconnecting. I will post the server code on how it is connecting below, but basically it would output ('resuming stream') when it was infact not resuming the stream.
At first I just messed with the state directly (tweetItems). I basically make an array called newArr = [...tweetItems, data]. Then I set the state below it like setTweetItems(newArr). This however just updated index 0 every single time it set the state. So it would always be an array of one single item. It would update the next tweet though.
Tried using the bearer token instead of using require('twitter') package and tried implementing it in the 'twitter docs way of streaming' I can insert that link if needed. The code is outdated and incorrect though as I came to find out. I literally just forked the entire project at one point and put in my token. It did not work. They also use body parser still so its a strong sign that it is outdated
Tried making a regular array without the state called items =[] this worked for a little bit , and I am not sure what I changed, but it eventually started to copy like 2 or 3 of the same item. like index 0 - 2 would all be the same twee 3 - 4 would all be the same tweet and so on.
Also when I performed a "reverse()" on this items array it would give me a fatal error telling me that items = [...items, data] can't be set. This was odd because the code to reverse items was below this but the error was saying that items can't be set I am assuming it was doing something then after it "reversed" it went to set the items again and was null. I used items = items.reverse() this is what caused the error
Tried making the streaming of the tweets async in the react(did nothing)
Tried slowing down each setState by 1 second(did nothing)
I have tried many more things, but hopefully this will give you an idea of the issue that I am having. Any help or tutorials on sockets would be awesome.
Here is the code that connects to the twitter API:
const Twitter = require('twitter')
module.exports = (app, io) => {
var client = new Twitter({
consumer_key: process.env.CONSUMER_KEY,
consumer_secret: process.env.CONSUMER_SECRET,
access_token_key: process.env.ACCESS_TOKEN_KEY,
access_token_secret: process.env.ACCESS_TOKEN_SECRET
});
let timeout = 0
let socketConnection;
let twitterStream;
app.locals.searchTerm = 'giveaway'
app.locals.showRetweets = false;
//twitter stream
const streamtweets = () => {
console.log("Resuming stream for: " + app.locals.searchTerm)
client.stream('statuses/filter', {track: app.locals.searchTerm, tweet_mode: 'extended', language: 'en'}, (streamData) => {
streamData.on('data', (data) => {
sendMessage(data)
});
streamData.on('error', (error) => {
console.log(error)
})
twitterStream = streamData
})
}
const sleep = async (delay) => {
return new Promise((resolve) => setTimeout(() => resolve(true), delay));
};
//twitter stream
io.on('connection', socket => {
console.log(socket.id)
socketConnection = socket;
streamtweets();
socket.on("connection", () => console.log("Client has connected to the server"))
socket.on("disconnect", () => {
console.log("Client has disconnected to the server")
// twitterStream.destroy()
// socket.off()
// reconnect(twitterStream, socket)
})
})
const reconnect = async (stream, socket) => {
timeout++;
stream.destroy()
await sleep(2 ** timeout * 1000);
// streamTweets(socket, token);
streamtweets()
};
/**
* Sets search term for twitter stream.
*/
app.post('/setSearchTerm', (req, res) => {
let term = req.body.term;
app.locals.searchTerm = term;
twitterStream.destroy();
streamtweets();
});
const sendMessage = (data) => {
if(data.text.includes('RT')){
return;
}
socketConnection.emit("tweets", data)
}
}

Related

how to allow multiple async routes in express.js

I'm fairly new to Node and Express and am struggling with creating a route that takes a user uploaded file and processes it into another file. The problem is that the second time the user sends a request, I am having to wait for the first process to complete to allow the user to upload a new file.
The basic structure of route that I have is below
THE PROBLEM is that the function convertFile below is a time taking process and it keeps the server busy from accepting new requests. How do I make it so that once the project is saved in mongo db at newProject.save() - the process to convertFile runs in the background while the server accepts new requests from the same route?
I'm sending a response back the user after newProject.save() and was hoping that would allow the user to send another request. And although this sends another request, the server doesn't accept it since its busy with the previous process to convertFile
router.post('/', upload.fields([{ name: 'zip' }]), async (req, res, next) => {
let data = {
title: req.body.title,
description: req.body.description,
}
if (req.files && req.files.zip) {
const newProject = new MongoProject(data);
newProject.save()
.then(async (project) => {
res.send(project);
console.log("Project created");
const uploadedFilePath = path.normalize(req.files.zip[0].path);
// below method - "convertFile" is a time taking method
const extractZipinfo = await convertFile(uploadedFilePath , data.masterFile).then((zipInfo) => {
console.log({ zipInfo })
data.zipInfo = {
sizes: zipInfo.sizes
}
})
})
.catch(err => {
console.log("Error creating project");
return res.send(err);
})
}
})
Below is the simplified version of code in convertFile function (code modified for brevity):
I know that this can be improvised a lot, but i'm struggling with getting it to function as expected first (allowing multiple routes)
async function convertFile(inputFilePath, outputInfo) {
const outputFilePath = "output.abc";
const jsonFilePath = "output.json";
const doc = new Document(); // this is a class to store all data of the output file that we will write at the end
const _FileAPI = new fileAPI();
const outputFinalData = await _FileAPI.Init() // this is an async method
.then(() => {
const dataClass = initiateClass(); // this is a class to store data in JSON format
const paragraphs = _FileAPI.GetallParagraphs(inputFilePath);
for (let i = 0, len = paragraphs.size(); i < len; i++) {
for (let j = 0, lenj = paragraphs.size(); j < lenj; j++) {
const para = paragraphs.get(j);
// read each para and Capitalize each word
dataClass.paragraphs.push(para);
}
}
fs.writeFileSync(jsonFilePath, JSON.stringify(dataClass, null, 2), 'utf-8');
console.log("then")
}).then(() => {
const io = new NodeIO(); // this class helps in writing the file in the desired output format
const outData = io.write(outputFilePath, doc).then(() => {
outputInfo.sizes.push(fs.statSync(outputFilePath).size);
return outputInfo;
});
return outData;
});
return outputFinalData;
}

Error to emit socket: internal / bootstrap / pre_execution.js: 308

internal/bootstrap/pre_execution.js:308
get() {
^
RangeError: Maximum call stack size exceeded
at get (internal/bootstrap/pre_execution.js:308:8)
at hasBinary (/home/spirit/Documents/ProjectsJS/ProjectBack/node_modules/has-binary2/index.js:44:3)
Hello I am trying to issue my socket but I have this error, and I can not receive my socket in the back end below all the code I used
io.on('connection', function (socket) {
setInterval(() => Queue.searching() , 1000);
sessionMap.set(socket.id,socket);
//ADD PLAYER TO QUEUE
socket.on('addPlayer-Queue', (result) => {
const player = {
id:result.id,
socketid: socket.id,
name: result.name,
mmr: result.mmr,
socket: socket
}
const nPlayer = new Player(player);
Queue.addPlayer(nPlayer);
/*
console.log(queue);
console.log(sessionMap.all());*/
//socket.emit('match', matches)
});
});
//
searching = () => {
const firstPlayer = this.getRandomPlayer();
const secondPlayer = this.players.find(
playerTwo =>
playerTwo.mmr < this.calculateLessThanPercentage(firstPlayer) &&
playerTwo.mmr > this.calculateGreaterThanPercentage(firstPlayer) &&
playerTwo.id != firstPlayer.id
);
if(secondPlayer){
const matchedPlayers = [firstPlayer, secondPlayer];
this.removePlayers(matchedPlayers);
Matches.configurePlayersForNewMatch(matchedPlayers);
}
}
//
getMatchConfigurationFor = players => {
console.log(sessionMap.all())
if(players){
const match = new Match(players);
const result = {
idMatch: match.id,
playerOne: match.players[0],
playerTwo:match.players[1]
}
return result;
}
}
configurePlayersForNewMatch = (matchedPlayers) => {
matchedPlayers.forEach(player =>
sessionMap.get(player.socketId)
.broadcast.to(player.socketId)
.emit('match',
this.getMatchConfigurationFor(matchedPlayers)));
}
so what I admit the problem is in this code snippet:
matchedPlayers.forEach(player =>
sessionMap.get(player.socketId)
.broadcast.to(player.socketId)
.emit('match',
this.getMatchConfigurationFor(matchedPlayers)));
The session map is a mapping I made of my sockets saving key (socket id) and socket to send by emit later. on console log about my session.map this is console . log on my socket player https://pastebin.com/XHDXH9ih

Deleting all messages in discord.js text channel

Ok, so I searched for a while, but I couldn't find any information on how to delete all messages in a discord channel. And by all messages I mean every single message ever written in that channel. Any clues?
Try this
async () => {
let fetched;
do {
fetched = await channel.fetchMessages({limit: 100});
message.channel.bulkDelete(fetched);
}
while(fetched.size >= 2);
}
Discord does not allow bots to delete more than 100 messages, so you can't delete every message in a channel. You can delete less then 100 messages, using BulkDelete.
Example:
const Discord = require("discord.js");
const client = new Discord.Client();
const prefix = "!";
client.on("ready" () => {
console.log("Successfully logged into client.");
});
client.on("message", msg => {
if (msg.content.toLowerCase().startsWith(prefix + "clearchat")) {
async function clear() {
msg.delete();
const fetched = await msg.channel.fetchMessages({limit: 99});
msg.channel.bulkDelete(fetched);
}
clear();
}
});
client.login("BOT_TOKEN");
Note, it has to be in a async function for the await to work.
Here's my improved version that is quicker and lets you know when its done in the console but you'll have to run it for each username that you used in a channel (if you changed your username at some point):
// Turn on Developer Mode under User Settings > Appearance > Developer Mode (at the bottom)
// Then open the channel you wish to delete all of the messages (could be a DM) and click the three dots on the far right.
// Click "Copy ID" and paste that instead of LAST_MESSAGE_ID.
// Copy / paste the below script into the JavaScript console.
var before = 'LAST_MESSAGE_ID';
var your_username = ''; //your username
var your_discriminator = ''; //that 4 digit code e.g. username#1234
var foundMessages = false;
clearMessages = function(){
const authToken = document.body.appendChild(document.createElement`iframe`).contentWindow.localStorage.token.replace(/"/g, "");
const channel = window.location.href.split('/').pop();
const baseURL = `https://discordapp.com/api/channels/${channel}/messages`;
const headers = {"Authorization": authToken };
let clock = 0;
let interval = 500;
function delay(duration) {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(), duration);
});
}
fetch(baseURL + '?before=' + before + '&limit=100', {headers})
.then(resp => resp.json())
.then(messages => {
return Promise.all(messages.map((message) => {
before = message.id;
foundMessages = true;
if (
message.author.username == your_username
&& message.author.discriminator == your_discriminator
) {
return delay(clock += interval).then(() => fetch(`${baseURL}/${message.id}`, {headers, method: 'DELETE'}));
}
}));
}).then(() => {
if (foundMessages) {
foundMessages = false;
clearMessages();
} else {
console.log('DONE CHECKING CHANNEL!!!')
}
});
}
clearMessages();
The previous script I found for deleting your own messages without a bot...
// Turn on Developer Mode under User Settings > Appearance > Developer Mode (at the bottom)
// Then open the channel you wish to delete all of the messages (could be a DM) and click the three dots on the far right.
// Click "Copy ID" and paste that instead of LAST_MESSAGE_ID.
// Copy / paste the below script into the JavaScript console.
// If you're in a DM you will receive a 403 error for every message the other user sent (you don't have permission to delete their messages).
var before = 'LAST_MESSAGE_ID';
clearMessages = function(){
const authToken = document.body.appendChild(document.createElement`iframe`).contentWindow.localStorage.token.replace(/"/g, "");
const channel = window.location.href.split('/').pop();
const baseURL = `https://discordapp.com/api/channels/${channel}/messages`;
const headers = {"Authorization": authToken };
let clock = 0;
let interval = 500;
function delay(duration) {
return new Promise((resolve, reject) => {
setTimeout(() => resolve(), duration);
});
}
fetch(baseURL + '?before=' + before + '&limit=100', {headers})
.then(resp => resp.json())
.then(messages => {
return Promise.all(messages.map((message) => {
before = message.id;
return delay(clock += interval).then(() => fetch(`${baseURL}/${message.id}`, {headers, method: 'DELETE'}));
}));
}).then(() => clearMessages());
}
clearMessages();
Reference: https://gist.github.com/IMcPwn/0c838a6248772c6fea1339ddad503cce
This will work on discord.js version 12.2.0
Just put this inside your client on message event
and type the command: !nuke-this-channel
Every message on channel will get wiped
then, a kim jong un meme will be posted.
if (msg.content.toLowerCase() == '!nuke-this-channel') {
async function wipe() {
var msg_size = 100;
while (msg_size == 100) {
await msg.channel.bulkDelete(100)
.then(messages => msg_size = messages.size)
.catch(console.error);
}
msg.channel.send(`<#${msg.author.id}>\n> ${msg.content}`, { files: ['http://www.quickmeme.com/img/cf/cfe8938e72eb94d41bbbe99acad77a50cb08a95e164c2b7163d50877e0f86441.jpg'] })
}
wipe()
}
This will work so long your bot has appropriate permissions.
module.exports = {
name: "clear",
description: "Clear messages from the channel.",
args: true,
usage: "<number greater than 0, less than 100>",
execute(message, args) {
const amount = parseInt(args[0]) + 1;
if (isNaN(amount)) {
return message.reply("that doesn't seem to be a valid number.");
} else if (amount <= 1 || amount > 100) {
return message.reply("you need to input a number between 1 and 99.");
}
message.channel.bulkDelete(amount, true).catch((err) => {
console.error(err);
message.channel.send(
"there was an error trying to prune messages in this channel!"
);
});
},
};
In case you didn't read the DiscordJS docs, you should have an index.js file that looks a little something like this:
const Discord = require("discord.js");
const { prefix, token } = require("./config.json");
const client = new Discord.Client();
client.commands = new Discord.Collection();
const commandFiles = fs
.readdirSync("./commands")
.filter((file) => file.endsWith(".js"));
for (const file of commandFiles) {
const command = require(`./commands/${file}`);
client.commands.set(command.name, command);
}
//client portion:
client.once("ready", () => {
console.log("Ready!");
});
client.on("message", (message) => {
if (!message.content.startsWith(prefix) || message.author.bot) return;
const args = message.content.slice(prefix.length).split(/ +/);
const commandName = args.shift().toLowerCase();
if (!client.commands.has(commandName)) return;
const command = client.commands.get(commandName);
if (command.args && !args.length) {
let reply = `You didn't provide any arguments, ${message.author}!`;
if (command.usage) {
reply += `\nThe proper usage would be: \`${prefix}${command.name} ${command.usage}\``;
}
return message.channel.send(reply);
}
try {
command.execute(message, args);
} catch (error) {
console.error(error);
message.reply("there was an error trying to execute that command!");
}
});
client.login(token);
Another approach could be cloning the channel and deleting the one with the messages you want deleted:
// Clears all messages from a channel by cloning channel and deleting old channel
async function clearAllMessagesByCloning(channel) {
// Clone channel
const newChannel = await channel.clone()
console.log(newChannel.id) // Do with this new channel ID what you want
// Delete old channel
channel.delete()
}
I prefer this method rather than the ones listed on this thread because it most likely takes less time to process and (I'm guessing) puts the Discord API under less stress. Also, channel.bulkDelete() is only able to delete messages that are newer than two weeks, which means you won't be able to delete every channel message in case your channel has messages that are older than two weeks.
The possible downside is the channel changing id. In case you rely on storing ids in a database or such, don't forget to update those documents with the id of the newly cloned channel!
Here's #Kiyokodyele answer but with some changes from #user8690818 answer.
(async () => {
let deleted;
do {
deleted = await channel.bulkDelete(100);
} while (deleted.size != 0);
})();

Why is Cloud Functions for Firebase taking 25 seconds?

For clarity I have other cloud functions that all run intermittently (i.e from 'cold' in around 2-6 seconds, and all use the same boilerplate set up of importing an admin instance and exporting the function as a module)
I've seen other similar posts but this is really bugging me. I have a cloud function like so:
const admin = require('../AdminConfig');
const { reportError } = require('../ReportError');
module.exports = (event) => {
const uid = event.params.uid;
const snapshot = event.data;
if (snapshot._newData === null ) {
return null;
}
console.log('Create org begin running: ', Date.now());
const organisation = event.data.val();
const rootRef = admin.database().ref();
const ref = rootRef.child('/organisations').push();
const oid = ref.key;
const userData = {
level: 'owner',
name: organisation.name,
};
const orgShiftInfo = {
name: organisation.name,
startDay: organisation.startDay || 'Monday',
};
const updatedData = {};
updatedData[`/users/${uid}/currentOrg`] = oid;
updatedData[`/users/${uid}/organisations/${oid}`] = userData;
updatedData[`/organisations/${oid}`] = organisation;
updatedData[`/org_shift_info/${oid}`] = orgShiftInfo;
rootRef.update(updatedData, (err) => {
if (err) {
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ error: true })
.then(() => {
console.log(`error adding organisation for ${uid}: `, err);
return reportError(err, { uid });
});
}
console.log('Create org wrote succesfully: ', Date.now());
return rootRef.child(`/users/${uid}/addOrgStatus`).set({ success: true });
});
}
I understand the 'cold start' thing but I think something is seriously wrong that it's taking 25 seconds. The logs don't return any error and are as so:
Is there some deeper way I can debug this to try and figure out why it's taking so long? It's unusable at the moment. Thanks a lot.
Solved:
Sorry,
I misunderstood the API a bit. I should have watched the promise video first!
I needed to put
return rootRef.update...
instead of
rootRef.update...

Use restful api to invoke nightmare scraping multi site with promise wrapper

I would like to scrap multi-site with one restful api, I use express to implement it.
But I only triggered nightmare successfully in first time with my api,
when I call again my api I can't trigger nightmare any more :(
Have any idea?
another question, in below case, I need to instantiate new Nightmare object individually , so that I can scrap three different site, have any smarter way to achieve that?
bellow getScrap is my apiControler function with express Router GET callback,
you also could check in gist:
https://gist.github.com/sevenLee/7091f8c56ccad3c0551b512f725af7da
import Nightmare from 'nightmare';
import cheerio from 'cheerio';
let nightmare = Nightmare({show: false});
let nightmare2 = Nightmare({show: false});
let nightmare3 = Nightmare({show: false});
const urlObject = {
site1: 'http://www.site1.com',
site2: 'http://www.site2.com',
site3: 'http://www.site3.com'
};
export function getScrap(req, res){
let result = {};
result.site1 = {
topList: []
};
result.site2 = {
topList: []
};
result.site3 = {
topList: []
};
const pro1 = Promise.resolve(
nightmare
.goto(urlObject.site1)
.wait(200)
.evaluate(() => {
console.log('site1 into evaluate');
return document.querySelector('.ninenine').innerHTML;
})
.end()
)
.then((html) => {
let $ = cheerio.load(html);
let tt = $('.horizontal-li');
let sections = $(".section-board-title");
sections.each((index, elm) => {
if($(elm).text() === 'TopList'){
$(elm).next('ul').find('li').each((index, elm_li) => {
let title =$(elm_li).find('.cabinet-instruction').text();
let price =$(elm_li).find('.cabinet-middle .price').text();
let imgSrc = $(elm_li).find('.cabinet-img').attr('data-temp-src');
if(title !== '' && price !==''){
result.site1.topList.push({
title,
price,
imgSrc
});
}
});
}
});
})
.catch((err) => {
console.log('site1 scrap err:', err);
return res.status(400).send({reason:'site1 scrap err'});
});
const pro2 = Promise.resolve(
nightmare2
.goto(urlObject.site2)
.wait(200)
.evaluate(() => {
return document.querySelector('.ninenine').innerHTML;
})
.end()
)
.then((html) => {
let $ = cheerio.load(html);
let tt = $('.horizontal-li');
let sections = $(".section-board-title");
sections.each((index, elm) => {
if($(elm).text() === 'TopList'){
$(elm).next('ul').find('li').each((index, elm_li) => {
let title =$(elm_li).find('.cabinet-instruction').text();
let price =$(elm_li).find('.cabinet-middle .price').text();
let imgSrc = $(elm_li).find('.cabinet-img').attr('data-temp-src');
if(title !== '' && price !==''){
result.site2.topList.push({
title,
price,
imgSrc
});
}
});
}
});
})
.catch((err) => {
console.log('site2 scrap err:', err);
return res.status(400).send({reason:'site2 scrap err'});
});
const pro3 = Promise.resolve(
nightmare3
.goto(urlObject.site3)
.wait(200)
.evaluate(() => {
return document.querySelector('#layout').innerHTML;
})
.end()
)
.then((html) => {
let $ = cheerio.load(html);
let sections = $(".pditem");
sections.each((index, elm) => {
let title = $(elm).find('.name').text();
let price = $(elm).find('.price').find('span').eq(1).text();
let imgSrc = ['www.site3.com',$(elm).find('li').eq(1).find('img').attr('src')].join('');
result.site3.topList.push({
title,
price,
imgSrc
});
});
})
.catch((err) => {
console.log('site3 scrap err:', err);
return res.status(400).send({reason:'site3 scrap err'});
});
Promise.all([pro1, pro2, pro3])
.then(values => {
res.json(result);
})
.catch((err) => {
return res.status(500).send({reason:err.toString()});
});
}
(From my original answer at segmentio/nightmare#715.)
But I only triggered nightmare successfully in first time with my api,
when I call again my api I can't trigger nightmare any more
It looks like you're defining your instances outside of getScrap(), then calling .end() inside of getScrap(), which will end and destroy the Nightmare/Electron instances. Once they are ended, they can no longer be used. Try moving the creation of your Nightmare instances inside of the getScrap() method.
another question, in below case, I need to instantiate new Nightmare object individually , so that I can scrap three different site, have any smarter way to achieve that?
Depends on what your use case is. You could use a single Nightmare instance and iterate over the URLs, but that will take more time as Nightmare execution must be sequential. If you're curious on how to do such a thing, this article from nightmare-examples might be worth reading.
Finally, it's probably worth pointing out that based on your above code, you don't have to use cheerio. You could use .evaluate() and CSS queries to accomplish what you want, I think.

Resources