Related
I've built a few functions to table channel & category information, one of which runs when the bot starts to make sure everything is synced.
The problem I've run into is that bot.channels.cache contains channels that no longer exist, or states of a channel that no longer exists. For example, I only have one channel in the server called "general". Yet, there are 3 separate entries for that channel by name, and only one contains the ID (711043006781849686) of the current "general" channel:
import Discord from 'discord.js'
import config from '../config.js'
const bot = new Discord.Client({ partials: ['MESSAGE', 'CHANNEL', 'REACTION'] })
bot.login(config.botToken)
bot.on('ready', async () => {
console.log(bot.channels.cache)
}
-- returns --
Collection(46) [Map] {
...
'711043006781849686' => <ref *16> TextChannel {
type: 'text',
deleted: false,
id: '711043006781849686',
name: 'general',
rawPosition: 10,
parentID: '711043007197216880',
permissionOverwrites: Collection(3) [Map] {
'711043006253367426' => [PermissionOverwrites],
'711043006295179347' => [PermissionOverwrites],
'861109585930747934' => [PermissionOverwrites]
},
topic: 'General chat channel.',
nsfw: false,
lastMessageID: '860794574707752980',
rateLimitPerUser: 0,
lastPinTimestamp: null,
guild: Guild {
members: [GuildMemberManager],
channels: [GuildChannelManager],
roles: [RoleManager],
presences: [PresenceManager],
voiceStates: [VoiceStateManager],
deleted: false,
available: true,
id: '711043006253367426',
shardID: 0,
name: 'Omegabox',
icon: null,
splash: null,
discoverySplash: null,
region: 'us-central',
memberCount: 5,
large: false,
features: [Array],
applicationID: null,
afkTimeout: 900,
afkChannelID: '711043009944223832',
systemChannelID: '711043006781849686',
embedEnabled: undefined,
premiumTier: 0,
premiumSubscriptionCount: 0,
verificationLevel: 'NONE',
explicitContentFilter: 'DISABLED',
mfaLevel: 0,
joinedTimestamp: 1589597389528,
defaultMessageNotifications: 'ALL',
systemChannelFlags: [SystemChannelFlags],
maximumMembers: 100000,
maximumPresences: null,
approximateMemberCount: null,
approximatePresenceCount: null,
vanityURLCode: null,
vanityURLUses: null,
description: null,
banner: null,
rulesChannelID: null,
publicUpdatesChannelID: null,
preferredLocale: 'en-US',
ownerID: '598729034867933195',
emojis: [GuildEmojiManager]
},
messages: MessageManager {
cacheType: [class LimitedCollection extends Collection],
cache: [LimitedCollection [Map]],
channel: [Circular *16]
},
_typing: Map(0) {}
},
...
'827343616678559757' => <ref *33> TextChannel {
type: 'text',
deleted: false,
id: '827343616678559757',
name: 'general',
rawPosition: 0,
parentID: '827343616678559755',
permissionOverwrites: Collection(0) [Map] {},
topic: null,
lastMessageID: '830245759152291860',
rateLimitPerUser: 0,
lastPinTimestamp: null,
guild: Guild {
members: [GuildMemberManager],
channels: [GuildChannelManager],
roles: [RoleManager],
presences: [PresenceManager],
voiceStates: [VoiceStateManager],
deleted: false,
available: true,
id: '827343616678559754',
shardID: 0,
name: 'Megabox Emojis 1',
icon: null,
splash: null,
discoverySplash: null,
region: 'us-west',
memberCount: 3,
large: false,
features: [],
applicationID: null,
afkTimeout: 300,
afkChannelID: null,
systemChannelID: '827343616678559757',
embedEnabled: undefined,
premiumTier: 0,
premiumSubscriptionCount: 0,
verificationLevel: 'NONE',
explicitContentFilter: 'DISABLED',
mfaLevel: 0,
joinedTimestamp: 1617380998194,
defaultMessageNotifications: 'ALL',
systemChannelFlags: [SystemChannelFlags],
maximumMembers: 100000,
maximumPresences: null,
approximateMemberCount: null,
approximatePresenceCount: null,
vanityURLCode: null,
vanityURLUses: null,
description: null,
banner: null,
rulesChannelID: null,
publicUpdatesChannelID: null,
preferredLocale: 'en-US',
ownerID: '598729034867933195',
emojis: [GuildEmojiManager]
},
messages: MessageManager {
cacheType: [class LimitedCollection extends Collection],
cache: [LimitedCollection [Map]],
channel: [Circular *33]
},
nsfw: false,
_typing: Map(0) {}
},
...
'827344454259703842' => <ref *34> TextChannel {
type: 'text',
deleted: false,
id: '827344454259703842',
name: 'general',
rawPosition: 0,
parentID: '827344454259703840',
permissionOverwrites: Collection(0) [Map] {},
topic: null,
lastMessageID: '827580681730261032',
rateLimitPerUser: 0,
lastPinTimestamp: null,
guild: Guild {
members: [GuildMemberManager],
channels: [GuildChannelManager],
roles: [RoleManager],
presences: [PresenceManager],
voiceStates: [VoiceStateManager],
deleted: false,
available: true,
id: '827344454259703838',
shardID: 0,
name: 'Megabox Emojis 2',
icon: null,
splash: null,
discoverySplash: null,
region: 'us-west',
memberCount: 3,
large: false,
features: [],
applicationID: null,
afkTimeout: 300,
afkChannelID: null,
systemChannelID: '827344454259703842',
embedEnabled: undefined,
premiumTier: 0,
premiumSubscriptionCount: 0,
verificationLevel: 'NONE',
explicitContentFilter: 'DISABLED',
mfaLevel: 0,
joinedTimestamp: 1617381010142,
defaultMessageNotifications: 'ALL',
systemChannelFlags: [SystemChannelFlags],
maximumMembers: 100000,
maximumPresences: null,
approximateMemberCount: null,
approximatePresenceCount: null,
vanityURLCode: null,
vanityURLUses: null,
description: null,
banner: null,
rulesChannelID: null,
publicUpdatesChannelID: null,
preferredLocale: 'en-US',
ownerID: '598729034867933195',
emojis: [GuildEmojiManager]
},
messages: MessageManager {
cacheType: [class LimitedCollection extends Collection],
cache: [LimitedCollection [Map]],
channel: [Circular *34]
},
nsfw: false,
_typing: Map(0) {}
}
}
I'm aware that there is a category named "General" (ID: 711043007197216880), but the casing is different which is maintained in an entry. I've filtered that one out, along with everything else that isn't "general" from the above block.
What am I missing here? Possible to sync things up?
Always have an epiphany right after finally making a post.
Been at this for hours and never realized that in this specific call I'm not specifying what guild, or rather what server. The bot is in multiple servers, and I've already got the current server's ID in the config I'm working with.
Changed bot.channels.cache to bot.guilds.cache.get(config.guildId).channels.cache.
I have an issue with slow data fetch. I have the following query to fetch the data
const query1 = this._ctx.signals.find({
user_id: user._id,
'spell.id': null,
'metadata.0.spell_id': { $in: spellsIds }
}).hint({user_id: 1, 'spell.id': 1, 'metadata.0.spell_id': 1}).explain('allPlansExecution')
And the execution time according to explain is 35ms. Here is explain object
{
queryPlanner: {
plannerVersion: 1,
namespace: 'gringotts.Signals',
indexFilterSet: false,
parsedQuery: { ... },
winningPlan: {
stage: 'FETCH',
inputStage: {
stage: 'IXSCAN',
keyPattern: {
user_id: 1,
'spell.id': 1
},
indexName: 'user_id_1_spell.id_1',
isMultiKey: false,
multiKeyPaths: {
user_id: [],
'spell.id': []
},
isUnique: false,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds: { ... }
}
},
rejectedPlans: []
},
executionStats: {
executionSuccess: true,
nReturned: 23866,
executionTimeMillis: 35,
totalKeysExamined: 23869,
totalDocsExamined: 23866,
executionStages: {
stage: 'FETCH',
nReturned: 23866,
executionTimeMillisEstimate: 1,
works: 23869,
advanced: 23866,
needTime: 2,
needYield: 0,
saveState: 23,
restoreState: 23,
isEOF: 1,
docsExamined: 23866,
alreadyHasObj: 0,
inputStage: {
stage: 'IXSCAN',
nReturned: 23866,
executionTimeMillisEstimate: 1,
works: 23869,
advanced: 23866,
needTime: 2,
needYield: 0,
saveState: 23,
restoreState: 23,
isEOF: 1,
keyPattern: {
user_id: 1,
'spell.id': 1
},
indexName: 'user_id_1_spell.id_1',
isMultiKey: false,
multiKeyPaths: {
user_id: [],
'spell.id': []
},
isUnique: false,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds: { ... },
keysExamined: 23869,
seeks: 3,
dupsTested: 0,
dupsDropped: 0
}
},
allPlansExecution: []
},
serverInfo: {
host: 'ip-192-168-1-98.ec2.internal',
port: 27017,
version: '4.4.4',
gitVersion: '8db30a63db1a9d84bdcad0c83369623f708e0397'
},
ok: 1
}
When I try to fetch data with the following piece of code I have execution time starting from 750ms to 900ms (21x). Average document size is 544.6135124888154 bytes.
console.time('q1-time')
const q1 = await this._ctx.signals.find({
user_id: user._id,
'spell.id': {
$in: spellsIds
}
// #ts-ignore
}).hint({
user_id: 1,
'spell.id': 1
})
const f = (q) => {
const result = []
return new Promise((res, rej) => {
q.stream().on('end', function() {
console.log('done processing stream')
res(result)
})
q.stream().on('data', (d) => {
result.push(d)
})
})
}
const data = await f(q1)
console.timeEnd('q1-time') -- > q1 - time 769.511 ms
I tried different approaches: .toArray, iteration via cursor and the one with streams (I posted above) is the fastest).
Why it takes so much longer to get the data? Can it be optimized somehow?
When I run concurrent MongoDb queries using Node.js, the second query always takes ~2 seconds to return. Using explain(), executionTimeMillis always returns 0ms, which is absolutely normal as my test collection has only 2 entries. Here's my reduced testcase:
'use strict'
const { MongoClient } = require('mongodb')
const main = async () => {
const client = new MongoClient('mongodb://admin:123456#localhost:27017/', {
useNewUrlParser: true,
useUnifiedTopology: true,
})
await client.connect()
const db = client.db('test')
const numbers = db.collection('numbers')
const promises = []
console.time()
for (let i = 0; i < 3; i++) {
promises.push(numbers.find({ number: i }).explain())
}
for (const promise of promises) {
console.log(await promise)
console.timeLog()
}
console.timeEnd()
await client.close()
}
main()
Output:
{
queryPlanner: {
plannerVersion: 1,
namespace: 'test.numbers',
indexFilterSet: false,
parsedQuery: { number: [Object] },
winningPlan: { stage: 'FETCH', inputStage: [Object] },
rejectedPlans: []
},
executionStats: {
executionSuccess: true,
nReturned: 1,
executionTimeMillis: 0,
totalKeysExamined: 1,
totalDocsExamined: 1,
executionStages: {
stage: 'FETCH',
nReturned: 1,
executionTimeMillisEstimate: 0,
works: 2,
advanced: 1,
needTime: 0,
needYield: 0,
saveState: 0,
restoreState: 0,
isEOF: 1,
invalidates: 0,
docsExamined: 1,
alreadyHasObj: 0,
inputStage: [Object]
},
allPlansExecution: []
},
serverInfo: {
host: 'DESKTOP-C7CAL9N',
port: 27017,
version: '4.0.10',
gitVersion: 'c389e7f69f637f7a1ac3cc9fae843b635f20b766'
},
ok: 1
}
default: 32.252ms
{
queryPlanner: {
plannerVersion: 1,
namespace: 'test.numbers',
indexFilterSet: false,
parsedQuery: { number: [Object] },
winningPlan: { stage: 'FETCH', inputStage: [Object] },
rejectedPlans: []
},
executionStats: {
executionSuccess: true,
nReturned: 1,
executionTimeMillis: 0,
totalKeysExamined: 1,
totalDocsExamined: 1,
executionStages: {
stage: 'FETCH',
nReturned: 1,
executionTimeMillisEstimate: 0,
works: 2,
advanced: 1,
needTime: 0,
needYield: 0,
saveState: 0,
restoreState: 0,
isEOF: 1,
invalidates: 0,
docsExamined: 1,
alreadyHasObj: 0,
inputStage: [Object]
},
allPlansExecution: []
},
serverInfo: {
host: 'DESKTOP-C7CAL9N',
port: 27017,
version: '4.0.10',
gitVersion: 'c389e7f69f637f7a1ac3cc9fae843b635f20b766'
},
ok: 1
}
default: 2042.929ms
{
queryPlanner: {
plannerVersion: 1,
namespace: 'test.numbers',
indexFilterSet: false,
parsedQuery: { number: [Object] },
winningPlan: { stage: 'FETCH', inputStage: [Object] },
rejectedPlans: []
},
executionStats: {
executionSuccess: true,
nReturned: 0,
executionTimeMillis: 0,
totalKeysExamined: 0,
totalDocsExamined: 0,
executionStages: {
stage: 'FETCH',
nReturned: 0,
executionTimeMillisEstimate: 0,
works: 1,
advanced: 0,
needTime: 0,
needYield: 0,
saveState: 0,
restoreState: 0,
isEOF: 1,
invalidates: 0,
docsExamined: 0,
alreadyHasObj: 0,
inputStage: [Object]
},
allPlansExecution: []
},
serverInfo: {
host: 'DESKTOP-C7CAL9N',
port: 27017,
version: '4.0.10',
gitVersion: 'c389e7f69f637f7a1ac3cc9fae843b635f20b766'
},
ok: 1
}
default: 2062.851ms
default: 2063.513ms
If I run queries consequentially, each query takes only some milliseconds to return. Then why is the 2 seconds response time?
Edit:
In the first for loop, I made/ran "concurrent" queries promises.push(numbers.find({ number: i }).explain()). In the second for loop, I wait for promises to resolve one after another but that doesn't mean that a promise must wait till the previous one resolved to begin its job.
To avoid misunderstandings, I've made a little changes to my code, replacing the two for loops with this:
for (let i = 0; i < 3; i++) {
promises.push(
numbers
.find({ number: i })
.explain()
.then(result => {
// console.log(result)
console.log('query index:', i)
console.timeLog()
})
)
}
await Promise.all(promises)
Output:
query index: 0
default: 22.040ms
query index: 2
default: 2032.921ms
query index: 1
default: 2034.682ms
default: 2035.260ms
Edit 2:
For further clarification, I use labels to denote timers.
for (let i = 0; i < 3; i++) {
console.time(`query index: ${ i }`)
promises.push(
numbers
.find({ number: i })
.explain()
.then(result => {
// console.log(result)
console.timeEnd(`query index: ${ i }`)
})
)
}
await Promise.all(promises)
Output:
query index: 0: 12.692ms
query index: 1: 2015.143ms
query index: 2: 2015.310ms
Set MongoClient's poolSize to 1.
I have a collection with more than 1 million user, I'm trying to update
users balance on some event.
while I'm trying to update e.g. 299 row it takes up to 15739.901ms
no high load on the sever, it's just mongo running. I'm storing the database on an SSD Samsung evo 860 but MongoDB installed on an HDD.
Here's my function:
async usersUpdate(usersToUpdate){
const updates = [];
return new Promise(async (resolve, reject) => {
users.forEach(user=>{
updates.push(
{ "updateOne": {
"filter": { "userID": user.userID, 'balance':user.userBalance },
"update": { "$set": { "user.$.userBalance": user.newBalance } , "$addToSet":{'orders.$.orderID':user.OrderID} }
}
});
}
console.log('total updates' , updates.length);
if (updates.length > 0){
const DbConnection = await getConnection();
const usersTable = DbConnection.collection('usersCollection');
transactionsTable.bulkWrite(updates, {"ordered": false, writeConcern : { w : 0 } }, function(err, result) {
// do something with result
if (err) return reject(err);
return resolve(result)
});
}else{
return resolve('Nothing to update');
}
});
}
both userID and userBalance are indexed, and I set writeconcern equals to false.
I don't know what's the wrong with code and why it's super slow.
What's the problem and how could I speed up the progress a bit?
Mongodb config file:
storage:
dbPath: "/ssd/mongodb"
journal:
enabled: false
Explain result:
{ queryPlanner:
{ plannerVersion: 1,
namespace: 'usersDB.usersCollection',
indexFilterSet: false,
parsedQuery:
{ '$and':
[ { userID:
{ '$eq': 'Kfasg3ffasg' } },
{ 'user.userBalance': { '$eq': 10 } } ] },
winningPlan:
{ stage: 'FETCH',
filter: { 'user.userBalance': { '$eq': 10 } },
inputStage:
{ stage: 'IXSCAN',
keyPattern: { userID: 1 },
indexName: 'userID_1',
isMultiKey: false,
multiKeyPaths: { userID: [] },
isUnique: true,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds:
{ userID:
[ '["Kfasg3ffasg", "Kfasg3ffasg"]' ] } } },
rejectedPlans: [] },
executionStats:
{ executionSuccess: true,
nReturned: 1,
executionTimeMillis: 24,
totalKeysExamined: 1,
totalDocsExamined: 1,
executionStages:
{ stage: 'FETCH',
filter: { 'user.userBalance': { '$eq': 10 } },
nReturned: 1,
executionTimeMillisEstimate: 0,
works: 2,
advanced: 1,
needTime: 0,
needYield: 0,
saveState: 0,
restoreState: 0,
isEOF: 1,
invalidates: 0,
docsExamined: 1,
alreadyHasObj: 0,
inputStage:
{ stage: 'IXSCAN',
nReturned: 1,
executionTimeMillisEstimate: 0,
works: 2,
advanced: 1,
needTime: 0,
needYield: 0,
saveState: 0,
restoreState: 0,
isEOF: 1,
invalidates: 0,
keyPattern: { userID: 1 },
indexName: 'userID_1',
isMultiKey: false,
multiKeyPaths: { userID: [] },
isUnique: true,
isSparse: false,
isPartial: false,
indexVersion: 2,
direction: 'forward',
indexBounds:
{ userID:
[ '["Kfasg3ffasg", "Kfasg3ffasg"]' ] },
keysExamined: 1,
seeks: 1,
dupsTested: 0,
dupsDropped: 0,
seenInvalidated: 0 } },
allPlansExecution: [] },
serverInfo:
{ }
i'm working with node-postgres and try to make the first example
And create a Class called Postgres with a method today
const conn = new pg.Pool({
user: 'user',
host: 'localhost',
database: 'test',
password: 'pass',
port: 5432,
});
class Postgres {
today() {
conn.query("SELECT NOW() as today",(err,res)=>{
if (err) throw err
//console.log(JSON.stringify(res.rows[0]));
return JSON.stringify(res.rows[0]);
conn.end();
})
}
}
The result is that rows: [ anonymous { today: 2017-08-14T00:08:52.643Z } ] and try with JSON.stringify(res.rows[0]) but not working i can't get the field
Result {
command: 'SELECT',
rowCount: 1,
oid: NaN,
rows: [ anonymous { today: 2017-08-14T00:08:52.643Z } ],
fields:
[ Field {
name: 'today',
tableID: 0,
columnID: 0,
dataTypeID: 1184,
dataTypeSize: 8,
dataTypeModifier: -1,
format: 'text' } ],
_parsers: [ [Function: parseDate] ],
RowCtor: [Function: anonymous],
rowAsArray: false,
_getTypeParser: [Function: bound ] }
Any sugestion be apreciated
Try res.rows[0].today this will work!