Retrieving an autocomplete value with cheerio - node.js

there's this website that conjugates korean verbs https://www.verbix.com/languages/korean. What i wanna do is type something in the input and then get the text of the suggestion that appears.
That blue part is the 'a' element's text
This is my code:
app.get("/conjugation", (req, res) => {
axios("https://www.verbix.com/languages/korean")
.then((response) => {
const htmlData = response.data;
const $ = cheerio.load(htmlData);
//Input Element
const input = $("#auto-complete-edit");
//Pass the value to it
input.attr("value", "먹어요");
//Get the selected element
const select = $(".selected");
select
.map((i, child) => {
//Get the 'a' element inner text
return $(child).find("a").text();
})
.get();
console.log(select);
})
.catch((err) => console.log(err));
});
And the console shows this
LoadedCheerio {
length: 0,
options: { xml: false, decodeEntities: true },
_root: <ref *1> LoadedCheerio {
'0': Document {
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array],
type: 'root',
'x-mode': 'no-quirks'
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *1]
},
prevObject: <ref *2> LoadedCheerio {
'0': Document {
parent: null,
prev: null,
next: null,
startIndex: null,
endIndex: null,
children: [Array],
type: 'root',
'x-mode': 'no-quirks'
},
length: 1,
options: { xml: false, decodeEntities: true },
_root: [Circular *2]
}
}

Related

Retrieve a document query with Email firebase firestore

Code:
At line number 57 I created a route to get the data according to user's email.
But I'm unable to get the specific data that this user added to the database. The response I'm getting is looking like this. I'm stuck here although I followed the instruction that is provided in the firebase firestore documentation.
QuerySnapshot {
_firestore: Firestore {
_authCredentials: FirebaseAuthCredentialsProvider {
authProvider: [Provider],
currentUser: [User],
tokenCounter: 0,
forceRefresh: false,
auth: null,
tokenListener: [Function (anonymous)]
},
_appCheckCredentials: FirebaseAppCheckTokenProvider {
appCheckProvider: [Provider],
forceRefresh: false,
appCheck: null,
tokenListener: [Function (anonymous)]
},
type: 'firestore',
_persistenceKey: '[DEFAULT]',
_settings: FirestoreSettingsImpl {
host: 'firestore.googleapis.com',
ssl: true,
credentials: undefined,
ignoreUndefinedProperties: false,
cacheSizeBytes: 41943040,
experimentalForceLongPolling: false,
experimentalAutoDetectLongPolling: false,
useFetchStreams: true
},
_settingsFrozen: true,
_app: FirebaseAppImpl {
_isDeleted: false,
_options: [Object],
_config: [Object],
_name: '[DEFAULT]',
_automaticDataCollectionEnabled: false,
_container: [ComponentContainer]
},
_databaseId: DatabaseId { projectId: 'parlour-1bcf7', database: '(default)' },
_queue: AsyncQueueImpl {
tail: [Promise],
retryableOps: [],
_isShuttingDown: false,
delayedOperations: [Array],
failure: null,
operationInProgress: true,
skipNonRestrictedTasks: false,
timerIdsToSkip: [],
backoff: [ExponentialBackoff],
visibilityHandler: [Function (anonymous)]
},
_firestoreClient: FirestoreClient {
authCredentials: [FirebaseAuthCredentialsProvider],
appCheckCredentials: [FirebaseAppCheckTokenProvider],
asyncQueue: [AsyncQueueImpl],
databaseInfo: [DatabaseInfo],
user: [User],
clientId: 'Hoh8WrPk3BFhkBsirvQx',
authCredentialListener: [Function (anonymous)],
appCheckCredentialListener: [Function (anonymous)],
offlineComponents: [MemoryOfflineComponentProvider],
onlineComponents: [OnlineComponentProvider]
}
},
_userDataWriter: ExpUserDataWriter {
firestore: Firestore {
_authCredentials: [FirebaseAuthCredentialsProvider],
_appCheckCredentials: [FirebaseAppCheckTokenProvider],
type: 'firestore',
_persistenceKey: '[DEFAULT]',
_settings: [FirestoreSettingsImpl],
_settingsFrozen: true,
_app: [FirebaseAppImpl],
_databaseId: [DatabaseId],
_queue: [AsyncQueueImpl],
_firestoreClient: [FirestoreClient]
}
},
_snapshot: ViewSnapshot {
query: QueryImpl {
path: [ResourcePath],
collectionGroup: null,
explicitOrderBy: [],
filters: [Array],
limit: null,
limitType: 'F',
startAt: null,
endAt: null,
memoizedOrderBy: [Array],
memoizedTarget: [TargetImpl]
},
docs: DocumentSet {
comparator: [Function (anonymous)],
keyedMap: [SortedMap],
sortedSet: [SortedMap]
},
oldDocs: DocumentSet {
comparator: [Function (anonymous)],
keyedMap: [SortedMap],
sortedSet: [SortedMap]
},
docChanges: [],
mutatedKeys: SortedSet { comparator: [Function: comparator], data: [SortedMap] },
fromCache: false,
syncStateChanged: true,
excludesMetadataChanges: false
},
metadata: SnapshotMetadata { hasPendingWrites: false, fromCache: false },
query: Query {
converter: null,
_query: QueryImpl {
path: [ResourcePath],
collectionGroup: null,
explicitOrderBy: [],
filters: [Array],
limit: null,
limitType: 'F',
startAt: null,
endAt: null,
memoizedOrderBy: [Array],
memoizedTarget: [TargetImpl]
},
type: 'query',
firestore: Firestore {
_authCredentials: [FirebaseAuthCredentialsProvider],
_appCheckCredentials: [FirebaseAppCheckTokenProvider],
type: 'firestore',
_persistenceKey: '[DEFAULT]',
_settings: [FirestoreSettingsImpl],
_settingsFrozen: true,
_app: [FirebaseAppImpl],
_databaseId: [DatabaseId],
_queue: [AsyncQueueImpl],
_firestoreClient: [FirestoreClient]
}
}
}
Replace lines 64 to 67 with the following snippet
for (const doc of querySnapshot.docs) {
console.log(doc.id, '=>', doc.data())
}

.patch() do not saves the parameter to the database in Objection.js with Knex

I am creating Express API and I am using Objection.js as ORM with Knex.js
I have created router for updating user password from the profile with 2 fields (old password and the new password),first it verifies the old password (protection from stealing JWT token). After it returns valid condition then I proceed to hash the new password with bcrypt and update it with .patch() otherwise it will return validation error the old password it is not the correct password. The problem is when I send the same exact request it goes through meaning that .patch not worked and did not save the new password to the database. Can anyone explain some solution to this problem or probably hit me with some documention on how to fix it
The code is bellow:
router.patch('/updatepassword', async (req, res, next) => {
const { id } = req.user;
const {
oldPassword,
newPassword,
} = req.body;
try {
await passwordSchema.validate({
oldPassword,
newPassword,
}, {
abortEarly: false
});
const UserOldPassword = await User.query().select('password').findById(id);
const validOldPassword = await bcrypt.compare(oldPassword, UserOldPassword.password);
if (validOldPassword) {
const hashedPassword = await bcrypt.hash(newPassword, 12);
const defi = User.query().patch({ password: hashedPassword }).where('id', id).returning('*')
.first();
console.log(defi);
res.status(200).json({
message: returnMessage.passwordUpdated
});
} else {
const error = new Error(returnMessage.invalidOldPassword);
res.status(403);
throw error;
}
} catch (error) {
next(error);
}
});
Console log:
QueryBuilder {
_modelClass: [Function: User],
_operations: [
UpdateOperation {
name: 'patch',
opt: [Object],
adderHookName: null,
parentOperation: null,
childOperations: [],
model: [User],
modelOptions: [Object]
},
KnexOperation {
name: 'where',
opt: {},
adderHookName: null,
parentOperation: null,
childOperations: [],
args: [Array]
},
ReturningOperation {
name: 'returning',
opt: {},
adderHookName: null,
parentOperation: null,
childOperations: [],
args: [Array]
},
FirstOperation {
name: 'first',
opt: {},
adderHookName: null,
parentOperation: null,
childOperations: []
}
],
_context: QueryBuilderContext {
userContext: QueryBuilderUserContext { [Symbol()]: [Circular] },
options: InternalOptions {
skipUndefined: false,
keepImplicitJoinProps: false,
isInternalQuery: false,
debug: false
},
knex: null,
aliasMap: null,
tableMap: null,
runBefore: [],
runAfter: [],
onBuild: []
},
_parentQuery: null,
_isPartialQuery: false,
_activeOperations: [],
_resultModelClass: null,
_explicitRejectValue: null,
_explicitResolveValue: null,
_modifiers: {},
_allowedGraphExpression: null,
_findOperationOptions: {},
_relatedQueryFor: null,
_findOperationFactory: [Function: findOperationFactory],
_insertOperationFactory: [Function: insertOperationFactory],
_updateOperationFactory: [Function: updateOperationFactory],
_patchOperationFactory: [Function: patchOperationFactory],
_relateOperationFactory: [Function: relateOperationFactory],
_unrelateOperationFactory: [Function: unrelateOperationFactory],
_deleteOperationFactory: [Function: deleteOperationFactory]
}
PATCH /v1/profile/updatepassword 200 1346.797 ms - 42
Solution: Do not forget to put await on async function.

How to read password protected PDF file in Nodejs and get it in buffer?

I tried using pdfjs-dist.
getting large json response.
var PDFJS=require('pdfjs-dist');
PDFJS.getDocument({ url: 'p1.pdf', password: '' }).then(function(pdf_doc)
{
console.log(pdf_doc);
}).catch(function(error) {
// incorrect password
// error is an object having 3 properties : name, message & code
});
Response
This is the whole response I am getting.
but I need response in buffer.
Can it be converted to buffer.
PDFDocumentProxy {
loadingTask:
{ _capability:
{ resolve: [Function], reject: [Function], promise: [Promise] },
_transport:
WorkerTransport {
messageHandler: [Object],
loadingTask: [Circular],
commonObjs: [Object],
fontLoader: [GenericFontLoader],
_params: [Object],
CMapReaderFactory: [DOMCMapReaderFactory],
destroyed: false,
destroyCapability: null,
_passwordCapability: null,
_networkStream: [PDFNodeStream],
_fullReader: [PDFNodeStreamFsFullReader],
_lastProgress: [Object],
pageCache: [],
pagePromises: [],
downloadInfoCapability: [Object],
numPages: 4,
pdfDocument: [Circular] },
_worker:
{ name: null,
destroyed: false,
postMessageTransfers: true,
verbosity: 1,
_readyCapability: [Object],
_port: [LoopbackPort],
_webWorker: null,
_messageHandler: [Object] },
docId: 'd0',
destroyed: false,
onPassword: null,
onProgress: null,
onUnsupportedFeature: null },
_pdfInfo:
{ numPages: 4,
fingerprint: '3432353738363537336c6e665361446f6f744f4a70' },
_transport:
WorkerTransport {
messageHandler:
{ sourceName: 'd0',
targetName: 'd0_worker',
comObj: [LoopbackPort],
callbackId: 1,
streamId: 1,
postMessageTransfers: true,
streamSinks: [Object],
streamControllers: [Object: null prototype] {},
callbacksCapabilities: [Object: null prototype] {},
actionHandler: [Object],
_onComObjOnMessage: [Function] },
loadingTask:
{ _capability: [Object],
_transport: [Circular],
_worker: [Object],
docId: 'd0',
destroyed: false,
onPassword: null,
onProgress: null,
onUnsupportedFeature: null },
commonObjs: { objs: [Object: null prototype] {} },
fontLoader:
GenericFontLoader {
docId: 'd0',
nativeFontFaces: [],
styleElement: null,
loadingContext: [Object],
loadTestFontId: 0 },
_params:
[Object: null prototype] {
url: 'p1.pdf',
password: '',
rangeChunkSize: 65536,
CMapReaderFactory: [Function: DOMCMapReaderFactory],
ignoreErrors: true,
pdfBug: false,
nativeImageDecoderSupport: 'none',
maxImageSize: -1,
isEvalSupported: true,
disableFontFace: true,
disableRange: false,
disableStream: false,
disableAutoFetch: false,
disableCreateObjectURL: false },
CMapReaderFactory: DOMCMapReaderFactory { baseUrl: null, isCompressed: false },
destroyed: false,
destroyCapability: null,
_passwordCapability: null,
_networkStream:
PDFNodeStream {
source: [Object],
url: [Url],
isHttp: false,
isFsUrl: true,
httpHeaders: {},
_fullRequest: [PDFNodeStreamFsFullReader],
_rangeRequestReaders: [Array] },
_fullReader:
PDFNodeStreamFsFullReader {
_url: [Url],
_done: false,
_storedError: null,
onProgress: [Function],
_contentLength: 112979,
_loaded: 112979,
_filename: null,
_disableRange: false,
_rangeChunkSize: 65536,
_isStreamingSupported: true,
_isRangeSupported: true,
_readableStream: [ReadStream],
_readCapability: [Object],
_headersCapability: [Object] },
_lastProgress: { loaded: 112979, total: 112979 },
pageCache: [],
pagePromises: [],
downloadInfoCapability:
{ resolve: [Function], reject: [Function], promise: [Promise] },
numPages: 4,
pdfDocument: [Circular] } }
*ignore below text*
efwrg rgsretg resgerstgh;ergh ;resjgysregh regjes powrjgu oiuueryoeq uieqroeqreqrilih ehr oiyeroeq ioiyeqroeq oieyqrioeq oieqyr oiyeqr oiyeqrp ioqyet oiehr oiyerh oieyreq oiyheqri iohereqk ioheqr qerioyqereq ioehqriheq rioqehriqeb ioeqrhpeq ioeqrhiqe ioqehriq ioqerhioq oirhqeipor oiqehrieq ioehqrq ioeqhrieq iohqerpq ieqhrpeq ioeqhrpeq iheqrpqe oiehrpqe ieqhrqierh ioeqhr ieqhr ioeqrh piqerh ieqhr iheqr piheqr ioheqr iheqr ioeqhrp ioqhre oieqhr oeqiyr qoeiryf pouqer poqure pouqr pouqre[q poquerq poqeur[q poqeur poqwuer poquer[ poqwur[wq poqr[ poqwhr powrq pow
You may open and read a password protected PDF like below. Working with your existing code:
var PDFJS = require('pdfjs-dist');
PDFJS.getDocument({ url: 'p1.pdf', password: '' }).then(function(pdf)
{
let text = [];
for(let i = 1; i <= pdf.numPages; i++) {
pdf.getPage(i).then(function(page) {
page.getTextContent().then(function(data) {
for(let j = 0; j < data.items.length; j++) {
text.push(data.items[j].str);
}
});
});
}
}).catch(function(error) {
// incorrect password
// error is an object having 3 properties : name, message & code
});

NodeJS + mongoose - user dynamic database globally

I am using NodeJS, ExpressJS, express-session, connect-mongo and mongoose. I want to create a database for every user. On login the database connection should be globally to use.
This is the: loginController
exports.loginPost = (req, res, next) => {
const email = req.body.email;
const pass = req.body.password;
const pin = req.body.pin;
if (email && pass && pin) {
User.findOne({ email })
.then(u => {
bcrypt.compare(pass, u.password, (err, result) => {
if (!err) {
bcrypt.compare(pin, u.pin, (err2, result2) => {
if (!err2) {
let msg = {};
global.userDB = null;
msg.success = [{ text: "You are now logged in..." }];
req.session.loggedIn = true;
req.session.userId = u._id;
req.session.role= u.role;
const Mongoose = require("mongoose").Mongoose;
const inst = new Mongoose();
let db = "user-" + orgid;
global.userDB = inst.connect(
"mongodb://localhost:27017?authSource=dbWithUserCredentials",
{
useNewUrlParser: true,
auth: { authSource: "admin" },
user: "root",
pass: "root",
dbName: db,
autoReconnect: true
}
);
return res.redirect("/");
} else {
console.log(err2);
}
});
} else {
return res.render("login", {
req,
msg: null
});
}
});
})
.catch(err => {
return res.render("login", {
req,
msg: err
});
});
}
};
This is the adressModel:
const aschema = global.userDB.Schema;
var AdressSchema = new aschema({
name: {
type: String
}
});
var Address = global.userDB.model("Adress", AdressSchema);
module.exports = Address;
When I'm running a page where I'm using this model, i see this on the console when I'm using this command: "console.log(global.userDB)";
NativeConnection {
base:
Mongoose {
connections: [ [Circular] ],
models: {},
modelSchemas: {},
options: { pluralization: true },
plugins: [ [Array], [Array], [Array] ] },
collections: {},
models: {},
config: { autoIndex: true },
replica: false,
hosts: null,
host: 'localhost',
port: 27017,
user: 'root',
pass: 'root',
name: 'user-undefined',
options:
{ pass: 'root',
user: 'root',
auth: { authSource: 'admin' },
useNewUrlParser: true,
db: { forceServerObjectId: false, w: 1 },
server: { socketOptions: {}, auto_reconnect: true },
replset: { socketOptions: {} },
mongos: undefined },
otherDbs: [],
states:
[Object: null prototype] {
'0': 'disconnected',
'1': 'connected',
'2': 'connecting',
'3': 'disconnecting',
'4': 'unauthorized',
'99': 'uninitialized',
disconnected: 0,
connected: 1,
connecting: 2,
disconnecting: 3,
unauthorized: 4,
uninitialized: 99 },
_readyState: 2,
_closeCalled: false,
_hasOpened: false,
_listening: false,
db:
Db {
_events: [Object: null prototype] {},
_eventsCount: 0,
_maxListeners: undefined,
s:
{ databaseName: 'user-undefined',
dbCache: {},
children: [],
topology: [Server],
options: [Object],
logger: [Logger],
bson: BSON {},
authSource: undefined,
readPreference: undefined,
bufferMaxEntries: -1,
parentDb: null,
pkFactory: undefined,
nativeParser: undefined,
promiseLibrary: [Function: Promise],
noListener: false,
readConcern: undefined },
serverConfig: [Getter],
bufferMaxEntries: [Getter],
databaseName: [Getter] } }
Can someone say, why i get an 'undefined' in the database-name, but when I console.log the 'db'-variable there is the correct user._id.
Thanks!

Selector with Cheerio fails to retrieve children

I believe this one is a bug.
I am trying to write a simple web scraper with request and cheerio.
How I tried to solve it:
Yes, I played with other ways to define a selector.
Yes, I have investigated other stackoverflow questions.
Yes, I have created an issue on cheerio github, here is the link: https://github.com/cheeriojs/cheerio/issues/1252
Yes, I am a professional web developer and this is not the first time I do node.js
Update:
After some people pointed out, the issue was that needed dom nodes were created after my page was parsed and traversed by cheerio.
So the part of the page I requested simply was not there.
Any Ideas how to bypass that?
I use versions:
{
"name": "discont",
"version": "1.0.0",
"description": "Find when the item is on sale",
"main": "index.js",
"license": "MIT",
"devDependencies": {
"express": "^4.16.4"
},
"dependencies": {
"cheerio": "^1.0.0-rc.2",
"ejs": "^2.6.1",
"request": "^2.88.0"
}
}
This is the HTML I am trying to scrape:
The link is here:
https://www.asos.com/new-look-wide-fit/new-look-wide-fit-court-shoe/prd/10675413?clr=oatmeal&SearchQuery=&cid=6461&gridcolumn=1&gridrow=9&gridsize=4&pge=1&pgesize=72&totalstyles=826
This is my code:
request(url, options, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html, { withDomLvl1: false });
// console.log("product-price", $("div.product-price")[0].attribs);
console.log("product-price", $("div#product-price > div"));
}
});
The console.log returns an empty array(unable to find nested div).
This is what I get in return:
initialize {
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
_root:
initialize {
'0':
{ type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: {},
'x-attribsNamespace': {},
'x-attribsPrefix': {},
children: [Array],
parent: null,
prev: null,
next: null },
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
length: 1,
_root: [Circular] },
length: 0,
prevObject:
initialize {
'0':
{ type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: {},
'x-attribsNamespace': {},
'x-attribsPrefix': {},
children: [Array],
parent: null,
prev: null,
next: null },
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
length: 1,
_root: [Circular] } }
but if I change my code to
request(url, options, function(error, response, html) {
if (!error) {
var $ = cheerio.load(html, { withDomLvl1: false });
// console.log("product-price", $("div.product-price")[0].attribs);
console.log("product-price", $("div#product-price"));
}
});
I get an array with a single element:
initialize {
'0':
{ type: 'tag',
name: 'div',
namespace: 'http://www.w3.org/1999/xhtml',
attribs:
{ class: 'product-price',
id: 'product-price',
'data-bind': 'component: { name: "product-price", params: {state: state, showGermanVatMessage: false }}' },
'x-attribsNamespace': { class: undefined, id: undefined, 'data-bind': undefined },
'x-attribsPrefix': { class: undefined, id: undefined, 'data-bind': undefined },
children: [],
parent:
{ type: 'tag',
name: 'div',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: [Object],
'x-attribsNamespace': [Object],
'x-attribsPrefix': [Object],
children: [Array],
parent: [Object],
prev: [Object],
next: [Object] },
prev:
{ type: 'text',
data: '\n ',
parent: [Object],
prev: [Object],
next: [Circular] },
next:
{ type: 'text',
data: '\n ',
parent: [Object],
prev: [Circular],
next: [Object] } },
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
_root:
initialize {
'0':
{ type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: {},
'x-attribsNamespace': {},
'x-attribsPrefix': {},
children: [Array],
parent: null,
prev: null,
next: null },
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
length: 1,
_root: [Circular] },
length: 1,
prevObject:
initialize {
'0':
{ type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: {},
'x-attribsNamespace': {},
'x-attribsPrefix': {},
children: [Array],
parent: null,
prev: null,
next: null },
options:
{ withDomLvl1: false,
normalizeWhitespace: false,
xml: false,
decodeEntities: true },
length: 1,
_root: [Circular] } }
yet, I am not able to see children of the element (the children array is empty), and I am not able to perform any methods on the object such as find() or text()
Any help is welcome!
Cheerio only has access to the DOM before any special things like XHRs have happened. You would need puppeteer or nightmarejs for the post-js-rendered DOM

Resources