I am new to node js. I am trying to do scraping some value and name from the select tab from a particular website. My code as follows,
const rp = require('request-promise');
const crio = require('cheerio');
const url = 'https://myurl';
const getOptions = function(optionName,ignoreOption) {
return rp(url)
.then(function(html) {
crio('[name='+optionName+']', html).find('option').each((i,op) => {
if(crio(op).text() != ignoreOption && crio(op).text() != '')
return {
name: crio(op).text(),
value: crio(op).val(),
};
})
})
.catch(function(err) {
//handle error
});
};
getOptions('test','-- Select company name --')
.then( data =>
console.log(data)
);
When I try to run this code, I am getting result as 'undefined'. How do I get the return json response from the getOptions function here?
You are not returning a value inside your then function that is using crio.
Change
crio('[name='+optionName+']', html).find('option').each((i,op) => {
to
return crio('[name='+optionName+']', html).find('option').map((i,op) => {
Note the use of map instead of each.
Related
I am new to Restful API development using NodeJS and SQL Server. I am trying to do a simple [post] operation where I am passing an array of objects to the API endpoint and then calling a SQL Server procedure with a table valued parameter. I am getting the below error
Cannot read properties of undefined (reading 'generateTypeInfo')
I was really shocked to see that there is not a single help topic found over Google regarding this error. I do not want to learn ASP.NET Core for this because JavaScript has an easy learning curve. Am I doing a mistake by developing a Rest API by using the combination of NodeJS and SQL Server? Below is my Related .JS file called in Post endpoint
const sql = require("mssql/msnodesqlv8");
const dataAccess = require("../DataAccess");
const fn_CreateProd = async function (product) {
let errmsg = "";
let connPool = null;
await sql
.connect(global.config)
.then((pool) => {
global.connPool = pool;
result = pool.request().query("select * from products where 1=2");
return result;
})
.then((retResult) => {
const srcTable = retResult.recordset.toTable("tvp_products");
let newsrcTable = Array.from(srcTable.columns);
console.log('Source table b4 mapping',srcTable)
newsrcTable = newsrcTable.map((i) => {
i.name = i.name.toUpperCase();
return i;
});
console.log('Source table after convert array with mapping',newsrcTable)
const prdTable = dataAccess.generateTable(
newsrcTable,
product,
"tvp_products"
);
console.log("Prepared TVp data", prdTable);
const newResult = dataAccess.execute(`sp3s_ins_products_tvp`, [
{ name: "tblprods", value: prdTable },
]);
console.log("Result of Execute Final procedure", newResult);
return newResult;
})
.then(result => {
console.log("Result of proc", result);
if (!result.errmsg) errmsg = "Products Inserted successfully";
else errmsg = result.errmsg;
})
.catch((err) => {
console.log("Enter catch of Posting prod", err.message);
errmsg = err.message;
})
.finally((resp) => {
sql.close();
});
return { retStatus: errmsg };
};
module.exports = fn_CreateProd;
and Content of Generatetable function are as below :
const generateTable = (columns, entities,tvpName) => {
const table = new mssql.Table(tvpName);
// const testobj = {type : [sql.numeric],name : 'Sanjay'}
// console.log('Columns testobj',testobj.type)
columns.forEach(column => {
// console.log('COlumn data for COlumn :',column)
if (column && typeof column === 'object' && column.name && column.type) {
let colOptions = {}
if (column.type==mssql.Numeric)
{
colOptions.scale=column.scale
colOptions.precision=column.precision
}
else
if (column.type==mssql.VarChar || column.type==mssql.Char )
{
colOptions.length = column.length
}
// console.log (`Column name type for column :${column.name} -${colType}-Actual :${column['type']}`)
if (column.hasOwnProperty('options')) {
table.columns.add(column.name.toUpperCase(), colType,column.options);
} else {
table.columns.add(column.name.toUpperCase(),colOptions)
}
}
});
console.log('Generated table',table)
const newEntities = entities.map(obj=>keystoUppercase(obj))
// console.log('New entities after uppercase',newEntities)
newEntities.forEach(entity => {
table.rows.add(...columns.map(i =>
entity[i.name]));
});
return table;
};
I have found the solution now. Actually, if you can see the code of generateTable function, I was adding the columns into the table but not mentioning the data type of the columns due to which this error was coming. I have added one more property [type] in the [colOptions] object being passed to columns.add command in the function [Generatetable]. Thanks a lot anyway to you for quick replies by Dale. K.
As an exercise, I'm creating a simple API that allows users to provide a search term to retrieve links to appropriate news articles across a collection of resources. The relevent function and the route handler that uses the function is as follows:
function GetArticles(searchTerm) {
const articles = [];
//Loop through each resource
resources.forEach(async resource => {
const result = await axios.get(resource.address);
const html = result.data;
//Use Cheerio: load the html document and create Cheerio selector/API
const $ = cheerio.load(html);
//Filter html to retrieve appropriate links
$(`a:contains(${searchTerm})`, html).each((i, el) => {
const title = $(el).text();
let url = $(el).attr('href');
articles.push(
{
title: title,
url: url,
source: resource.name
}
);
})
})
return articles; //Empty array is returned
}
And the route handler that uses the function:
app.get('/news/:searchTerm', async (req, res) => {
const searchTerm = req.params.searchTerm;
const articles = await GetArticles(searchTerm);
res.json(articles);
})
The problem I'm getting is that the returned "articles" array is empty. However, if I'm not "looping over each resource" as commented in the beginning of GetArticles, but instead perform the main logic on just a single "resource", "articles" is returned with the requested data and is not empty. In other words, if the function is the following:
async function GetArticles(searchTerm) {
const articles = [];
const result = await axios.get(resources[0].address);
const html = result.data;
const $ = cheerio.load(html);
$(`a:contains(${searchTerm})`, html).each((i, el) => {
const title = $(el).text();
let url = $(el).attr('href');
articles.push(
{
title: title,
url: url,
source: resources[0].name
}
);
})
return articles; //Populated array
}
Then "articles" is not empty, as intended.
I'm sure this has to do with how I'm dealing with the asynchronous nature of the code. I've tried refreshing my knowledge of asynchronous programming in JS but I still can't quite fix the function. Clearly, the "articles" array is being returned before it's populated, but how?
Could someone please help explain why my GetArticles function works with a single "resource" but not when looping over an array of "resources"?
Try this
function GetArticles(searchTerm) {
return Promise.all(resources.map(resource => axios.get(resource.address))
.then(responses => responses.flatMap(result => {
const html = result.data;
//Use Cheerio: load the html document and create Cheerio selector/API
const $ = cheerio.load(html);
let articles = []
//Filter html to retrieve appropriate links
$(`a:contains(${searchTerm})`, html).each((i, el) => {
const title = $(el).text();
let url = $(el).attr('href');
articles.push(
{
title: title,
url: url,
source: resource.name
}
);
})
return articles;
}))
}
The problem in your implementation was here
resources.forEach(async resource...
You have defined your function async but when result.foreach get executed and launch your async functions it doesn't wait.
So your array will always be empty.
I have a module that looks like follows:
calculate-average.js
const fetch = require('node-fetch')
const stats = require('stats-lite')
const BASE_URL = 'https://www.example.com/api'
const calculateAverage = async(numApiCalls) => {
const importantData = []
for (let i = 0; i < numApiCalls; i++) {
const url = `${BASE_URL}/${i}` // will make requests to https://www.example.com/api/0, https://www.example.com/api/1 and so on....
const res = await fetch(url)
const jsonRes = await res.json()
importantData.push(jsonRes.importantData)
}
return stats.mean(importantData)
}
module.exports = calculateAverage
I tried testing it along the following lines but I am clearly way off from the solution:
calculate-average.test.js
const calculateAverage = require('../calculate-average')
jest.mock(
'node-fetch',
() => {
return jest.fn(() => {})
}
)
test('Should calculate stats for liquidation open interest delatas', async() => {
const stats = await calculateAverage(100) // Should make 100 API calls.
console.log(stats)
})
What I need to do is the following:
Be able to specify custom varied responses for each API call. For example, I should be able to specify that a call to https://www.example.com/api/0 returns { importantData: 0 }, a call to https://www.example.com/api/1 returns { importantData: 1 } and so on...
If a request is made to a url that I have not specified a response for, a default response is provided. For example if a response is made to https://www.example.com/api/101, then a default response of { importantData: 1000 } is sent.
I would preferably like to do this only using Jest without depending on modules like mock-fetch and jest-mock-fetch. However, if the solution without using is way too complex, then I would be happy to use them. Just don't want to create unnecessary dependencies if not required.
Sure you can! You can use mock function mockResolvedValueOnce method to return a result for a specific call and mockResolvedValue to return the default result.
jest.mock('node-fetch', () => {
const generateResponse = (value) => {
return { json: () => ({ importantData: value }) };
};
return jest
.fn()
.mockResolvedValue(generateResponse(1000)) // default response
.mockResolvedValueOnce(generateResponse(0)) // response for first call
.mockResolvedValueOnce(generateResponse(1)) // response for second call
.mockResolvedValueOnce(generateResponse(2)); // response for third call
});
Note that we are returning an object with the json property so that it returns the json data when you call res.json() in calculate-average.js.
If you want to return a specific response based on the url parameter, you will have to mock the desired behaviour in the returned mock function for node-fetch. The following example will mock the returned value so that for URLs where the counter is greater than 100 it will return 1000. Otherwise, it will return the same value present in the url:
jest.mock('node-fetch', () => {
return jest.fn((url) => {
// Get and parse the URL parameter.
const value = parseInt(url.split('/').slice(-1)[0], 10);
return Promise.resolve({
json: () => ({ importantData: value > 100 ? 1000 : value })
});
});
});
I am trying to call one async function from inside a loop run by another async function. These functions call APIs and I am using request-promise using nodeJS.
functions.js file
const rp = require("request-promise");
// function (1)
async email_views: emailId => {
let data = {};
await rp({
url: 'myapiurl',
qs: { accessToken: 'xyz', emailID: emailId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above JSON looks like this:
...
data:{
records: [
{
...
contactID: 123456,
...
},
{
...
contactID: 456789,
...
}
]
}
...
I am running a loop to get individual record, where I am getting a contactID associated with each of them.
// function#2 (also in functions.js file)
async contact_detail: contactId => {
let data = {};
await rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above function takes one contactId as parameter and gets that contact's detail calling another API endpoint.
Both functions work fine when they are called separately. But I am trying to do it inside a loop like this:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId); // function#2
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
The problem is I am only getting the first contact back from the above code block, i.e. even I have 20 records from function#1, in the loop when I am calling contact_detail (function#2) for each contactID (cId), I get contact detail once, i.e. for the first cId only. For rest I get nothing!
What is the correct way to achieve this using nodeJs?
UPDATE:
const { App } = require("jovo-framework");
const { Alexa } = require("jovo-platform-alexa");
const { GoogleAssistant } = require("jovo-platform-googleassistant");
const { JovoDebugger } = require("jovo-plugin-debugger");
const { FileDb } = require("jovo-db-filedb");
const custom = require("./functions");
const menuop = require("./menu");
const stateus = require("./stateus");
const alexaSpeeches = require("./default_speech");
const app = new App();
app.use(new Alexa(), new GoogleAssistant(), new JovoDebugger(), new FileDb());
let sp = "";
async EmailViewsByContactIntent() {
try {
const viewEmailId =
this.$session.$data.viewEmailIdSessionKey != null
? this.$session.$data.viewEmailIdSessionKey
: this.$inputs.view_email_Id_Number.value;
let pageIndex =
this.$session.$data.viewEmailPageIndex != null
? this.$session.$data.viewEmailPageIndex
: 1;
const result = await custom.email_views_by_emailId(
viewEmailId,
pageIndex
);
const records = result.data.records;
if (records.length > 0) {
const totalRecords = result.data.paging.totalRecords;
this.$session.$data.viewEmailTotalPages = totalRecords;
sp = `i have found a total of ${totalRecords} following view records. `;
if (totalRecords > 5) {
sp += `i will tell you 5 records at a time. for next 5 records, please say, next. `;
this.$session.$data.viewEmailIdSessionKey = this.$inputs.view_email_Id_Number.value;
this.$session.$data.viewEmailPageIndex++;
}
for (let i = 0; i < records.length; i++) {
const r = records[i];
/* Here I want to pass r.contactID as contactId in the function contact_detail like this: */
const contact = await custom.contact_detail(r.contactID);
const contact_name = contact.data.firstName + " " + contact.data.lastName;
/* The above two lines of code fetch contact_name for the first r.contactID and for the rest I get an empty string only. */
const formatted_date = r.date.split(" ")[0];
sp += `contact ID ${spellOut_speech_builder(
r.contactID
)} had viewed on ${formatted_date} from IP address ${
r.ipAddress
}. name of contact is, ${contact_name}. `;
}
if (totalRecords > 5) {
sp += ` please say, next, for next 5 records. `;
}
} else {
sp = ``;
}
this.ask(sp);
} catch (e) {
this.tell(e);
}
}
I am building an alexa skill using JOVO framework and nodeJS.
UPDATE #2
As a test, I only returned the contactId which I am passing to the contact_detail function and I am getting the correct value back to the above code under my first UPDATE.
async contact_detail: contactId => {
return contactId;
}
It seems even after getting the value right, the function is somehow failing to execute. However, the same contact_detail function works perfectly OK, when I am calling it from another place. Only doesn't not work inside a loop.
What could be the reason?
I must be missing something but don't know what!
You are mixing async await and promises together which is causing you confusion. You typically would use one of the other(as async await effectivly provides syntax sugar so you can avoid dealing with the verbose promise code) in a given location.
Because you mixed the two you are in a weird area where the behavior is harder to nail down.
If you want to use async await your functions should look like
async contact_detail: contactId => {
try {
const body = await rp({
url: 'myapiurl2',
qs: { ... }
});
return JSON.parse(body);
} catch(e) {
console.log(e);
//This will return undefined in exception cases. You may want to catch at a higher level.
}
};
or with promises
async contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { ... }
})
.then( body => JSON.parse(body))
.catch( error => {
console.log(error);
//This will return undefined in exception cases. You probably dont want to catch here.
});
};
Keep in mind your current code executing the function will do each call in series. If you want to do them in parallel you will need to call the function a bit differently and use something like Promise.all to resolve the result.
Here you go:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
await Promise.all(records.map(async record => {
let cId = record.contactID;
let contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
});
console.log(names);
...
I'm posting this as an answer only because I need to show you some multi-line code as part of throubleshooting this. Not sure this solves your issue yet, but it is a problem.
Your contact_detail() function is not properly returning errors. Instead, it eats the error and resolves with an empty object. That could be what is causing your blank names. It should just return the promise directly and if you want to log the error, then it needs to rethrow. Also, there's no reason for it to be declared async or to use await. You can just return the promise directly. You can also let request-promise parts the JSON response for you too.
Also, I notice, there appears to be a syntax error in your .catch() which could also be part of the problem.
contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
json: true,
method: 'GET'
}).catch( error => {
// log error and rethrow so any error propagates
console.log(error);
throw error;
});
};
Then, you would call this like you originally were (note you still use await when calling it because it returns a promise):
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
I have a nodejs serverless backend which takes an interger and then execute some code. My problem is that if I send a string, I still get a 200 code response with null data.
To stop the function and send an error message I'm trying to check if the request is an interger with if and else statements but its not working. So, is it possible to do it like this, or is there a better way to stop the execution and send an error?
app.post('/xrp', (req, res) => {
axios.get('https://www.surbtc.com/api/v2/markets/btc-clp/ticker')
.then(response => {
var clpbtc = parseFloat(response.data.ticker.min_ask[0]);
if (typeof clpbtc === 'number') {
coinTicker('bittrex', 'XRP_BTC')
.then(response => {
var xrpbtc = response.rawData.Bid;
var exchangeRate = clpbtc * xrpbtc;
if (xrpbtc) {
var cantidadCLP = req.body.numero;
var cantidadXRP = parseFloat(req.body.numero) / exchangeRate;
var fee = 0.025;
var oppositeFee = 0.975
var totalCompraXRP = cantidadXRP * oppositeFee;
res.status(200).json({ "cantidadCLP" : cantidadCLP, "cantidadXRP" : cantidadXRP, "fee": fee, "oppositeFee": oppositeFee, "totalCompraXRP": totalCompraXRP})
} else {
res.sendStatus(500).send("No válido2")
}
})
.catch(error => {
res.sendStatus(500).json(error);
})
} else {
res.sendStatus(500).send("No válido")
}
})
.catch(error => {
res.sendStatus(500).json(error);
});
});
Your problem is that you're parsing the parameter as a float anyway so even if you send a string, it will convert to a number and your query will simply return no data, hence the null response.
parseFloat('This is a string')
// => NaN
typeof parseFloat('This is a string')
// => 'number'
This is a weird property of javascript, where typeof NaN == 'number'
To accommodate for this, change your code to the following:
.then(response => {
var clpbtc = parseFloat(response.data.ticker.min_ask[0]);
if (!isNaN(clpbtc)) {
coinTicker('bittrex', 'XRP_BTC')
Now, if you pass in a string, you should get a 500 response.