NodeJS get Google Sheet list - node.js

I have a google sheet with a list (dropdown, or data validation - list from ranges, as Google Sheets call it), like so:
Image of sheet
Imagine that in the list I have 4 values to select from. My goal is not only to get all table values, but also all values that constitute the list ["Beer","Wine","Rum","Martini"].
I've tried 2 different ways to retrieve the info and the list:
a) With sheets.spreadsheets.values.get, I get the table values in a digestible way, but not the content of the dropdown. Instead, the cell comes in as blank ("") [Comment: on Apps Script, you would get this information]
b) With sheets.spreadsheets.getByDataFilter, I get much more than I need and in a horrible format. However, I do not get the dropdown content as an array (as I'd want), but rather as a refence: (userEnteredValue: "=Input!$F$5:$F$7")
The question is, how do I get only the table, including the dropdown content as an array? I know it is possible and easy to do in Google Apps Script (I have it implemented), but not on Node.
Below the code as a reference for other programmers.
var {google} = require("googleapis");
let privatekey = require("./client_secret.json");
// configure a JWT auth client
let jwtClient = new google.auth.JWT(
privatekey.client_email,
null,
privatekey.private_key,
['https://www.googleapis.com/auth/spreadsheets',
]);
//authenticate request
jwtClient.authorize(function (err, tokens) {
if (err) {
console.log(err);
return;
} else {
console.log("Successfully connected!");
}
});
//Google Sheets API
let spreadsheetId = '<SPREADSHEET ID>';
let sheetName = 'Input!A1:B4'
let sheets = google.sheets('v4');
exports.fetch = (req, res) => {
sheets.spreadsheets.values.get({
auth: jwtClient,
spreadsheetId: spreadsheetId,
range: sheetName,
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err);
} else {
res.json(response);
}
});
// OR
sheets.spreadsheets.getByDataFilter({
auth: jwtClient,
spreadsheetId: spreadsheetId,
"includeGridData": true,
}, function (err, response) {
if (err) {
console.log('The API returned an error: ' + err);
} else {
res.json(response);
}
});
}

What you are interested in is dataValidation
In order to retrieve it, you can use the method spreadsheets.get, setting the parameter fields to sheets/data/rowData/values/dataValidation
If your data validation is set as List of items, the response will look like:
{
"sheets": [
{
"data": [
{
"rowData": [
{
"values": [
{
"dataValidation": {
"condition": {
"type": "ONE_OF_LIST",
"values": [
{
"userEnteredValue": "Beer"
},
{
"userEnteredValue": "Wine"
},
{
"userEnteredValue": "Martini"
}
]
},
"showCustomUi": true
}
}
]
}
]
}
]
}
]
}
If your data validation is set as List form a range the response will look like:
{
"sheets": [
{
"data": [
{
"rowData": [
{
"values": [
{
{
"dataValidation": {
"condition": {
"type": "ONE_OF_RANGE",
"values": [
{
"userEnteredValue": "=Sheet1!$B$1:$B$3"
}
]
},
"showCustomUi": true
}
}
]
}
]
}
]
}
]
}
In the latter case you would need to call subsequently the method spreadsheets.values.get on =Sheet1!$B$1:$B$3 - that is the range returned as userEnteredValue within the object values nested within the object dataValidation.

Related

Elasticsearch node js point in time search_phase_execution_exception

const body = {
query: {
geo_shape: {
geometry: {
relation: 'within',
shape: {
type: 'polygon',
coordinates: [$polygon],
},
},
},
},
pit: {
id: "t_yxAwEPZXNyaS1wYzYtMjAxN3IxFjZxU2RBTzNyUXhTUV9XbzhHSk9IZ3cAFjhlclRmRGFLUU5TVHZKNXZReUc3SWcAAAAAAAALmpMWQkNwYmVSeGVRaHU2aDFZZExFRjZXZwEWNnFTZEFPM3JReFNRX1dvOEdKT0hndwAA",
keep_alive: "1m",
},
};
Query fails with search_phase_execution_exception at onBody
Without pit query works fine but it's needed to retrieve more than 10000 hits
Well, using PIT in NodeJS ElasticSearch's client is not clear, or at least is not well documented. You can create a PIT using the client like:
const pitRes = await elastic.openPointInTime({
index: index,
keep_alive: "1m"
});
pit_id = pitRes.body.id;
But there is no way to use that pit_id in the search method, and it's not documented properly :S
BUT, you can use the scroll API as follows:
const scrollSearch = await elastic.helpers.scrollSearch({
index: index,
body: {
"size": 10000,
"query": {
"query_string": {
"fields": [ "vm_ref", "org", "vm" ],
"query": organization + moreQuery
},
"sort": [
{ "utc_date": "desc" }
]
}
}});
And then read the results as follows:
let res = [];
try {
for await (const result of scrollSearch) {
res.push(...result.body.hits.hits);
}
} catch (e) {
console.log(e);
}
I know that's not the exact answer to your question, but I hope it helps ;)
The usage of point-in-time for pagination of search results is now documented in ElasticSearch. You can find more or less detailed explanations here: Paginate search results
I prepared an example that may give an idea about how to implement the workflow, described in the documentation:
async function searchWithPointInTime(cluster, index, chunkSize, keepAlive) {
if (!chunkSize) {
chunkSize = 5000;
}
if (!keepAlive) {
keepAlive = "1m";
}
const client = new Client({ node: cluster });
let pointInTimeId = null;
let searchAfter = null;
try {
// Open point in time
pointInTimeId = (await client.openPointInTime({ index, keep_alive: keepAlive })).body.id;
// Query next chunk of data
while (true) {
const size = remained === null ? chunkSize : Math.min(remained, chunkSize);
const response = await client.search({
// Pay attention: no index here (because it will come from the point-in-time)
body: {
size: chunkSize,
track_total_hits: false, // This will make query faster
query: {
// (1) TODO: put any filter you need here (instead of match_all)
match_all: {},
},
pit: {
id: pointInTimeId,
keep_alive: keepAlive,
},
// Sorting should be by _shard_doc or at least include _shard_doc
sort: [{ _shard_doc: "desc" }],
// The next parameter is very important - it tells Elastic to bring us next portion
...(searchAfter !== null && { search_after: [searchAfter] }),
},
});
const { hits } = response.body.hits;
if (!hits || !hits.length) {
break; // No more data
}
for (hit of hits) {
// (2) TODO: Do whatever you need with results
}
// Check if we done reading the data
if (hits.length < size) {
break; // We finished reading all data
}
// Get next value for the 'search after' position
// by extracting the _shard_doc from the sort key of the last hit
searchAfter = hits[hits.length - 1].sort[0];
}
} catch (ex) {
console.error(ex);
} finally {
// Close point in time
if (pointInTime) {
await client.closePointInTime({ body: { id: pointInTime } });
}
}
}

Error when trying to purchase multiple items with one transaction

I have gotten the PayPal API to work with one item and I'm now trying to get it to work with a whole "shopping-cart". I have encountered an error that I don't know how to solve. I suspect it might have to do something with the payment-jsons total value that represents the total cost of the whole transaction. However I don't know what to do about it.
Here is the error:
Error: Response Status : 400
at IncomingMessage.<anonymous> (E:\Users\willi\Documents\Node\Store\node_modules\paypal-rest-sdk\lib\client.js:130:23)
at IncomingMessage.emit (events.js:327:22)
at endReadableNT (internal/streams/readable.js:1327:12)
at processTicksAndRejections (internal/process/task_queues.js:80:21) {
response: {
name: 'MALFORMED_REQUEST',
message: 'Incoming JSON request does not map to API request',
information_link: 'https://developer.paypal.com/webapps/developer/docs/api/#MALFORMED_REQUEST',
debug_id: '9e8898a463ee3',
httpStatusCode: 400
},
httpStatusCode: 400
}
And here is the code in question
const pay = (req, res) => {
async function f() {
items = [];
req_items = req.body.body
let itemsProcessed = 0
req_items.forEach(item => {
console.log(item.id)
const param = item.id
Item.find({ _id: param })
.then((result) => {
const item_body = {
"name": result[0].title,
"sku": "001",
"price": parseFloat(result[0].price),
"currency": "EUR",
"quantity": item.amount
}
items.push(item_body)
itemsProcessed = itemsProcessed + 1
})
.catch((err) => {
console.log(err)
})
})
let promise = new Promise((resolve, reject) => {
setTimeout(() => resolve("done!"), 1000)
});
let result = await promise; // wait until the promise resolves (*)
console.log(items)
const create_payment_json = {
"intent": "sale",
"payer": {
"payment_method": "paypal"
},
"redirect_urls": {
"return_url": "http://localhost:3000/success",
"cancel_url": "http://localhost:3000/cancel"
},
"transactions": [{
"item_list": {
"items": [items]
},
"amount": {
"currency": "EUR",
"total": parseFloat(req.body.subtotal) // 25
},
"description": "Purcahsed from the Store"
}]
};
// console.log(req.body)
// console.log(create_payment_json.transactions[0])
paypal.payment.create(create_payment_json, function (error, payment) {
if (error) {
throw error;
} else {
for(let i = 0;i < payment.links.length;i++){
if(payment.links[i].rel === 'approval_url'){
res.redirect(payment.links[i].href);
}
}
}
});
}
f();
}
API deprecation notice
You are integrating the deprecated v1/payments PayPal API. You shouldn't be doing so for a new integration; the current API is v2/checkout/orders, documented here.
Typically you'll want to create two routes on your own server, 'Create Order' and 'Capture Order', which return their own JSON when called. Then you can pair those two routes with the following approval flow: https://developer.paypal.com/demo/checkout/#/pattern/server
But as for your problem, debugging an issue like this is much simpler if you simply log your request JSON to see what the problem with it is.
If you do so, you will see that the "items" array you are sending has an array inside of an array of only one item (the other array). That array shouldn't be there.
This seems the culprit:
"items": [items]
Here you decided to make an array, which was useful when "items" was a single item (no array). But when items is already an array, you shouldn't be putting the array into a new array -- the resulting JSON won't map to an API request, and PayPal will return an error.
What you should do is get rid of those brackets and ensure that at this point in the code execution, "items" is already an array (if it wasn't before).

javascript object in lambda function

I am working on AWS Lambda and creating method by using node.js.
I need an object like this:
[
{
"TeamName" : "Sales",
"2020-01-01": "90",
"2020-01-02": "92",
"2020-01-03": "95",
"2020-01-04": "90",
"2020-01-05": "56",
"2020-01-06": "70",
"2020-01-07": "73"
},
]
but my current response is this:
[
{
"TeamName": "Billing",
"DateTime": "2020-06-13T00:00:00.000Z",
"Score": 9
},
{
"TeamName": "Billing",
"DateTime": "2020-06-13T00:00:00.000Z",
"Score": 9
},
{
"TeamName": "Billing",
"DateTime": "2020-06-11T00:00:00.000Z",
"Score": 5
},
]
Here is my Lambda method. I am not good at creating javascript object so please help me to make a response like this, Thanks.
exports.handler = (event, context, callback) => {
console.log('Events:',event);
let UserHierarchyGroupID = event['hierarchyGroupId'];
let team = [];
// allows for using callbacks as finish/error-handlers
context.callbackWaitsForEmptyEventLoop = false;
pool.getConnection(function(err, connection) {
if (err) throw err;
let sql = `SELECT date(Feedback.DateTime) as datetime,Feedback.Score,UserHierarchy.Layer5
FROM ctrData2.Feedback
LEFT OUTER JOIN ctrData2.CallDetail ON CallDetail.ContactId = Feedback.FeedbackID
LEFT OUTER JOIN ctrData2.UserTable ON UserTable.UserID = CallDetail.UserID
LEFT OUTER JOIN ctrData2.UserHierarchy ON UserTable.UserID = UserHierarchy.UserID
WHERE UserTable.UserHierarchyGroupID=?`;
let field = [UserHierarchyGroupID];
connection.query(sql,field, function (err, result, fields) {
if (err) throw err;
// console.log(result);
connection.release();
var date;
var score;
if(result.length>0){
result.forEach((item)=>{
team.push({
"TeamName": item.Layer5,
"DateTime": item.datetime,
"Score": item.Score
});
});
}else{
callback(null,{
status: 404,
Body: "Not found"
});
}
callback(null,team);
// FomratObjects(result,(formattedResponse)=>{
// // console.log(formattedResponse);
// callback(formattedResponse);
// });
});
});
};
Its doesn't look possible to create an object exactly like you mentioned but you can do this to assign value to every single date.
Hope it will be helpful.
function formatData(data){
var nObject = {};
data.forEach(d=>{
nObject[moment(d.datetime).format('MM-DD-YYYY')]=d.Score;
});
return nObject;
}

Unable to write item(s) to DynamoDB table utilizing DocumentClient - Nodejs

I'm absolutely brand new to DynamoDb and I'm trying to simply write an object from a NodeJS Lambda. Based on what I've read and researched I should probably be using DocumentClient from the aws-sdk. I also found the following question here regarding issues with DocumentClient, but it doesn't seem to address my specific issue....which I can't really find/pinpoint unfortunately. I've set up a debugger to help with SAM local development, but it appears to be only providing some of the errors.
The code's implementation is shown here.
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": {"S": randstring.generate(9)},
"School":{"S": team_name},
"Seed": {"S": seed},
"ESPN_Id": {"S": espn_id}
}
}
console.log(JSON.stringify(params))
dynamodb.put(params, (error,data) => {
if (error) {
console.log("Error ", error)
} else {
console.log("Success! ", data)
}
})
Basically I'm scrubbing a website utilizing cheerio library and cherry picking values from the DOM and saving them into the json object shown below.
{
"TableName": "March-Madness-Teams",
"Item": {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
When I attempt to push this json object to Dynamo, I get errors says
Error MultipleValidationErrors: There were 2 validation errors:
* MissingRequiredParameter: Missing required key 'TableName' in params
* MissingRequiredParameter: Missing required key 'Item' in params
The above error is all good in well....I assume it didn't like the fact that I had wrapped those to keys in strings, so I removed the quotes and sent the following
{
TableName: "March-Madness-Teams",
Item: {
"Id": {
"S": "ED311Oi3N"
},
"School": {
"S": "BAYLOR"
},
"Seed": {
"S": "1"
},
"ESPN_Id": {
"S": "239"
}
}
}
However, when I do that...I kind of get nothing.
Here is a larger code snippet.
return new Promise((resolve,reject) => {
axios.get('http://www.espn.com/mens-college-basketball/bracketology')
.then(html => {
const dynamodb = new aws.DynamoDB.DocumentClient()
let $ = cheerio.load(html.data)
$('.region').each(async function(index, element){
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(async function(index2, element2){
var seed = $(element2).siblings('span.rank').text()
if (seed.length > 2){
seed = $(element2).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element2).attr('href').split('/').slice(-2)[0]
var team_name = $(element2).text()
var params = {
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School":team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
}
console.log(JSON.stringify(params))
// dynamodb.put(params)
// .then(function(data) {
// console.log(`Success`, data)
// })
})
})
})
})
Can you try without the type?
Instead of
"School":{"S": team_name},
for example, use
"School": team_name,
From your code, I can see the mis promise on the dynamodb request. Try to change your lines :
dynamodb.put(params).then(function(data) {
console.log(`Success`, data)
})
to be :
dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
you can combine with await too :
await dynamodb.put(params).promise().then(function(data) {
console.log(`Success`, data)
})
exports.lambdaHandler = async (event, context) => {
const html = await axios.get('http://www.espn.com/mens-college-basketball/bracketology')
let $ = cheerio.load(html.data)
const schools = buildCompleteSchoolObject(html, $)
try {
await writeSchoolsToDynamo(schools)
return { statusCode: 200 }
} catch (error) {
return { statusCode: 400, message: error.message }
}
}
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(async school => {
await dynamodb.put(school).promise()
})
await Promise.all(promises)
}
const buildCompleteSchoolObject = (html, $) => {
const schools = []
$('.region').each(loopThroughSubRegions(schools, $))
return schools
}
const loopThroughSubRegions = (schools, $) => {
return (index, element) => {
var preregion = $(element).children('h3,b').text()
var region = preregion.substr(0, preregion.indexOf('(') - 1)
$(element).find('a').each(populateSchoolObjects(schools, $))
}
}
const populateSchoolObjects = (schools, $) => {
return (index, element) => {
var seed = $(element).siblings('span.rank').text()
if (seed.length > 2) {
seed = $(element).siblings('span.rank').text().substring(0, 2)
}
var espn_id = $(element).attr('href').split('/').slice(-2)[0]
var team_name = $(element).text()
schools.push({
TableName: "March-Madness-Teams",
Item: {
"Id": randstring.generate(9),
"School": team_name,
"Seed": seed,
"ESPN_Id": espn_id
}
})
}
}
I know this is drastically different from what I started with but I did some more digging and kind of kind of worked to this...I'm not sure if this is the best way, but I seemed to get it to work...Let me know if something should change!
Oh I understand what you want.
Maybe you can see the code above works, but there is one concept you have to improve here about async - await and promise especially on lambda function.
I have some notes here from your code above, maybe can be your consideration to improve your lambda :
Using await for every promise in lambda is not the best approach because we know the lambda time limitation. But sometimes we can do that for other case.
Maybe you can change the dynamodb.put method to be dynamodb.batchWriteItem :
The BatchWriteItem operation puts or deletes multiple items in one or more tables.
Or If you have to use dynamodb.put instead, try to get improve the code to be like so :
const writeSchoolsToDynamo = async (schools) => {
const promises = schools.map(school => {
dynamodb.put(school).promise()
})
return Promise.all(promises)
}

Array full of regex, match to string, return path

Still very much a beginner, Just editing to make a little more sense
Here is an example payload.
{
"Status":
{
"result":[
{
"LastUpdate":"2017-09-07 06:47:09",
"Type":"2",
"Value":"' s the inside temperature",
"idx":"4"
}
],
"status":"OK",
"title":"GetUserVariable"
},
"Devices":
{
"28":
{
"rid":"28",
"regex":"(the )?(AC|(Air( )?(Con)?(ditioner)?))( Power)?( )?$/i"
},
"71":
{
"rid":"71",
"regex":"(the )?inside temp/i"
}
}
}
I want to filter the "Devices" array down to entires that match Status.result[0].Value.
I have the follow code working, but it is in working in reverse, it returns the matching string, not the filtered array, just not sure how to reverse it.
var devices = msg.payload.Devices;
var request = [ msg.payload.Status.result[0].Value ];
var matches = request.filter(function (text) {
return devices.some(function (regex) {
var realregex = new RegExp(regex, "i");
return realregex.test(text);
});
});
msg = { topic:"Inputs", payload: { devices:devices, request:request } };
msg2 = { topic:"Output", payload: { matches:matches } };
return [ [ msg, msg2 ] ];
Thanks,
Wob

Resources