Google Apps Script: How to retrieve values from deeply nested object having nulls and empty elements? - object
Trying to retrieve {raw} values from all {type} keys in the object of this URL. #Tanaike made this code which worked well when all of {type} keys have clean data without any missing or null. But it didn't work when some of {type} keys have null elements or are empty, getting an error:
TypeError: Cannot read properties of null (reading 'reportedValue')
In #Tanaike's code below, I like to achieve something like if {type} key doesn't have {raw} value, array = [[type, ''], ....], and remove any elements of null. What should be changed in this line? Thank you!
var array = obj.timeseries.result.map(o => types.flatMap(type => o[type] ? [type, ...o[type].map(({ reportedValue: { raw } }) => raw)] : []));
function test() {
var url = 'https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/CRWD?lang=en-US®ion=US&symbol=CRWD&padTimeSeries=true&type=annualTaxProvision,trailingTaxProvision,annualPretaxIncome,trailingPretaxIncome,annualInterestExpenseNonOperating,trailingInterestExpenseNonOperating,annualLongTermDebt,quarterlyLongTermDebt,annualCurrentDebt,quarterlyCurrentDebt,annualCashCashEquivalentsAndShortTermInvestments,quarterlyCashCashEquivalentsAndShortTermInvestments,annualFreeCashFlow,trailingFreeCashFlow&merge=false&period1=493590046&period2=1672980169&corsDomain=finance.yahoo.com'
var obj = UrlFetchApp.fetch(url, { muteHttpExceptions: true }).getContentText();
obj = JSON.parse(obj); //Convert strings to object
var types = obj.timeseries.result.flatMap(({ meta: { type } }) => type);
var array = obj.timeseries.result.map(o => types.flatMap(type => o[type] ? [type, ...o[type].map(({ reportedValue: { raw } }) => raw)] : []));
array = Object.keys(array).map(k => [...array[k]]); // Convert object to array
console.log(array)
}
Looks like there's two different errors that could occur. One for a series missing all of its values and one where the series has null entries.
// A pull of the data at asker's url
var obj = '{"timeseries":{"result":[{"meta":{"symbol":["CRWD"],"type":["annualLongTermDebt"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualLongTermDebt":[null,null,{"dataId":23123,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":7.38029E8,"fmt":"738.03M"}},null,{"dataId":23123,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":7.39517E8,"fmt":"739.52M"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["trailingFreeCashFlow"]},"timestamp":[1667174400],"trailingFreeCashFlow":[{"dataId":26185,"asOfDate":"2022-10-31","periodType":"TTM","currencyCode":"USD","reportedValue":{"raw":5.93996E8,"fmt":"594.00M"}}]},{"meta":{"symbol":["CRWD"],"type":["trailingInterestExpenseNonOperating"]},"timestamp":[1667174400],"trailingInterestExpenseNonOperating":[{"dataId":20064,"asOfDate":"2022-10-31","periodType":"TTM","currencyCode":"USD","reportedValue":{"raw":2.5269E7,"fmt":"25.27M"}}]},{"meta":{"symbol":["CRWD"],"type":["annualFreeCashFlow"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualFreeCashFlow":[{"dataId":26185,"asOfDate":"2019-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":-6.5613E7,"fmt":"-65.61M"}},{"dataId":26185,"asOfDate":"2020-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1.2456E7,"fmt":"12.46M"}},{"dataId":26185,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":2.92723E8,"fmt":"292.72M"}},null,{"dataId":26185,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":4.41095E8,"fmt":"441.10M"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["trailingTaxProvision"]},"timestamp":[1667174400],"trailingTaxProvision":[{"dataId":20145,"asOfDate":"2022-10-31","periodType":"TTM","currencyCode":"USD","reportedValue":{"raw":3.067E7,"fmt":"30.67M"}}]},{"meta":{"symbol":["CRWD"],"type":["trailingPretaxIncome"]},"timestamp":[1667174400],"trailingPretaxIncome":[{"dataId":20136,"asOfDate":"2022-10-31","periodType":"TTM","currencyCode":"USD","reportedValue":{"raw":-1.44422E8,"fmt":"-144.42M"}}]},{"meta":{"symbol":["CRWD"],"type":["annualTaxProvision"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualTaxProvision":[{"dataId":20145,"asOfDate":"2019-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1367000.0,"fmt":"1.37M"}},{"dataId":20145,"asOfDate":"2020-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1997000.0,"fmt":"2.00M"}},{"dataId":20145,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":4760000.0,"fmt":"4.76M"}},null,{"dataId":20145,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":7.2355E7,"fmt":"72.36M"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["annualPretaxIncome"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualPretaxIncome":[{"dataId":20136,"asOfDate":"2019-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":-1.3871E8,"fmt":"-138.71M"}},{"dataId":20136,"asOfDate":"2020-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":-1.39782E8,"fmt":"-139.78M"}},{"dataId":20136,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":-8.7869E7,"fmt":"-87.87M"}},null,{"dataId":20136,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":-1.60023E8,"fmt":"-160.02M"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["quarterlyCashCashEquivalentsAndShortTermInvestments"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"quarterlyCashCashEquivalentsAndShortTermInvestments":[null,null,null,{"dataId":23033,"asOfDate":"2021-10-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":1.907508E9,"fmt":"1.91B"}},{"dataId":23033,"asOfDate":"2022-01-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":1.996633E9,"fmt":"2.00B"}},{"dataId":23033,"asOfDate":"2022-04-30","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":2.152736E9,"fmt":"2.15B"}},{"dataId":23033,"asOfDate":"2022-07-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":2.318858E9,"fmt":"2.32B"}},{"dataId":23033,"asOfDate":"2022-10-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":2.466551E9,"fmt":"2.47B"}}]},{"meta":{"symbol":["CRWD"],"type":["annualCashCashEquivalentsAndShortTermInvestments"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualCashCashEquivalentsAndShortTermInvestments":[{"dataId":23033,"asOfDate":"2019-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1.91655E8,"fmt":"191.66M"}},{"dataId":23033,"asOfDate":"2020-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":9.12064E8,"fmt":"912.06M"}},{"dataId":23033,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1.918608E9,"fmt":"1.92B"}},null,{"dataId":23033,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1.996633E9,"fmt":"2.00B"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["quarterlyLongTermDebt"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"quarterlyLongTermDebt":[null,null,null,{"dataId":23123,"asOfDate":"2021-10-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":7.39145E8,"fmt":"739.14M"}},{"dataId":23123,"asOfDate":"2022-01-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":7.39517E8,"fmt":"739.52M"}},{"dataId":23123,"asOfDate":"2022-04-30","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":7.39889E8,"fmt":"739.89M"}},{"dataId":23123,"asOfDate":"2022-07-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":7.40261E8,"fmt":"740.26M"}},{"dataId":23123,"asOfDate":"2022-10-31","periodType":"3M","currencyCode":"USD","reportedValue":{"raw":7.40633E8,"fmt":"740.63M"}}]},{"meta":{"symbol":["CRWD"],"type":["annualInterestExpenseNonOperating"]},"timestamp":[1548892800,1580428800,1612051200,1635638400,1643587200,1651276800,1659225600,1667174400],"annualInterestExpenseNonOperating":[{"dataId":20064,"asOfDate":"2019-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":428000.0,"fmt":"428.00k"}},{"dataId":20064,"asOfDate":"2020-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":442000.0,"fmt":"442.00k"}},{"dataId":20064,"asOfDate":"2021-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":1559000.0,"fmt":"1.56M"}},null,{"dataId":20064,"asOfDate":"2022-01-31","periodType":"12M","currencyCode":"USD","reportedValue":{"raw":2.5231E7,"fmt":"25.23M"}},null,null,null]},{"meta":{"symbol":["CRWD"],"type":["quarterlyCurrentDebt"]}},{"meta":{"symbol":["CRWD"],"type":["annualCurrentDebt"]}}],"error":null}}'
obj = JSON.parse(obj); //Convert strings to object
// for each object in `obj.timeseries.result`, `.meta.type` names a key in
// that object that contains the data array. That key may not exist. Some
// entries in the data array may be null.
var mangled = obj.timeseries.result.map(o => {
// pull the key
const type = o.meta.type[0];
// if there's no attribute of that name, return an empty array of values
if (!(type in o)) return [type, []];
// otherwise return an array of values, being careful of `null`s
return [
type,
o[type].map(value => value == null ? null : value.reportedValue.raw),
];
});
console.log(mangled)
Related
Get rows if key has a specific value in array of arrays mongodb
I'm not able to get the rows on condition.
The condition is based on a ID, but this ID is a value of a key inside an object that is inside an array, also the array is inside another array.
So I have
Cinema (ARRAY) -> ARRAYS (0,1,2,3....) -> OBJECTS inside each ARRAY
I need the object where cinema_id_db is matched.
For example this code return the data of
cinema[0][0]['cinema_id_db'] = ObjectId("5b44a78b38be3aeb31f092cb")
code
{ "cinema.0.0.cinema_id_db" : ObjectId("5b44a78b38be3aeb31f092cb")}
I need something like
cinema[x][y]['cinema_id_db'] = ObjectId("5b44a78b38be3aeb31f092cb")
This is how the db looks
you can get this value you want to get, this way
let cinema_id_db = cinema[0][0].cinema_id_db
OR
var ciname_ids_db = [];
var index = 0;
cinema.forEach(element => {
element.forEach(elem => {
ciname_ids_db[index++] = elem.cinema_id_db
})
})
Multiple types in query string in nodejs
I am creating a get api in nodejs.I am requesting the following url
http://localhost:8080/api?id=20&condition1=true&arr=[{prop1:1}]&obj={a:1,b:2}
And I am getting the request query object as follows-
req.query = {
arr:"[{prop1:1}]",
condition1:"true",
id:"20",
obj:"{a:1,b:2}"
}
I want to convert the query object keys to appropriate types.My query object should be converted to
req.query = {
arr:[{prop1:1}], // Array
condition1:true, // Boolean
id:20, // Number
obj: {a:1,b:2} //Object
}
req.query object is dynamic, it can contain any number of objects, array, boolean , number or strings. Is there any way to do it?
This functionality doesn't come out of the box with express and query parameters.
The problem is that in order for the query string parser to know if "true" is actual boolean true or the string "true" it needs some sort of Schema for the query object to help parsing the string.
Option A
What I can recommend is using Joi.
In your case it will look like :
const Joi = require( "joi" );
const querySchema = {
arr: Joi.array(),
condition1: Joi.boolean(),
id: Joi.number(),
obj: {
a: Joi.number(),
b: Joi.number()
}
}
Having this schema you can attach it to your express method and use Joi.validate To validate it.
function getFoo( req, res, next ) {
const query = req.query; // query is { condition1: "true" // string, ... }
Joi.validate( query, querySchema, ( err, values ) => {
values.condition1 === true // converted to boolean
} );
}
Option B
Another way of having properly typed GET requests would be to trick the query parameters and just provide a stringified JSON.
GET localhost/foo?data='{"foo":true,"bar":1}'
This will give you the possibility to just parse the request query
function getFoo( req, res, next ) {
const data = JSON.parse( req.query.data )
data.foo // boolean
data.bar // number
}
Cloud Functions & Firestore: Iterate over document.data
The database structure looks like this:
User {id}
Settings (Collection)
device_uids(document)
{device_uid_1}: Boolean
(...)
{device_uid_n}: Boolean
I want to get the document and access all of the device_uids within that document.
I tried like this, however the console logs, that forEach is not definded:
const settings_ref = admin.firestore().collection('User').doc(uid).collection('Settings').doc('device_uids');
settings_ref.get()
.then(snap =>{
let uids = snap.data();
uids.array.forEach(element => {
let device = element.key;
if(device != device_uid){
//GO ON
}
});
})
How can I access the values individually?
You don't have a field called array in your document, so uids.array will always be undefined. If you just want to iterate all the properties of the document, it's just like iterating all the properties of a plain old JavaScript object:
const data = snap.data();
for (const key in data) {
const value = data[key];
// now key and value are the property name and value
}
array manipulation in node js and lodash
I have two arrays
typeArr = [1010111,23342344]
infoArr={'name':'jon,'age':25}
I am expecting following
[{'name:'jone','age':25,'type':1010111,'default':'ok'},{'name:'jone','age':25,'type':23342344,'default':'nok'}]
Code :
updaterecord(infoArr,type)
{
infoArr.type=type;
response = calculate(age);
if(response)
infoArr.default = 'ok';
else
infoArr.default = 'nok';
return infoArr;
}
createRecord(infoArr,typeArr)
{
var data = _.map(typeArr, type => {
return updaterecord(infoArr,type);
});
return (data);
}
var myData = createRecord(infoArr,typeArr);
I am getting
[{'name:'jone,'age':25.'type':23342344,'default':nok},{'name:'jone,'age':25.'type':23342344,'default':nok}]
with some reason the last record updates the previous one. I have tried generating array using index var but not sure what's wrong it keep overriding the previous item.
how can I resolve this
You are passing the entire infoArr array to your updaterecord() function, but updaterecord() looks like it's expecting a single object. As a result it is adding those properties to the array rather than individual members of the array.
It's not really clear what is supposed to happen because typeArr has two elements and infoArr has one. Do you want to add another to infoArr or should infoArr have the same number of elements as typeArr.
Assuming it should have the same number you would need to use the index the _map gives you to send each item from infoArr:
function createRecord(infoArr,typeArr) {
var data = _.map(typeArr, (type, i) => {
// use infoArr[i] to send one element
return updaterecord(infoArr[i],type);
});
return (data);
}
Edit:
I'm not sure how you are calculating default since it's different in your expected output, but based on one number. To get an array of objects based on infoArray you need to copy the object and add the additional properties the you want. Object.assign() is good for this:
let typeArr = [1010111,23342344]
let infoArr={'name':'jon','age':25}
function updaterecord(infoArr,type){
var obj = Object.assign({}, infoArr)
return Object.assign(obj, {
type: type,
default: infoArr.age > 25 ? 'ok' : 'nok' //or however your figuring this out
})
}
function createRecord(infoArr,typeArr) {
return _.map(typeArr, type => updaterecord(infoArr,type));
}
Result:
[ { name: 'jon', age: 25, type: 1010111, default: 'nok' },
{ name: 'jon', age: 25, type: 23342344, default: 'nok' } ]
Mongoose dynamic query
Query parsed from URL, example :
?year=2014&cat=sonny
Or it can be
?year=2014&id=223&something=high&cat=sonny
I could do
Model.find({year: 2014}).where('cat').equals('sonny')
But what if there a second example? How can I make it dynamic?
You can set the query to a variable and add multiple conditions:
var query = Model.find();
query.where('year').equals('2014');
query.where('cat').equals('sonny');
query.where('id').equals('223');
query.where('something').equals('high');
query.exec(callback);
For dynamic, just pass the query to a for loop and iterate through an array of your filter objects:
var query = Model.find();
var filters = [
{fieldName: "year", value: "2014"},
{fieldName: "cat", value: "sonny"}
...
];
for (var i = 0; i < filters.length; i++) {
query.where(filters[i].fieldName).equals(filters[i].value)
}
query.exec(callback);
Building on the cdbajorin's answer - I suspect many coders are trying to take input from a form and dynamically build a Mongoose filter from the end users input. (or at least that was my scenario).
If you 'name' the html input fields the same as your Mongoose Schema name
<input type='text' name='person.address'>
Then in your code you can use the req.body object
var query = Model.find();
for (var fieldName in req.body)
{
if(req.body.hasOwnProperty(fieldName)) //no inherited properties
{
if(req.body[fieldName]) //get rid of empty fields
{
query.where(fieldName).equals(req.body[fieldName]);
}
}
}
query.exec(function(err,data){console.log('QUERY EXECUTE : ' + err, data, data.length);});