Aggregation of aggregation on Google Bigquery - struct

My Data looks like this
WITH test AS (
SELECT * FROM UNNEST([
STRUCT('2019-10-26' as date,'1.8025137' AS article_id, 'Digital Paying' as user_type,'open' as openmode, '123' as uid),
('2019-10-26','1.8025137' , 'Digital Paying','close', '523'),
('2019-10-26','1.8025137' , 'Anonymous','open', '321'),
('2019-10-26','1.8025137' , 'Registered','close', '231'),
('2019-10-26','1.8025137' , 'Registered','open', '431'),
('2019-10-26','1.8025137' , 'Digital Paying','close', '132'),
('2019-10-26','1.8025137' , 'Anonymous','close', '111')
])
),
-- first level of aggregation, prepare for fine tuning
date_article as (
SELECT
date,
article_id,
ARRAY_AGG(struct(user_type,openmode, uid)) AS ut
FROM test
GROUP BY 1,2
)
(SELECT
date,
article_id,
-- feed sub-query output into an array "action"
array(SELECT AS STRUCT
user_type as user_type, -- re-group data within the array by field "action"
array_agg(struct(openmode as openmode,uid as uid) ) op
FROM UNNEST(ut)
GROUP BY 1
) as user_types
FROM date_article)
My goal is to aggregate the user_types.op.openmode and user_types.op.uid by
user_types.user_type without create any duplicates as :

I think you are looking for below
#standardSQL
WITH test AS (
SELECT * FROM UNNEST([
STRUCT('2019-10-26' AS DATE,'1.8025137' AS article_id, 'Digital Paying' AS user_type,'open' AS openmode, '123' AS uid),
('2019-10-26','1.8025137' , 'Digital Paying','close', '523'),
('2019-10-26','1.8025137' , 'Anonymous','open', '321'),
('2019-10-26','1.8025137' , 'Registered','close', '231'),
('2019-10-26','1.8025137' , 'Registered','open', '431'),
('2019-10-26','1.8025137' , 'Digital Paying','close', '132'),
('2019-10-26','1.8025137' , 'Anonymous','close', '111')
])
), users_agg AS (
SELECT DATE, article_id, user_type, openmode, COUNT(DISTINCT uid) AS uids
FROM test GROUP BY 1,2,3,4
), modes_agg AS (
SELECT DATE, article_id, user_type, ARRAY_AGG(STRUCT(openmode, uids)) AS modes
FROM users_agg GROUP BY 1,2,3
), types_agg AS (
SELECT DATE, article_id, ARRAY_AGG(STRUCT(user_type, modes)) types
FROM modes_agg GROUP BY 1,2
), article_agg AS (
SELECT DATE, ARRAY_AGG(STRUCT(article_id, types)) articles
FROM types_agg GROUP BY 1
)
SELECT *
FROM article_agg
with result

You were making it a bit more complicated than necessary. If possible, do your 'normal' SQL first and then format into arrays/structs afterwards.
WITH test AS (
SELECT * FROM UNNEST([
STRUCT('2019-10-26' as date,'1.8025137' AS article_id, 'Digital Paying' as user_type,'open' as openmode, '123' as uid),
('2019-10-26','1.8025137' , 'Digital Paying','close', '523'),
('2019-10-26','1.8025137' , 'Anonymous','open', '321'),
('2019-10-26','1.8025137' , 'Registered','close', '231'),
('2019-10-26','1.8025137' , 'Registered','open', '431'),
('2019-10-26','1.8025137' , 'Digital Paying','close', '132'),
('2019-10-26','1.8025137' , 'Anonymous','close', '111')
])
),
agg as (
select
date,
article_id,
user_type,
openmode,
count(distinct uid) as uids
from test
group by 1,2,3,4
),
final as (
select
date,
article_id,
user_type,
array_agg(struct(openmode, uids)) as subfields
from agg
group by 1,2,3
)
select * from final

Related

How to SortByColumns, Search, Filter in powerapps

I got it to search for one column, but how to I add more items? ie "Material", "Destination",etc
SortByColumns(
Search(
Filter(
'Transfer Request',
"Not-Completed" in Putaway.Value
),
Search_SC.Text,
"Material"
),
"ID",
If(
SortDescending1,
Ascending,
Descending
)
)
The Search function can search over multiple columns:
SortByColumns(
Search(
Filter( 'Transfer Request', "Not-Completed" in Putaway.Value ),
Search_SC.Text,
"Material", "Destination", "OtherColumn" ),
"ID",
If( SortDescending1, SortOrder.Ascending, SortOrder.Descending ) )

filter a list of tuples using filter() function and lambda expression

I am learning Python programming;i'm trying to filter a list of tuples using filter() function and lambda expression. but there is something wrong.
The code I have tried is as follows:
state_and_population = [
( 'Uttar Pradesh' , 199812341 ),
( 'Maharashtra' , 112372972 ),
( 'Bihar' , 103804637 ),
( 'West Bengal' , 91347736 ),
( 'Madhya Pradesh' , 72597565 ),
( 'Tamil Nadu' , 72138958 ),
( 'Rajasthan' , 68621012 ),
( 'Karnataka' , 61130704 ),
( 'Gujarat' , 60383628 ),
( 'Andhra Pradesh' , 49386799 )
]
filtered_object = filter( lambda s_a_p: [ s_a_p[0] , s_a_p[1] < 0 ] ,state_and_population )
print( list( filtered_object ) )
The list should print an empty list according to my given condition but it is showing all the elements
[('Uttar Pradesh', 199812341), ('Maharashtra', 112372972), ('Bihar', 103804637), ('West Bengal', 91347736), ('Madhya Pradesh', 72597565), ('Tamil Nadu', 72138958), ('Rajasthan', 68621012), ('Karnataka', 61130704), ('Gujarat', 60383628), ('Andhra Pradesh', 49386799)]
Your lambda lambda s_a_p: [ s_a_p[0] , s_a_p[1] < 0 ] returns a non-empty list which, in Python, is considered a True boolean that, in turn, causes filter to accept the corresponding item from the iterator.
Write the lambda as follows: lambda e: e[1] < 0

How to Automate SQL code using Python in efficient way

I have a sql code which looks like this:
Database: RedShift
WITH
X as
(
SELECT distinct pn , pg , ic, sr , cm, fq , m1 , m2 , m3 , m4
FROM table1 ORDER BY 1,2,3
),
table2 AS
(
Select g,p,t , avg(ss) as ss , avg(ce) as ce , sum(av) as ps
from
(
select distinct ic AS g , pn AS p , cm AS t , ss , cast((sum_m1/nullif(sum_m2,0)) as decimal(3,2)) as
ce , av
from
(
select *
, cast((sum(m3) over (partition by ic, pn,cm)) as decimal) as ss
, sum(m1) over (partition by ic, pn,cm) as sum_m1
, sum(m2) over (partition by ic, pn,cm) as sum_m2
, cast((avg(m2) over (partition by ic, pn,cm)) as decimal) as av
from X
ORDER BY 1,2,3
)
order by 1,2,3
)
where ss is not null
group by 1,2,3
order by 1,2,3
)
The group by value g,p,t changes every time and so it creates table for every new combination of g,p,t values.
One way to automate is dump this sql code in Python which might be inefficient:
Here is my approach-> I replace all values in list by {} braces
Example say:
I store all possible group by values in a list.
G=[g1,g2,g3]
P=[p1,p2,p3]
T=[t1,t2,t3]
Connection to databse:
c= psycopg2.connect(database=db,host=host,port=port,user=user,password=password,sslmode='require')
data2={}
for g in G:
for p in P:
for t in T:
sqlstr=( """ WITH
X as
(
SELECT distinct pn , pg , ic, sr , cm, fq , m1 , m2 , m3 , m4
FROM table1 ORDER BY 1,2,3
),
table2 AS
Select {},{},{} , avg(ss) as ss , avg(ce) as ce , sum(av) as ps
from
(
select distinct ic AS g , pn AS p , cm AS t , ss , cast((sum_m1/nullif(sum_m2,0)) as decimal(3,2)) as
ce , av
from
(
select *
, cast((sum(m3) over (partition by ic, pn,cm)) as decimal) as ss
, sum(m1) over (partition by ic, pn,cm) as sum_m1
, sum(m2) over (partition by ic, pn,cm) as sum_m2
, cast((avg(m2) over (partition by ic, pn,cm)) as decimal) as av
from X
ORDER BY 1,2,3
)
order by 1,2,3
)
where ss is not null
group by 1,2,3
order by 1,2,3
), select * from table2 """).format(g,p,t)
data2[g+"_"+p+"_"+t] = pd.read_sql_query(sqlstr, c)
Is there any better way to pass parameters as in above code sequence of {} should be maintained to pass parameters in order ??
Can we use some other approach other than SQL? In pythonic way?

Recursive Relationship Query

I am looking to implement graph tables to map the role hierarchy for my application in Azure SQL. The graph will look like a tree if it is laid out. With the parent being able to manage any role that falls beneath it on the tree.
So I have a roles node table and a canmanage edge table.
I am familiar with querying the first level and the second level of relationships, however I need to have a query where I can put in any role and receive a list of all the children that fall under it.
I am familiar with this sort of thing in NEO4J, but I have not found any documentation on how to accomplish this in Azure SQL.
How do I go about running a recursive query to get all the child roles give a specific role name or id?
This is possible from SQL Server 2017 and Azure SQL DB using the new graph database capabilities and the new MATCH clause to model this type of relationship. Unfortunately in v1 polymorphism and transitive closure are not natively included but are possible using recursive queries. If you look at the last query, it keep the parameter you input as the top-level manager and iterates over the rest.
A sample script:
USE tempdb
GO
-- NODES
DROP TABLE IF EXISTS dbo.roles
-- EDGES
DROP TABLE IF EXISTS dbo.canManage
DROP TABLE IF EXISTS dbo.isManagedBy
GO
CREATE TABLE dbo.roles (
roleId INT PRIMARY KEY,
roleName VARCHAR(20) UNIQUE NOT NULL
) AS NODE
CREATE TABLE dbo.canManage AS EDGE;
CREATE TABLE dbo.isManagedBy AS EDGE;
GO
-- Populate node table
INSERT INTO dbo.roles ( roleId, roleName )
VALUES
( 1, 'CEO' ),
( 2, 'VP 1' ),
( 3, 'VP 2' ),
( 4, 'Sales Manager 1' ),
( 5, 'Sales Manager 2' ),
( 6, 'Ops Manager 1' ),
( 7, 'Ops Manager 2' ),
( 8, 'Sales Lead 1' ),
( 9, 'Salesperson 1' ),
( 10, 'Salesperson 2' ),
( 11, 'Salesperson 3' )
GO
-- Populate edge table
INSERT INTO dbo.canManage ( $from_id, $to_id )
SELECT ceo.$node_id, VPs.$node_id
FROM dbo.roles ceo
CROSS JOIN dbo.roles VPs
WHERE ceo.roleName = 'CEO'
AND VPs.roleName Like 'VP%'
-- VP 1 manages Sales Managers
INSERT INTO dbo.canManage ( $from_id, $to_id )
SELECT a.$node_id, b.$node_id
FROM dbo.roles a
CROSS JOIN dbo.roles b
WHERE a.roleName = 'VP 1'
AND b.roleName Like 'Sales Manager%'
-- VP 2 manages Ops Managers
INSERT INTO dbo.canManage ( $from_id, $to_id )
SELECT a.$node_id, b.$node_id
FROM dbo.roles a
CROSS JOIN dbo.roles b
WHERE a.roleName = 'VP 2'
AND b.roleName Like 'Ops Manager%'
-- Sales Manger 1 manages Sales Leads
INSERT INTO dbo.canManage ( $from_id, $to_id )
SELECT a.$node_id, b.$node_id
FROM dbo.roles a
CROSS JOIN dbo.roles b
WHERE a.roleName = 'Sales Manager 1'
AND b.roleName Like 'Sales Lead%'
-- Sales Leads 1 manages all salespersons
INSERT INTO dbo.canManage ( $from_id, $to_id )
SELECT a.$node_id, b.$node_id
FROM dbo.roles a
CROSS JOIN dbo.roles b
WHERE a.roleName = 'Sales Lead 1'
AND b.roleName Like 'Salesperson%'
-- Create the inverse edge / relationship
INSERT INTO dbo.isManagedBy ( $from_id, $to_id )
SELECT $to_id, $from_id
FROM dbo.canManage
GO
-- Now write the graph queries:
-- Manages
SELECT FORMATMESSAGE( '%s manages %s', r1.roleName, r2.roleName ) manages
FROM dbo.roles r1, dbo.canManage canManage, dbo.roles r2
WHERE MATCH ( r1-(canManage)->r2 )
-- Same manager
SELECT FORMATMESSAGE( '%s and %s have the same manager %s', r1.roleName, r3.roleName, r2.roleName )
FROM dbo.roles r1, dbo.isManagedBy m1, dbo.roles r2, dbo.isManagedBy m2, dbo.roles r3
WHERE MATCH ( r1-(m1)->r2<-(m2)-r3 )
AND r1.$node_id < r3.$node_id
-- Recursive
-- walk the tree ... CEO manages everyone ...
;WITH cte AS (
SELECT 1 xlevel, r1.roleName manager, r2.roleName managed
FROM dbo.roles r1, dbo.canManage canManage, dbo.roles r2
WHERE MATCH ( r1-(canManage)->r2 )
AND r1.roleName = 'CEO'
UNION ALL
SELECT c.xlevel + 1, r1.roleName, r2.roleName
FROM cte c, dbo.roles r1, dbo.canManage canManage, dbo.roles r2
WHERE c.managed = r1.roleName
AND MATCH ( r1-(canManage)->r2 )
)
SELECT *
FROM cte
ORDER BY xlevel, manager, managed
;WITH cte AS (
SELECT 1 xlevel, r1.roleName manager, r2.roleName managed
FROM dbo.roles r1, dbo.canManage canManage, dbo.roles r2
WHERE MATCH ( r1-(canManage)->r2 )
AND r1.roleName = 'CEO'
UNION ALL
SELECT c.xlevel + 1, c.manager, r2.roleName
FROM cte c, dbo.roles r1, dbo.canManage canManage, dbo.roles r2
WHERE c.managed = r1.roleName
AND MATCH ( r1-(canManage)->r2 )
)
SELECT *
FROM cte
ORDER BY xlevel, manager, managed

Add extra fields in manage customer and edit customer section in magento

I am using Magento ver. 1.9.1.0
So please help me according to the version of magento. I am new in this.
I am facing the problem is that I want to use extra (Lat Long field in) manage customer->add new customer in magento backend, so that user can add latlong of the address(any latlong). Now I have tried the following steps from the website.
http://excellencemagentoblog.com/blog/2011/10/02/customer-registration-fields-magento1-6/
https://magento.stackexchange.com/questions/45620/how-to-add-extra-fields-in-registration-form
http://www.silksoftware.com/magento-module-creator/#instructions
but I can't solve the problem. Somewhere showing extra field for my account in frontend, but I want to use the backend extra field on manage customer as an address and save it.
So if possible please provide me the right way to do and if you have the right tutorial please reply me.
I am trying this from few days but still no luck.
Also I have tried to install some extension from mangeto connect but giving error
Thank you all, I have done this by the following queries.
INSERT INTO `sta_eav_attribute` (
attribute_id ,
entity_type_id ,
attribute_code ,
attribute_model ,
backend_model ,
backend_type ,
backend_table ,
frontend_model ,
frontend_input ,
frontend_label ,
frontend_class ,
source_model ,
is_required ,
is_user_defined ,
default_value ,
is_unique ,
note
)
VALUES (
NULL , '2', 'latlong', NULL , NULL , 'varchar', NULL , NULL , 'text', 'Lat Long', NULL , '', '1', '0', '0', '0', ''
);
INSERT INTO sta_eav_entity_attribute (
entity_attribute_id ,
entity_type_id ,
attribute_set_id ,
attribute_group_id ,
attribute_id ,
sort_order
)
VALUES (
NULL , '2', '2', '2', 'your id max of sta_eav_attribute', '200'
);
INSERT INTO sta_customer_eav_attribute (
attribute_id ,
is_visible ,
input_filter ,
multiline_count ,
validate_rules ,
is_system ,
sort_order ,
data_model
)
VALUES (
'your id max of sta_eav_attribute', '1', NULL , '1', NULL , '0', '0', NULL
);
INSERT INTO sta_customer_form_attribute (
form_code ,
attribute_id
)
VALUES (
'adminhtml_customer_address', 'your id max of sta_eav_attribute'
), (
'customer_address_edit', 'your id max of sta_eav_attribute'
), (
'customer_register_address', 'your id max of sta_eav_attribute'
);

Resources