improving query - subquery

IS there anyway i can improve this query?
SELECT DISTINCT adb_product_type.strproduct_type AS strproduct_type, adb_product_cat.strproduct_cat AS strproduct_cat, adb_product.strproduct AS strproduct,
CASE #campaignCount:=
(SELECT COUNT(DISTINCT adb_campaign_1.campaign_id)
FROM adb_camp_media AS adb_camp_media_1
LEFT JOIN adb_campaign AS adb_campaign_1 ON adb_campaign_1.campaign_id = adb_camp_media_1.campaign_id
LEFT JOIN adb_camp_media_prod AS adb_camp_media_prod_1 ON adb_camp_media_prod_1.media_num = adb_camp_media_1.media_num
WHERE
(adb_camp_media_1.inttotal_per_store = 1 AND adb_camp_media_1.lngproduct = adb_product.lngproduct OR adb_camp_media_1.inttotal_per_store > 1 AND adb_camp_media_prod_1.lngproduct = adb_product.lngproduct)
AND (
(
adb_campaign_1.start_date
BETWEEN '2011-01-01'
AND '2011-01-31'
)
OR (
adb_campaign_1.end_date
BETWEEN '2011-01-01'
AND '2011-01-31'
)
OR (
adb_campaign_1.start_date < '2011-01-01'
AND adb_campaign_1.end_date > '2011-01-31'
)
)
)
WHEN 0 THEN 'No' ELSE 'Yes' END AS 'YesNo',
CASE #campaignCount WHEN 0 THEN '' ELSE #campaignCount END AS 'CampaignCount',
CASE #campaignCount WHEN 0 THEN '' ELSE
(SELECT GROUP_CONCAT(DISTINCT adb_media_1.name ORDER BY adb_media_1.name SEPARATOR ', ' )
FROM adb_camp_media AS adb_camp_media_1
LEFT JOIN adb_campaign AS adb_campaign_1 ON adb_campaign_1.campaign_id = adb_camp_media_1.campaign_id
LEFT JOIN adb_camp_media_prod AS adb_camp_media_prod_1 ON adb_camp_media_prod_1.media_num = adb_camp_media_1.media_num
LEFT JOIN adb_media AS adb_media_1 ON adb_media_1.media_id = adb_camp_media_1.media_id
WHERE
(adb_camp_media_1.inttotal_per_store = 1 AND adb_camp_media_1.lngproduct = adb_product.lngproduct OR adb_camp_media_1.inttotal_per_store > 1 AND adb_camp_media_prod_1.lngproduct = adb_product.lngproduct)
AND (
(
adb_campaign_1.start_date
BETWEEN '2011-01-01'
AND '2011-01-31'
)
OR (
adb_campaign_1.end_date
BETWEEN '2011-01-01'
AND '2011-01-31'
)
OR (
adb_campaign_1.start_date < '2011-01-01'
AND adb_campaign_1.end_date > '2011-01-31'
)
)
)
END AS mediaName , adb_product.lngproduct
FROM adb_product_type
LEFT JOIN adb_product_cat ON adb_product_cat.lngproduct_type = adb_product_type.lngproduct_type
LEFT JOIN adb_product ON adb_product.lngproduct_cat = adb_product_cat.lngproduct_cat
LEFT JOIN adb_camp_media ON adb_camp_media.lngproduct = adb_product.lngproduct
LEFT JOIN adb_media ON adb_media.media_id = adb_camp_media.media_id
LEFT JOIN adb_camp_media_prod ON adb_camp_media_prod.media_num = adb_camp_media.media_num
LEFT JOIN adb_campaign ON adb_campaign.campaign_id = adb_camp_media.campaign_id
WHERE 1=1
ORDER BY YesNo DESC , strproduct_type, strproduct_cat, strproduct
TABLE STRUCTURE
-- Table structure for table adb_camp_media
CREATE TABLE adb_camp_media (
media_num int(11) NOT NULL auto_increment,
campaign_id int(11) NOT NULL default '0',
media_id int(11) NOT NULL default '0',
inttotal_per_store int(4) NOT NULL default '0',
units_per_item int(2) default NULL,
lngproduct int(11) NOT NULL default '0',
rental_cost double(10,2) default NULL,
checkers_rental double default NULL,
hyper_rental double default NULL,
usave_rental double default NULL,
ls_rental double(10,2) default '0.00',
spar_rental double(10,2) default '0.00',
backboard tinyint(1) NOT NULL,
PRIMARY KEY (media_num),
KEY campaign_id (campaign_id,media_id,lngproduct),
KEY lngproduct (lngproduct)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table adb_camp_media_prod
CREATE TABLE adb_camp_media_prod (
media_prod_id int(11) NOT NULL auto_increment,
media_num int(4) NOT NULL,
lngproduct int(4) NOT NULL,
PRIMARY KEY (media_prod_id),
KEY media_num (media_num,lngproduct),
KEY lngproduct (lngproduct)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table adb_campaign
CREATE TABLE adb_campaign (
campaign_id int(11) NOT NULL auto_increment,
client_id int(11) NOT NULL default '0',
campaign_name varchar(50) NOT NULL default '',
description text,
position_desc text,
start_date date NOT NULL default '0000-00-00',
end_date date NOT NULL default '0000-00-00',
approved tinyint(1) NOT NULL default '0',
created_date date NOT NULL default '0000-00-00',
updated_date datetime default NULL,
user_id int(11) NOT NULL default '0',
pinno varchar(50) NOT NULL default '',
jobno varchar(30) NOT NULL default '',
quoteno varchar(30) NOT NULL default '',
sales_ae_id int(11) default NULL,
artwork_specno varchar(30) NOT NULL default '',
artwork_jobno varchar(30) NOT NULL default '',
agency tinyint(1) NOT NULL default '0',
brand_name varchar(50) NOT NULL default '',
paypercycle tinyint(1) NOT NULL default '0',
agency_comm double default NULL,
variant varchar(100) default NULL,
pack_size varchar(100) default NULL,
discount double default NULL,
discount_val double default NULL,
sales_comm double NOT NULL default '0',
cancelled tinyint(1) default '0',
onhold tinyint(1) default '0',
campaign_status_id int(4) default NULL,
PRIMARY KEY (campaign_id),
UNIQUE KEY jobno (jobno),
KEY start_date (start_date),
KEY end_date (end_date),
KEY campaign_name (campaign_name),
KEY user_id (user_id,cancelled,onhold)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table adb_product
CREATE TABLE adb_product (
lngproduct int(11) NOT NULL auto_increment,
strproduct varchar(100) NOT NULL default '',
lngproduct_cat int(11) NOT NULL default '0',
PRIMARY KEY (lngproduct),
KEY lngproduct_cat (lngproduct_cat)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table adb_product_cat
CREATE TABLE adb_product_cat (
lngproduct_cat int(11) NOT NULL auto_increment,
strproduct_cat varchar(100) NOT NULL default '',
lngproduct_type int(11) NOT NULL default '0',
PRIMARY KEY (lngproduct_cat),
KEY lngproduct_type (lngproduct_type)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
--
-- Table structure for table adb_product_type
CREATE TABLE adb_product_type (
lngproduct_type int(11) NOT NULL auto_increment,
strproduct_type varchar(100) NOT NULL default '',
PRIMARY KEY (lngproduct_type)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;

Related

importing multiple excel files with multiple sheets and different column headers

Need help please.
I have excel files where some sheets are as follows:
enter image description here
enter image description here
I need to insert into the db table the matching columns from the excel file but for example WID and eename are both under column J and status column is in N but some sheets end at M.
When I print column names this is what I get:
ToPay,CompEntity,ProjectName,Details,Amount,Invoicenumber,PayInCurrency,PayOutCurrency,Paymentdate,PaidFrom,Addedby,Unnamed:13,Status
def readfile(self):
filepath = 'MatchPoin EUR - PT123.xlsx'
df1 = pd.read_excel(filepath, sheet_name=None, usecols="A,B,C,E,F,G,H,I,K,L,M,N")
df2 = pd.concat(df1, ignore_index=True)
return df2
def insert_to_db(self, df):
conn = self.db[0]
cursor = self.db[1]
col_list = df.columns.str.replace(' ', '')
insert_columns = ",".join(col for col in col_list)
print(insert_columns)
index_list = []
for _ in col_list:
index_list.append("%s")
index_str = ",".join(index for index in index_list)
values_list = df.values
insert_values_list = [tuple(values) for values in values_list]
num_rows_inserted = 0
# query = f"""insert into suppay_payments_stagingtable ({insert_columns}) values({index_str})"""
# print (insert_values_list)
for k, v in enumerate(insert_values_list):
try:
cursor.callproc('ins_beneficiaryfromfiles', v)
num_rows_inserted = num_rows_inserted + cursor.rowcount
conn.commit()
except Exception as e:
conn.rollback()
print(e)
logging.error(e)
My database table:
CREATE TABLE `suppay_payments_stagingtable` (
`payID` int(11) NOT NULL AUTO_INCREMENT,
`toPay` varchar(100) DEFAULT NULL,
`papayaEntityName` varchar(200) DEFAULT NULL,
`projectID` varchar(100) DEFAULT NULL,
`projectName` varchar(200) DEFAULT NULL,
`details` varchar(200) DEFAULT NULL,
`amount` decimal(10,3) DEFAULT NULL,
`invoiceNumber` varchar(50) DEFAULT NULL,
`payInCurrency` varchar(5) DEFAULT NULL,
`payOutCurrency` varchar(5) DEFAULT NULL,
`paymentDate` varchar(200) DEFAULT NULL,
`paidFrom` varchar(500) DEFAULT NULL,
`addedBy` varchar(200) DEFAULT NULL,
`WID` varchar(50) DEFAULT NULL,
`status` varchar(50) DEFAULT NULL,
`eename` varchar(50) DEFAULT NULL,
`bookKeepingNotes` varchar(200) DEFAULT NULL,
`noteDateBookeeping` date DEFAULT NULL,
`underOverPayment` varchar(200) DEFAULT NULL,
`opsNotes` varchar(200) DEFAULT NULL,
`opsNoteDate` varchar(200) DEFAULT NULL,
`sourcefilename` varchar(1000) DEFAULT NULL,
`sheetname` varchar(20) DEFAULT NULL,
`beneficiaryname` varchar(200) DEFAULT NULL,
`beneficiarycode` varchar(20) DEFAULT NULL,
`createdAt` datetime DEFAULT current_timestamp(),
`updateAt` datetime DEFAULT current_timestamp(),
`deposit` varchar(100) DEFAULT NULL,
PRIMARY KEY (`payID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

How do I insert records in a node-pg-migrate migration?

I'm trying to use node-pg-migrate to handle migrations for an ExpressJS app. I can translate most of the SQL dump into pgm.func() type calls, but I can't see any method for handling actual INSERT statements for initial data in my solution's lookup tables.
It is possible using the pgm.sql catch all:
pgm.sql(`INSERT INTO users (username, password, created, forname, surname, department, reviewer, approver, active) VALUES
('rd#example.com', 'salty', '2019-12-31 11:00:00', 'Richard', 'Dyce', 'MDM', 'No', 'No', 'Yes');`)
Note the use of backtick (`) to allow breaking the SQL statement across multiple lines.
You can use raw sql if you needed.
Create a migration file with the extension .sql and write usual requests.
This article has a great example.
My example:
-- Up Migration
CREATE TABLE users
(
id BIGSERIAL NOT NULL PRIMARY KEY,
username VARCHAR(50) NOT NULL,
email VARCHAR(50) NOT NULL,
password VARCHAR(50) NOT NULL,
class_id INTEGER NOT NULL,
created_at DATE NOT NULL,
updated_at DATE NOT NULL
);
CREATE TABLE classes
(
id INTEGER NOT NULL PRIMARY KEY,
name VARCHAR(50) NOT NULL,
health INTEGER NOT NULL,
damage INTEGER NOT NULL,
attack_type VARCHAR(50) NOT NULL,
ability VARCHAR(50) NOT NULL,
created_at DATE NOT NULL,
updated_at DATE NOT NULL
);
INSERT INTO classes (id,
name,
health,
damage,
attack_type,
ability,
created_at,
updated_at)
VALUES (0,
'Thief',
100,
25,
'Archery Shot',
'Run Away',
NOW(),
NOW());
-- Down Migration
DROP TABLE users;
DROP TABLE classes;

memsql show slow performance response using order by

I have the following query:
SELECT * FROM scheme.table cont WHERE game_id = 'some-game-id' AND event_arrival_time BETWEEN '2019-12-02 00:00:00' AND '2019-12-31 23:59:59' ORDER BY event_arrival_time
I get response time of almost 30 seconds
for the same query if I remove ORDER BY event_arrival_time i get the response in few seconds
This is the create table query:
CREATE TABLE `cont_event` ( `event_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `action` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `correlation_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `status` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `event_arrival_time` datetime DEFAULT NULL, `create_time` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, `create_ts` bigint(20) DEFAULT NULL, `operator_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `game_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `player_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `segment_code` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `bet_amount_original` decimal(15,4) DEFAULT NULL, `bet_amount_converted` decimal(15,4) DEFAULT NULL, `cont_amount_player` decimal(15,4) DEFAULT NULL, `cont_amount_operator` decimal(15,4) DEFAULT NULL, `cont_amount_total` decimal(15,4) DEFAULT NULL, `operator_income` decimal(15,4) DEFAULT NULL, `cont_amount_jackpot` decimal(15,4) DEFAULT NULL, `original_currency` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `base_currency` varchar(20) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `currency_rate` decimal(20,6) DEFAULT NULL, `operator_game_code` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `funnel_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `segment_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `operator_game_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `description` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `extra_fields` longtext CHARACTER SET utf8 COLLATE utf8_general_ci, `jackpot_game_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `game_version` varchar(16) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, `event_type` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, `event` JSON COLLATE utf8_bin, KEY `operator_id` (`event_arrival_time`,`action`,`operator_id`,`game_id`,`correlation_id`) /*!90619 USING CLUSTERED COLUMNSTORE */ /*!90618 , SHARD KEY () */ ) /*!90621 AUTOSTATS_ENABLED=TRUE */
i do have index on: event_arrival_time, action, operator_id, game_id, correlation_id
when doing memsql profile i can see the filter works fine(see attached):
iam not sure what am I missing? any suggestions for optimization?
Here is your first query repeated:
SELECT *
FROM scheme.table cont
WHERE
game_id = 'some-game-id' AND
event_arrival_time BETWEEN '2019-12-02 00:00:00' AND '2019-12-31 23:59:59'
ORDER BY
event_arrival_time
The index you have currently defined looks either wrong to me, or at least sub-optimal. Try the following index instead:
CREATE INDEX idx ON cont (event_arrival_time, game_id);
Ideally, MemSQL would be able to scan the above index, in the order of the event arrival time, in the range you defined, and retrieve all matching records.

Cassandra - query not returning result with frozen type

I have created a cassandra table using few primary datatypes and a frozen type address_type.
CREATE TYPE address_type (
first_name text,
last_name text,
address_line1 text,
address_line2 text
);
CREATE TABLE user (
id text,
active_profile boolean,
addresses frozen<address_type>,
PRIMARY KEY (id)
);
And indexed the columns addresses because I want to select few resultset based on address_type.first_name.
CREATE INDEX ON user (addresses) ;
Finally this is my query which returns 0 rows.
select * from user where addresses = {first_name:'test2'};
When I tried
select * from "user" where addresses > {first_name:'test2'};
Which resulted in
code=2200 [Invalid query] message="No secondary indexes on the restricted columns support the provided operators: 'addresses > <value>'"
Can someone help me? Where I am going wrong here?
Let's insert some data :
cqlsh:test> INSERT INTO user (id , addresses) VALUES ('user_0', {first_name:'Ashraful', last_name:'Islam'});
cqlsh:test> INSERT INTO user (id , addresses) VALUES ('user_1', {first_name:'Ashraful'});
cqlsh:test> SELECT * FROM user ;
id | active_profile | addresses
--------+----------------+----------------------------------------------------------------------------------------
user_1 | null | {first_name: 'Ashraful', last_name: null, address_line1: null, address_line2: null}
user_0 | null | {first_name: 'Ashraful', last_name: 'Islam', address_line1: null, address_line2: null}
Since addresses is frozen type. you can't query with a piece of a frozen field. You have to provide full value of addresses
Example :
cqlsh:test> SELECT * FROM user WHERE addresses = {first_name:'Ashraful', last_name:'Islam'} ;
id | active_profile | addresses
--------+----------------+----------------------------------------------------------------------------------------
user_0 | null | {first_name: 'Ashraful', last_name: 'Islam', address_line1: null, address_line2: null}
(1 rows)
cqlsh:test> SELECT * FROM user WHERE addresses = {first_name: 'Ashraful'} ;
id | active_profile | addresses
--------+----------------+-------------------------------------------------------------------------------------
user_1 | null | {first_name: 'Ashraful', last_name: null, address_line1: null, address_line2: null}
(1 rows)

Sequelizejs HasMany ternary relationship

I want to make in Sequelize a ternary relationship as seen in the picture below
Image
https://www.dropbox.com/s/v8bgsir2qqw6ccv/relationship%20.jpg
If I apply this code in sequelize
A.hasMany(B);
A.hasMany(C);
B.hasMany(C);
C.hasMany(A);
C.hasMany(B);
The resulting SQL code is as follows
CREATE TABLE IF NOT EXISTS `a_b_` (
PRIMARY KEY (`BId`,`AId`)
)
CREATE TABLE IF NOT EXISTS `a_c_` (
`CId` int(11) NOT NULL DEFAULT '0',
`AId` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`CId`,`AId`)
)
CREATE TABLE IF NOT EXISTS `b_c_` (
PRIMARY KEY (`CId`,`BId`)
)
But the result should be
CREATE TABLE IF NOT EXISTS `a_b_` (
PRIMARY KEY (`BId`,`AId`)
)
CREATE TABLE IF NOT EXISTS `a_b_c_` (
`AId` int(11) NOT NULL DEFAULT '0',
`BId` int(11) NOT NULL DEFAULT '0',
`CId` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`AId`,`BId`, 'CId')
)
I just can not get create a table with pk (AId, BId, CId) someone could indicate me which way to go or what I can do.
Thanks so much.

Resources