How to improve write to synapse in Spark? - apache-spark

Question

I have a script to do write a dataframe to synapse, the flow of my script is Read from Synapse > Transform data on Databricks (GROUP BY CUBE) > Write to Synapse, the data that I read from synapse has 150 million row (Fact Table), I do GROUP BY CUBE to transform the fact table. Now, I have a problem here, the process from write so slowly, I still didn't know what's the problem here.
I have tried some solution:
- Auto optimize
%sql set spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite = true;
set spark.databricks.delta.properties.defaults.autoOptimize.autoCompact = true;
- Repartition
df.repartition(10000)
- Adaptive Query Execution (AQE)
spark.conf.set("spark.sql.adaptive.optimizeSkewedJoin.enabled", "true")
spark.conf.set("spark.sql.adaptive.enabled", "true")
- Write Semantics
spark.conf.set("spark.databricks.sqldw.writeSemantics", "copy")
Nothing from them solved my problem, and it still write slowly.
This is my time process. , as you can see, some of the job skipped, why does it happen?
This is my specification:
This is my read from synapse script:
df = (spark.read
.format("com.databricks.spark.sqldw")
.option("url", url)
.option("tempDir", tempDir)
.option("forwardSparkAzureStorageCredentials", "true")
.option("query", sql)
.load()
)
This is my write to synapse script:
df.write
.format("com.databricks.spark.sqldw")
.option("tableOptions", "CLUSTERED COLUMNSTORE INDEX, DISTRIBUTION = ROUND_ROBIN") # Added at 20200121
.option("url", url)
.option("dbtable", table)
.option("forward_spark_azure_storage_credentials","True")
.option("tempdir", tempDir)
.mode(write_mode)
.save()
This is my transform script:
cube_department_read = cube_department_df.cube(cube_department_df["YEAR"], cube_department_df["WeekOfYear"], cube_department_df["Month"],
cube_department_df["department_groups"], cube_department_df["category_name"],
cube_department_df["subcategory_name"], cube_department_df["section_name"]) \
.agg(F.max('last_date_of_week').alias('last_date_of_week'),
F.countDistinct('internal_tranx_key').alias('sales_basket'),
F.sum('SalesAmt').alias('sales_amt'),
F.sum('SalesQty').alias('sales_qty'),
F.sum('SalesQtyPro').alias('SalesQtyPro'),
F.sum('SalesAmtPro').alias('SalesAmtPro'),
F.countDistinct('membership_id').alias('member_count'),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_Member"),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_Member"),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
1).otherwise(0)).alias("Basket_Count_Member"),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
0).otherwise(cube_department_df["SalesQty"])).alias("SalesQty_NonMember"),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
0).otherwise(cube_department_df["SalesAmt"])).alias("SalesAmt_NonMember"),
F.sum(F.when(cube_department_df["membership_id"].isNotNull(),
0).otherwise(1)).alias("Basket_Count_NonMember"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_MMDS_Promotion"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_MMDS_Promotion"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
1).otherwise(0)).alias("Basket_Count_MMDS_Promotion"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
0).otherwise(cube_department_df["SalesAmt"])).alias("SalesAmt_Non_MMDS_Promotion"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
0).otherwise(cube_department_df["SalesQty"])).alias("SalesQty_Non_MMDS_Promotion"),
F.sum(F.when(cube_department_df["promotion_flag"] == 'Y',
0).otherwise(1)).alias("Basket_Count_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNotNull()),
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNotNull()),
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNotNull()),
1).otherwise(0)).alias("Basket_Count_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNull()),
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_Non_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNull()),
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_Non_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'Y') & (cube_department_df["membership_id"].isNull()),
1).otherwise(0)).alias("Basket_Count_Non_Member_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNotNull()),
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_Member_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNotNull()),
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_Member_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNotNull()),
1).otherwise(0)).alias("Basket_Count_Member_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNull()),
cube_department_df["SalesAmt"]).otherwise(0)).alias("SalesAmt_Non_Member_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNull()),
cube_department_df["SalesQty"]).otherwise(0)).alias("SalesQty_Non_Member_Non_MMDS_Promotion"),
F.sum(F.when((cube_department_df["promotion_flag"] == 'N') & (cube_department_df["membership_id"].isNull()),
1).otherwise(0)).alias("Basket_Count_Non_Member_Non_MMDS_Promotion"),
F.when((F.sum(cube_department_df["SalesQty"]) < 0) & (F.sum(cube_department_df["SalesAmt"]) < 0),
(F.sum(cube_department_df["SalesAmt"]) / F.sum(cube_department_df["SalesQty"])) * -1) \
.when((F.sum(cube_department_df["SalesQty"]) == 0) | (F.sum(cube_department_df["SalesAmt"]) == 0),
0).otherwise(F.sum(cube_department_df["SalesAmt"]) / F.sum(cube_department_df["SalesQty"])).alias("sales_per_unit"),
F.when((F.sum(cube_department_df["SalesQty"]) < 0) & (F.sum(cube_department_df["SalesAmt"]) < 0),
(F.sum(cube_department_df["SalesAmt"]) / F.countDistinct(cube_department_df["internal_tranx_key"])) * -1) \
.when((F.sum(cube_department_df["SalesQty"]) == 0) | (F.sum(cube_department_df["SalesAmt"]) == 0),
0).otherwise(F.sum(cube_department_df["SalesAmt"]) / F.countDistinct(cube_department_df["internal_tranx_key"])).alias("sales_per_basket"),
F.when((F.sum(cube_department_df["SalesQty"]) < 0) & (F.sum(cube_department_df["SalesAmt"]) < 0),
(F.sum(cube_department_df["SalesQty"]) / F.countDistinct(cube_department_df["internal_tranx_key"])) * -1) \
.when((F.sum(cube_department_df["SalesQty"]) == 0) | (F.sum(cube_department_df["SalesAmt"]) == 0),
0).otherwise(F.sum(cube_department_df["SalesQty"]) / F.countDistinct(cube_department_df["internal_tranx_key"])).alias("unit_per_basket"),
F.when((F.countDistinct(cube_department_df["membership_id"]) < 0) & (F.sum(cube_department_df["SalesAmt"]) < 0),
(F.sum(cube_department_df["SalesAmt"]) / F.countDistinct(cube_department_df["membership_id"])) * -1) \
.when((F.countDistinct(cube_department_df["membership_id"]) == 0) | (F.sum(cube_department_df["SalesAmt"]) == 0),
0).otherwise(F.sum(cube_department_df["SalesAmt"]) / F.countDistinct(cube_department_df["membership_id"])).alias("spend_per_customer")) \
.select("YEAR","WeekOfYear","Month","department_groups","category_name","subcategory_name","section_name",
"last_date_of_week","sales_basket","sales_amt","sales_qty","SalesQtyPro","SalesAmtPro",
"member_count","SalesQty_Member","SalesAmt_Member", "Basket_Count_Member",
"SalesQty_NonMember","SalesAmt_NonMember", "Basket_Count_NonMember",
"SalesAmt_MMDS_Promotion", "SalesQty_MMDS_Promotion", "Basket_Count_MMDS_Promotion",
"SalesAmt_Non_MMDS_Promotion","SalesQty_Non_MMDS_Promotion", "Basket_Count_Non_MMDS_Promotion",
"SalesAmt_Member_MMDS_Promotion","SalesQty_Member_MMDS_Promotion","Basket_Count_Member_MMDS_Promotion",
"SalesAmt_Non_Member_MMDS_Promotion","SalesQty_Non_Member_MMDS_Promotion","Basket_Count_Non_Member_MMDS_Promotion",
"SalesAmt_Member_Non_MMDS_Promotion","SalesQty_Member_Non_MMDS_Promotion","Basket_Count_Member_Non_MMDS_Promotion",
"SalesAmt_Non_Member_Non_MMDS_Promotion","SalesQty_Non_Member_Non_MMDS_Promotion","Basket_Count_Non_Member_Non_MMDS_Promotion",
"sales_per_unit","sales_per_basket","unit_per_basket", "spend_per_customer") \
.orderBy(F.col("YEAR").asc(),
F.col("WeekOfYear").asc(),
F.col("Month").asc(),
F.col("department_groups").asc(),
F.col("category_name").asc(),
F.col("subcategory_name").asc(),
F.col("section_name").asc())