how to separate codegen for special operator? - apache-spark

Question

I have many explode function in one compute. It oom.heap dump
I found explode function will codegen in one class, the intermediate result will be buffered in memory.
spark web ui
How can i avoid this promble? separate codegen?
public Object generate(Object[] references) {
return new GeneratedIteratorForCodegenStage1(references);
}
/*wsc_codegenStageId*/
final class GeneratedIteratorForCodegenStage1 extends org.apache.spark.sql.execution.BufferedRowIterator {
private Object[] references;
private scala.collection.Iterator[] inputs;
private scala.collection.Iterator scan_input_0;
private boolean generate_generate_isNull_3_0;
private boolean generate_generate_isNull_14_0;
private org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[] scan_mutableStateArray_0 = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter[10];
public GeneratedIteratorForCodegenStage1(Object[] references) {
this.references = references;
}
public void init(int index, scala.collection.Iterator[] inputs) {
partitionIndex = index;
this.inputs = inputs;
scan_input_0 = inputs[0];
scan_mutableStateArray_0[0] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32);
scan_mutableStateArray_0[1] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(1, 32);
scan_mutableStateArray_0[2] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64);
scan_mutableStateArray_0[3] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64);
scan_mutableStateArray_0[4] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(2, 64);
scan_mutableStateArray_0[5] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 96);
scan_mutableStateArray_0[6] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 96);
scan_mutableStateArray_0[7] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(3, 96);
scan_mutableStateArray_0[8] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(4, 128);
scan_mutableStateArray_0[9] = new org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(4, 128);
}
private void generate_doConsume_2(UTF8String generate_expr_0_2, boolean generate_exprIsNull_0_2, UTF8String generate_expr_1_1, UTF8String generate_expr_2_0) throws java.io.IOException {
ArrayData generate_value_26 = null;
generate_value_26 = new org.apache.spark.sql.catalyst.util.GenericArrayData(generate_expr_1_1.split(((UTF8String) references[7] /* literal */), -1));
int generate_numElements_2 = false ? 0 : generate_value_26.numElements();
for (int generate_index_2 = 0; generate_index_2 < generate_numElements_2; generate_index_2++) {
((org.apache.spark.sql.execution.metric.SQLMetric) references[8] /* numOutputRows */).add(1);
// common sub-expressions
UTF8String generate_col_2 = generate_value_26.getUTF8String(generate_index_2);
scan_mutableStateArray_0[9].reset();
scan_mutableStateArray_0[9].zeroOutNullBytes();
if (generate_exprIsNull_0_2) {
scan_mutableStateArray_0[9].setNullAt(0);
} else {
scan_mutableStateArray_0[9].write(0, generate_expr_0_2);
}
scan_mutableStateArray_0[9].write(1, generate_expr_1_1);
scan_mutableStateArray_0[9].write(2, generate_expr_2_0);
scan_mutableStateArray_0[9].write(3, generate_col_2);
append((scan_mutableStateArray_0[9].getRow()).copy());
}
}
private void generate_doConsume_1(UTF8String generate_expr_0_1, boolean generate_exprIsNull_0_1, UTF8String generate_expr_1_0) throws java.io.IOException {
generate_generate_isNull_14_0 = true;
UTF8String generate_value_14 = null;
do {
if (!generate_exprIsNull_0_1) {
generate_generate_isNull_14_0 = false;
generate_value_14 = generate_expr_0_1;
continue;
}
if (!false) {
generate_generate_isNull_14_0 = false;
generate_value_14 = ((UTF8String) references[4] /* literal */);
continue;
}
} while (false);
ArrayData generate_value_13 = null;
generate_value_13 = new org.apache.spark.sql.catalyst.util.GenericArrayData(generate_value_14.split(((UTF8String) references[5] /* literal */), -1));
int generate_numElements_1 = false ? 0 : generate_value_13.numElements();
for (int generate_index_1 = 0; generate_index_1 < generate_numElements_1; generate_index_1++) {
((org.apache.spark.sql.execution.metric.SQLMetric) references[6] /* numOutputRows */).add(1);
// common sub-expressions
UTF8String generate_col_1 = generate_value_13.getUTF8String(generate_index_1);
generate_doConsume_2(generate_expr_0_1, generate_exprIsNull_0_1, generate_expr_1_0, generate_col_1);
}
}
protected void processNext() throws java.io.IOException {
while (scan_input_0.hasNext()) {
InternalRow scan_row_0 = (InternalRow) scan_input_0.next();
((org.apache.spark.sql.execution.metric.SQLMetric) references[0] /* numOutputRows */).add(1);
boolean scan_isNull_0 = scan_row_0.isNullAt(0);
UTF8String scan_value_0 = scan_isNull_0 ?
null : (scan_row_0.getUTF8String(0));
generate_doConsume_0(scan_value_0, scan_isNull_0);
if (shouldStop()) return;
}
}
private void generate_doConsume_0(UTF8String generate_expr_0_0, boolean generate_exprIsNull_0_0) throws java.io.IOException {
generate_generate_isNull_3_0 = true;
UTF8String generate_value_3 = null;
do {
if (!generate_exprIsNull_0_0) {
generate_generate_isNull_3_0 = false;
generate_value_3 = generate_expr_0_0;
continue;
}
if (!false) {
generate_generate_isNull_3_0 = false;
generate_value_3 = ((UTF8String) references[1] /* literal */);
continue;
}
} while (false);
ArrayData generate_value_2 = null;
generate_value_2 = new org.apache.spark.sql.catalyst.util.GenericArrayData(generate_value_3.split(((UTF8String) references[2] /* literal */), -1));
int generate_numElements_0 = false ? 0 : generate_value_2.numElements();
for (int generate_index_0 = 0; generate_index_0 < generate_numElements_0; generate_index_0++) {
((org.apache.spark.sql.execution.metric.SQLMetric) references[3] /* numOutputRows */).add(1);
// common sub-expressions
UTF8String generate_col_0 = generate_value_2.getUTF8String(generate_index_0);
generate_doConsume_1(generate_expr_0_0, generate_exprIsNull_0_0, generate_col_0);
}
}
}
explain output
== Physical Plan ==
AdaptiveSparkPlan isFinalPlan=false
+- Project [__fcol_4#345 AS table#482, __fcol_5#346 AS table-aaa#483, __fcol_7#335 AS table-bbb#484, __fcol_12#423 AS table-1#485]
+- Project [__fcol_4#345, __fcol_5#346, __fcol_7#335, UDF(SPLITPART($0$, $1$, 1), , [$1$,$0$], [string_type,string_type], [], array(__fcol_11#417, __fcol_4#345), [], []) AS __fcol_12#423]
+- Project [__fcol_1#330 AS __fcol_4#345, __fcol_3#323 AS __fcol_5#346, __fcol_7#335, UDF(",", , [], [], [], [], [], []) AS __fcol_11#417]
+- Generate explode(split(coalesce(__fcol_1#330, ), :, -1)), [__fcol_1#330, __fcol_3#323], false, [__fcol_7#335]
+- Project [__fcol_0#320 AS __fcol_1#330, __fcol_3#323]
+- Generate explode(split(coalesce(__fcol_0#320, ), :, -1)), [__fcol_0#320], false, [__fcol_3#323]
+- Exchange SinglePartition, ENSURE_REQUIREMENTS, [id=#210]
+- Project [table#252 AS __fcol_0#320]
+- Scan [table#252] PushedFilters: [], ReadSchema: struct<table:string>
I try to separate codegen to avoid this oom.
But i don't know how to do this, or there is a better way to solve this.