Apache Beam Metrics Counter giving incorrect count using SparkRunner - apache-spark

Question

I am having source and target csv files with 10 million records with 250 columns.
I am running an apache beam pipeline which joins all columns from source and target file.
When, I run this on spark cluster the pipeline executes correctly with no exceptions but,
The join beam metrics counter returns double count when the following spark property is used.
-- executor-memory "2g"
But, When I increase the excutor-memory to 11g then it returns the correct count.
I have tried following example,
Pipeline pipeline = Pipeline.create(options);
final TupleTag<String> eventInfoTag = new TupleTag<>();
final TupleTag<String> countryInfoTag = new TupleTag<>();
PCollection<KV<String, String>> eventInfo =
eventsTable.apply(ParDo.of(new ExtractEventDataFn()));
PCollection<KV<String, String>> countryInfo =
countryCodes.apply(ParDo.of(new ExtractCountryInfoFn()));
PCollection<KV<String, CoGbkResult>> kvpCollection =
KeyedPCollectionTuple.of(eventInfoTag, eventInfo)
.and(countryInfoTag, countryInfo)
.apply(CoGroupByKey.create());
PCollection<KV<String, String>> finalResultCollection =
kvpCollection.apply(
"Process",
ParDo.of(
new DoFn<KV<String, CoGbkResult>, KV<String, String>>() {
#ProcessElement
public void processElement(ProcessContext c) {
KV<String, CoGbkResult> e = c.element();
String countryCode = e.getKey();
String countryName = "none";
countryName = e.getValue().getOnly(countryInfoTag);
for (String eventInfo : c.element().getValue().getAll(eventInfoTag)) {
Metrics.counter("count", "errorcount").inc();
c.output(
KV.of(
countryCode,
"Country name: " + countryName + ", Event info: " + eventInfo));
}
}
}));
final PipelineResult result = pipeline.run();
MetricQueryResults metrics =
result
.metrics()
.queryMetrics(
MetricsFilter.builder()
.addNameFilter(MetricNameFilter.inNamespace("count"))
.build());
Iterable<MetricResult<Long>> counters = metrics.getCounters();
for (MetricResult<Long> counter : counters) {
System.out.println("Hi >> "+counter.getName().getName() + " : " + counter.getAttempted() + " " + counter.getCommittedOrNull());
}
I need help with this.
Thank you

Answer 1

public static void main(String[] args) {
Configuration hadoopConf = new Configuration();
hadoopConf.set("fs.defaultFS", args[13]);
hadoopConf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
hadoopConf.set("fs.file.impl",org.apache.hadoop.fs.LocalFileSystem.class.getName());
final TupleTag<Row> sourceDataInfoTag = new TupleTag<Row>(){};
final TupleTag<Row> targetDataInfoTag = new TupleTag<Row>(){};
HadoopFileSystemOptions options = PipelineOptionsFactory.as(HadoopFileSystemOptions.class);
options.setRunner(SparkRunner.class);
options.setHdfsConfiguration(Collections.singletonList(hadoopConf));
Pipeline pipeline = Pipeline.create(options);
PCollection<String> sourceData = pipeline.apply(TextIO.read().from(args[14]).withDelimiter("\n".getBytes()));
PCollection<KV<Row, Row>> sourceDataRows = sourceData.apply(ParDo.of(new ExtractFunction()));
PCollection<String> targetData = pipeline.apply(TextIO.read().from(args[23]).withDelimiter("\n".getBytes()));
PCollection<KV<Row, Row>> targetDataRows = targetData.apply(ParDo.of(new ExtractFunction()));
PCollection<KV<Row, CoGbkResult>> kvpCollection = KeyedPCollectionTuple
.of(sourceDataInfoTag, sourceDataRows.setCoder(KvCoder.of(RowCoder.of(SOURCE_JOIN_RECORD_TYPE),RowCoder.of(SOURCE_RECORD_TYPE))))
.and(targetDataInfoTag, targetDataRows.setCoder(KvCoder.of(RowCoder.of(TARGET_JOIN_RECORD_TYPE),RowCoder.of(TARGET_RECORD_TYPE))))
.apply(CoGroupByKey.<Row>create());
PCollection<GenericRecord> finalResultCollections = kvpCollection.apply("process",ParDo.of(new DoFn<KV<Row, CoGbkResult>, GenericRecord>() {
#ProcessElement
public void processElement(ProcessContext context) {
KV<Row, CoGbkResult> element = context.element();
Iterator<Row> srcIter = element.getValue().getAll(sourceDataInfoTag).iterator();
Iterator<Row> trgIter = element.getValue().getAll(targetDataInfoTag).iterator();
Metrics.counter("count", "count").inc();
GenericRecordBuilder builder = new GenericRecordBuilder(SCHEMA);
boolean done = false;
boolean captureError = false;
while (!done)
{
// Some iterator data here.
.
.
builder.set(colName, data);
if(captureError){
GenericRecord record = builder.build();
context.output(record);
}
}
}
})).setCoder(AvroCoder.of(GenericRecord.class, SCHEMA));
finalResultCollections.apply("writeText",FileIO.<GenericRecord>write()
.via(ParquetIO.sink(SCHEMA))
.withSuffix(".parquet")
.withPrefix("part")
.to("hdfs://temp/"));
final PipelineResult result = pipeline.run();
State state = result.waitUntilFinish();
MetricQueryResults metrics =
result
.metrics()
.queryMetrics(
MetricsFilter.builder()
.addNameFilter(MetricNameFilter.inNamespace("count"))
.build());
Iterable<MetricResult<Long>> counters = metrics.getCounters();
for (MetricResult<Long> counter : counters) {
System.out.println("Count >> "+counter.getName().getName() + " : " + counter.getAttempted() + " " + counter.getCommittedOrNull());
}
}

Answer 2

In your code, when you do Metrics.counter("count", "errorcount") you define the counter. But it is defined in a loop which is also in sort of a loop (processElement). You should define your counter as a field in the DoFn. Don't worry the DoFn is reused for processing the bundle. Such as: private final Counter counter = Metrics.counter(MyClass.class, COUNTER_NAME);
Also you showed only part of the code but I don't see done boolean set to true. But that is just out of curiosity.
And last but not least, you should try spark runner on master branch of Beam because there was a fix merged yesterday about metrics (metrics not reset when running multiple pipelines inside the same JVM). I don't know if it matches your use case but it's worth trying.