Spark AccumulatorV2 with HashMap - apache-spark

Question

I am trying to create a custom AccumulatorV2 with a hash map, the input would be hashmap and output would be a map of HashMap,
My intention is to have a K -> K1,V, where the value will increment. I am confused by the scala syntax for overriding AccumulatorV2 for Map, did anyone had a luck with this.
class CustomAccumulator extends AccumulatorV2[java.util.Map[String, String], java.util.Map[String,java.util.Map[String, Double]]]

Answer 1

I'm assuming that this is the scenario that needs to be implemented.
Input:
HashMap<String, String>
Output:
Should output a HashMap<String, HashMap<String, Double>>, where the second hashmap contains the count of values corresponding to the keys.
Example:
Inputs:(Following HashMaps are added to the accumulator)
Input HashMap1 -> {"key1", "value1"}, {"key2", "value1"}, {"key3", "value3"}
Input HashMap2 -> {"key1", "value1"}, {"key2", "value1"}
Input HashMap3 -> {"key2", "value1"}
Output:
{"key1", {"value1", 2}}, {"key2", {"value1", 3}}, {"key3", {"value3", 1}}
Code below:
import java.util
import java.util.Map.Entry
import java.util.{HashMap, Map}
import java.util.function.{BiFunction, Consumer}
import scala.collection.JavaConversions._
import org.apache.spark.util.AccumulatorV2
import org.datanucleus.store.rdbms.fieldmanager.OldValueParameterSetter
class CustomAccumulator extends AccumulatorV2[Map[String, String], Map[String, Map[String,Double]]] {
private var hashmap : Map[String, Map[String, Double]] = new HashMap[String, Map[String, Double]];
override def isZero: Boolean = {
return hashmap.size() == 0
}
override def copy(): AccumulatorV2[util.Map[String, String], util.Map[String, util.Map[String, Double]]] = {
var customAccumulatorcopy = new CustomAccumulator()
customAccumulatorcopy.merge(this)
return customAccumulatorcopy
}
override def reset(): Unit = {
this.hashmap = new HashMap[String, Map[String, Double]];
}
override def add(v: util.Map[String, String]): Unit = {
v.foreach(kv => {
val unitValueDouble : Double = 1;
if(this.hashmap.containsKey(kv._1)){
val innerMap = this.hashmap.get(kv._1)
innerMap.merge(kv._2, unitValueDouble, addFunction)
}
else {
val innerMap : Map[String, Double] = new HashMap[String, Double]()
innerMap.put(kv._2, unitValueDouble)
this.hashmap.put(kv._1, innerMap)
}
}
)
}
override def merge(otherAccumulator: AccumulatorV2[util.Map[String, String], util.Map[String, util.Map[String, Double]]]): Unit = {
otherAccumulator.value.foreach(kv => {
this.hashmap.merge(kv._1, kv._2, mergeMapsFunction)
})
}
override def value: util.Map[String, util.Map[String, Double]] = {
return this.hashmap
}
val mergeMapsFunction = new BiFunction[Map[String, Double], Map[String, Double], Map[String, Double]] {
override def apply(oldMap: Map[String, Double], newMap: Map[String, Double]): Map[String, Double] = {
newMap.foreach(kv => {
oldMap.merge(kv._1, kv._2, addFunction);
})
oldMap
}
}
val addFunction = new BiFunction[Double, Double, Double] {
override def apply(oldValue: Double, newValue: Double): Double = oldValue + newValue
}
}
Thanks!!!