How can we change the filename of spark output [sparkJava] - apache-spark

Question

I wanted to change spark output filename , is the anyway to do that, Spark is writting in AWS S3.

Answer 1

Check below code will work for HDFS & also S3.
Created rename function which will take path & name to be changed, if directory contains multiple files, This will just append sequence number at the end of file name like json_data_1.json
import org.apache.hadoop.fs.{FileSystem, Path, RemoteIterator}
import org.apache.hadoop.fs._
// For converting to scala Iterator
implicit def convertToScalaIterator[T](remoteIterator: RemoteIterator[T]): Iterator[T] = {
case class wrapper(remoteIterator: RemoteIterator[T]) extends Iterator[T] {
override def hasNext: Boolean = remoteIterator.hasNext
override def next(): T = remoteIterator.next()
}
wrapper(remoteIterator)
}
import java.net.URI
def fs(path: String) = FileSystem.get(URI.create(path),spark.sparkContext.hadoopConfiguration)
// Rename files
def rename(path: String,name: String) = {
fs(path)
.listFiles(new Path(path),true)
.toList
.filter(_.isFile)
.map(_.getPath)
.filterNot(_.toString.contains("_SUCCESS"))
.zipWithIndex
.map(p => fs(p._1.toString).rename(p._1,new Path(s"${p._1.getParent}/${name}_${p._2}.${p._1.toString.split("\\.")(1)}")))
}
scala> val path = "/tmp/samplea"
path: String = /tmp/samplea
scala> df.repartition(5).write.format("json").mode("overwrite").save(path)
scala> s"ls -ltr ${path}".!
total 8
-rw-r--r-- 1 sriniva wheel 0 Jun 6 13:57 part-00000-607ffd5e-7d28-4331-9a69-de36254c80b1-c000.json
-rw-r--r-- 1 sriniva wheel 282 Jun 6 13:57 part-00001-607ffd5e-7d28-4331-9a69-de36254c80b1-c000.json
-rw-r--r-- 1 sriniva wheel 0 Jun 6 13:57 _SUCCESS
scala> rename(path,"json_data")
res193: List[Boolean] = List(true, true)
scala> s"ls -ltr ${path}".!
total 8
-rw-r--r-- 1 sriniva wheel 0 Jun 6 13:57 json_data_0.json
-rw-r--r-- 1 sriniva wheel 282 Jun 6 13:57 json_data_1.json
-rw-r--r-- 1 sriniva wheel 0 Jun 6 13:57 _SUCCESS