{"error":"RPC is disabled. This version of Bitcoin Cash Node is old and
may be out of consensus with the network. It is recommended that you
upgrade. To proceed without upgrading, and re-enable the RPC interface,
restart the node with the configuration option expire=0."}
I am getting this Error while sending BCH using the Trust wallet Library also I updated the latest version (implementation 'com.trustwallet:wallet-core:2.6.3'). Kindly help me with this issue
Here is my BCH Transaction code
private fun getUtox(receiveAddress: String, finalreceive: String) {
showProgress()
if (bchutoxlist.size > 0) {
val seed1 = UtilsDefault.getSharedPreferenceString(Constants.SEED)
val wallet = HDWallet(seed1, "")
val coinBch = CoinType.BITCOINCASH
val address = wallet.getAddressForCoin(coinBch)
val value = finalreceive.toDouble()
val vv = value * 100000000
val myvalue = vv.toLong()
val secretPrivateKeyBtc = wallet.getKeyForCoin(coinBch)
val toAddress = receiveAddress
val changeAddress = address
val script = BitcoinScript.lockScriptForAddress(address, coinBch).data()
val input = Bitcoin.SigningInput.newBuilder().apply {
this.amount = myvalue
this.hashType = BitcoinScript.hashTypeForCoin(coinBch)
this.toAddress = toAddress
this.useMaxAmount = fullAmount
this.changeAddress = changeAddress
this.byteFee = btcgas.toLong()
}
for (i in 0..bchutoxlist.size.minus(1)) {
val txid = bchutoxlist[i].txid
val vout = bchutoxlist[i].vout
val satoshi = bchutoxlist[i].satoshis
val utox = Numeric.hexStringToByteArray(txid)
val utxoTxId = utox
val outPoint = Bitcoin.OutPoint.newBuilder().apply {
this.hash = ByteString.copyFrom(utxoTxId.reversedArray())
this.sequence = Long.MAX_VALUE.toInt()
this.index = vout!!
}.build()
val utxo = Bitcoin.UnspentTransaction.newBuilder().apply {
this.amount = satoshi!!
this.outPoint = outPoint
this.script = ByteString.copyFrom(script)
}.build()
input.addUtxo(utxo)
input.addPrivateKey(ByteString.copyFrom(secretPrivateKeyBtc.data()))
}
val output = AnySigner.sign(input.build(), coinBch, Bitcoin.SigningOutput.parser())
val signedTransaction = output.encoded?.toByteArray()
val hex = signedTransaction?.toHexString()
if (hex != "") {
sendBch(hex!!, address,myvalue)
} else {
runOnUiThread {
hideProgress()
Toast.makeText(
this#ConfirmDepositActivity,
getString(R.string.withdraw_failed),
Toast.LENGTH_SHORT
).show()
finish()
}
}
} else {
runOnUiThread {
hideProgress()
Toast.makeText(
this#ConfirmDepositActivity,
"No Unspent Amount",
Toast.LENGTH_SHORT
).show()
finish()
}
}
}
Related
1. I use a custom sparkSQL plugin to start a spark-shell terminal and execute the following command in it
import org.apache.spark.sql.{DataFrame, DataFrameReader, Row, SparkSession}
import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType}
sc.setLogLevel("WARN")
val spark: SparkSession = SparkSession.builder().appName("Test").config("parquet.enable.dictionary","false").getOrCreate()
val res = spark.sql("SELECT USER_ID FROM TEST_PARQUET_10G where USER_ID >= 0 and HOUR_ID > 0")
res.collect()
2. This is my table creation statement
CREATE
TABLE
TEST_PARQUET_10G(
USER_ID BIGINT,
SERIAL_NUMBER BIGINT,
KAFKA_TIME BIGINT,
ACCT_DATE BIGINT,
HOUR_ID BIGINT
)
COMMENT 'Long类型'
STORED AS PARQUET
TBLPROPERTIES('parquet.compression'='none')
LOCATION
'/user/hive/warehouse/hanlei/Long_PARQUET/TEST_PARQUET_10G'
3. I customized a sql plug-in, in which I rewritten the rules for reading paquet files this is my reader, in this step, I set USER_ID ADN HOUR_ID in my reader
this is my reader code
private def createParquetFileReader(file: PartitionedFile): ParquetFileReader = {
val conf = broadcastedConf.value.value
val filePath = new Path(file.filePath)
val split = new FileSplit(filePath, file.start, file.length, Array.empty[String])
val reader = ParquetFileReader.open(HadoopInputFile.fromPath(filePath,conf))
val requiredNameArr = readDataSchema.fieldNames
val requiredSchemaName:Set[String] = requiredNameArr.toSet
val fields: util.List[Type] = reader.getFileMetaData.getSchema.getFields
fields.removeIf{
field =>
!requiredSchemaName.contains(field.getName)
}
requiredSchema = new MessageType("requiredSchema", fields)
reader.setRequestedSchema(requiredSchema)
reader
}
4. But when I call the readNextRowGroup method of the reader, the program reports this error
code from here
class ParquetDataPagesPartitionReader(
parquetReader: ParquetFileReader,
readDataSchema: StructType,
requiredSchema: MessageType
) extends PartitionReader[ColumnarBatch]
with ScanWithMetrics
with Arm
with Logging {
private var batch: Option[ColumnarBatch] = None
//private val columns: Iterator[String] = readDataSchema.fieldNames.iterator
var rowGroup: PageReadStore = _
val columns: util.List[ColumnDescriptor] = requiredSchema.getColumns
override def next(): Boolean = {
batch.foreach(_.close())
rowGroup = parquetReader.readNextRowGroup()
batch = if ( rowGroup != null) {
readBatch()
} else {
None
}
batch.isDefined
}
// one batch is all columnChunk's dataPages from a rowGroup
private def readBatch(): Option[ColumnarBatch] = {
val starTime = System.currentTimeMillis()
// val starTime = System.nanoTime()
logError(s"startTime ${starTime}")
val dpuColumnVectors: Array[DpuColumnVector] = new Array[DpuColumnVector](columns.size())
for (i <- 0 until readDataSchema.length) {
val raceDType = DpuColumnVector.getRaceDataType(readDataSchema(i).dataType)
val raceVec = new RaceColumnVector(raceDType, DpuBatchUtils.DPU_MAX_ROWS)
logDebug(s"try to malloc ${i} column")
raceVec.setRowSizeAndMalloc()
logDebug(s"successful malloc ${i} column")
dpuColumnVectors(i) = new DpuColumnVector(readDataSchema(i).dataType, raceVec)
}
var useTime: Long = 0
for (index <- 0 until readDataSchema.length) {
var offset: Int = 0
val pageReader = rowGroup.getPageReader(columns.get(index))
var page: DataPageV1 = pageReader.readPage.asInstanceOf[DataPageV1]
logDebug(s"appending data to column ${index}")
while (page != null && offset < DpuBatchUtils.DPU_MAX_ROWS) {
dpuColumnVectors(index).dataType() match {
case LongType =>
val startReadingIntoMemory = System.currentTimeMillis()
val byteArray: Array[Byte] = page.getBytes.toByteArray
val bytes: Array[Byte] = byteArray.slice(8, byteArray.length)
val endReadIntoMemory = System.currentTimeMillis()
//logError(s"successful read into memory column: ${columns.get(index).getPath.apply(0)} use time: ${endReadIntoMemory - startReadingIntoMemory}")
dpuColumnVectors(index).getRaceColumnVector
// todo: The byte array here is not decoded
.appendValuesFromBuffer(bytes, 0, page.getValueCount)
val endAppendToRace = System.currentTimeMillis()
//logError(s"successful read into race, appended ${page.getValueCount} rows from ${offset} use time ${endAppendToRace - endReadIntoMemory}")
case _ => throw new RuntimeException(RaceConstant.UNSUPPORTED_DATA_TYPE)
}
offset += page.getValueCount
page = pageReader.readPage.asInstanceOf[DataPageV1]
}
}
val endTime = System.currentTimeMillis()
// val endTime = System.nanoTime()
// useTime = useTime + endTime - starTime
val columnarBatch = new ColumnarBatch(dpuColumnVectors.toArray, rowGroup.getRowCount.toInt)
logError(s"endTime: ${endTime}")
logError(s"read batch success, write to race successful,batchRows: ${rowGroup.getRowCount}, column number: ${columns.size()}, use time: ${endTime - starTime}")
Some(columnarBatch)
}
override def get(): ColumnarBatch = {
val ret = batch.getOrElse(throw new NoSuchElementException)
batch = None
ret
}
override def close(): Unit = {
if (parquetReader != null) parquetReader.close()
batch.foreach(_.close())
batch = None
}
}
5. In this reader, I have the schema I injected, but the subscript is mapped according to the original schema
code injected HOUR_ID and USER_ID
6. Here is my call stack info
stack info
How do I read parquet's columnChunk through a custom mapping？
Hello everyone please help me i am try to modify link but it's not working. it's working on java but recently i convert java to kotlin and getting this error.
am trying to change my link http://www.offertoro.com/click_track/api?offer_id=419393&pub_id=14&pub_app_id=5&USER_ID=[USER_ID]&GAID=[your_GAID] in [USER_ID] with current login user email but getting error.
Check Screenshot
Screenshot
Error
None of the following functions can be called with the arguments supplied.
CharSequence.replace(Regex, String) defined in kotlin.text
String.replace(String, String, Boolean = ...) defined in kotlin.text
My code
fun modifyOfferLink() {
val id = mAuth!!.currentUser!!.email
// Modifying Offer Link Acording to Offer Partner
when (partner) {
"ogads" -> Finallink = link + "&aff_sub5=" + mAuth!!.currentUser!!.email
"offertoro" -> Finallink = link.replace("[USER_ID]", mAuth!!.currentUser!!.email)
"none" -> {
Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
else -> Finallink = link.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
}
OfferActivity.kt
package com.sgamer.creditsk.Activity
import android.content.Intent
import android.content.pm.PackageInfo
import android.content.pm.PackageManager
import android.graphics.Color
import android.graphics.drawable.ColorDrawable
import android.os.Build
import android.os.Bundle
import android.util.Log
import android.view.View
import android.view.Window
import android.view.WindowManager
import android.widget.ImageView
import android.widget.LinearLayout
import android.widget.TextView
import androidx.appcompat.app.AppCompatActivity
import androidx.appcompat.widget.Toolbar
import com.bumptech.glide.Glide
import com.bumptech.glide.request.RequestOptions
import com.google.firebase.auth.FirebaseAuth
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.loopj.android.http.AsyncHttpClient
import com.loopj.android.http.AsyncHttpResponseHandler
import com.loopj.android.http.RequestParams
import com.sgamer.creditsk.Activity.AndyConstants.ServiceType
import com.sgamer.creditsk.Activity.OfferDetailsActivity
import com.sgamer.creditsk.R
import com.sgamer.creditsk.Utils.*
import cz.msebera.android.httpclient.Header
import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
class OfferDetailsActivity constructor() : AppCompatActivity() {
var Finallink: String? = null
var package_id: String? = null
var uniq_id: String? = null
var offerid: String? = null
var app_name: String? = null
var description: String? = null
var icon_url: String? = null
var bg_image_url: String? = null
var amount: String? = null
var OriginalAmount: String? = null
var link: String? = null
var partner: String? = null
var insTitle: String? = null
var first_text: String? = null
var second_text: String? = null
var third_text: String? = null
var fourth_text: String? = null
var webview: Boolean? = null
var ClickId: String? = null
var ctx: OfferDetailsActivity? = null
var later: TextView? = null
var status_image: ImageView? = null
var mAuth: FirebaseAuth? = null
private val bannerAdManager: BannerAdManager_SK? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_offer_details)
val toolbar: Toolbar = findViewById<View>(R.id.toolbar) as Toolbar
val adContainer: LinearLayout = findViewById<View>(R.id.adView) as LinearLayout
setSupportActionBar(toolbar)
val bannerAdManager_SK: BannerAdManager_SK = BannerAdManager_SK(this#OfferDetailsActivity, adContainer)
bannerAdManager_SK.BannerAds()
ctx = this
mAuth = FirebaseAuth.getInstance()
getSupportActionBar()!!.setTitle(R.string.offer_details)
getSupportActionBar()!!.setDisplayHomeAsUpEnabled(true)
getSupportActionBar()!!.setBackgroundDrawable(ColorDrawable(getResources().getColor(android.R.color.transparent)))
getSupportActionBar()!!.setElevation(0f)
if (Build.VERSION.SDK_INT >= 21) {
getWindow().getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_LAYOUT_STABLE or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN)
}
changeStatusBarColor()
initViews()
modifyOfferLink()
}
private fun changeStatusBarColor() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
val window: Window = getWindow()
window.addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS)
window.setStatusBarColor(Color.TRANSPARENT)
}
}
fun initViews() {
val title: TextView = findViewById(R.id.title)
val desc: TextView = findViewById(R.id.description)
val instructionsTitle: TextView = findViewById(R.id.instructions)
val first: TextView = findViewById(R.id.first)
val second: TextView = findViewById(R.id.second)
val third: TextView = findViewById(R.id.third)
val fourth: TextView = findViewById(R.id.fourth)
val des: TextView = findViewById(R.id.des)
val complete_button: TextView = findViewById(R.id.complete_button)
val button: TextView = findViewById(R.id.button)
later = findViewById(R.id.later)
val comSpace: LinearLayout = findViewById(R.id.comSpace)
val offer_icon: ImageView = findViewById(R.id.offer_icon)
val bg_image: ImageView = findViewById(R.id.bg_image)
status_image = findViewById(R.id.status_image)
uniq_id = getIntent().getStringExtra("uniq_id")
offerid = getIntent().getStringExtra("offerid")
app_name = getIntent().getStringExtra("app_name")
package_id = getIntent().getStringExtra("package_id")
description = getIntent().getStringExtra("description")
icon_url = getIntent().getStringExtra("icon_url")
bg_image_url = getIntent().getStringExtra("bg_image_url")
amount = getIntent().getStringExtra("amount")
OriginalAmount = getIntent().getStringExtra("OriginalAmount")
link = getIntent().getStringExtra("link")
partner = getIntent().getStringExtra("partner")
first_text = getIntent().getStringExtra("first_text")
insTitle = getIntent().getStringExtra("instructionsTitle")
second_text = getIntent().getStringExtra("second_text")
third_text = getIntent().getStringExtra("third_text")
fourth_text = getIntent().getStringExtra("fourth_text")
webview = getIntent().getBooleanExtra("webview", false)
if (getIntent().hasExtra("description")) {
des.setText(getIntent().getStringExtra("description"))
} else {
des.setText(getIntent().getStringExtra("description"))
}
title.setText(app_name)
desc.setText(getString(R.string.earn) + " " + amount + " " + getString(R.string.app_currency) + " " + getString(R.string.on_this_offer))
Glide.with(this).load(icon_url)
.apply(RequestOptions().placeholder(R.drawable.placeholder_image).error(R.drawable.placeholder_image))
.into(offer_icon)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
comSpace.setElevation(20f)
}
instructionsTitle.setText(insTitle)
first.setText(first_text)
second.setText(second_text)
third.setText(third_text)
fourth.setText(fourth_text)
complete_button.setText(getResources().getString(R.string.complete_offer))
if (!bg_image_url!!.isEmpty()) {
Glide.with(this).load(bg_image_url).into(bg_image)
} else {
}
// On click Listners
later!!.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
finish()
}
})
complete_button.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
if (!App.isVPNConnected()) {
addoffer(amount, app_name + " Offer Credit", offerid)
}
AppUtils.parse(this#OfferDetailsActivity, Finallink)
}
})
button.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
val launchIntent: Intent? = getPackageManager().getLaunchIntentForPackage((package_id)!!)
startActivity(launchIntent)
}
})
isAppExist
if (isAppExist) {
complete_button.setVisibility(View.GONE)
button.setVisibility(View.VISIBLE)
} else {
button.setVisibility(View.GONE)
complete_button.setVisibility(View.VISIBLE)
}
}
fun addoffer(points: String?, Activity: String?, offerid: String?) {
val client: AsyncHttpClient = AsyncHttpClient()
val params: RequestParams = RequestParams()
val jsObj: JsonObject = Gson().toJsonTree(API()) as JsonObject
jsObj.addProperty("method_name", "user_offeradd")
jsObj.addProperty("offer_id", offerid)
jsObj.addProperty("email", mAuth!!.getCurrentUser()!!.getEmail())
jsObj.addProperty("points", points)
jsObj.addProperty("firebase_id", mAuth!!.getCurrentUser()!!.getUid())
jsObj.addProperty("Activity", Activity)
params.put("data", API.toBase64(jsObj.toString()))
client.post(Javaaescipher.decrypt(), params, object : AsyncHttpResponseHandler() {
public override fun onSuccess(statusCode: Int, headers: Array<Header>, responseBody: ByteArray) {
Log.d("Response", String(responseBody))
val res: String = String(responseBody)
try {
val jsonObject: JSONObject = JSONObject(res)
val jsonArray: JSONArray = jsonObject.getJSONArray("ANDROID_REWARDS_APP")
for (i in 0 until jsonArray.length()) {
val `object`: JSONObject = jsonArray.getJSONObject(i)
val success: String = `object`.getString("success")
val msg: String = `object`.getString("msg")
if ((success == "1")) {
// Toast.makeText(OfferDetailsActivity.this, msg, Toast.LENGTH_LONG).show();
} else {
// Toast.makeText(OfferDetailsActivity.this, msg, Toast.LENGTH_LONG).show();
}
}
} catch (e: JSONException) {
e.printStackTrace()
}
}
public override fun onFailure(statusCode: Int, headers: Array<Header>, responseBody: ByteArray, error: Throwable) {
Log.d("error", error.toString())
}
})
}
private val isAppExist: Boolean
private get() {
val pm: PackageManager = getPackageManager()
try {
val info: PackageInfo = pm.getPackageInfo((package_id)!!, PackageManager.GET_META_DATA)
} catch (e: PackageManager.NameNotFoundException) {
return false
}
return true
}
fun modifyOfferLink() {
val id = mAuth!!.currentUser!!.email
// Modifying Offer Link Acording to Offer Partner
when (partner) {
"ogads" -> Finallink = link + "&aff_sub5=" + mAuth!!.currentUser!!.email
"offertoro" -> Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
"none" -> {
Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
else -> Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
}
}
It is likely there is no sub-string that is exactly [USER_ID]. Are you sure is it not ["USER_ID"] ?
Can you print the value of link before you call the replace. This may let us and you see the problem.
I have went through multiple sites like
https://spark.apache.org/docs/latest/streaming-programming-guide.html
https://data-flair.training/blogs/spark-streaming-checkpoint/
https://docs.databricks.com/spark/latest/rdd-streaming/developing-streaming-applications.html
Some links talk about the how we can code but it's so abstract that I needed a lot of time to figure out how this actually works
After a long fight I am able to setup the streaming code with checkpoint, adding here to help others
import java.util.concurrent.Executors
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
import com.amazonaws.regions.Regions
import com.amazonaws.services.sqs.model.Message
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.log4j.LogManager
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}
object StreamingApp extends scala.Serializable {
#transient private final val mapper = new ObjectMapper
#transient private final val LOG = LogManager.getLogger(getClass)
#transient private final val executor = Executors.newFixedThreadPool(Runtime.getRuntime.availableProcessors)
var s3 = "s3"
private var shutdownMarker: String = _
private var stopFlag: Boolean = false
def main(args: Array[String]): Unit = {
val queueName = args(0)
val region = args(1)
val fetchMaxMessage = args(2).toInt
val visibilityTimeOutSeconds = args(3).toInt
val waitTimeoutInMillis = args(4).toLong
val isLocal = args(5).toBoolean
val bucket = args(6)
if (args.length >= 10)
shutdownMarker = args(9)
val sparkConf = initialiseSparkConf(isLocal)
sparkConf.set(Constants.QUEUE_NAME, queueName)
sparkConf.set(Constants.REGION, region)
sparkConf.set(Constants.FETCH_MAX_MESSAGE, fetchMaxMessage.toString)
sparkConf.set(Constants.VISIBILITY_TIMEOUT_SECONDS, visibilityTimeOutSeconds.toString)
sparkConf.set(Constants.WAIT_TIMEOUT_IN_MILLIS, waitTimeoutInMillis.toString)
shutdownMarker = s"$s3://$bucket/streaming/shutdownmarker"
val checkpointDirectory = s"$s3://$bucket/streaming/checkpoint/"
var context: StreamingContext = null
try {
context = StreamingContext.getOrCreate(checkpointDirectory, () => createContext(sparkConf, waitTimeoutInMillis, checkpointDirectory, args))
context.start
val checkIntervalMillis = 10000
var isStopped = false
while (!isStopped) {
println("calling awaitTerminationOrTimeout")
isStopped = context.awaitTerminationOrTimeout(checkIntervalMillis)
if (isStopped)
println("confirmed! The streaming context is stopped. Exiting application...")
checkShutdownMarker(context.sparkContext)
if (!isStopped && stopFlag) {
println("stopping ssc right now")
context.stop(stopSparkContext = true, stopGracefully = true)
println("ssc is stopped!!!!!!!")
}
}
}
finally {
LOG.info("Exiting the Application")
if (context != null && org.apache.spark.streaming.StreamingContextState.STOPPED != context.getState) {
context.stop(stopSparkContext = true, stopGracefully = true)
}
if (!executor.isShutdown)
executor.shutdown()
}
}
def checkShutdownMarker(sparkContext: SparkContext): Unit = {
if (!stopFlag) {
stopFlag = isFileExists(shutdownMarker, sparkContext)
}
println(s"Stop marker $shutdownMarker file found: $stopFlag at time ${System.currentTimeMillis()}")
}
def isFileExists(path: String, sparkContext: SparkContext): Boolean = {
isValidPath(isDir = false, path, getFileSystem(path,sparkContext))
}
def getFileSystem(path: String, sparkContext: SparkContext): FileSystem = {
FileSystem.get(URI.create(path), sparkContext.hadoopConfiguration)
}
def isValidPath(isDir: Boolean, path: String, fileSystem: FileSystem): Boolean = {
LOG.info("Validating path {}", path)
if (path.startsWith(Constants.S3) || path.startsWith(Constants.HDFS) || path.startsWith(Constants.FILE)) {
val fsPath = new Path(path)
if (isDir) {
fileSystem isDirectory fsPath
} else {
fileSystem isFile fsPath
}
} else {
Files.exists(Paths.get(path))
}
}
def createContext(sparkConf: SparkConf, waitTimeoutInMillis: Long, checkpointDirectory: String, args: Array[String]): StreamingContext = {
val context = new StreamingContext(sparkConf, Duration(waitTimeoutInMillis + 1000))
processMessage(context, args)
context.checkpoint(checkpointDirectory) // set checkpoint directory
context
}
def processMessage(context: StreamingContext, args: Array[String]): Unit = {
val bucket = args(6)
val wgPath = args(7)
var stagingPath = args(8)
val waitTimeoutInMillis = args(4).toLong
if (context != null) {
if (!stagingPath.endsWith("/")) {
stagingPath = s"$stagingPath/"
}
val outputPrefix = s"$s3://$bucket/$stagingPath"
LOG.info(s"Number of cores for driver: ${Runtime.getRuntime.availableProcessors}")
val sparkContext: SparkContext = context.sparkContext
val broadcasts = BroadCaster.getInstance(sparkContext, s"$s3://$bucket/$wgPath")
val input = context.receiverStream(broadcasts(Constants.SQS_RECEIVER).value.asInstanceOf[SQSReceiver])
//input.checkpoint(interval = Seconds(60))
LOG.info(s"Scheduling mode ${sparkContext.getSchedulingMode.toString}")
input.foreachRDD(r => {
val sparkSession = SparkSession.builder.config(r.sparkContext.getConf).getOrCreate()
val messages = r.collect().map(message => mapper.readValue(message, classOf[Message]))
val broadcasts = BroadCaster.getInstance(r.sparkContext, s"$s3://$bucket/$wgPath")
//Application logic
})
}
}
def initialiseSparkConf(local: Boolean): SparkConf = {
val sparkConf = new SparkConf()
.setAppName("Spark Streaming")
.set("spark.scheduler.mode", "FAIR")
.set("spark.sql.parquet.filterpushdown", "true")
.set("spark.executor.hearbeatInterval", "20")
.set("spark.streaming.driver.writeAheadLog.closeFileAfterWrite", "true")
.set("spark.streaming.receiver.writeAheadLog.closeFileAfterWrite", "true")
.set("spark.streaming.receiver.writeAheadLog.enable", "true")
.set("spark.streaming.stopGracefullyOnShutdown", "true")
.set("spark.streaming.backpressure.enabled","true")
.set("spark.streaming.backpressure.pid.minRate","10") //SQS support batch of 10
if (local) {
s3 = "s3a"
sparkConf.setMaster("local[*]")
} else {
sparkConf.set("hive.metastore.client.factory.class",
"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory")
}
}
}
object BroadCaster {
#volatile private var instance: Map[String, Broadcast[Any]] = _
def getInstance(sparkContext: SparkContext, wgPath: String): Map[String, Broadcast[Any]] = {
if (instance == null) {
synchronized {
if (instance == null) {
instance = Utils.createBroadcastObjects(wgPath, sparkContext)
instance += (Constants.SQS_RECEIVER -> sparkContext.broadcast(getSQSReceiver(sparkContext.getConf)))
}
}
}
instance
}
private def getSQSReceiver(conf: SparkConf): SQSReceiver = {
val javaSQSReceiver = new SQSReceiver(conf.get(Constants.QUEUE_NAME)).withRegion(Regions.fromName(conf.get(Constants.REGION))).withCredential(new DefaultAWSCredentialsProviderChain())
.withFetchMaxMessage(conf.getInt(Constants.FETCH_MAX_MESSAGE, 10)).withVisibilityTimeOutSeconds(conf.getInt(Constants.VISIBILITY_TIMEOUT_SECONDS, 1800)).withWaitTimeoutinMillis(conf.getLong(Constants.WAIT_TIMEOUT_IN_MILLIS, 1000))
javaSQSReceiver
}
}
import java.util.List;
import org.apache.log4j.Logger;
import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.receiver.Receiver;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
import com.amazonaws.services.sqs.model.DeleteMessageRequest;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SQSReceiver extends Receiver<String> {
private String queueName;
private transient AWSCredentialsProvider credential;
private Regions region = Regions.US_EAST_1;
private Long waitTimeoutinMillis = 0L;
private ObjectMapper mapper= new ObjectMapper();
private transient Logger logger = Logger.getLogger(SQSReceiver.class);
private boolean deleteOnReceipt = false;
private int fetchMaxMessage = 100;
private int visibilityTimeOutSeconds = 60;
private String sqsQueueUrl;
private transient AmazonSQS amazonSQS;
public SQSReceiver(String queueName) {
this(queueName, false);
}
public SQSReceiver(String queueName, boolean deleteOnReceipt) {
super(StorageLevel.MEMORY_AND_DISK_SER());
this.queueName = queueName;
this.deleteOnReceipt = deleteOnReceipt;
setupSQS(queueName);
}
private void setupSQS(String queueName) {
AmazonSQSClientBuilder amazonSQSClientBuilder = AmazonSQSClientBuilder.standard();
if (credential != null) {
amazonSQSClientBuilder.withCredentials(credential);
}
amazonSQSClientBuilder.withRegion(region);
amazonSQS = amazonSQSClientBuilder.build();
sqsQueueUrl = amazonSQS.getQueueUrl(queueName).getQueueUrl();
}
public void onStart() {
new Thread(this::receive).start();
}
public void onStop() {
// There is nothing much to do as the thread calling receive()
// is designed to stop by itself if isStopped() returns false
}
private void receive() {
try {
setupSQS(queueName);
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(sqsQueueUrl).withMaxNumberOfMessages(fetchMaxMessage).withVisibilityTimeout(visibilityTimeOutSeconds)
.withWaitTimeSeconds(20); //https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/examples-sqs-long-polling.html
receiveMessagesFromSQS(amazonSQS, sqsQueueUrl, receiveMessageRequest);
} catch (Throwable e) {
stop("Error encountered while initializing", e);
}
}
private void receiveMessagesFromSQS(final AmazonSQS amazonSQS, final String sqsQueueUrl,
ReceiveMessageRequest receiveMessageRequest) {
try {
while (!isStopped()) {
List<Message> messages = amazonSQS.receiveMessage(receiveMessageRequest).getMessages();
if (deleteOnReceipt) {
String receiptHandle = messages.get(0).getReceiptHandle();
messages.forEach(m -> store(m.getBody()));
amazonSQS.deleteMessage(new DeleteMessageRequest(sqsQueueUrl, receiptHandle));
} else {
messages.forEach(this::storeMessage);
}
if (waitTimeoutinMillis > 0L)
Thread.sleep(waitTimeoutinMillis);
}
restart("Trying to connect again");
} catch (IllegalArgumentException | InterruptedException e) {
restart("Could not connect", e);
} catch (Throwable e) {
restart("Error Receiving Data", e);
}
}
private void storeMessage(Message m) {
try {
if (m != null)
store(mapper.writeValueAsString(m));
} catch (JsonProcessingException e) {
logger.error("Unable to write message to streaming context");
}
}
public SQSReceiver withVisibilityTimeOutSeconds(int visibilityTimeOutSeconds) {
this.visibilityTimeOutSeconds = visibilityTimeOutSeconds;
return this;
}
public SQSReceiver withFetchMaxMessage(int fetchMaxMessage) {
if (fetchMaxMessage > 10) {
throw new IllegalArgumentException("FetchMaxMessage can't be greater than 10");
}
this.fetchMaxMessage = fetchMaxMessage;
return this;
}
public SQSReceiver withWaitTimeoutinMillis(long waitTimeoutinMillis) {
this.waitTimeoutinMillis = waitTimeoutinMillis;
return this;
}
public SQSReceiver withRegion(Regions region) {
this.region = region;
return this;
}
public SQSReceiver withCredential(AWSCredentialsProvider credential) {
this.credential = credential;
return this;
}
public void deleteMessages(DeleteMessageBatchRequest request) {
request.withQueueUrl(sqsQueueUrl);
amazonSQS.deleteMessageBatch(request);
}
}
I am trying to write an updateStateWithKey for getting the first values on an input. When I try to use it on a case class, I get an error
import org.apache.log4j.{Level, Logger}
import org.apache.spark._
import org.apache.spark.streaming._
case class Persons(name : String, school : String)
object StatefulNetworkWordCount {
def getPerson (str : String) : Persons = {
val splitArray = str.split(",")
val name = splitArray(0)
val school = splitArray(1)
Persons(name, school)
}
//Now, newValues is the new set of values
//runningCount is the existing values for each key
def updateFunction(newValues: Seq[Int], runningCount: Option[Int]): Option[Int] = {
val newCount = runningCount.getOrElse(0) + newValues.sum
Some(newCount)
}
def updateFunctionFrist(newValues: Seq[String], runningCount: Option[String]): Option[String] =
{
val newWord = if (runningCount.getOrElse("") == "")
{
val str = newValues.head.toString //Use existing values
Some(str)
}
else
{
val str = runningCount.getOrElse(newValues.head.toString)
Some(str)
}
newWord
}
def updateFunctionFirstPerson(newValues: Seq[Person], state: Option[Person]): Option[Person] =
{
val newWord = if (state.getOrElse("") == "") //If running count is empty
{
val str = newValues.head.asInstanceOf[Person]
Some(str)
}
else
{
val str = state.getOrElse(newValues.head.asInstanceOf[Person])
Some(str)
}
newWord
}
def main(args: Array[String]) {
Logger.getLogger("org").setLevel(Level.ERROR)
val conf = new SparkConf().setMaster("local[8]").setAppName("StatefulNetworkWordCount")
val ssc = new StreamingContext(conf, Seconds(10))
// Set checkpoint directory
ssc.checkpoint(".")
// Create a DStream that will connect to hostname:port, like localhost:9999
val lines = ssc.socketTextStream("localhost", 9999)
// Split each line into words
val words = lines.flatMap(_.split(" "))
// Count each word in each batch
val pairs = words.map(word => (word.hashCode, word))
val runningCounts = pairs.updateStateByKey[Persons](updateFunctionFirstPerson _)
runningCounts.print()
ssc.start() // Start the computation
ssc.awaitTermination() // Wait for the computation to terminate
}
}
The line
val runningCounts = pairs.updateStateByKey[Persons](updateFunctionFirstPerson _)
throws an error, but if I use the
val runningCounts = pairs.updateStateByKey[String](updateFunctionFirst _)
to get the first value for a key it works fine. Can we use custom class in updateStateBykey? How can i use it?
I'm trying to read a excel file using poi SXSSF. For some reason sheet.rowIterator is returning empty iterator even though there are rows in the sheet. Here is the code I have
import java.io.File
import org.apache.poi.xssf.streaming.SXSSFWorkbook
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import scala.collection.JavaConverters._
class ExcelReader {
final val fileName = "c:\\temp\\data-200.xlsx"
def read(): Iterator[Contact] = {
val file = new File(fileName)
val workBook = new SXSSFWorkbook(new XSSFWorkbook(file),100)
val sheet = workBook.getSheetAt(0) //this works gets sheet name
Console.println(s"Sheet Name: ${sheet.getSheetName()}")
val rowItr = sheet.rowIterator().asScala // this is empty iterator
for (e <- rowItr) yield Contact(e.getCell(0).getStringCellValue(),
e.getCell(1).getStringCellValue(),
e.getCell(2).getStringCellValue(),
e.getCell(3).getStringCellValue(),
e.getCell(4).getStringCellValue())
}
}
Not sure what I'm doing wrong.
Here is a simple example of reading excel file that I have tried.
val myFile = new File("/home/sakoirala/Downloads/eu-historical-price-series_en.xls")
val fis = new FileInputStream(myFile)
val myWorkbook = new HSSFWorkbook(fis)
val mySheet = myWorkbook.getSheetAt(0)
val rowIterator = mySheet.iterator()
while(rowIterator.hasNext){
val row = rowIterator.next()
val cellIterator = row.cellIterator()
while(cellIterator.hasNext) {
val cell = cellIterator.next()
cell.getCellType match {
case Cell.CELL_TYPE_STRING => {
print(cell.getStringCellValue + "\t")
}
case Cell.CELL_TYPE_NUMERIC => {
print(cell.getNumericCellValue + "\t")
}
case Cell.CELL_TYPE_BOOLEAN => {
print(cell.getBooleanCellValue + "\t")
}
case Cell.CELL_TYPE_BLANK => {
print("null" + "\t")
}
case _ => throw new RuntimeException(" this error occured when reading ")
// case Cell.CELL_TYPE_FORMULA => {print(cell.getF + "\t")}
}
}
println("")
}
Hope this helps!