Spark streaming SQS with checkpoint enable - apache-spark

I have went through multiple sites like
https://spark.apache.org/docs/latest/streaming-programming-guide.html
https://data-flair.training/blogs/spark-streaming-checkpoint/
https://docs.databricks.com/spark/latest/rdd-streaming/developing-streaming-applications.html
Some links talk about the how we can code but it's so abstract that I needed a lot of time to figure out how this actually works

After a long fight I am able to setup the streaming code with checkpoint, adding here to help others
import java.util.concurrent.Executors
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
import com.amazonaws.regions.Regions
import com.amazonaws.services.sqs.model.Message
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.log4j.LogManager
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.SparkSession
import org.apache.spark.streaming.{Duration, Seconds, StreamingContext}
object StreamingApp extends scala.Serializable {
#transient private final val mapper = new ObjectMapper
#transient private final val LOG = LogManager.getLogger(getClass)
#transient private final val executor = Executors.newFixedThreadPool(Runtime.getRuntime.availableProcessors)
var s3 = "s3"
private var shutdownMarker: String = _
private var stopFlag: Boolean = false
def main(args: Array[String]): Unit = {
val queueName = args(0)
val region = args(1)
val fetchMaxMessage = args(2).toInt
val visibilityTimeOutSeconds = args(3).toInt
val waitTimeoutInMillis = args(4).toLong
val isLocal = args(5).toBoolean
val bucket = args(6)
if (args.length >= 10)
shutdownMarker = args(9)
val sparkConf = initialiseSparkConf(isLocal)
sparkConf.set(Constants.QUEUE_NAME, queueName)
sparkConf.set(Constants.REGION, region)
sparkConf.set(Constants.FETCH_MAX_MESSAGE, fetchMaxMessage.toString)
sparkConf.set(Constants.VISIBILITY_TIMEOUT_SECONDS, visibilityTimeOutSeconds.toString)
sparkConf.set(Constants.WAIT_TIMEOUT_IN_MILLIS, waitTimeoutInMillis.toString)
shutdownMarker = s"$s3://$bucket/streaming/shutdownmarker"
val checkpointDirectory = s"$s3://$bucket/streaming/checkpoint/"
var context: StreamingContext = null
try {
context = StreamingContext.getOrCreate(checkpointDirectory, () => createContext(sparkConf, waitTimeoutInMillis, checkpointDirectory, args))
context.start
val checkIntervalMillis = 10000
var isStopped = false
while (!isStopped) {
println("calling awaitTerminationOrTimeout")
isStopped = context.awaitTerminationOrTimeout(checkIntervalMillis)
if (isStopped)
println("confirmed! The streaming context is stopped. Exiting application...")
checkShutdownMarker(context.sparkContext)
if (!isStopped && stopFlag) {
println("stopping ssc right now")
context.stop(stopSparkContext = true, stopGracefully = true)
println("ssc is stopped!!!!!!!")
}
}
}
finally {
LOG.info("Exiting the Application")
if (context != null && org.apache.spark.streaming.StreamingContextState.STOPPED != context.getState) {
context.stop(stopSparkContext = true, stopGracefully = true)
}
if (!executor.isShutdown)
executor.shutdown()
}
}
def checkShutdownMarker(sparkContext: SparkContext): Unit = {
if (!stopFlag) {
stopFlag = isFileExists(shutdownMarker, sparkContext)
}
println(s"Stop marker $shutdownMarker file found: $stopFlag at time ${System.currentTimeMillis()}")
}
def isFileExists(path: String, sparkContext: SparkContext): Boolean = {
isValidPath(isDir = false, path, getFileSystem(path,sparkContext))
}
def getFileSystem(path: String, sparkContext: SparkContext): FileSystem = {
FileSystem.get(URI.create(path), sparkContext.hadoopConfiguration)
}
def isValidPath(isDir: Boolean, path: String, fileSystem: FileSystem): Boolean = {
LOG.info("Validating path {}", path)
if (path.startsWith(Constants.S3) || path.startsWith(Constants.HDFS) || path.startsWith(Constants.FILE)) {
val fsPath = new Path(path)
if (isDir) {
fileSystem isDirectory fsPath
} else {
fileSystem isFile fsPath
}
} else {
Files.exists(Paths.get(path))
}
}
def createContext(sparkConf: SparkConf, waitTimeoutInMillis: Long, checkpointDirectory: String, args: Array[String]): StreamingContext = {
val context = new StreamingContext(sparkConf, Duration(waitTimeoutInMillis + 1000))
processMessage(context, args)
context.checkpoint(checkpointDirectory) // set checkpoint directory
context
}
def processMessage(context: StreamingContext, args: Array[String]): Unit = {
val bucket = args(6)
val wgPath = args(7)
var stagingPath = args(8)
val waitTimeoutInMillis = args(4).toLong
if (context != null) {
if (!stagingPath.endsWith("/")) {
stagingPath = s"$stagingPath/"
}
val outputPrefix = s"$s3://$bucket/$stagingPath"
LOG.info(s"Number of cores for driver: ${Runtime.getRuntime.availableProcessors}")
val sparkContext: SparkContext = context.sparkContext
val broadcasts = BroadCaster.getInstance(sparkContext, s"$s3://$bucket/$wgPath")
val input = context.receiverStream(broadcasts(Constants.SQS_RECEIVER).value.asInstanceOf[SQSReceiver])
//input.checkpoint(interval = Seconds(60))
LOG.info(s"Scheduling mode ${sparkContext.getSchedulingMode.toString}")
input.foreachRDD(r => {
val sparkSession = SparkSession.builder.config(r.sparkContext.getConf).getOrCreate()
val messages = r.collect().map(message => mapper.readValue(message, classOf[Message]))
val broadcasts = BroadCaster.getInstance(r.sparkContext, s"$s3://$bucket/$wgPath")
//Application logic
})
}
}
def initialiseSparkConf(local: Boolean): SparkConf = {
val sparkConf = new SparkConf()
.setAppName("Spark Streaming")
.set("spark.scheduler.mode", "FAIR")
.set("spark.sql.parquet.filterpushdown", "true")
.set("spark.executor.hearbeatInterval", "20")
.set("spark.streaming.driver.writeAheadLog.closeFileAfterWrite", "true")
.set("spark.streaming.receiver.writeAheadLog.closeFileAfterWrite", "true")
.set("spark.streaming.receiver.writeAheadLog.enable", "true")
.set("spark.streaming.stopGracefullyOnShutdown", "true")
.set("spark.streaming.backpressure.enabled","true")
.set("spark.streaming.backpressure.pid.minRate","10") //SQS support batch of 10
if (local) {
s3 = "s3a"
sparkConf.setMaster("local[*]")
} else {
sparkConf.set("hive.metastore.client.factory.class",
"com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory")
}
}
}
object BroadCaster {
#volatile private var instance: Map[String, Broadcast[Any]] = _
def getInstance(sparkContext: SparkContext, wgPath: String): Map[String, Broadcast[Any]] = {
if (instance == null) {
synchronized {
if (instance == null) {
instance = Utils.createBroadcastObjects(wgPath, sparkContext)
instance += (Constants.SQS_RECEIVER -> sparkContext.broadcast(getSQSReceiver(sparkContext.getConf)))
}
}
}
instance
}
private def getSQSReceiver(conf: SparkConf): SQSReceiver = {
val javaSQSReceiver = new SQSReceiver(conf.get(Constants.QUEUE_NAME)).withRegion(Regions.fromName(conf.get(Constants.REGION))).withCredential(new DefaultAWSCredentialsProviderChain())
.withFetchMaxMessage(conf.getInt(Constants.FETCH_MAX_MESSAGE, 10)).withVisibilityTimeOutSeconds(conf.getInt(Constants.VISIBILITY_TIMEOUT_SECONDS, 1800)).withWaitTimeoutinMillis(conf.getLong(Constants.WAIT_TIMEOUT_IN_MILLIS, 1000))
javaSQSReceiver
}
}
import java.util.List;
import org.apache.log4j.Logger;
import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.receiver.Receiver;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequest;
import com.amazonaws.services.sqs.model.DeleteMessageRequest;
import com.amazonaws.services.sqs.model.Message;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SQSReceiver extends Receiver<String> {
private String queueName;
private transient AWSCredentialsProvider credential;
private Regions region = Regions.US_EAST_1;
private Long waitTimeoutinMillis = 0L;
private ObjectMapper mapper= new ObjectMapper();
private transient Logger logger = Logger.getLogger(SQSReceiver.class);
private boolean deleteOnReceipt = false;
private int fetchMaxMessage = 100;
private int visibilityTimeOutSeconds = 60;
private String sqsQueueUrl;
private transient AmazonSQS amazonSQS;
public SQSReceiver(String queueName) {
this(queueName, false);
}
public SQSReceiver(String queueName, boolean deleteOnReceipt) {
super(StorageLevel.MEMORY_AND_DISK_SER());
this.queueName = queueName;
this.deleteOnReceipt = deleteOnReceipt;
setupSQS(queueName);
}
private void setupSQS(String queueName) {
AmazonSQSClientBuilder amazonSQSClientBuilder = AmazonSQSClientBuilder.standard();
if (credential != null) {
amazonSQSClientBuilder.withCredentials(credential);
}
amazonSQSClientBuilder.withRegion(region);
amazonSQS = amazonSQSClientBuilder.build();
sqsQueueUrl = amazonSQS.getQueueUrl(queueName).getQueueUrl();
}
public void onStart() {
new Thread(this::receive).start();
}
public void onStop() {
// There is nothing much to do as the thread calling receive()
// is designed to stop by itself if isStopped() returns false
}
private void receive() {
try {
setupSQS(queueName);
ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(sqsQueueUrl).withMaxNumberOfMessages(fetchMaxMessage).withVisibilityTimeout(visibilityTimeOutSeconds)
.withWaitTimeSeconds(20); //https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/examples-sqs-long-polling.html
receiveMessagesFromSQS(amazonSQS, sqsQueueUrl, receiveMessageRequest);
} catch (Throwable e) {
stop("Error encountered while initializing", e);
}
}
private void receiveMessagesFromSQS(final AmazonSQS amazonSQS, final String sqsQueueUrl,
ReceiveMessageRequest receiveMessageRequest) {
try {
while (!isStopped()) {
List<Message> messages = amazonSQS.receiveMessage(receiveMessageRequest).getMessages();
if (deleteOnReceipt) {
String receiptHandle = messages.get(0).getReceiptHandle();
messages.forEach(m -> store(m.getBody()));
amazonSQS.deleteMessage(new DeleteMessageRequest(sqsQueueUrl, receiptHandle));
} else {
messages.forEach(this::storeMessage);
}
if (waitTimeoutinMillis > 0L)
Thread.sleep(waitTimeoutinMillis);
}
restart("Trying to connect again");
} catch (IllegalArgumentException | InterruptedException e) {
restart("Could not connect", e);
} catch (Throwable e) {
restart("Error Receiving Data", e);
}
}
private void storeMessage(Message m) {
try {
if (m != null)
store(mapper.writeValueAsString(m));
} catch (JsonProcessingException e) {
logger.error("Unable to write message to streaming context");
}
}
public SQSReceiver withVisibilityTimeOutSeconds(int visibilityTimeOutSeconds) {
this.visibilityTimeOutSeconds = visibilityTimeOutSeconds;
return this;
}
public SQSReceiver withFetchMaxMessage(int fetchMaxMessage) {
if (fetchMaxMessage > 10) {
throw new IllegalArgumentException("FetchMaxMessage can't be greater than 10");
}
this.fetchMaxMessage = fetchMaxMessage;
return this;
}
public SQSReceiver withWaitTimeoutinMillis(long waitTimeoutinMillis) {
this.waitTimeoutinMillis = waitTimeoutinMillis;
return this;
}
public SQSReceiver withRegion(Regions region) {
this.region = region;
return this;
}
public SQSReceiver withCredential(AWSCredentialsProvider credential) {
this.credential = credential;
return this;
}
public void deleteMessages(DeleteMessageBatchRequest request) {
request.withQueueUrl(sqsQueueUrl);
amazonSQS.deleteMessageBatch(request);
}
}

Related

.AuthenticationViewModel .collect and .it problem

This is my project. This is AuthenticationViewModel part
the last .collect and = it code don't work. Please help me - where is the problem?
I can't find solution
package com.sondilek.presentation.Authentication
import androidx.compose.runtime.State
import androidx.compose.runtime.mutableStateOf
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import com.sondilek.domain.use_cases.AuthenticationUseCases
import com.sondilek.util.Response
import dagger.hilt.android.lifecycle.HiltViewModel
import kotlinx.coroutines.flow.collect
import kotlinx.coroutines.launch
import javax.inject.Inject
#HiltViewModel
class AuthenticationViewModel #Inject constructor(
private val authUseCases : AuthenticationUseCases
) : ViewModel(){
val isUserAuthenticated get() = authUseCases.isUserAuthenticated()
private val _signInState = mutableStateOf<Response<Boolean>>(Response.Success(false))
val signInState : State<Response<Boolean>> = _signInState
private val _signUpState = mutableStateOf<Response<Boolean>>(Response.Success(false))
val signUpState : State<Response<Boolean>> = _signUpState
private val _signOutState = mutableStateOf<Response<Boolean>>(Response.Success(false))
val signOutState : State<Response<Boolean>> = _signOutState
private val _firebaseAuthState = mutableStateOf<Boolean>(false)
val firebaseAuthState : State<Boolean> = _firebaseAuthState
fun signOut(){
viewModelScope.launch {
authUseCases.firebaseSignOut().collect{
_signOutState.value = it
if(it==Response.Success(true)){
_signInState.value = Response.Success(false)
}
}
}
}
fun signIn(email:String,password:String){
viewModelScope.launch {
authUseCases.firebaseSignIn(email = email, password = password).collect{
_signInState.value = it
}
}
}
fun signUp(email:String,password:String,username:String){
viewModelScope.launch {
authUseCases.firebaseSignUp(email = email, password = password, userName = username)
.collect{
_signUpState.value = it
}
}
}
fun getFirebaseAuthState(){
viewModelScope.launch {
authUseCases.firebaseAuthState().collect {
_firebaseAuthState.value = it
}
}
}
}

How can an App write and read file in Documents folder?

I wrote an App, in Kotlin with Android Studio that write some strings to a file.
All work, I can write and read inside the App, but I can't see the file looking in Documents folder.
How can I use the folder Documents as a storage space?
Thank you
These are the function I use:
fun saveTextFile(view: View, nomeFile: String, testo: String, contesto: Context){
var fileStream: FileOutputStream
try {
fileStream = contesto.openFileOutput(nomeFile, MODE_APPEND) // OK esegue append
fileStream.write(testo.toByteArray())
fileStream.close()
}catch (e: Exception){
e.printStackTrace()
}
}
fun readTextFile(view: View, nomeFile: String, contesto: Context): String{
var fileInputStream: FileInputStream? = null
fileInputStream = contesto.openFileInput(nomeFile)
var inputStreamReader: InputStreamReader = InputStreamReader(fileInputStream)
val bufferedReader: BufferedReader = BufferedReader(inputStreamReader)
val stringBuilder: StringBuilder = StringBuilder()
var text: String? = null
while ({ text = bufferedReader.readLine(); text }() != null) {
stringBuilder.append(text)
}
inputStreamReader.close();
return(stringBuilder.toString())
}
Thank you, Livio
For writing in Documents folder of your device , you just need to make use of MediaStore for the same. You can take input for this function anything that you want like String , bitmap , PdfDocument and other's too .
For Your UseCase you can do the following ,
Global Variable :
private var imageUri: Uri? = null
override suspend fun saveDocument(context : Context, text : String) {
withContext(Dispatchers.IO) {
try {
val collection =
MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY)
val dirDest = File(
Environment.DIRECTORY_DOCUMENTS,
context.getString(R.string.app_name)
)
val date = System.currentTimeMillis()
val fileName = "$date.txt"
val contentValues = ContentValues().apply {
put(MediaStore.MediaColumns.DISPLAY_NAME, fileName)
put(MediaStore.MediaColumns.RELATIVE_PATH,
"$dirDest${File.separator}")
put(MediaStore.Files.FileColumns.IS_PENDING, 1)
}
}
val imageUri = context.contentResolver.insert(collection, contentValues)
withContext(Dispatchers.IO) {
imageUri?.let { uri ->
context.contentResolver.openOutputStream(uri, "w").use { out -> out?.write(text.toByteArray())
}
contentValues.clear()
contentValues.put(MediaStore.Files.FileColumns.IS_PENDING, 0)
context.contentResolver.update(uri, contentValues, null, null)
}
}
} catch (e: FileNotFoundException) {
null
}
}
}
For Updating the already existing file , do the following . After creating file for the first time I have saved the imageUri in a global variable (If you want to store it permanently / or for a while you can use Jetpack Datastore / Shared Preference to save the same ):
suspend fun updateData(context: Context,text : String){
withContext(Dispatchers.IO) {
try {
val collection =
MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY)
val dirDest = File(
Environment.DIRECTORY_DOCUMENTS,
context.getString(R.string.app_name)
)
val fileName = "test.txt"
val contentValues = ContentValues().apply {
put(MediaStore.MediaColumns.DISPLAY_NAME, fileName)
put(
MediaStore.MediaColumns.RELATIVE_PATH,
"$dirDest${File.separator}"
)
put(MediaStore.Files.FileColumns.IS_PENDING, 1)
}
withContext(Dispatchers.IO) {
imageUri?.let { uri ->
context.contentResolver.openOutputStream(uri, "wa").use { out ->
out?.write(text.toByteArray())
}
contentValues.clear()
contentValues.put(MediaStore.Files.FileColumns.IS_PENDING, 0)
context.contentResolver.update(uri, contentValues, null, null)
}
}
} catch (e: FileNotFoundException) {
null
}
}
}
For Reading the File , Do the following :
suspend fun read(context: Context, source: Uri): String = withContext(Dispatchers.IO) {
val resolver: ContentResolver = context.contentResolver
resolver.openInputStream(source)?.use { stream -> stream.readText() }
?: throw IllegalStateException("could not open $source")
}
private fun InputStream.readText(charset: Charset = Charsets.UTF_8): String =
readBytes().toString(charset)
This is how the final code looks like :
class MainActivity : AppCompatActivity() {
private lateinit var btn: Button
private var imageUri: Uri? = null
private lateinit var btn2: Button
private lateinit var btn3 : Button
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_main)
btn = findViewById(R.id.btnAdd)
btn2 = findViewById(R.id.getText)
btn3 = findViewById(R.id.updateText)
btn.setOnClickListener {
lifecycleScope.launch {
saveDocument(applicationContext, "Original ")
}
}
btn3.setOnClickListener {
lifecycleScope.launch {
updateData(applicationContext,"Appended")
}
}
btn2.setOnClickListener {
lifecycleScope.launch {
imageUri?.let { it1 ->
val data = read(applicationContext, it1)
Toast.makeText(applicationContext, "The data is $data ", Toast.LENGTH_LONG)
.show()
}
}
}
}
suspend fun saveDocument(context: Context, text: String) {
withContext(Dispatchers.IO) {
try {
val collection =
MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY)
val dirDest = File(
Environment.DIRECTORY_DOCUMENTS,
context.getString(R.string.app_name)
)
val date = System.currentTimeMillis()
val fileName = "test.txt"
val contentValues = ContentValues().apply {
put(MediaStore.MediaColumns.DISPLAY_NAME, fileName)
put(
MediaStore.MediaColumns.RELATIVE_PATH,
"$dirDest${File.separator}"
)
put(MediaStore.Files.FileColumns.IS_PENDING, 1)
}
imageUri = context.contentResolver.insert(collection, contentValues)
withContext(Dispatchers.IO) {
imageUri?.let { uri ->
context.contentResolver.openOutputStream(uri, "w").use { out ->
out?.write(text.toByteArray())
}
contentValues.clear()
contentValues.put(MediaStore.Files.FileColumns.IS_PENDING, 0)
context.contentResolver.update(uri, contentValues, null, null)
}
}
} catch (e: FileNotFoundException) {
null
}
}
}
suspend fun updateData(context: Context, text: String) {
withContext(Dispatchers.IO) {
try {
val collection =
MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY)
val dirDest = File(
Environment.DIRECTORY_DOCUMENTS,
context.getString(R.string.app_name)
)
val fileName = "test.txt"
val contentValues = ContentValues().apply {
put(MediaStore.MediaColumns.DISPLAY_NAME, fileName)
put(
MediaStore.MediaColumns.RELATIVE_PATH,
"$dirDest${File.separator}"
)
put(MediaStore.Files.FileColumns.IS_PENDING, 1)
}
withContext(Dispatchers.IO) {
imageUri?.let { uri ->
context.contentResolver.openOutputStream(uri, "wa").use { out ->
out?.write(text.toByteArray())
}
contentValues.clear()
contentValues.put(MediaStore.Files.FileColumns.IS_PENDING, 0)
context.contentResolver.update(uri, contentValues, null, null)
}
}
} catch (e: FileNotFoundException) {
null
}
}
}
suspend fun read(context: Context, source: Uri): String = withContext(Dispatchers.IO) {
val resolver: ContentResolver = context.contentResolver
resolver.openInputStream(source)?.use { stream -> stream.readText() }
?: throw IllegalStateException("could not open $source")
}
private fun InputStream.readText(charset: Charset = Charsets.UTF_8): String =
readBytes().toString(charset)
I have three buttons . With the first I create a file , then the uri gets stored in the global variable . Then onClick of second button I add to the already existing file and then read the file using the third button using the same imageUri stored in the global variable
This is the demo for the same . Check when the buttons are being pressed and the output in the form of Toast at the bottom .

Kotlin string replace function not working for me?

Hello everyone please help me i am try to modify link but it's not working. it's working on java but recently i convert java to kotlin and getting this error.
am trying to change my link http://www.offertoro.com/click_track/api?offer_id=419393&pub_id=14&pub_app_id=5&USER_ID=[USER_ID]&GAID=[your_GAID] in [USER_ID] with current login user email but getting error.
Check Screenshot
Screenshot
Error
None of the following functions can be called with the arguments supplied.
CharSequence.replace(Regex, String) defined in kotlin.text
String.replace(String, String, Boolean = ...) defined in kotlin.text
My code
fun modifyOfferLink() {
val id = mAuth!!.currentUser!!.email
// Modifying Offer Link Acording to Offer Partner
when (partner) {
"ogads" -> Finallink = link + "&aff_sub5=" + mAuth!!.currentUser!!.email
"offertoro" -> Finallink = link.replace("[USER_ID]", mAuth!!.currentUser!!.email)
"none" -> {
Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
else -> Finallink = link.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
}
OfferActivity.kt
package com.sgamer.creditsk.Activity
import android.content.Intent
import android.content.pm.PackageInfo
import android.content.pm.PackageManager
import android.graphics.Color
import android.graphics.drawable.ColorDrawable
import android.os.Build
import android.os.Bundle
import android.util.Log
import android.view.View
import android.view.Window
import android.view.WindowManager
import android.widget.ImageView
import android.widget.LinearLayout
import android.widget.TextView
import androidx.appcompat.app.AppCompatActivity
import androidx.appcompat.widget.Toolbar
import com.bumptech.glide.Glide
import com.bumptech.glide.request.RequestOptions
import com.google.firebase.auth.FirebaseAuth
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.loopj.android.http.AsyncHttpClient
import com.loopj.android.http.AsyncHttpResponseHandler
import com.loopj.android.http.RequestParams
import com.sgamer.creditsk.Activity.AndyConstants.ServiceType
import com.sgamer.creditsk.Activity.OfferDetailsActivity
import com.sgamer.creditsk.R
import com.sgamer.creditsk.Utils.*
import cz.msebera.android.httpclient.Header
import org.json.JSONArray
import org.json.JSONException
import org.json.JSONObject
class OfferDetailsActivity constructor() : AppCompatActivity() {
var Finallink: String? = null
var package_id: String? = null
var uniq_id: String? = null
var offerid: String? = null
var app_name: String? = null
var description: String? = null
var icon_url: String? = null
var bg_image_url: String? = null
var amount: String? = null
var OriginalAmount: String? = null
var link: String? = null
var partner: String? = null
var insTitle: String? = null
var first_text: String? = null
var second_text: String? = null
var third_text: String? = null
var fourth_text: String? = null
var webview: Boolean? = null
var ClickId: String? = null
var ctx: OfferDetailsActivity? = null
var later: TextView? = null
var status_image: ImageView? = null
var mAuth: FirebaseAuth? = null
private val bannerAdManager: BannerAdManager_SK? = null
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_offer_details)
val toolbar: Toolbar = findViewById<View>(R.id.toolbar) as Toolbar
val adContainer: LinearLayout = findViewById<View>(R.id.adView) as LinearLayout
setSupportActionBar(toolbar)
val bannerAdManager_SK: BannerAdManager_SK = BannerAdManager_SK(this#OfferDetailsActivity, adContainer)
bannerAdManager_SK.BannerAds()
ctx = this
mAuth = FirebaseAuth.getInstance()
getSupportActionBar()!!.setTitle(R.string.offer_details)
getSupportActionBar()!!.setDisplayHomeAsUpEnabled(true)
getSupportActionBar()!!.setBackgroundDrawable(ColorDrawable(getResources().getColor(android.R.color.transparent)))
getSupportActionBar()!!.setElevation(0f)
if (Build.VERSION.SDK_INT >= 21) {
getWindow().getDecorView().setSystemUiVisibility(View.SYSTEM_UI_FLAG_LAYOUT_STABLE or View.SYSTEM_UI_FLAG_LAYOUT_FULLSCREEN)
}
changeStatusBarColor()
initViews()
modifyOfferLink()
}
private fun changeStatusBarColor() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
val window: Window = getWindow()
window.addFlags(WindowManager.LayoutParams.FLAG_DRAWS_SYSTEM_BAR_BACKGROUNDS)
window.setStatusBarColor(Color.TRANSPARENT)
}
}
fun initViews() {
val title: TextView = findViewById(R.id.title)
val desc: TextView = findViewById(R.id.description)
val instructionsTitle: TextView = findViewById(R.id.instructions)
val first: TextView = findViewById(R.id.first)
val second: TextView = findViewById(R.id.second)
val third: TextView = findViewById(R.id.third)
val fourth: TextView = findViewById(R.id.fourth)
val des: TextView = findViewById(R.id.des)
val complete_button: TextView = findViewById(R.id.complete_button)
val button: TextView = findViewById(R.id.button)
later = findViewById(R.id.later)
val comSpace: LinearLayout = findViewById(R.id.comSpace)
val offer_icon: ImageView = findViewById(R.id.offer_icon)
val bg_image: ImageView = findViewById(R.id.bg_image)
status_image = findViewById(R.id.status_image)
uniq_id = getIntent().getStringExtra("uniq_id")
offerid = getIntent().getStringExtra("offerid")
app_name = getIntent().getStringExtra("app_name")
package_id = getIntent().getStringExtra("package_id")
description = getIntent().getStringExtra("description")
icon_url = getIntent().getStringExtra("icon_url")
bg_image_url = getIntent().getStringExtra("bg_image_url")
amount = getIntent().getStringExtra("amount")
OriginalAmount = getIntent().getStringExtra("OriginalAmount")
link = getIntent().getStringExtra("link")
partner = getIntent().getStringExtra("partner")
first_text = getIntent().getStringExtra("first_text")
insTitle = getIntent().getStringExtra("instructionsTitle")
second_text = getIntent().getStringExtra("second_text")
third_text = getIntent().getStringExtra("third_text")
fourth_text = getIntent().getStringExtra("fourth_text")
webview = getIntent().getBooleanExtra("webview", false)
if (getIntent().hasExtra("description")) {
des.setText(getIntent().getStringExtra("description"))
} else {
des.setText(getIntent().getStringExtra("description"))
}
title.setText(app_name)
desc.setText(getString(R.string.earn) + " " + amount + " " + getString(R.string.app_currency) + " " + getString(R.string.on_this_offer))
Glide.with(this).load(icon_url)
.apply(RequestOptions().placeholder(R.drawable.placeholder_image).error(R.drawable.placeholder_image))
.into(offer_icon)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
comSpace.setElevation(20f)
}
instructionsTitle.setText(insTitle)
first.setText(first_text)
second.setText(second_text)
third.setText(third_text)
fourth.setText(fourth_text)
complete_button.setText(getResources().getString(R.string.complete_offer))
if (!bg_image_url!!.isEmpty()) {
Glide.with(this).load(bg_image_url).into(bg_image)
} else {
}
// On click Listners
later!!.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
finish()
}
})
complete_button.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
if (!App.isVPNConnected()) {
addoffer(amount, app_name + " Offer Credit", offerid)
}
AppUtils.parse(this#OfferDetailsActivity, Finallink)
}
})
button.setOnClickListener(object : View.OnClickListener {
public override fun onClick(view: View) {
val launchIntent: Intent? = getPackageManager().getLaunchIntentForPackage((package_id)!!)
startActivity(launchIntent)
}
})
isAppExist
if (isAppExist) {
complete_button.setVisibility(View.GONE)
button.setVisibility(View.VISIBLE)
} else {
button.setVisibility(View.GONE)
complete_button.setVisibility(View.VISIBLE)
}
}
fun addoffer(points: String?, Activity: String?, offerid: String?) {
val client: AsyncHttpClient = AsyncHttpClient()
val params: RequestParams = RequestParams()
val jsObj: JsonObject = Gson().toJsonTree(API()) as JsonObject
jsObj.addProperty("method_name", "user_offeradd")
jsObj.addProperty("offer_id", offerid)
jsObj.addProperty("email", mAuth!!.getCurrentUser()!!.getEmail())
jsObj.addProperty("points", points)
jsObj.addProperty("firebase_id", mAuth!!.getCurrentUser()!!.getUid())
jsObj.addProperty("Activity", Activity)
params.put("data", API.toBase64(jsObj.toString()))
client.post(Javaaescipher.decrypt(), params, object : AsyncHttpResponseHandler() {
public override fun onSuccess(statusCode: Int, headers: Array<Header>, responseBody: ByteArray) {
Log.d("Response", String(responseBody))
val res: String = String(responseBody)
try {
val jsonObject: JSONObject = JSONObject(res)
val jsonArray: JSONArray = jsonObject.getJSONArray("ANDROID_REWARDS_APP")
for (i in 0 until jsonArray.length()) {
val `object`: JSONObject = jsonArray.getJSONObject(i)
val success: String = `object`.getString("success")
val msg: String = `object`.getString("msg")
if ((success == "1")) {
// Toast.makeText(OfferDetailsActivity.this, msg, Toast.LENGTH_LONG).show();
} else {
// Toast.makeText(OfferDetailsActivity.this, msg, Toast.LENGTH_LONG).show();
}
}
} catch (e: JSONException) {
e.printStackTrace()
}
}
public override fun onFailure(statusCode: Int, headers: Array<Header>, responseBody: ByteArray, error: Throwable) {
Log.d("error", error.toString())
}
})
}
private val isAppExist: Boolean
private get() {
val pm: PackageManager = getPackageManager()
try {
val info: PackageInfo = pm.getPackageInfo((package_id)!!, PackageManager.GET_META_DATA)
} catch (e: PackageManager.NameNotFoundException) {
return false
}
return true
}
fun modifyOfferLink() {
val id = mAuth!!.currentUser!!.email
// Modifying Offer Link Acording to Offer Partner
when (partner) {
"ogads" -> Finallink = link + "&aff_sub5=" + mAuth!!.currentUser!!.email
"offertoro" -> Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
"none" -> {
Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
else -> Finallink = link!!.replace("[USER_ID]", mAuth!!.currentUser!!.email)
}
}
}
It is likely there is no sub-string that is exactly [USER_ID]. Are you sure is it not ["USER_ID"] ?
Can you print the value of link before you call the replace. This may let us and you see the problem.

how to fix conversion error when converting from java to kotlin

I am a student, new to kotlin, so I am converting java codes to kotlin to learn and see how it works, but I didnt understand what the error says.
private val _songs = ArrayList<SongInfo>()
internal lateinit var recyclerView: RecyclerView
internal lateinit var seekBar: SeekBar
internal lateinit var songAdapter: SongAdapter
internal var mediaPlayer: MediaPlayer? = null
private val myHandler = Handler()
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContentView(R.layout.activity_main)
recyclerView = findViewById(R.id.recyclerView) as RecyclerView
seekBar = findViewById(R.id.seekBar) as SeekBar
songAdapter = SongAdapter(this, _songs)
recyclerView.adapter = songAdapter
val linearLayoutManager = LinearLayoutManager(this)
val dividerItemDecoration = DividerItemDecoration(recyclerView.context,
linearLayoutManager.orientation)
recyclerView.layoutManager = linearLayoutManager
recyclerView.addItemDecoration(dividerItemDecoration)
songAdapter.setOnItemClickListener { b, view, obj, position ->
if (b.text == "Stop") {
mediaPlayer!!.stop()
mediaPlayer!!.reset()
mediaPlayer!!.release()
mediaPlayer = null
b.text = "Play"
} else {
val runnable = Runnable {
try {
mediaPlayer = MediaPlayer()
mediaPlayer!!.setDataSource(obj.songUrl)
mediaPlayer!!.prepareAsync()
mediaPlayer!!.setOnPreparedListener { mp ->
mp.start()
seekBar.progress = 0
seekBar.max = mediaPlayer!!.duration
Log.d("Prog", "run: " + mediaPlayer!!.duration)
}
b.text = "Stop"
} catch (e: Exception) {
}
}
myHandler.postDelayed(runnable, 100)
}
}
checkUserPermission()
val t = runThread()
t.start()
}
inner class runThread : Thread() {
override fun run() {
while (true) {
try {
Thread.sleep(1000)
} catch (e: InterruptedException) {
e.printStackTrace()
}
Log.d("Runwa", "run: " + 1)
if (mediaPlayer != null) {
seekBar.post { seekBar.progress = mediaPlayer!!.currentPosition }
Log.d("Runwa", "run: " + mediaPlayer!!.currentPosition)
}
}
}
}
private fun checkUserPermission() {
if (Build.VERSION.SDK_INT >= 23) {
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.READ_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) {
requestPermissions(arrayOf(Manifest.permission.READ_EXTERNAL_STORAGE), 123)
return
}
}
loadSongs()
}
override fun onRequestPermissionsResult(requestCode: Int, permissions: Array<String>, grantResults: IntArray) {
when (requestCode) {
123 -> if (grantResults[0] == PackageManager.PERMISSION_GRANTED) {
loadSongs()
} else {
Toast.makeText(this, "Permission Denied", Toast.LENGTH_SHORT).show()
checkUserPermission()
}
else -> super.onRequestPermissionsResult(requestCode, permissions, grantResults)
}
}
private fun loadSongs() {
val uri = MediaStore.Audio.Media.EXTERNAL_CONTENT_URI
val selection = MediaStore.Audio.Media.IS_MUSIC + "!=0"
val cursor = contentResolver.query(uri, null, selection, null, null)
if (cursor != null) {
if (cursor.moveToFirst()) {
do {
val name = cursor.getString(cursor.getColumnIndex(MediaStore.Audio.Media.DISPLAY_NAME))
val artist = cursor.getString(cursor.getColumnIndex(MediaStore.Audio.Media.ARTIST))
val url = cursor.getString(cursor.getColumnIndex(MediaStore.Audio.Media.DATA))
val s = SongInfo(name, artist, url)
_songs.add(s)
} while (cursor.moveToNext())
}
cursor.close()
songAdapter = SongAdapter(this#MainActivity, _songs)
}
}
}
This is the error:
"Error:(46, 44) Type mismatch: inferred type is (???, ???, ???, ???)
-> Any but SongAdapter.OnItemClickListener was expected Error:(46, 46) Cannot infer a type for this parameter. Please specify it explicitly."
Batch conversion to Kotlin is not the best way to learn the language. I suggest you to re-implement your Android component in Kotlin manually, to get the feeling of language.
The error you see says: "I can not understand how this lambda with 4 parameters can be an instance of SongAdapter.OnItemClickListener, please help". You can try anonymous class in this place.

How to read a composite column that might/might not be existing, get its value to update it and save it back?

I have a composite column (Int32Type,BytesType,AsciiType) that I need to read its value (based on criteria), modify it and save it back (something like manual counter column).
The composite column that I'm querying might exist or it may not.
What is the best way to do that in Hector?
I cannot vouch the following solution is the best but it does the basic functionality like creating composite columns. It basically does reading and writing which essentially inline to "I need to read its value (based on criteria), modify it and save it back (something like manual counter column)." . But I think with this sample codes, it should be able to serve as a basic and improve here and there and so that it become the best. :-) Will test it thoroughly when there is free time. With that said, the following is my suggestion.
package com.hector.dataTypes;
import java.util.Iterator;
import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
import me.prettyprint.cassandra.serializers.CompositeSerializer;
import me.prettyprint.cassandra.serializers.IntegerSerializer;
import me.prettyprint.cassandra.serializers.StringSerializer;
import me.prettyprint.cassandra.service.CassandraHostConfigurator;
import me.prettyprint.cassandra.service.ColumnSliceIterator;
import me.prettyprint.cassandra.service.ThriftCluster;
import me.prettyprint.hector.api.Cluster;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.Composite;
import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.ddl.ColumnFamilyDefinition;
import me.prettyprint.hector.api.ddl.ColumnType;
import me.prettyprint.hector.api.ddl.ComparatorType;
import me.prettyprint.hector.api.exceptions.HectorException;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.SliceQuery;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.utils.ByteBufferUtil;
import com.google.common.base.Joiner;
/**
*
* #author jasonw
*
*/
public class CompositeExample
{
private String m_node;
private String m_keyspace;
private String m_column_family;
private ThriftCluster m_cassandraCluster;
private CassandraHostConfigurator m_cassandraHostConfigurator;
private Mutator<String> mutator;
private SliceQuery<String, Composite, String> sliceQuery;
public CompositeExample(String p_node, String p_keyspace, String p_column_family, String p_cluster)
{
m_node = p_node;
m_keyspace = p_keyspace;
m_column_family = p_column_family;
m_cassandraHostConfigurator = new CassandraHostConfigurator(m_node);
m_cassandraCluster = new ThriftCluster(p_cluster, m_cassandraHostConfigurator);
Cluster cluster = HFactory.getOrCreateCluster(p_cluster, m_cassandraHostConfigurator);
Keyspace keyspace = HFactory.createKeyspace(m_keyspace, cluster);
mutator = HFactory.createMutator(keyspace, StringSerializer.get());
sliceQuery = HFactory.createSliceQuery(keyspace, StringSerializer.get(), CompositeSerializer.get(), StringSerializer.get());
}
public boolean createCompositeColumn(String... p_new_columns)
{
try
{
ColumnFamilyDefinition cfDef = HFactory.createColumnFamilyDefinition(m_keyspace, m_column_family, ComparatorType.COMPOSITETYPE);
cfDef.setColumnType(ColumnType.STANDARD);
cfDef.setComparatorTypeAlias("(".concat(Joiner.on(",").join(p_new_columns)).concat(")"));
cfDef.setKeyValidationClass("UTF8Type");
cfDef.setDefaultValidationClass("UTF8Type");
m_cassandraCluster.addColumnFamily(cfDef, true);
return true;
}
catch (HectorException e)
{
e.printStackTrace();
}
return false;
}
public boolean saveColumn(String p_field_one, String p_field_two, String p_field_three)
{
try
{
Composite c = new Composite();
c.addComponent(Int32Type.instance.fromString(p_field_one), ByteBufferSerializer.get());
c.addComponent(ByteBufferUtil.bytes(p_field_two), ByteBufferSerializer.get());
c.addComponent(p_field_three, StringSerializer.get());
HColumn<Composite, String> col = HFactory.createColumn(c, "composite_value", CompositeSerializer.get(), StringSerializer.get());
mutator.addInsertion("key", m_column_family, col);
mutator.execute();
return true;
}
catch (HectorException e)
{
e.printStackTrace();
}
return false;
}
public boolean readColumn(String p_key, int p_column_number, ComponentEquality p_equality, int p_value)
{
if (p_column_number < 0 || p_column_number > 2)
{
return false;
}
try
{
sliceQuery.setColumnFamily(m_column_family);
sliceQuery.setKey(p_key);
Composite start = new Composite();
start.addComponent(0, p_value, p_equality);
Composite end = new Composite();
end.addComponent(0, p_value, ComponentEquality.GREATER_THAN_EQUAL);
sliceQuery.setRange(start, end, false, 1000);
QueryResult<ColumnSlice<Composite, String>> qr = sliceQuery.execute();
System.out.println("size = " + qr.get().getColumns().size());
Iterator<HColumn<Composite, String>> iter = qr.get().getColumns().iterator();
while (iter.hasNext())
{
HColumn<Composite, String> column = iter.next();
System.out.print(column.getName().get(0, IntegerSerializer.get()));
System.out.print(":");
System.out.print(column.getName().get(1, StringSerializer.get()));
System.out.print(":");
System.out.print(column.getName().get(2, StringSerializer.get()));
System.out.println("=" + column.getValue());
}
return true;
}
catch (HectorException e)
{
e.printStackTrace();
}
catch (Exception why)
{
why.printStackTrace();
}
return false;
}
public static void main(String[] args)
{
boolean isSuccess = false;
String node_ip = "192.168.0.1";
String keyspace_name = "mykeyspace";
String column_family_name = "compositecf";
String cluster_name = "Test Cluster";
CompositeExample test1 = new CompositeExample(node_ip, keyspace_name, column_family_name, cluster_name);
isSuccess = test1.createCompositeColumn("Int32Type", "BytesType", "AsciiType");
if (!isSuccess)
{
System.err.println("failed to create cf");
System.exit(-1);
}
isSuccess = test1.saveColumn("1027", "blablabla", "this is ascii field");
if (!isSuccess)
{
System.err.println("failed to write");
System.exit(-1);
}
isSuccess = test1.readColumn("key", 0, ComponentEquality.EQUAL, 1027);
if (!isSuccess)
{
System.err.println("failed to read");
System.exit(-1);
}
}
}
Composite col = new Composite(yourInt, yourBytes, yourString);
ColumnSlice<Composite, valueType> result = HFactory.createSliceQuery(keyspace, keySerializer, compositeSerializer, intSerializer)
.setColumnFamily(columnFamily)
.setKey(key)
.setRange(col, col, false, 1)
.execute()
.get();
if (result.getColumns().isEmpty()) {
// do whatever you need to do if there's no value
} else {
int value = result.getColumns().get(0).getValue();
int newValue = //some modification to value
Mutator<keyType> mutator = HFactory.createMutator(keyspace, keySerializer);
HColumn<Composite, int> column = HFactory.createColumn(col, newValue, CompositeSerializer, intSerializer);
mutator.addInsertion(key, columnFamily, column);
mutator.execute();
}

Resources