real time inference on Spark Streaming - apache-spark

Question

I'm trying to run real-time inference on a spark structured stream, first I trained the model
#model_creation
model.fit()
model.predict([ 33.26, 68.51, 1012.49, 52.68])
#create spark df from kafka stream
df = spark.readstream.format("kafka").....
#inference
def predict(input):
#extract json from input and conver to list of doubles
#model.predict(input_array)
#result = model.predict(input_list)
#return result
spark.udf.register("lr_predict", predict ,StringType())
df3 = df2.withColumn('predict_response',predict(col('value')))
display(df3)
I'm not sure how to extract the json input from sql spark dataframe and run it in model, I've been trying things since yesterday nothing seem to stick.

Answer 1

import json
def predict(input):
""" I am creating the function with following assumptions
please tell me if those are not correct
the model outputs a string value
the model need numeric data type as input
"""
#extract json from input and conver to list of doubles
#model.predict(input_array)
#result = model.predict(input_list)
#return result
input_proccesed=json.loads(input)
input_features_array=input_proccesed['input']
##casting to float just to be sure
input_features_array=[float(x) for x in input_features_array]
#predicting the output
result= model.predict(input_proccesed['input'])
return result
##if the output is string like - "GOOD" /"BAD" the use StringType()
##if the output is numeric like - 0.92 ,1028.384 the use DecimalType()
from pyspark.sql.types import StringType,DecimalType,FloatType,DoubleType
from pyspark.sql.functions import udf,col
lr_prediction=udf(predict,DoubleType())
df3 = df2.withColumn('predict_response_as_double',lr_prediction(col('value')))
lr_prediction=udf(predict,StringType())
df4 = df2.withColumn('predict_response_as_string',lr_prediction(col('value')))