In Pyspark, whenever i read a json file with an empty set element. The entire element is ignored in the resultant DataFrame.
Sample json :
{logs :[],pagination:{}}
And it only ignores the second element, i.e pagination in the above example. is there anyway to read the json with proper schema.?
Yes, you can perform in two ways with schema and without schema:
Reading Json with schema:
from pyspark.sql.functions import from_json, col
from pyspark.sql.types import StructType, StructField, StringType, IntegerType,LongType
schema = StructType([StructField('email', StringType(), True),
StructField('first_name', StringType(), True),
StructField('gender', StringType(), True),
StructField('id', LongType(), True),
StructField('last_name', StringType(), True)])
df = spark.read.schema(schema).json(r'dbfs:/FileStore/MOCK_DATA__1_.json')
Reading Json Without schema
d1 = spark.read.json(r'dbfs:/FileStore/MOCK_DATA__1_.json')
d1.show()
Related
I currently have a database called "bronze" with one table inside it that was created using almost the same code as below (just changing the TABLE_NAME and SCHEMA).
import findspark
findspark.init()
import delta
import os
from pyspark.sql.types import StructType, StructField, IntegerType, StringType, BooleanType, FloatType
from pyspark.sql import SparkSession, window
from pyspark.sql import functions as F
from os.path import abspath
def upsertToDelta(df, batchId):
'''
In order to guarantee there aren't any duplicated matches, a Window is used to filter matches based on its GameId and UpdatedUtc.
The GameId is used as a group by and UpdatedUtc is used as an order by.
If it's found a duplicated match, the duplicate will be not be saved.
'''
windowSpec = window.Window.partitionBy("GameId").orderBy("UpdatedUtc") # .orderBy(1)
df_new = df.withColumn("row_number", F.row_number().over(windowSpec)).filter("row_number = 1")
( bronzeDeltaTable.alias("bronze")
.merge(df_new.alias("raw"), "bronze.GameId = raw.GameId")
.whenMatchedUpdateAll()
.whenNotMatchedInsertAll()
.execute()
)
warehouse_location = abspath('spark-warehouse')
builder = SparkSession.builder \
.master('local[*]') \
.config("spark.sql.warehouse.dir", warehouse_location) \
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
.config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
spark = delta.configure_spark_with_delta_pip(builder) \
.getOrCreate()
leaderboards_schema = StructType([
StructField("PlayerId", IntegerType(), False),
StructField("TeamId", IntegerType(), False),
StructField("Name", StringType(), False),
StructField("MatchName", StringType(), False),
StructField("Team", StringType(), False),
StructField("IsClosed", BooleanType(), False),
StructField("GameId", IntegerType(), False),
StructField("OpponentId", IntegerType(), False),
StructField("Opponent", StringType(), False),
StructField("Day", StringType(), True),
StructField("DateTime", StringType(), True),
StructField("Updated", StringType(), True),
StructField("UpdatedUtc", StringType(), True),
StructField("Games", StringType(), True),
StructField("Maps", FloatType(), True),
StructField("FantasyPoints", FloatType(), True),
StructField("Kills", FloatType(), True),
StructField("Assists", FloatType(), True),
StructField("Deaths", FloatType(), True),
StructField("Headshots", FloatType(), True),
StructField("AverageDamagePerRound", FloatType(), True),
StructField("Kast", FloatType(), True),
StructField("Rating", FloatType(), True),
StructField("EntryKills", FloatType(), True),
StructField("QuadKills", FloatType(), True),
StructField("Aces", FloatType(), True),
StructField("Clutch1v2s", FloatType(), True),
StructField("Clutch1v3s", FloatType(), True),
StructField("Clutch1v4s", FloatType(), True),
StructField("Clutch1v5s", FloatType(), True),
])
map_schema = StructType([
StructField("Number", IntegerType(), True),
StructField("Name", StringType(), True),
StructField("Status", StringType(), True),
StructField("CurrentRound", IntegerType(), True),
StructField("TeamAScore", IntegerType(), True),
StructField("TeamBScore", IntegerType(), True),
])
SCHEMAS = {
"tb_leaderboards": leaderboards_schema,
"tb_maps": map_schema
}
if "spark-warehouse" not in os.listdir():
spark.sql("CREATE DATABASE bronze")
try:
for TABLE_NAME in list(SCHEMAS.keys()):
'''
Full load
'''
if TABLE_NAME not in os.listdir('spark-warehouse/bronze.db'):
df = spark.read.parquet(f"raw/{TABLE_NAME}")
windowSpec = window.Window.partitionBy("GameId").orderBy("UpdatedUtc") # .orderBy(1)
df_new = df.withColumn("row_number", F.row_number().over(windowSpec)).filter("row_number = 1").drop("row_number")
df_new.write.mode("overwrite").format("delta").saveAsTable(f"bronze.{TABLE_NAME}") # overwriting it's not overwrititng because it creates a different file name
# df_new.write.format("delta").saveAsTable(name=f"{warehouse_location}.bronze.{TABLE_NAME}", mode="overwrite")
# df_new.write.mode("overwrite").format("delta").saveAsTable(f"bronze.{TABLE_NAME}")
bronzeDeltaTable = delta.tables.DeltaTable.forPath(spark, f"spark-warehouse/bronze.db/{TABLE_NAME}") #"bronze"
'''
When new matches lands in raw, a stream is responsible for saving these new matches in bronze.
'''
df_stream = ( spark.readStream
.format("parquet")
.schema(SCHEMAS[TABLE_NAME])
.load(f"raw/{TABLE_NAME}")
)
stream = ( df_stream.writeStream
.foreachBatch(upsertToDelta)
.option("checkpointLocation", f"spark-warehouse/bronze.db/{TABLE_NAME}_checkpoint")
.outputMode("update")
.start()
)
stream.processAllAvailable()
stream.stop()
finally:
spark.stop()
But when I execute the code above I'm getting the error pyspark.sql.utils.AnalysisException: Database 'bronze' not found. The error occurs when trying to execute df_new.write.mode("overwrite").format("delta").saveAsTable(f"bronze.{TABLE_NAME}")
This is the current directory structure
I've already tried to include "spark-warehouse." before "bronze" as also add backquotes on "spark-warehouse", "bronze" and "{TABLE_NAME}" but nothing seems to work.
I'm running the code on Windows 10 with PySpark 3.3.1, Hadoop 3, delta-spark 2.2.0 and Java 11.0.16, but I also tested on Ubuntu 22.04 with the same config.
------------
Edit #1:
Asking ChatGPT for a solution to my problem, it suggested to use save() instead of saveAsTable(). So, changing df_new.write.mode("overwrite").format("delta").saveAsTable(f"bronze.{TABLE_NAME}") to df_new.write.mode("overwrite").format("delta").save(f"spark-warehouse/bronze.db/{TABLE_NAME}") actually saves inside bronze database folder. However, if I run spark.sql("USE bronze") it still gives the same AnalysisException: Database 'bronze' not found error. Also, spark.sql("SHOW DATABASES").show() doesn't show bronze database, it only shows default.
------------
Any solutions to my problem ?
If anyone wants to test in your local machine, here's the repository.
Not too sure, but I think for saveAsTable, you need to set the write mode inside the method as an argument (pyspark.sql.DataFrameWriter.saveAsTable).
Try this:
df.write.format("delta").saveAsTable(
name=f"bronze.{TABLE_NAME}",
mode="overwrite"
)
I am trying to read a json file, convert it to CSV in PySpark as below.
df = spark.read.json(inputdir')
I have the below schema which I am imposing on my dataframe.
mechanic_schema = StructType([
StructField("name", StringType(), True),
StructField("some_other_column", StringType(), True),
StructField("url", StringType(), True),
StructField("image", StringType(), True),
StructField("startTime", StringType(), True),
StructField("recipeYield", StringType(), True),
StructField("datePublished", StringType(), True),
StructField("endTime", StringType(), True),
StructField("description", StringType(), True)
])
I am saving the dataframe: df in an output directory as below.
df.select(mechanic_schema.names).write.format('csv').option("header","true").save(''/Users/bobby/Desktop/output/', header='true')
This is how the output looks like:
df.show()
Now in another script, I am reading the same csv file that I saved in output path of df as below:
df = spark.read.format('csv').option('header', True).load('/Users/bobby/Desktop/output/')
df.show()
But strangely, the output contains so many columns as nulls which looks like this:
So I checked my output CSV file and the data looks exactly fine there.
I have never come across this phenomenon until now and don't understand what did I do wrong here.
Could anyone let me know what is causing this issue and how can fix this problem ?
Any help is appreciated.
Syntax for creating a schema in PySpark.
data.csv
id,name
1,sam
2,smith
val schema = new StructType().add("id", IntType).add("name", StringType)
val ds = spark.read.schema(schema).option("header", "true").csv("data.csv")
ds.show
define StructType with StructField(name, dataType, nullable=True)
from pyspark.sql.types you can import datatypes
from pyspark.sql.types import StructType, StructField, IntegerType, StringType,FloatType,BooleanType
schema = StructType([
StructField("col_a", StringType(), True),
StructField("col_b", IntegerType(), True),
StructField("col_c", FloatType(), True),
StructField("col_d", BooleanType(), True)
])
I have a pyspark to load data from a TSV file and save it as parquet file as well save it as a persistent SQL table.
When I run it line by line through pyspark CLI, it works exactly like expected. When I run it as as an application using spark-submit it runs without any errors but I get strange results: 1. the data is overwritten instead of appended. 2. When I run SQL queries against it I get no data returned even though the parquet files are several gigabytes in size (what I expect). Any suggestions?
Code:
from pyspark import SparkContext, SparkConf
from pyspark.sql.types import *
from pyspark.sql.functions import *
csv_file = '/srv/spark/data/input/ipfixminute2018-03-28.tsv'
parquet_dir = '/srv/spark/data/parquet/ipfixminute'
sc = SparkContext(appName='import-ipfixminute')
spark = SQLContext(sc)
fields = [StructField('time_stamp', TimestampType(), True),
StructField('subscriberId', StringType(), True),
StructField('sourceIPv4Address', StringType(), True),
StructField('destinationIPv4Address', StringType(), True),
StructField('service',StringType(), True),
StructField('baseService',StringType(), True),
StructField('serverHostname', StringType(), True),
StructField('rat', StringType(), True),
StructField('userAgent', StringType(), True),
StructField('accessPoint', StringType(), True),
StructField('station', StringType(), True),
StructField('device', StringType(), True),
StructField('contentCategories', StringType(), True),
StructField('incomingOctets', LongType(), True),
StructField('outgoingOctets', LongType(), True),
StructField('incomingShapingDrops', IntegerType(), True),
StructField('outgoingShapingDrops', IntegerType(), True),
StructField('qoeIncomingInternal', DoubleType(), True),
StructField('qoeIncomingExternal', DoubleType(), True),
StructField('qoeOutgoingInternal', DoubleType(), True),
StructField('qoeOutgoingExternal', DoubleType(), True),
StructField('incomingShapingLatency', DoubleType(), True),
StructField('outgoingShapingLatency', DoubleType(), True),
StructField('internalRtt', DoubleType(), True),
StructField('externalRtt', DoubleType(), True),
StructField('HttpUrl',StringType(), True)]
schema = StructType(fields)
df = spark.read.load(csv_file, format='csv',sep='\t',header=True,schema=schema,timestampFormat='yyyy-MM-dd HH:mm:ss')
df = df.drop('all')
df = df.withColumn('date',to_date('time_stamp'))
df.write.saveAsTable('test2',mode='append',partitionBy='date',path=parquet_dir)
As #user8371915 suggested it is similar to this:
Spark can access Hive table from pyspark but not from spark-submit
I needed to replace
from pyspark.sql import SQLContext
sqlContext = SQLContext(sc)
with
from pyspark.sql import HiveContext
sqlContext = HiveContext(sc)
This resolved this issue.
I have data in a list and want to convert it to a spark dataframe with one of the column names containing a "."
I wrote the below code which ran without any errors.
input_data = [('retail', '2017-01-03T13:21:00', 134),
('retail', '2017-01-03T13:21:00', 100)]
rdd_schema = StructType([StructField('business', StringType(), True), \
StructField('date', StringType(), True), \
StructField("`US.sales`", FloatType(), True)])
input_mock_df = spark.createDataFrame(input_mock_rdd_map, rdd_schema)
The below code returns the column names
input_mock_df.columns
But any operations on this dataframe is giving error for example
input_mock_df.count()
How do I make a valid spark dataframe which contains a "."?
Note:
I dont give "." in the column name the code works perfectly.
I want to solve it using native spark and not use pandas etc
I have ran the below code
input_data = [('retail', '2017-01-03T13:21:00', 134),
('retail', '2017-01-03T13:21:00', 100)]
rdd_schema = StructType([StructField('business', StringType(), True), \
StructField('date', StringType(), True), \
StructField("US.sales", IntegerType(), True)])
input_mock_df = sqlContext.createDataFrame(input_data, rdd_schema)
input_mock_df.count()
and it works fine returning the count as 2. Please try and reply