Extract StructType from JSON schema definition - apache-spark

Question

I have a schema definition generated by Hackolade. It looks like this:
{
"properties": {
"data": {
"isActivated": true,
"type": "object",
"properties": {
"value01": {
"isActivated": true,
"type": "string",
"readOnly": true,
"description": "description 01",
"pattern": "^([a-z _-]*)$",
"minLength": 4,
"examples": [
"example01",
"example02"
],
"$comment": "comment 01."
},
"start_timestamp": {
"isActivated": true,
"type": "string",
"format": "date-time",
"pattern": "^(\\d{4}(?!\\d{2}\\b))((-?)((0[1-9]|1[0-2])(\\3([12]\\d|0[1-9]|3[01]))?|W([0-4]\\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\\d|[12]\\d{2}|3([0-5]\\d|6[1-6])))([T\\s]((([01]\\d|2[0-3])((:?)[0-5]\\d)?|24\\:?00)([\\.,]\\d+(?!:))?)?(\\17[0-5]\\d?)?([zZ]|([\\+-])([01]\\d|2[0-3]):?([0-5]\\d)?)?)?)?(.\\d{6})?(\\+00:00)$",
"readOnly": true,
"examples": [
"2022-08-15T12:50:25.456789+00:00"
],
"description": "timestamp description",
"maxLength": 33,
"minLength": 27
}
},
"additionalProperties": false,
"description": "data description",
"readOnly": true,
"required": [
"value01",
"start_timestamp"
]
}
} }
I want to convert this schema definition to a StructType schema to use with PySpark DF:
df_with_json = df.withColumn("col_with_schema", f.from_json(f.col(value), schema))
Where value column is a dict string:
'{"data":{"value01":"example01", "start_timestamp":"2021-05-12T12:42:56.236123+00:00"}}'
I tried something like: schema = f.schema_of_json(f.lit(str(json_schema_definition))) but it didn't work.

Answer 1

The following will convert your JSON string into a StructType object. The issue is that you have regexes as values in your JSON, so before conversion you must do a small replace.
json_str = """
{
"properties": {
"data": {
"isActivated": true,
"type": "object",
"properties": {
"value01": {
"isActivated": true,
"type": "string",
"readOnly": true,
"description": "description 01",
"pattern": "^([a-z _-]*)$",
"minLength": 4,
"examples": [
"example01",
"example02"
],
"$comment": "comment 01."
},
"start_timestamp": {
"isActivated": true,
"type": "string",
"format": "date-time",
"pattern": "^(\\d{4}(?!\\d{2}\\b))((-?)((0[1-9]|1[0-2])(\\3([12]\\d|0[1-9]|3[01]))?|W([0-4]\\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\\d|[12]\\d{2}|3([0-5]\\d|6[1-6])))([T\\s]((([01]\\d|2[0-3])((:?)[0-5]\\d)?|24\\:?00)([\\.,]\\d+(?!:))?)?(\\17[0-5]\\d?)?([zZ]|([\\+-])([01]\\d|2[0-3]):?([0-5]\\d)?)?)?)?(.\\d{6})?(\\+00:00)$",
"readOnly": true,
"examples": [
"2022-08-15T12:50:25.456789+00:00"
],
"description": "timestamp description",
"maxLength": 33,
"minLength": 27
}
},
"additionalProperties": false,
"description": "data description",
"readOnly": true,
"required": [
"value01",
"start_timestamp"
]
}
} }
"""
schema = spark.read.json(sc.parallelize([json_str.replace('\\', '\\\\\\\\')])).schema
print(schema)
# StructType([StructField('properties', StructType([StructField('data', StructType([StructField('additionalProperties', BooleanType(), True), StructField('description', StringType(), True), StructField('isActivated', BooleanType(), True), StructField('properties', StructType([StructField('start_timestamp', StructType([StructField('description', StringType(), True), StructField('examples', ArrayType(StringType(), True), True), StructField('format', StringType(), True), StructField('isActivated', BooleanType(), True), StructField('maxLength', LongType(), True), StructField('minLength', LongType(), True), StructField('pattern', StringType(), True), StructField('readOnly', BooleanType(), True), StructField('type', StringType(), True)]), True), StructField('value01', StructType([StructField('$comment', StringType(), True), StructField('description', StringType(), True), StructField('examples', ArrayType(StringType(), True), True), StructField('isActivated', BooleanType(), True), StructField('minLength', LongType(), True), StructField('pattern', StringType(), True), StructField('readOnly', BooleanType(), True), StructField('type', StringType(), True)]), True)]), True), StructField('readOnly', BooleanType(), True), StructField('required', ArrayType(StringType(), True), True), StructField('type', StringType(), True)]), True)]), True)])