how to define bigquery schema when build apache beam data pipeline - python-3.x

Question

I create a data pipeline with apache beam, but it can not insert the data to bigquery.
I use beam.ParDo to process the data, and yield the data row by row,
below is the code.
project = 'project_name'
dataset = 'XXX'
class parser_data(beam.DoFn):
def process(self, data):
ZZ = [{"NN":d["NNN"], "descrip":d} for d in data["colZ"]]
ret = pd.DataFrame(data['colD'])
ret["colA"] = data["colA"]
ret["colB"] = data["colB"]
ret["colC"] = data["colC"]
ret = pd.merge(ret, pd.DataFrame(ZZ), on=["NN"], how="left")
ret = ret[["colA", "colB", "colC", "NN", "sample", "descrip"]]
print(ret)
ret_dict = ret.to_dict("records")
print(ret_dict)
for i in range(len(ret_dict)):
yield ret_dict[i]
options = PipelineOptions(
runner = 'DirectRunner',
region = 'us-west1',
project = project,
job_name = "test-tmp",
streaming = False,
setup_file = './setup.py',
subnetwork = "XXXXXXX",
service_account_email = "XXXXXX",
temp_location='XXXXXX',
staging_location="XXXXXX",
use_public_ips = False
)
d = {
'colA': '1',
'colB': 'Strawberry',
'colC': 2,
'colD': [{"NN":"AA", "sample":1}, {"NN":"AA", "sample":2}, {"NN":"BB", "sample":3}, {"NN":"CC", "sample":4}, {"NN":"CC", "sample":5}],
'colZ': [{"NNN":"AA", "name":"123", "timeperiod":"152"}, {"NNN":"BB", "name":"1212513", "timeperiod":"1952"}, {"NNN":"CC", "name":"13", "timeperiod":"14152"}],
}
schema = {
'fields':[
{'name': 'colA', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'colB', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'colC', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'NN', 'type': 'STRING', 'mode': 'REQUIRED'},
{'name': 'sample', 'type': 'STRING', 'mode': 'REQUIRED'},
{
'name': 'descrip', 'type': 'RECORD', 'mode': 'NULLABLE',
'fields':[
{"name": "NNN", "type": "STRING", 'mode': 'NULLABLE'},
{"name": "name", "type": "STRING", 'mode': 'NULLABLE'},
{"name": "timeperiod", "type": "STRING", 'mode': 'NULLABLE'},
]
},
]
}
with beam.Pipeline(options=options) as pipeline:
data = (
pipeline | 'get data' >> beam.Create([d])
)
ret_A = (
data | "Process A data " >> beam.ParDo(parser_data())
| "Insert data into BQ" >> beam.io.WriteToBigQuery(
f"{project}:{dataset}.TestJsonData",
schema=schema,
create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND
)
)
The error is below
RuntimeError: BigQuery job beam_bq_job_LOAD_testtmp_LOAD_STEP_820_9672b886a985a9a36a9c3805cee3be5e_3f26019c07d746ef92c0893574156f5b failed. Error Result: <ErrorProto
location: 'gs://XXXXXXXX/dataflow_temp/bq_load/db11e8430c10470382be2565136d53fb/{project}.{dataset}.TestJsonData/39e0d645-8484-4033-a1c4-3e4a825d6fee'
message: 'Error while reading data, error message: JSON table encountered too many errors, giving up. Rows: 1; errors: 1. Please look into the errors[] collection for more details.'
reason: 'invalid'> [while running '[25]: Insert data into BQ/BigQueryBatchFileLoads/WaitForDestinationLoadJobs']
Also the print function show the data, so I think the problem is in the bigquery schema, but I can not find it
Anyone have any idea?