I using configuration file as below :
"trial":{
"stage_table": "trial_stg",
"folder_location": "Trial",
"column_mapping": [
{
"source_column": "(split(INPUT__FILE__NAME, '\\/')[11])",
"source_datatype": "text",
"target_column": "indication",
"target_datatype": "text",
"transform_type": "expression",
"validate": false
}
I am trying to get file name using INPUT__FILE__NAME function in pyspark but I am getting issue.
Below is the code after reading this config file :
def query_expression_builder(mapping):
print("Inside query_expression_builder")
print("mapping :",mapping)
def match_transform_type(map_col):
print("Inside match_transform_type")
if map_col.get('transform_type') is None:
print("transform_type is",map_col.get('transform_type'))
print("map_col inside if :",map_col)
return f"`{map_col['source_column']}` AS {map_col['target_column']}"
elif str(map_col.get('transform_type')).__eq__('expression'):
print("transform_type is",map_col.get('transform_type'))
print("map_col inside elif :",map_col)
return f"{map_col['source_column']} AS {map_col['target_column']}"
else:
print("transform_type is",map_col.get('transform_type'))
print("map_col inside else :",map_col)
return f"`{map_col['source_column']}` AS {map_col['target_column']}"
if mapping is None:
print("Check for mapping is None")
return []
else:
print("Mapping is not None")
return list(map(lambda col_mapping: match_transform_type(map_col=col_mapping), mapping))
def main():
query = query_expression_builder\
(mapping=config['file_table_mapping'][tbl]['column_mapping'])
print(f"Table = {tbl} Executing query {query}")
file_path = f"s3://{config['raw_bucket']}/{config['landing_directory']}/{config['file_table_mapping'][tbl]['folder_location']}/{config_audit['watermark_timestamp']}*.csv"
write_df = spark.read.csv(path=file_path, header=True,\
inferSchema=False).selectExpr(query) \
.withColumn("prcs_run_id", func.lit(config_audit['prcs_run_id']))\
.withColumn("job_run_id",\
func.lit(config_audit['job_run_id']))\
.withColumn("ins_ts", func.lit(ins_ts))\
.withColumn("rec_crt_user", func.lit(config["username"]))
write_df.show()
Below is the error I am getting :
"cannot resolve '`INPUT__FILE__NAME`' given input columns: [Pediatric Patients included (Y/N), Trial registry number, Number of patients, Sponsor, Number of treatment arms, Multicenter study, Trial Conclusion, Clinical Phase, Study Population, Country Codes, Exclusion criteria, Trial ID, Trial AcronymDerived, Comments, Countries, Trial registry name, Sample size calculation details, Randomisation, Blinding, Trial Comments, Trial start year, Trial end year, Inclusion criteria, Study treatment, Trial design, Controlled trial, Trial Acronym, Trial Control, Asymptomatic patients, Analysis method details]; line 1 pos 7;\n'Project ['split('INPUT__FILE__NAME, /)[11] AS indication#4346, Trial ID#4286 AS trial_id#4347, Trial Acronym#4287 AS trial_acronym#4348, Trial AcronymDerived#4288 AS trial_acronym_derived#4349, Sponsor#4289 AS sponsor#4350, Asymptomatic patients#4290 AS asymptomatic_patients#4351, Pediatric Patients included (Y/N)#4291 AS pediatric_patients_included#4352, Number of patients#4292 AS num_of_patients#4353, Number of treatment arms#4293 AS num_of_treatment_arms#4354, Trial start year#4294 AS trial_strt_yr#4355, Trial end year#4295 AS trial_end_yr#4356, Clinical Phase#4296 AS clinical_phase#4357, Study Population#4297 AS study_population#4358, Study treatment#4298 AS study_treatment#4359, Randomisation#4299 AS randomization#4360, Controlled trial#4300 AS controlled_trial#4361, Trial Control#4301 AS trial_control#4362, Blinding#4302 AS blinding#4363, Trial registry name#4303 AS trial_registry_name#4364, Trial registry number#4304 AS trial_registry_num#4365, Countries#4305 AS countries#4366, Country Codes#4306 AS country_codes#4367, Trial design#4307 AS trial_design#4368, Multicenter study#4308 AS multicenter_study#4369, ... 7 more fields]\n+- Relation[Trial ID#4286,Trial Acronym#4287,Trial AcronymDerived#4288,Sponsor#4289,Asymptomatic patients#4290,Pediatric Patients included (Y/N)#4291,Number of patients#4292,Number of treatment arms#4293,Trial start year#4294,Trial end year#4295,Clinical Phase#4296,Study Population#4297,Study treatment#4298,Randomisation#4299,Controlled trial#4300,Trial Control#4301,Blinding#4302,Trial registry name#4303,Trial registry number#4304,Countries#4305,Country Codes#4306,Trial design#4307,Multicenter study#4308,Inclusion criteria#4309,... 6 more fields] csv\n"
Traceback (most recent call last):
File "/mnt/yarn/usercache/root/appcache/application_1594568207850_0001/container_1594568207850_0001_01_000001/pyspark.zip/pyspark/sql/utils.py", line 63, in deco
return f(*a, **kw)
File "/mnt/yarn/usercache/root/appcache/application_1594568207850_0001/container_1594568207850_0001_01_000001/py4j-0.10.7-src.zip/py4j/protocol.py", line 328, in get_return_value
format(target_id, ".", name), value)
py4j.protocol.Py4JJavaError: An error occurred while calling o529.selectExpr.
: org.apache.spark.sql.AnalysisException: cannot resolve '`INPUT__FILE__NAME`' given input columns:
How can I use INPUT__FILE__NAME function? I have already enabled hive support in my code. Or is there any other way to do this? I cannot find anything on net on how to use this function.
Try by using single underscore(_) in input_file_name() instead of double underscore.
Example:
from pyspark.sql.functions import *
sql("select *,input_file_name() from tmp")
#or
df.withColumn("filename",input_file_name()).show()
For this project, I was trying to make a list that contained all the distances between all the points that I had in a list(I have a list of x coordinates and one of y)
for da in range(len(meta.values())):
for db in range(len(meta.values())):
dis.append(math.sqrt((x[db] - x[da])**2 + (y[db] - y[da])**2)
print(dis)
However, this part of it either gives me an "unexpected EOF while parsing" or an "invalid syntax" by the print statement. I cant see the mistake here can someone please help me?
You have an unmatched parantheses in dis.append(...
it should be:
dis.append(math.sqrt((x[db] - x[da])**2 + (y[db] - y[da])**2))
this is my code
url = "E:\dataset\state_dataset\drug.csv"
dataframe = read_csv(url)
df=dataframe.loc[:,['Product Name','Number of Prescriptions','Total Amount Reimbursed','Medicaid Amount Reimbursed']]
df[(df.Number of Prescriptions >= 100)]
and I faced the error
File "", line 10
df[(df.Number of Prescriptions >= 100)]
^
SyntaxError: invalid syntax
please how can I fixed this error
Using MBasic on an old CP/M trying to get a simple input which seems to not be working
10 INPUT "Your Name:", NAME$
20 PRINT "Hello, " + NAME$ + ","
I am just working threw this, never used it before. Its a program for an old CPM
When i Run this i get
Syntax error in 10
The correct syntax is LINE INPUT with a semi-colon, i.e.
10 LINE INPUT "Your name:"; NAME$
Microsoft BASIC Compiler 1980
I am trying to create a program which tells me if a number's square has different digits.
I have an "Index Error" in the really long "if" line. How do I fix it?
a= 4486659
f= (a**2)
s= str(f)
for num in range (1089):
if s[6]==s[7] or s[6]==s[8] or s[6]==s[9] or s[6]==s[10] or s[6]==s[11] or s[6]==s[12] or s[6]==s[13] or s[6]==s[14] or s[6]==s[15] or s[7]==s[8] or s[7]==s[9] or s[7]==s[10] or s[7]==s[11] or s[7]==s[12] or s[7]==s[13] or s[7]==s[14] or s[7]==s[15] or s[8]==s[9] or s[8]==s[10] or s[8]==s[11] or s[8]==s[12] or s[8]==s[13] or s[8]==s[14] or s[9]==s[10] or s[9]==s[11] or s[9]==s[12] or s[9]==s[13] or s[9]==s[14] or s[9]==s[15] or s[10]==s[11] or s[10]==s[12] or s[10]==s[13] or s[10]==s[14] or s[10]==s[15] or s[11]==s[12] or s[11]==s[13] or s[11]==s[14] or s[11]==s[15] or s[12]==s[13] or s[12]==s[14] or s[12]==s[15] or s[13]==s[14] or s[13]==s[15] or s[14]==s[15]:
a= a+1
else:
print(a)
The index error is because your string s is only 14 chars long, but you try to access s[14] and s[15]
Since you cannot call a nonexistent item in a string, Try this:
if s[6]==s[7] or s[6]==s[8] or s[6]==s[9] or s[6]==s[10] or s[6]==s[11] or s[6]==s[12] or s[6]==s[13] or s[6]==s[14] or s[7]==s[8] or s[7]==s[9] or s[7]==s[10] or s[7]==s[11] or s[7]==s[12] or s[7]==s[13] or s[7]==s[14] or s[8]==s[9] or s[8]==s[10] or s[8]==s[11] or s[8]==s[12] or s[8]==s[13] or s[8]==s[14] or s[9]==s[10] or s[9]==s[11] or s[9]==s[12] or s[9]==s[13] or s[9]==s[14] or s[10]==s[11] or s[10]==s[12] or s[10]==s[13] or s[10]==s[14] or s[11]==s[12] or s[11]==s[13] or s[11]==s[14] or s[12]==s[13] or s[12]==s[14] or s[13]==s[14]: