How to convert for loop generated data into Data Frame? - python-3.x

Question

I have extracted data from a table in a website with for-loop using
selenium automation webdriver. How to convert that data into dataframe
and export to csv file. I tried to assign the 'value' within pandas dataframe but it is throwing error.
from selenium import webdriver
url = "https://www.jambalakadi.info/status/"
driver = webdriver.Chrome(executable_path="chromedriver.exe")
driver.get(url)
row_count = len(driver.find_elements_by_xpath(" //*[#id='main_table_countries_today']/tbody[1]/tr "))
col_count = len(driver.find_elements_by_xpath(" //*[#id='main_table_countries_today']/tbody[1]/tr[1]/td "))
print('Number of row counts:', row_count)
print("Number of column counts:", col_count)
for r in range(2, row_count+1):
for c in range(1, col_count+1):
value = driver.find_element_by_xpath(" //*[#id='main_table_countries_today']/tbody[1]/tr["+str(r)+"]/td["+str(c)+"] ").text
print(value, end=" ")
print(" ")
when I run the for-loop the 'value' variable printing the data but I'm
not able to create data frame and export it to CSV file using pandas.
I updated the code is it correctly formatted?
my_data = []
for r in range(2, row_count+1):
for c in range(1, col_count+1):
value = driver.find_element_by_xpath(" //*[#id='main_table_countries_today']/tbody[1]/tr["+str(r)+"]/td["+str(c)+"] ").text
print(value, end=" ")
for line in value:
my_data.append(line[0],line[1],line[2])
pd.DataFrame.from_records(my_data, columns=column).to_csv('output.csv')
print(" ")

Answer 1

Here is the code using pandas to get data in dataframe and then import to csv.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from bs4 import BeautifulSoup
driver=webdriver.Chrome(executable_path="chromedriver.exe")
driver.get("https://yourwebsitename.com")
WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CSS_SELECTOR,"table#main_table_countries_today")))
html=driver.page_source
soup=BeautifulSoup(driver.page_source,'html.parser')
table=soup.find('table',attrs={"id":"main_table_countries_today"})
df=pd.read_html(str(table))
print(df[0])
df[0].to_csv('output.csv',index=False)
Updated:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
driver=webdriver.Chrome(executable_path = "chromedriver.exe")
driver.get("https://yourwebsitename.com")
element=WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CSS_SELECTOR,"table#main_table_countries_today")))
table=driver.execute_script("return arguments[0].outerHTML;",element)
df=pd.read_html(str(table))
print(df[0])
df[0].to_csv('output.csv',index=False)

Answer 2

You need to use the function pd.DataFrame.from_records()
Use case:
import pandas as pd
#Reading the data
my_data = []
for line in my_database:
#preprocess the line (say you get 3 columns date,customer,price)
#say you use line.split(" "), now your line is actually an array of values (line = line.split(" ")
my_data.append([line[0],line[1],line[2]]) #each index corresponds to date, customer and price respectively
pd.DataFrame.from_records(my_data, columns=['date','customer','price']).to_csv('output.csv')