I have to process some xlsx received from external source. Is there a more straightforward way to load a xlsx in pandas while also skipping rows with strikethrough?
Currently I have to do something like this:
import pandas as pd, openpyxl
working_file = r"something.xlsx"
working_wb = openpyxl.load_workbook(working_file, data_only=True)
working_sheet = working_wb.active
empty = []
for row in working_sheet.iter_rows("B", row_offset=3):
for cell in row:
if cell.font.strike is True:
p_id = working_sheet.cell(row=cell.row, column=37).value
empty.append(p_id)
df = pd.read_excel(working_file, skiprows=3)
df = df[~df["ID"].isin(empty)]
...
Which works but only by going through every excel sheet twice.
Ended up subclassing pd.ExcelFile and _OpenpyxlReader. It was easier than I thought :)
import pandas as pd
from pandas.io.excel._openpyxl import _OpenpyxlReader
from pandas._typing import Scalar
from typing import List
from pandas.io.excel._odfreader import _ODFReader
from pandas.io.excel._xlrd import _XlrdReader
class CustomReader(_OpenpyxlReader):
def get_sheet_data(self, sheet, convert_float: bool) -> List[List[Scalar]]:
data = []
for row in sheet.rows:
first = row[1] # I need the strikethrough check on this cell only
if first.value is not None and first.font.strike: continue
else:
data.append([self._convert_cell(cell, convert_float) for cell in row])
return data
class CustomExcelFile(pd.ExcelFile):
_engines = {"xlrd": _XlrdReader, "openpyxl": CustomReader, "odf": _ODFReader}
With the custom classes set, now just pass the files like a normal ExcelFile, specify the engine to openpyxl and voila! Rows with strikethrough cells are gone.
excel = CustomExcelFile(r"excel_file_name.xlsx", engine="openpyxl")
df = excel.parse()
print (df)
In this case I would not use Pandas. Just use openpyxl, work from the end of the worksheet and delete rows accordingly. Working backwards from the end of the worksheet means you don't suffer with side-effects when deleting rows.
Related
This drives me crazy.
I have the following csv file:
Short name;Calculation;29221
peter;foster;1,755345
karin;paris;0,2343543
john;dee;0
lisa;long;1,434534
lauren;lovely;0,123124
linda;loss;0,0234
I read this file in pandas, print it and everything looks fine in pandas.
Then I write it to an existing excel workbook and the values are partly corrupted.
THis is my code
import pandas as pd
import xlwings as xw
#öffne csv
QTH = pd.read_csv(r"C:/Users/A692517/PhytonStuff/testCSVtoExcel.csv",sep = ';')#,
# engine = 'python')
for idx, row in QTH.iterrows():
#c=QoSFTTH[row[2]].at[idx]
myString = str(row[2])
row[2]=myString
#ziel workbook
fn="C:/Users/A692517/PhytonStuff/myClist.xlsx"
wb = xw.Book(fn)
ws = wb.sheets["Tabelle1"]
#schreibe QoSFTTH dataframe in zielworkbook
ws["A1"].options(pd.DataFrame, header=1, index=False, expand='table').value = QTH
wb.save(fn)
wb.close()
When I export the Excel result in a new csv(;) you see what I mean:
Short name;Calculation;29221,00
peter;foster;1755345,00
karin;paris;0,2343543
john;dee;0,00
lisa;long;1434534,00
lauren;lovely;0,123124
linda;loss;0,0234
You may have stumbled on a pd.read_csv bug found via this stack question. Change the engine to engine = c and try thousands=','.
pd.read_csv('path', sep=';', thousands=',', engine='c')
I am using Excelwriter with openpyxl engine as I want to open excel file in append mode.
I am using append mode so that I would be able to clear previous sheets in workbook while every re run. But I m getting this error while using he syntax as below for adding formats to the excel :-AttributeError: 'Workbook' object has no attribute 'add_format'
How do I make it work with openpyxl engine
def write_dataframes_to_excel_sheet(dataframes, dir, name,writer):
#with pd.ExcelWriter(f'{dir}/{name}.xlsx', engine='xlsxwriter') as writer:
workbook = writer.book
worksheet = workbook.create_sheet(str(id))
writer.sheets[str(id)] = worksheet
COLUMN = 0
row = 0
for df in dataframes:
#worksheet.write_string(row, COLUMN, df.name)
row += 1
df.to_excel(writer, sheet_name=str(id),
startrow=row, startcol=COLUMN,index=False)
header_format= workbook.add_format({'bold':True,'fg_color' :'00C0C0C0','border': 1})
for col_num,value in enumerate(df.columns.values):
worksheet.write(0,col_num,value,header_format)
column_len=df[value].astype(str).str.len().max()
column_len=max(column_len,len(value))+3
worksheet.set_column(col_num,col_num,column_len)
row += df.shape[0] + 3
with pd.ExcelWriter(input_filename, engine='openpyxl',mode='a') as writer:
write_dataframes_to_excel_sheet(df_array, 'C:/Users/path',input_filename,writer)
AttributeError: 'Workbook' object has no attribute 'add_format'
The add_format() method is an xlsxwriter method so that won't work with openpyxl. You will need to use the equivalent openpyxl method.
You can find all the info here.
I was searching for the same thing, so let me give you a snippet. xlsxwriter seems so much easier though.
from openpyxl import Workbook
wb = Workbook(write_only = True)
ws = wb.create_sheet('test')
from openpyxl.cell import WriteOnlyCell
from openpyxl.styles import Font
for row in ws.iter_rows(min_row=1, max_col=3, max_row=2):
for cell in row:
cell = WriteOnlyCell(ws, value="hello world")
cell.font = Font(bold=True, color='00C0C0C0')
cell.border = Border(left=Side(border_style='Thin',
color='FF000000'),
right=Side(border_style='Thin',
color='FF000000'),
top=Side(border_style='Thin',
color='FF000000'),
bottom=Side(border_style='Thin',
color='FF000000')
You should find here all the border styles you would like to add.
Hope this helps!
I have an xlsx file with 1 sheet.
I am trying to open it using python 3 (xlrd lib), but I get an empty file!
I use this code:
file_errors_location = "C:\\Users\\atheelm\\Documents\\python excel mission\\errors1.xlsx"
workbook_errors = xlrd.open_workbook(file_errors_location)
and I have no errors, but when I type:
workbook_errors.nsheets
I get "0", even the file has some sheets... when I type:
workbook_errors
I get:
xlrd.book.Book object at 0x2..
any help? thanks
You can use Pandas pandas.read_excel just like pandas.read_csv:
import pandas as pd
file_errors_location = 'C:\\Users\\atheelm\\Documents\\python excel mission\\errors1.xlsx'
df = pd.read_excel(file_errors_location)
print(df)
There are two modules for reading xls file : openpyxl and xlrd
This script allow you to transform a excel data to list of dictionnaries using xlrd
import xlrd
workbook = xlrd.open_workbook('C:\\Users\\atheelm\\Documents\\python excel mission\\errors1.xlsx')
workbook = xlrd.open_workbook('C:\\Users\\atheelm\\Documents\\python excel mission\\errors1.xlsx', on_demand = True)
worksheet = workbook.sheet_by_index(0)
first_row = [] # The row where we stock the name of the column
for col in range(worksheet.ncols):
first_row.append( worksheet.cell_value(0,col) )
# tronsform the workbook to a list of dictionnary
data =[]
for row in range(1, worksheet.nrows):
elm = {}
for col in range(worksheet.ncols):
elm[first_row[col]]=worksheet.cell_value(row,col)
data.append(elm)
print data
Unfortunately, the python engine 'xlrd' that is required to read the Excel docs has explicitly removed support for anything other than xls files.
So here's how you can do it now -
Install openpyxl:
https://openpyxl.readthedocs.io/en/stable/
Change your pandas code to:
pandas.read_excel('cat.xlsx', engine='openpyxl')
Note: This worked for me with the latest version of Pandas (i.e. 1.1.5). Previously, I was using version 0.24.0 and it didn't work so I had to update to latest version.
Another way to do it:
import openpyxl
workbook_errors = openpyxl.Workbook()
workbook_errors = openpyxl.load_workbook(file_errors_location)
I have an Excel spreadsheet that I am reading into a Pandas DataFrame:
df = pd.read_excel("file.xls")
However, one of the columns of the spreadsheet contains text which have a hyperlink associated with it. How do I access the underlying hyperlink in Pandas?
This can be done with openpyxl, I'm not sure its possible with Pandas at all. Here's how I've done it:
import openpyxl
wb = openpyxl.load_workbook('yourfile.xlsm')
ws = wb.get_sheet_by_name('Sheet1')
print(ws.cell(row=2, column=1).hyperlink.target)
You can also use iPython, and set a variable equal to the hyperlink object:
t = ws.cell(row=2, column=1).hyperlink
then do t. and press tab to see all the options for what you can do with or access from the object.
Quick monkey patching, without converters or anything like this, if you would like to treat ALL cells with hyperlinks as hyperlinks, more sophisticated way, I suppose, at least be able to choose, what columns treat as hyperlinked or gather data, or save somehow both data and hyperlink in same cell at dataframe. And using converters, dunno. (BTW I played also with data_only, keep_links, did not helped, only changing read_only resulted ok, I suppose it can slow down your code speed).
P.S.: Works only with xlsx, i.e., engine is openpyxl
P.P.S.: If you reading this comment in the future and issue https://github.com/pandas-dev/pandas/issues/13439 still Open, don't forget to see changes in _convert_cell and load_workbook at pandas.io.excel._openpyxl and update them accordingly.
import pandas
from pandas.io.excel._openpyxl import OpenpyxlReader
import numpy as np
from pandas._typing import FilePathOrBuffer, Scalar
def _convert_cell(self, cell, convert_float: bool) -> Scalar:
from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC
# here we adding this hyperlink support:
if cell.hyperlink and cell.hyperlink.target:
return cell.hyperlink.target
# just for example, you able to return both value and hyperlink,
# comment return above and uncomment return below
# btw this may hurt you on parsing values, if symbols "|||" in value or hyperlink.
# return f'{cell.value}|||{cell.hyperlink.target}'
# here starts original code, except for "if" became "elif"
elif cell.is_date:
return cell.value
elif cell.data_type == TYPE_ERROR:
return np.nan
elif cell.data_type == TYPE_BOOL:
return bool(cell.value)
elif cell.value is None:
return "" # compat with xlrd
elif cell.data_type == TYPE_NUMERIC:
# GH5394
if convert_float:
val = int(cell.value)
if val == cell.value:
return val
else:
return float(cell.value)
return cell.value
def load_workbook(self, filepath_or_buffer: FilePathOrBuffer):
from openpyxl import load_workbook
# had to change read_only to False:
return load_workbook(
filepath_or_buffer, read_only=False, data_only=True, keep_links=False
)
OpenpyxlReader._convert_cell = _convert_cell
OpenpyxlReader.load_workbook = load_workbook
And after adding this above in your python file, you will be able to call df = pandas.read_excel(input_file)
After writing all this stuff it came to me, that maybe it would be easier and cleaner just use openpyxl by itself ^_^
as commented by slaw it doesnt grab the hyperlink but only the text
here text.xlsx contains links in the 9th column
from openpyxl import load_workbook
workbook = load_workbook('test.xlsx')
worksheet = workbook.active
column_indices = [9]
for row in range(2, worksheet.max_row + 1):
for col in column_indices:
filelocation = worksheet.cell(column=col, row=row) # this is hyperlink
text = worksheet.cell(column=col + 1, row=row) # thi is your text
worksheet.cell(column=col + 1, row=row).value = '=HYPERLINK("' + filelocation.value + '","' + text.value + '")'
workbook.save('test.xlsx')
You cannot do that in pandas. You can try with other libraries designed to deal with excel files.
import string
import xlrd
import xlsxwriter
workbook = xlsxwriter.Workbook('C:\T\file.xlsx')
worksheet = workbook.add_worksheet()
book = open_workbook(r'C:\T\test.xls','r')
sheet = book.sheet_by_index(0)
for row_index in range(sheet.nrows):
for col_index in range(sheet.ncols):
print sheet.cell(row_index,0).value
x = sheet.cell(row_index,0).value
worksheet.write_string(row_index,col_index,x)
workbook.close()
I'm a skiddy to python. Here i'm trying to read the xls file with xlrd for data and copy it to another xlsx file through xlsxwriter module. but the data won't get pasted in the created xlsx sheet. Please guide me through this. Above is my exact code. Please correct me if any wrong.
A volley of Thanks in advance.
Your example program almost works. Mainly it needs the open_workbook() method to be prefixed with a class and it is better to use XlsxWriter write() instead of write_string() unless you are sure that all the data you are reading is of a string type. Also, the program was only reading values from column 0.
Here is the same example with those changes in place. I've also renamed the variables in_ and out_ to make it clearer which module is calling which method:
import xlrd
import xlsxwriter
out_workbook = xlsxwriter.Workbook('file.xlsx')
out_worksheet = out_workbook.add_worksheet()
in_workbook = xlrd.open_workbook(r'test.xls', 'r')
in_worksheet = in_workbook.sheet_by_index(0)
for row_index in range(in_worksheet.nrows):
for col_index in range(in_worksheet.ncols):
cell_value = in_worksheet.cell(row_index, col_index).value
out_worksheet.write(row_index, col_index, cell_value)
print cell_value
out_workbook.close()