Python code slowing down when wrapped in a function - python-3.x

Question

I'm reading and processing a file (using the same bit of code) which runs at two very different speeds: 1. scripted (50K+ iterations per second) and 2. wrapped in function (~300 iterations per second). I really can't figure out why there are such massive differences in reading time consumption.
The module structure (unused and irrelevant files omitted. The code is at the end.):
| experiments/
|--| experiment_runner.py
|
| module/
|--| shared/
|--|--| dataloaders.py
|--|--| data.py
In data.py we have the method (load, the class wrapping the method inherits from torch.utils.data.Dataset) actually loading the file. In dataloaders.py I prepare the arguments to pass to load, wrapped in a function for each dataset that I'm using. This is then passed to a loader function, which handles splitting the dataset and so on.
In experiment_runner then is where the differences in speed happen. If I use the dataset functions in dataloaders.py the loading happens at around 300 iterations/second. If I copy the code from the function and throw it directly into the experiment_runner, still using the loader function from dataloaders.py (so, not wrapped in a function for each dataset), the loading happens at roughly 50000 iterations / second. I am at a complete loss as to why wrapping code in a function would alter the speed of it that drastically.
Now the actual code:
data.py:
def load(self, dataset: str = 'train', skip_header = True, **kwargs) -> None:
fp = open(self.data_files[dataset])
if skip_header:
next(fp)
data = []
for line in tqdm(self.reader(fp), desc = f'loading {self.name} ({dataset})'):
data_line, datapoint = {}, base.Datapoint()
for field in self.train_fields:
idx = field.index if self.ftype in ['CSV', 'TSV'] else field.cname
data_line[field.name] = self.process_doc(line[idx].rstrip())
data_line['original'] = line[idx].rstrip()
for field in self.label_fields:
idx = field.index if self.ftype in ['CSV', 'TSV'] else field.cname
if self.label_preprocessor:
data_line[field.name] = self.label_preprocessor(line[idx].rstrip())
else:
data_line[field.name] = line[idx].rstrip()
for key, val in data_line.items():
setattr(datapoint, key, val)
data.append(datapoint)
fp.close()
if self.length is None:
# Get the max length
lens = []
for doc in data:
for f in self.train_fields:
lens.append(len([tok for tok in getattr(doc, getattr(f, 'name'))]))
self.length = max(lens)
if dataset == 'train':
self.data = data
elif dataset == 'dev':
self.dev = data
elif dataset == 'test':
self.test = data
dataloaders.py:
def loader(args: dict, **kwargs):
"""Loads the dataset.
:args (dict): Dict containing arguments to load dataaset.
:returns: Loaded and splitted dataset.
"""
dataset = GeneralDataset(**args)
dataset.load('train', **kwargs)
if (args['dev'], args['test']) == (None, None): # Only train set is given.
dataset.split(dataset.data, [0.8, 0.1, 0.1], **kwargs)
elif args['dev'] is not None and args['test'] is None: # Dev set is given, test it not.
dataset.load('dev', **kwargs)
dataset.split(dataset.data, [0.8], **kwargs)
elif args['dev'] is None and args['test'] is not None: # Test is given, dev is not.
dataset.split(dataset.data, [0.8], **kwargs)
dataset.dev_set = dataset.test
dataset.load('test', **kwargs)
else: # Both dev and test sets are given.
dataset.load('dev', **kwargs)
dataset.load('test', **kwargs)
return dataset
def binarize(label: str) -> str:
if label in ['0', '1']:
return 'pos'
else:
return 'neg'
def datal(path: str, cleaners: base.Callable, preprocessor: base.Callable = None):
args = {'data_dir': path,
'ftype': 'csv',
'fields': None,
'train': 'dataset.csv', 'dev': None, 'test': None,
'train_labels': None, 'dev_labels': None, 'test_labels': None,
'sep': ',',
'tokenizer': lambda x: x.split(),
'preprocessor': preprocessor,
'transformations': None,
'length': None,
'label_preprocessor': binarize,
'name': 'First dataset.'
}
ignore = base.Field('ignore', train = False, label = False, ignore = True)
d_text = base.Field('text', train = True, label = False, ignore = False, ix = 6, cname = 'text')
d_label = base.Field('label', train = False, label = True, cname = 'label', ignore = False, ix = 5)
args['fields'] = [ignore, ignore, ignore, ignore, ignore, d_label, d_text]
return loader(args)
And for the purposes:
experiment_runner.py
from module.dataloaders import datal, loader
dataset = datal() # Slow: 300-ish iterations/second
# Fast version: 50000 iter/second
def binarize(label: str) -> str:
if label in ['0', '1']:
return 'pos'
else:
return 'neg'
args = {'data_dir': path,
'ftype': 'csv',
'fields': None,
'train': 'dataset.csv', 'dev': None, 'test': None,
'train_labels': None, 'dev_labels': None, 'test_labels': None,
'sep': ',',
'tokenizer': lambda x: x.split(),
'preprocessor': preprocessor,
'transformations': None,
'length': None,
'label_preprocessor': binarize,
'name': 'First dataset.'
}
ignore = base.Field('ignore', train = False, label = False, ignore = True)
d_text = base.Field('text', train = True, label = False, ignore = False, ix = 6, cname = 'text')
d_label = base.Field('label', train = False, label = True, cname = 'label', ignore = False, ix = 5)
args['fields'] = [ignore, ignore, ignore, ignore, ignore, d_label, d_text]
dataset = loader(args)
I would ideally prefer to keep the dataset functions (e.g. datal) wrapped to keep the logic separate but with this speed decrease, that's not feasible.