I am running T5-base-grammar-correction for grammer correction on my dataframe with text column
from happytransformer import HappyTextToText
from happytransformer import TTSettings
from tqdm.notebook import tqdm
tqdm.pandas()
happy_tt = HappyTextToText("T5", "./t5-base-grammar-correction")
beam_settings = TTSettings(num_beams=5, min_length=1, max_length=30)
def grammer_pipeline(text):
text = "gec: " + text
result = happy_tt.generate_text(text, args=beam_settings)
return result.text
df['new_text'] = df['original_text'].progress_apply(grammer_pipeline)
Pandas apply function, though runs and provides required results, but runs quite slow.
Also I get the below warning while executing the code
/home/.local/lib/python3.6/site-packages/transformers/pipelines/base.py:908: UserWarning: You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset
UserWarning,
I have access to GPU. Can somebody provide some pointers to speed up the execution and utilising full capabilities of GPU
--------------------------------EDIT---------------------------------
I tried using pytorch Dataset in the below way, but still the processing is slow:
class CustomD(Dataset):
def __init__(self, text):
self.text = text
self.len = text.shape[0]
def __len__(self):
return self.len
def __getitem__(self, idx):
text = self.text[idx]
text = "gec: " + text
result = happy_tt.generate_text(text, args=beam_settings)
return result.text
TD = GramData(df.original_text)
final_data = DataLoader(dataset=TD,
batch_size=10,
shuffle=False
)
import itertools
list_modified=[]
for (idx, batch) in enumerate(final_data):
list_modified.append(batch)
flat_list = [item for sublist in list_modified for item in sublist]
df["new_text"]=flat_list
Related
I'm trying to train my own model, but for some reason, my gpu is not fully utilized. Is there any way to solve this?
Here is the snipet of my Dataset.
I have already tried pin_memory=True, still not working.
environment: torch1.8 + cuda11.1 + RTX3090
class ImageDataset(Dataset):
def __init__(self, data_path, transform=None, image_size=512):
self.train_data_path = data_path['train']
self.train_label_path = data_path['label']
self.train_lables = os.listdir(self.train_label_path)
self.train_data = os.listdir(self.train_data_path)
self.transform = transform
self.image_size = to_2tuple(image_size)
def __len__(self):
return len(self.train_data)
def __getitem__(self, indx):
if indx >= len(self.train_data):
raise Exception("Index should be less than {}".format(len(self.train_data)))
image = Image.open(os.path.join(self.train_data_path, self.train_data[indx]))
final_label = Image.open(os.path.join(self.train_label_path, self.train_lables[indx]))
image = self.transform(image)
final_label = self.transform(final_label)
return image, final_label
I am working with a sklearn pipeline and would like to have a feature selection step that could possibly be set to no feature selection. Is there a sklearn.feature_selection.SelectorMixin object that does nothing?
EDIT: or is there at least a template to develop one, like there can be an estimator?
End of the day I went with something like this which seems to fit my current purpose. Not sure if it would be validated as a proper sklearn.feature_selection selector though:
import numpy as np
import pandas as pd
from sklearn.utils.validation import check_is_fitted
class PassThroughSelector():
"""
Simply selects all columns of the dataframe, allowing
to have the equivalent of no selector without changing
too much of the structure.
Args:
"""
def __init__(self):
pass
def fit(self, x, y=None): # pylint:disable=unused-argument, arguments-differ
"""
Stores a list of selected columns.
Args:
x: training data
y: training y (no effect)
Returns:
self
"""
self.check_x(x)
mask = np.where(np.ones(len(x.columns)), True, False)
self.support_ = mask
self.selected_features_ = x.columns[self.support_].tolist()
return self
def get_support(self, indices = False) -> np.ndarray:
"""Provides a boolean mask of the selected features."""
check_is_fitted(self)
if indices == True:
return np.array([i for i in range(len(self.support_)) if self.support_[i]==True])
else:
return self.support_
def transform(self, x: pd.DataFrame):
"""Selects the features selected in `fit` from a provided dataframe."""
check_is_fitted(self)
self.check_x(x)
return x.loc[:, self.support_]
def get_params(self, deep = True):
return {}
I am trying to implement a parallelized function for Geopandas that takes a single vector data (i.e.: a Shapefile containing a Multipolygon data type), and converts it to a standard celular grid with cell x and y sizes defined by the user.
As this function may result in serious Memory issues (i.e.: caused by too high spatial resolution), I was wondering whether it would be possible to save the data iteratively in the given destinated file. That way, as each parallel process runs the "GRID" function, the same process can save the data iteratively in appended mode. That way, I believe that one wouldn't have Memory issues.
Here is my "SHP_to_GRID_Function". Note that the code below still requires that the whole data generated by the multiprocessing be handled by memory directly (so overflow is more than certain for large datasets).
import pandas as pd
import numpy as np
import geopandas as gpd
from shapely.geometry import Polygon
from multiprocessing import Pool
import os
from functools import partial
def info(title):
print(title)
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
def parallelize_df(gdf, func, n_cores, dx=100, dy=100, verbose=False):
Geometries= gdf.loc[:, 'geometry'].values
pool = Pool(processes=n_cores)
func_partial=partial(func, dx, dy, verbose) # prod_x has only one argument x (y is fixed to 10)
results = pool.map(func_partial, Geometries)
pool.close()
pool.join()
print(np.shape(results))
GRID = gpd.GeoSeries(np.array(results).ravel())
print("GRID well created")
return GRID
def generate_grid_from_Poligon(dx=100, dy=100, verbose=False, polygon=None):
if verbose == True:
info('function parallelize_df')
else:
None
xmin,ymin,xmax,ymax = polygon.bounds
lenght = dx
wide = dy
cols = list(np.arange(int(np.floor(xmin)), int(np.ceil(xmax)), wide))
rows = list(np.arange(int(np.floor(ymin)), int(np.ceil(ymax)), lenght))
rows.reverse()
subpolygons = []
for x in cols:
for y in rows:
subpolygons.append( Polygon([(x,y), (x+wide, y), (x+wide, y-lenght), (x, y-lenght)]) )
return subpolygons
def main(GDF, n_cores='standard', dx=100, dy=100, verbose= False):
"""
GDF: geodataframe
n_cores: use standard or a positive numerical (int) value. It will set the number of cores to use in the multiprocessing
args: (dx: dimension in the x coordinate to make the grid
dy: dimenion in the y coordinate to make the grid)
"""
if isinstance(n_cores, str):
import multiprocessing
N_cores = multiprocessing.cpu_count() -1
elif isinstance(n_cores, int):
N_cores =n_cores
GRID_GDF = parallelize_df(GDF, generate_grid_from_Poligon, n_cores=N_cores, dx=dx, dy=dy, verbose=verbose)
return GRID_GDF
I thank you for you time,
Sincerely yours,
Philipe Leal
I finally have come across a solution for my question. It is not perfect, since it requires several writing processes and one final concatenation process over all temporary files created during the run.
Feel free to suggest alternatives.
Here is the solution I found.
import numpy as np
import geopandas as gpd
import pandas as pd
from shapely.geometry import Polygon
from multiprocessing import Pool, Lock, freeze_support
import os
from functools import partial
import time
def info(time_value):
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
print("Time spent: ", time.time() - time_value)
def init(l):
global lock
lock=l
def Data_Arranger(to_filename):
"""This function concatenates and deletes temporary files. It is an arranger
of the multicessing data results"
"""
Base = os.path.join(os.path.dirname(to_filename), 'temp')
Strings = [file for file in os.listdir(Base)]
Strings = [os.path.join(Base, S) for S in Strings]
if not os.path.exists(os.path.dirname(to_filename)):
os.mkdir(os.path.dirname(to_filename))
Sq = [S for S in Strings if S.endswith('.shp')]
gpd.GeoDataFrame(pd.concat([gpd.read_file(sq1) for sq1 in Sq]), crs=GDF.crs).to_file(to_filename)
for sq1 in Sq:
os.remove(sq1)
import shutil
shutil.rmtree(Base, ignore_errors=True)
def parallelize_df(gdf, func, n_cores, dx=100, dy=100, verbose=False, to_filename=None):
Geometries= gdf.loc[:, 'geometry'].values
crs = gdf.crs
pool = Pool(processes=n_cores, initializer=init, initargs=(Lock(), ) )
func_partial=partial(func, dx, dy, verbose, to_filename, crs) # prod_x has only one argument x (y is fixed to 10)
pool.map(func_partial, Geometries)
pool.close()
pool.join()
def generate_grid_from_gdf(dx=100, dy=100, verbose=False, to_filename=None, crs=None, polygon=None):
if verbose == True:
info(time.time())
else:
None
xmin,ymin,xmax,ymax = polygon.bounds
lenght = dx
wide = dy
cols = list(np.arange(int(np.floor(xmin)), int(np.ceil(xmax)), wide))
rows = list(np.arange(int(np.floor(ymin)), int(np.ceil(ymax)), lenght))
rows.reverse()
subpolygons = []
for x in cols:
for y in rows:
subpolygons.append( Polygon([(x,y), (x+wide, y), (x+wide, y-lenght), (x, y-lenght)]) )
lock.acquire()
print('parent process: ', os.getppid(), ' has activated the Lock')
GDF = gpd.GeoDataFrame(geometry=subpolygons, crs=crs)
to_filename = os.path.join(os.path.dirname(to_filename), 'temp', str(os.getpid()) + '_' + str(time.time()) + '.' + os.path.basename(to_filename).split('.')[-1])
if not os.path.exists(os.path.dirname(to_filename)):
os.mkdir(os.path.dirname(to_filename))
try:
print("to_filename: ", to_filename)
GDF.to_file(to_filename)
except:
print("error in the file saving")
lock.release()
print('parent process: ', os.getppid(), ' has unlocked')
def main(GDF, n_cores='standard', dx=100, dy=100, verbose= False, to_filename=None):
"""
GDF: geodataframe
n_cores: use standard or a positive numerical (int) value. It will set the number of cores to use in the multiprocessing
dx: dimension in the x coordinate to make the grid
dy: dimenion in the y coordinate to make the grid)
verbose: whether or not to show info from the processing. Appliable only if applying the function not
in Windows (LINUX, UBUNTU, etc.), or when running in separte console in Windows.
to_filename: the path which will be used to save the resultant file.
"""
if isinstance(n_cores, str):
import multiprocessing
N_cores = multiprocessing.cpu_count() -1
elif isinstance(n_cores, int):
N_cores =n_cores
parallelize_df(GDF, generate_grid_from_gdf, n_cores=N_cores, dx=dx, dy=dy, verbose=verbose, to_filename=to_filename)
Data_Arranger(to_filename)
####################################################################################
if "__main__" == __name__:
freeze_support()
GDF = gpd.read_file("Someone's_file.shp")
to_filename = "To_file_directory/To_file_name.shp"
dx = 500 # resampling to 500 units. Ex: assuming the coordinate reference system is in meters, this function will return polygons of the given geometries in 500m for the longitudinal dimension.
dy = 500 # same here. Assuming CRS is in meters units, the resultant file will be have polygons of 500m in latitudinal dimension
main(GDF, dx=dx, dy=dy, verbose=True, to_filename=to_filename)
I thank you for your time.
Philipe Leal
I run an evaluation at the end of each epoch and need to show an image calculated from the features and labels arguments of the model function model_fn. Including a tf.summary.image(name, image) in evaluation part of the model function does not help and it looks to me that the only way to do so is to pass the correct eval_metric_ops to construct the EstimatorSpec for mode EVAL. So I first sub-class Estimator so that it considers images. The following code is mostly from estimator.py; the only change is the few lines marked by "my change" inside _write_dict_to_summary:
import logging
import io
import numpy as np
import matplotlib.pyplot as plt
import six
from google.protobuf import message
import tensorflow as tf
from tensorflow.python.training import evaluation
from tensorflow.python import ops
from tensorflow.python.estimator.estimator import _dict_to_str, _write_checkpoint_path_to_summary
from tensorflow.core.framework import summary_pb2
from tensorflow.python.framework import tensor_util
from tensorflow.python.summary.writer import writer_cache
def dump_as_image(a):
vmin = np.min(a)
vmax = np.max(a)
img = np.squeeze((img - vmin) / (vmax - vmin) * 255).astype(np.uint8)
s = io.BytesIO()
plt.imsave(s, img, format='png', vmin=0, vmax=255, cmap='gray')
return s.getvalue()
# see https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/estimator/estimator.py
def _write_dict_to_summary(output_dir, dictionary, current_global_step):
logging.info('Saving dict for global step %d: %s', current_global_step, _dict_to_str(dictionary))
summary_writer = writer_cache.FileWriterCache.get(output_dir)
summary_proto = summary_pb2.Summary()
for key in dictionary:
if dictionary[key] is None:
continue
if key == 'global_step':
continue
if (isinstance(dictionary[key], np.float32) or
isinstance(dictionary[key], float)):
summary_proto.value.add(tag=key, simple_value=float(dictionary[key]))
elif (isinstance(dictionary[key], np.int64) or
isinstance(dictionary[key], np.int32) or
isinstance(dictionary[key], int)):
summary_proto.value.add(tag=key, simple_value=int(dictionary[key]))
elif isinstance(dictionary[key], six.binary_type):
try:
summ = summary_pb2.Summary.FromString(dictionary[key])
for i, img_bytes in enumerate(summ.value):
summ.value[i].tag = '%s/%d' % (key, i)
summary_proto.value.extend(summ.value)
except message.DecodeError:
logging.warn('Skipping summary for %s, cannot parse string to Summary.', key)
continue
elif isinstance(dictionary[key], np.ndarray):
value = summary_proto.value.add()
value.tag = key
value.node_name = key
array = dictionary[key]
# my change begins
if array.ndim == 2:
buffer = dump_as_image(array)
value.image.encoded_image_string = buffer
# my change ends
else:
tensor_proto = tensor_util.make_tensor_proto(array)
value.tensor.CopyFrom(tensor_proto)
logging.info(
'Summary for np.ndarray is not visible in Tensorboard by default. '
'Consider using a Tensorboard plugin for visualization (see '
'https://github.com/tensorflow/tensorboard-plugin-example/blob/master/README.md'
' for more information).')
else:
logging.warn(
'Skipping summary for %s, must be a float, np.float32, np.int64, '
'np.int32 or int or np.ndarray or a serialized string of Summary.',
key)
summary_writer.add_summary(summary_proto, current_global_step)
summary_writer.flush()
class ImageMonitoringEstimator(tf.estimator.Estimator):
def __init__(self, *args, **kwargs):
tf.estimator.Estimator._assert_members_are_not_overridden = lambda self: None
super(ImageMonitoringEstimator, self).__init__(*args, **kwargs)
def _evaluate_run(self, checkpoint_path, scaffold, update_op, eval_dict, all_hooks, output_dir):
eval_results = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
master=self._config.evaluation_master,
scaffold=scaffold,
eval_ops=update_op,
final_ops=eval_dict,
hooks=all_hooks,
config=self._session_config)
current_global_step = eval_results[ops.GraphKeys.GLOBAL_STEP]
_write_dict_to_summary(
output_dir=output_dir,
dictionary=eval_results,
current_global_step=current_global_step)
if checkpoint_path:
_write_checkpoint_path_to_summary(
output_dir=output_dir,
checkpoint_path=checkpoint_path,
current_global_step=current_global_step)
return eval_results
the model function is like --
def model_func(features, labels, mode):
# calculate network_output
if mode == tf.estimator.ModeKeys.TRAIN:
# training
elif mode == tf.estimator.ModeKeys.EVAL:
# make_image consists of slicing and concatenations
images = tf.map_fn(make_image, (features, network_output, labels), dtype=features.dtype)
eval_metric_ops = images, tf.no_op() # not working
return tf.estimator.EstimatorSpec(mode, loss=loss)
eval_metric_ops={'images': eval_metric_ops})
else:
# prediction
And the main part --
# mon_features and mon_labels are np.ndarray
estimator = ImageMonitoringEstimator(model_fn=model_func,...)
mon_input_func = tf.estimator.inputs.numpy_input_fn(mon_features,
mon_labels,
shuffle=False,
num_epochs=num_epochs,
batch_size=len(mon_features))
for _ in range(num_epochs):
estimator.train(...)
estimator.evaluate(input_fn=mon_input_func)
The code above will give a warning (later an error):
WARNING:tensorflow:An OutOfRangeError or StopIteration exception is
raised by the code in FinalOpsHook. This typically means the Ops
running by the FinalOpsHook have a dependency back to some input
source, which should not happen. For example, for metrics in
tf.estimator.Estimator, all metrics functions return two Ops:
value_op and update_op. Estimator.evaluate calls the update_op
for each batch of the data in input source and, once it is exhausted,
it call the value_op to get the metric values. The value_op here
should have dependency back to variables reading only, rather than
reading another batch from input. Otherwise, the value_op, executed
by FinalOpsHook, triggers another data reading, which ends
OutOfRangeError/StopIteration. Please fix that.
Looks like I didn't set the eval_metric_ops correctly. I guess tf.map_fn touches another batch as the warning message hints; maybe I need some stacking operation as the update_op to build the images used for monitoring incrementally? But I am not sure how to do that. So how to add an image to summary during evaluation when using Estimator?
The way I make it work is by passing a tf.train.SummarySaverHook under the evaluation mode and then declaring it to the tf.estimator.EstimatorSpec at evaluation_hooks=.
images is a list of the desired tf.summary.image you want to print during evaluation.
example:
eval_summary_hook = tf.train.SummarySaverHook(output_dir=params['eval_save_path'], summary_op=images, save_secs=120)
spec = tf.estimator.EstimatorSpec(mode=mode, predictions=y_pred, loss=loss, eval_metric_ops=eval_metric_ops,
evaluation_hooks=[eval_summary_hook])
I want to make a batch request getting campaigns for a specific ad account. I created a simple code based on this issue
but I've used some global arrays and I don't know if time.sleep(2) is necessary for this code. My code is as below:
from facebookads import FacebookAdsApi
from facebookads.api import FacebookRequest
import pandas as pd
import time
batch_body_responses = []
list_of_artists = [1]
def success_callback(response):
try:
pair = [response.json()['data']]
next = [response.json()['paging']['next']]
batch_body_responses.append(pair)
batch_body_responses.append(next)
except IndexError:
pass
except UnicodeEncodeError:
pass
def error_callback(response):
pass
def generate_batches(iterable, batch_size_limit):
# This function can be found in examples/batch_utils.py
batch = []
for item in iterable:
if len(batch) == batch_size_limit:
yield batch
batch = []
batch.append(item)
if len(batch):
yield batch
def get_id_list(art_search_list):
batches = []
your_app_id = '756885'
your_app_secret = '123456789'
your_access_token = 'EAA.....'
api = FacebookAdsApi.init(your_app_id, your_app_secret, your_access_token)
batch_limit = 25
for batch in generate_batches(art_search_list, batch_limit):
next_batch = api.new_batch()
for artt in batch:
requestss = [FacebookRequest(node_id='act_1234/campaigns',method="GET",endpoint="?fields=id,name")]
for req in requestss:
next_batch.add_request(req, success_callback, error_callback)
batches.append(next_batch)
for batch_request in batches:
batch_request.execute()
time.sleep(2)
print(batch_body_responses)
return batch_body_responses
df = pd.DataFrame(get_id_list(list_of_artists))
How can this code optimized by not using global arrays and how to execute without sleep statement and why it is needed sleep?