I am making data windows (input, output pairs of windows) from time series data. I have already converted my time series to a tf dataset, where each batch has the number of time steps equal to the total window sizes I need.
def make_dataset(data=train_df[0]):
ds = tf.keras.preprocessing.timeseries_dataset_from_array(
data=data,
targets=None,
sequence_length=total_window_size,
sequence_stride=1,
shuffle=True,
batch_size=32
)
return ds
Example of the shape returned:
for example in tensor.take(1):
print(f'shape: {example.shape}')
shape: (32, 48, 18)
What I need to do now is split the time dimension into my input-output pairs, and I have a function to do this, however, when I try to map this function to my 'ds' in the above function I get the following error:
'BatchDataset' object is not subscriptable
I am hoping someone can help me understand where I am going wrong? I am pretty new to tensorflow... My code is below, in this example 'input slice' and 'label_slice' are 0 and 24 respectively. So my aim is to split my batches into input-output pairs of length 24 each.
def split_window(features):
inputs = features[:, input_slice, :]
labels = features[:, labels_slice, :]
inputs.set_shape([None, input_width, None])
labels.set_shape([None, label_width, None])
return inputs, labels
def make_dataset(data=train_df[0]):
data = np.array(data, dtype=np.float32)
ds = tf.keras.preprocessing.timeseries_dataset_from_array(
data=data,
targets=None,
sequence_length=total_window_size,
sequence_stride=1,
shuffle=True,
batch_size=32
)
ds = ds.map(split_window(ds))
return ds
tensor = make_dataset()
tensor
'BatchDataset' object is not subscriptable
Your snippet of code looks similar to the tutorial of Time Series in Tensorflow. Based on that, I modified the main class WindowGenerator() (excluded the parts of train/val/test datasets and output-labels selection) to a simpler class suitable to your question.
class WindowGenerator():
def __init__(self, input_width, label_width, shift):
self.input_width = input_width
self.label_width = label_width
self.shift = shift
self.total_window_size = input_width + shift
self.input_slice = slice(0, input_width)
self.input_indices = np.arange(self.total_window_size[self.input_slice]
self.label_start = self.total_window_size - self.label_width
self.labels_slice = slice(self.label_start, None)
self.label_indices = np.arange(self.total_window_size [self.labels_slice]
def split_window(self, features):
inputs = features[:, self.input_slice, :]
labels = features[:, self.labels_slice, :]
inputs.set_shape([None, self.input_width, None])
labels.set_shape([None, self.label_width, None])
return inputs, labels
def make_dataset(self, data):
data = np.array(data, dtype=np.float32)
ds = tf.keras.utils.timeseries_dataset_from_array(
data=data,
targets=None,
sequence_length=self.total_window_size,
sequence_stride=1,
shuffle=True,
batch_size=batch_size,)
ds = ds.map(self.split_window)
return ds
input_width=24
label_width=24
total_windth = input_width + label_width
batch_size = 32
window = WindowGenerator(input_width=input_width, label_width=label_width, shift=1)
dataset = window.make_dataset(train_df[0])
I would recommend, though, to use Dataset.window(). It is simpler and more intuitive.
dataset = tf.data.Dataset.from_tensor_slices(train_df[0])
dataset = dataset.window(total_windth, shift=1, drop_remainder=True)
dataset = dataset.flat_map(lambda window: window.batch(batch_size))
dataset = dataset.map(lambda window: (window[:-label_width], window[-input_width:]))
I want to compute a Bayesian Search with [skopt] (https://scikit-optimize.github.io/stable/auto_examples/bayesian-optimization.html).
My dataset is a Time series, and t is my time step.
But i have a error:
recursionerror: maximum recursion depth exceeded in comparison
This is my code :
def Grid_search_class(X_train=X_train[:,0:t+1,:]
,y_train=y_train
,X_test=X_test[:,0:t+1,:],
y_test=y_test
,n_calls=20,
print_score=False,t=t):
""" INPUTS : Train Test data
n_calls Number of calls to func"""
import tensorflow as tf
Adam = tf.keras.optimizers.Adam(learning_rate=0.007)
Adagrad = tf.keras.optimizers.Adagrad(learning_rate=0.007)
dim_num_input_text = Categorical([16,32,64,128,256,512,1024,2048], name='num_dense_layers_text')
dim_num_dense_text = Integer(low=0, high=5, name='num_HLD_nodes_text')
dim_drop_text = Categorical([0.01,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4], name='drop_text')
dim_num_input_temp = Categorical([16,32,64,128,256,512,1024,2048], name='num_dense_layers_temp')
dim_num_dense_temp = Integer(low=0, high=5, name='num_HLD_nodes_temp')
dim_drop_temp = Categorical([0.01,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4], name='drop_temp')
dim_num_input_fixe = Categorical([16,32,64,128,256,512,1024,2048], name='num_dense_layers_fixe')
dim_num_dense_fixe = Integer(low=0, high=5, name='num_HLD_nodes_fixe')
dim_drop_fixe = Categorical([0.01,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4], name='drop_fixe')
dim_num_input_merge = Categorical([16,32,64,128,256,512,1024,2048], name='num_dense_layers_merge')
dim_num_dense_merge = Integer(low=0, high=5, name='num_HLD_nodes_merge')
dim_drop_merge = Categorical([0.01,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4], name='drop_merge')
dim_optim=Categorical([Adam,Adagrad], name='optim')
dimensions = [dim_num_input_text,
dim_num_dense_text,
dim_drop_text,
dim_num_input_temp,
dim_num_dense_temp,
dim_drop_temp,
dim_num_input_fixe,
dim_num_dense_fixe,
dim_drop_fixe,
dim_num_input_merge,
dim_num_dense_merge,
dim_drop_merge,
dim_optim
]
default_parameters = [512,0,0.1,512,0,0.1,512,0,0.1,512,0,0.1,Adam]
def create_model(num_dense_layers_text,num_HLD_nodes_text,drop_text,
num_dense_layers_temp,num_HLD_nodes_temp,drop_temp,
num_dense_layers_fixe,num_HLD_nodes_fixe,drop_fixe,
num_dense_layers_merge,num_HLD_nodes_merge,drop_merge,optim,t=t):
x_text = model_text.layers[ind_list[-1]-1].output
if num_dense_layers_text>0:
for i in range(num_dense_layers_text):
x_text =tf.keras.layers.Dense(num_HLD_nodes_text,activation='relu')(x_text)
x_text=tf.keras.layers.Dropout(drop_text)(x_text)
x_temp = model_temp[t].layers[ind_list[t]].output
if num_dense_layers_temp>0:
for i in range(num_dense_layers_temp):
x_temp =tf.keras.layers.Dense(num_HLD_nodes_temp,activation='relu')(x_temp)
x_temp=tf.keras.layers.Dropout(drop_temp)(x_temp)
x_fixe= model_fixe.layers[1].output
if num_dense_layers_fixe>0:
for i in range(num_dense_layers_fixe):
x_fixe =tf.keras.layers.Dense(num_HLD_nodes_fixe,activation='relu')(x_fixe)
x_fixe=tf.keras.layers.Dropout(drop_fixe)(x_fixe)
merge = tf.keras.layers.concatenate([x_text,x_temp,x_fixe])
if num_dense_layers_merge>0:
for i in range(num_dense_layers_merge):
merge =tf.keras.layers.Dense(num_HLD_nodes_merge,activation='relu')(merge)
merge=tf.keras.layers.Dropout(drop_merge)(merge)
#add our classification layer.
predictions = tf.keras.layers.Dense(3,activation='softmax')(merge)
model = tf.keras.Model(inputs = [model_text.input,model_temp[t].input,model_fixe.input], outputs = predictions)
#setup our optimizer and compile
model.compile(optimizer=optim, loss=ncce,
metrics=[ tf.keras.metrics.Precision(name='precision'),
tf.keras.metrics.Recall(name='recall'),F1Score(num_classes=3,name='F1',average='macro')])
return model
score='val_F1'
#use_named_args(dimensions=dimensions)
def fitness(num_dense_layers_text,num_HLD_nodes_text,drop_text,
num_dense_layers_temp,num_HLD_nodes_temp,drop_temp,
num_dense_layers_fixe,num_HLD_nodes_fixe,drop_fixe,
num_dense_layers_merge,num_HLD_nodes_merge,drop_merge,optim):
print(num_dense_layers_text,num_HLD_nodes_text,drop_text,
num_dense_layers_temp,num_HLD_nodes_temp,drop_temp,
num_dense_layers_fixe,num_HLD_nodes_fixe,drop_fixe,
num_dense_layers_merge,num_HLD_nodes_merge,drop_merge,optim)
model = create_model(num_dense_layers_text=num_dense_layers_text,
num_HLD_nodes_text=num_HLD_nodes_text,drop_text=drop_text,
num_dense_layers_temp=num_dense_layers_temp,
num_HLD_nodes_temp=num_HLD_nodes_temp,drop_temp=drop_temp,
num_dense_layers_fixe=num_dense_layers_fixe,
num_HLD_nodes_fixe=num_HLD_nodes_fixe,drop_fixe=drop_fixe,
num_dense_layers_merge=num_dense_layers_merge,drop_merge=drop_merge,
num_HLD_nodes_merge=num_HLD_nodes_merge,optim=optim,t=t)
callback=tf.keras.callbacks.EarlyStopping(
monitor=score, min_delta=0.01, patience=1, verbose=0, mode='auto',
baseline=0, restore_best_weights=False
)
#named blackbox becuase it represents the structure
blackbox = model.fit(x=X_train,
y=y_train,verbose=1,
epochs=2,
batch_size=32,
validation_data=(X_test,y_test)
)
#return the validation accuracy for the last epoch.
val_loss = blackbox.history[score][-1]
if score=='val_F1':
val_loss=-val_loss
# Print the classification accuracy.
if print_score :
print()
print("val_score: {}".format(val_loss))
print()
# Delete the Keras model with these hyper-parameters from memory.
del model
# Clear the Keras session, otherwise it will keep adding new
# models to the same TensorFlow graph each time we create
# a model with a different set of hyper-parameters.
tf.keras.backend.clear_session()
tf.compat.v1.reset_default_graph()
# the optimizer aims for the lowest score, so we return our negative accuracy
return -val_loss
gp_result = gp_minimize(fitness,
dimensions=dimensions,
n_calls=n_calls,n_random_starts=7,
noise= 0.01,
x0=default_parameters)
a=pd.concat([pd.DataFrame(gp_result.x_iters, columns = ["dense layers text","HLD nodes text","drop text",
"dense layers temp","HLD nodes temp","drop temp",
"dense layers fixe","HLD nodes fixe","drop fixe",
"dense layers merge","HLD nodes merge","drop merge",
"optim","batch size"]),
(pd.Series(gp_result.func_vals*-1, name="val_loss"))], axis=1)
a.sort_values(by=['val_loss'], inplace=True,ascending=False)
print(a.iloc[:10])
return a
This step is looking for the best parameters a step t
def Run_Grid_search_temp(j=0,n_calls=25):
while j<X_train.shape[1] :
temp=Grid_search_class(t=j,n_calls=n_calls)
print(temp)
j+=1
return
And this one is a loop on the step.
sys.setrecursionlimit(10000)
seems to resolve my problem.
I have time-series sequences which I needed to keep the length of sequences fixed to a number by padding zeroes into matrix and using keras.layers.Masking in keras I could neglect those padded zeros for further computations, I am wondering how could it be done in Pytorch?
Either I need to do the padding in pytroch and pytorch can't handle the sequences with varying lengths what is the equivalent to Masking layer of keras in pytorch, or if pytorch handles the sequences with varying lengths, how could it be done?
You can use PackedSequence class as equivalent to keras masking. you can find more features at torch.nn.utils.rnn
Here putting example from packing for variable-length sequence inputs for rnn
import torch
import torch.nn as nn
from torch.autograd import Variable
batch_size = 3
max_length = 3
hidden_size = 2
n_layers =1
# container
batch_in = torch.zeros((batch_size, 1, max_length))
#data
vec_1 = torch.FloatTensor([[1, 2, 3]])
vec_2 = torch.FloatTensor([[1, 2, 0]])
vec_3 = torch.FloatTensor([[1, 0, 0]])
batch_in[0] = vec_1
batch_in[1] = vec_2
batch_in[2] = vec_3
batch_in = Variable(batch_in)
seq_lengths = [3,2,1] # list of integers holding information about the batch size at each sequence step
# pack it
pack = torch.nn.utils.rnn.pack_padded_sequence(batch_in, seq_lengths, batch_first=True)
>>> pack
PackedSequence(data=Variable containing:
1 2 3
1 2 0
1 0 0
[torch.FloatTensor of size 3x3]
, batch_sizes=[3])
# initialize
rnn = nn.RNN(max_length, hidden_size, n_layers, batch_first=True)
h0 = Variable(torch.randn(n_layers, batch_size, hidden_size))
#forward
out, _ = rnn(pack, h0)
# unpack
unpacked, unpacked_len = torch.nn.utils.rnn.pad_packed_sequence(out)
>>> unpacked
Variable containing:
(0 ,.,.) =
-0.7883 -0.7972
0.3367 -0.6102
0.1502 -0.4654
[torch.FloatTensor of size 1x3x2]
more you would find this article useful. [Jum to Title - "How the PackedSequence object works"] - link
You can use a packed sequence to mask a timestep in the sequence dimension:
batch_mask = ... # boolean mask e.g. (seq x batch)
# move `padding` at right place then it will be cut when packing
compact_seq = torch.zeros_like(x)
for i, seq_len in enumerate(batch_mask.sum(0)):
compact_seq[:seq_len, i] = x[batch_mask[:,i],i]
# pack in sequence dimension (the number of agents)
packed_x = pack_padded_sequence(compact_seq, batch_mask.sum(0).cpu().numpy(), enforce_sorted=False)
packed_scores, rnn_hxs = nn.GRU(packed_x, rnn_hxs)
# restore sequence dimension
scores, _ = pad_packed_sequence(packed_scores)
# restore order, moving padding in its place
scores = torch.zeros((*batch_mask.shape,scores.size(-1))).to(scores.device).masked_scatter(batch_mask.unsqueeze(-1), scores)
instead use a mask select/scatter to mask in the batch dimension:
batch_mask = torch.any(x, -1).unsqueeze(-1) # boolean mask (batch,1)
batch_x = torch.masked_select(x, batch_mask).reshape(-1, x.size(-1))
batch_rnn_hxs = torch.masked_select(rnn_hxs, batch_mask).reshape(-1, rnn_hxs.size(-1))
batch_rnn_hxs = nn.GRUCell(batch_x, batch_rnn_hxs)
rnn_hxs = rnn_hxs.masked_scatter(batch_mask, batch_rnn_hxs) # restore batch
Note that using scatter function is safe for gradient backpropagation
I have to use tf.data.Dataset for creating a input pipeline for an RNN model in tensorflow. I am providing a basic code, by which I need to pad the data in batch with a pad token and use it for further manipulation.
import pandas as pd
import numpy as np
import tensorflow as tf
import functools
total_data_size = 10000
embedding_dimension = 25
max_len = 17
varying_length = np.random.randint(max_len, size=(10000)) # varying length data
X = np.array([np.random.randint(1000, size=(value)).tolist()for index, value in enumerate(varying_length)]) # data of arying length
Y = np.random.randint(2, size=(total_data_size)).astype(np.int32) # target binary
embedding = np.random.uniform(-1,1,(1000, embedding_dimension)) # word embedding
def gen():
for index in range(len(X)):
yield X[index] , Y[index]
dataset = tf.data.Dataset.from_generator(gen,(tf.int32,tf.int32))
dataset = dataset.batch(batch_size=25)
padded_shapes = (tf.TensorShape([None])) # sentence of unknown size
padding_values = (tf.constant(-111)) # the value with which pad index needs to be filled
dataset = (dataset
.padded_batch(25, padded_shapes=padded_shapes, padding_values=padding_values)
)
iter2 = dataset.make_initializable_iterator()
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(iter2.initializer)
print(sess.run(iter2.get_next()))
I hope the code is self explanatory with comments. But I am getting following error,
InvalidArgumentError (see above for traceback): Cannot batch tensors with different shapes in component 0. First element had shape [11] and element 1 had shape [12].
[[Node: IteratorGetNext = IteratorGetNext[output_shapes=[[?,?], [?]], output_types=[DT_INT32, DT_INT32], _device="/job:localhost/replica:0/task:0/device:CPU:0"](Iterator)]]
I believe that since your generator yields two outputs, your padded_shapes and padded_values tuples must have a length of two. For me, this works:
dataset = tf.data.Dataset.from_generator(gen, (tf.int32, tf.int32))
dataset = dataset.batch(batch_size=25)
padded_shapes = (tf.TensorShape([None]), tf.TensorShape([None])) # sentence of unknown size
padding_values = (tf.constant(-111), tf.constant(-111)) # the value with which pad index needs to be filled
dataset = (dataset
.padded_batch(25, padded_shapes=padded_shapes, padding_values=padding_values)
)
iter2 = dataset.make_initializable_iterator()
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(iter2.initializer)
Finally got the answer. The issue was for the second padded shapes instead of Tensorshape([None]), we should provide [], because the second item returned by the generator is a scalar. If using Tensorshape([None]),, make sure we are returning a vector
import pandas as pd
import numpy as np
import tensorflow as tf
import functools
total_data_size = 10000
embedding_dimension = 25
max_len = 17
varying_length = np.random.randint(max_len, size=(10000)) # varying length data
X = np.array([np.random.randint(1000, size=(value)).tolist()for index, value in enumerate(varying_length)]) # data of arying length
Y = np.random.randint(2, size=(total_data_size)).astype(np.int32) # target binary
embedding = np.random.uniform(-1,1,(1000, embedding_dimension)) # word embedding
def gen():
for index in range(len(X)):
yield X[index] , Y[index]
dataset = tf.data.Dataset.from_generator(gen, (tf.int32, tf.int32), (tf.TensorShape([None]), []))
padded_shapes = (tf.TensorShape([None]), []) # sentence of unknown size
dataset = (dataset
.padded_batch(25, padded_shapes=padded_shapes, padding_values=(-111, 0))
)
iter2 = dataset.make_initializable_iterator()
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(iter2.initializer)
sess.run(iter2.get_next())