I am running Python 3.8 on Windows 10 Pro. I am attempting to fit a multinomial fixed effects model to some data. All the packages appear to load fine, the data loads and can be manipulated and viewed. But I get an error when I call a module that my collaborator wrote to fit the model (called contactai).
Here is the code for contactai:
Note: tt stands for theano.tensor and pm stands for pymc3
def fixed_effects_model(X, y):
Parameters
X : array contains covariates
Design matrix n x p
y : array or shared theano variable
Multinomial response variable, n x 3
Returns
pymc3 model object containing fitted model
yarray = y.eval()
else:
yarray = y
if type(X) == tt.sharedvar.TensorSharedVariable:
Xmat = X.eval()
else:
Xmat = X
model = pm.Model()
with model:
# Transformed coefficients
betaD = pm.Normal('betad', mu=0, sd=5, shape=(Xmat.shape[1], 1))
betaI = pm.Normal('betai', mu=0, sd=5, shape=(Xmat.shape[1], 1))
# Softmax transformation with no contact as baseline
etad = tt.exp(tt.dot(X, betaD))
etai = tt.exp(tt.dot(X, betaI))
p_d = etad / (1 + etad + etai)
p_i = etai / (1 + etad + etai)
p_n = 1 - p_d - p_i
ps = tt.stack([p_n, p_d, p_i], axis=1).reshape((len(yarray), 3)) # Stack column-wise
Y_obs = pm.Multinomial("Y_obs", n=1, p=ps, observed=y)
return(model)
When I call contaciai.py in my jupyter notebook using
test_mod = contactai.fixed_effects_model(Xmat1, Yres.values)
I get this error message:
ImportError Traceback (most recent call last)
<ipython-input-6-275ca67d4b63> in <module>
1 # Run Fixed Effects model
----> 2 test_mod = contactai.fixed_effects_model(Xmat1, Yres.values)
3 niter = 1000
4
5 with test_mod:
~\dropbox\Brooke\ai_multinomial_model\contactai.py in fixed_effects_model(X, y)
50 # Transformed coefficients, what are D and I? In other examples this is where priors are
51 #specified.
---> 52 betaD = pm.Normal('betad', mu=0, sd=5, shape=(Xmat.shape[1], 1))
53 betaI = pm.Normal('betai', mu=0, sd=5, shape=(Xmat.shape[1], 1))
54 # First argument is the name of the random variable
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\pymc3\distributions\distribution.py in __new__(cls, name, *args, **kwargs)
44 raise TypeError("observed needs to be data but got: {}".format(type(data)))
45 total_size = kwargs.pop('total_size', None)
---> 46 dist = cls.dist(*args, **kwargs)
47 return model.Var(name, dist, data, total_size)
48 else:
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\pymc3\distributions\distribution.py in dist(cls, *args, **kwargs)
55 def dist(cls, *args, **kwargs):
56 dist = object.__new__(cls)
---> 57 dist.__init__(*args, **kwargs)
58 return dist
59
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\pymc3\distributions\continuous.py in __init__(self, mu, sigma, tau, sd, **kwargs)
468
469 self.mean = self.median = self.mode = self.mu = mu = tt.as_tensor_variable(floatX(mu))
--> 470 self.variance = 1. / self.tau
471
472 assert_negative_support(sigma, 'sigma', 'Normal')
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\tensor\var.py in __rtruediv__(self, other)
204
205 def __rtruediv__(self, other):
--> 206 return theano.tensor.basic.true_div(other, self)
207
208 def __rfloordiv__(self, other):
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\op.py in __call__(self, *inputs, **kwargs)
667
668 # compute output value once with test inputs to validate graph
--> 669 thunk = node.op.make_thunk(node, storage_map, compute_map,
670 no_recycling=[])
671 thunk.inputs = [storage_map[v] for v in node.inputs]
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\op.py in make_thunk(self, node, storage_map, compute_map, no_recycling, impl)
952 compute_map=compute_map, impl='c')
953 try:
--> 954 return self.make_c_thunk(node, storage_map, compute_map,
955 no_recycling)
956 except (NotImplementedError, utils.MethodNotDefined):
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\op.py in make_c_thunk(self, node, storage_map, compute_map, no_recycling)
855 raise NotImplementedError("float16")
856 _logger.debug('Trying CLinker.make_thunk')
--> 857 outputs = cl.make_thunk(input_storage=node_input_storage,
858 output_storage=node_output_storage)
859 thunk, node_input_filters, node_output_filters = outputs
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cc.py in make_thunk(self, input_storage, output_storage, storage_map, keep_lock)
1213 """
1214 init_tasks, tasks = self.get_init_tasks()
-> 1215 cthunk, module, in_storage, out_storage, error_storage = self.__compile__(
1216 input_storage, output_storage, storage_map,
1217 keep_lock=keep_lock)
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cc.py in __compile__(self, input_storage, output_storage, storage_map, keep_lock)
1151 input_storage = tuple(input_storage)
1152 output_storage = tuple(output_storage)
-> 1153 thunk, module = self.cthunk_factory(error_storage,
1154 input_storage,
1155 output_storage,
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cc.py in cthunk_factory(self, error_storage, in_storage, out_storage, storage_map, keep_lock)
1621 for node in self.node_order:
1622 node.op.prepare_node(node, storage_map, None, 'c')
-> 1623 module = get_module_cache().module_from_key(
1624 key=key, lnk=self, keep_lock=keep_lock)
1625
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cmodule.py in module_from_key(self, key, lnk, keep_lock)
1187 try:
1188 location = dlimport_workdir(self.dirname)
-> 1189 module = lnk.compile_cmodule(location)
1190 name = module.__file__
1191 assert name.startswith(location)
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cc.py in compile_cmodule(self, location)
1518 try:
1519 _logger.debug("LOCATION %s", str(location))
-> 1520 module = c_compiler.compile_str(
1521 module_name=mod.code_hash,
1522 src_code=src_code,
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cmodule.py in compile_str(module_name, src_code, location, include_dirs, lib_dirs, libs, preargs, py_module, hide_symbols)
2418 open(os.path.join(location, "__init__.py"), 'w').close()
2419 assert os.path.isfile(lib_filename)
-> 2420 return dlimport(lib_filename)
2421
2422
c:\users\webblab\appdata\local\programs\python\python38-32\lib\site-packages\theano\gof\cmodule.py in dlimport(fullpath, suffix)
315 warnings.filterwarnings("ignore",
316 message="numpy.ndarray size changed")
--> 317 rval = __import__(module_name, {}, {}, [module_name])
318 t1 = time.time()
319 import_time += t1 - t0
ImportError: DLL load failed while importing m54e1ce4760689129b813fce15fee7f8d468afbe167bc036205eea6bd301309d7: The specified module could not be found.
My collaborator does not get this error when running identical code on their macbook so the issue is not with the code. I installed all packages with pip. Any help would be greatly appreciated, this has brought me to a standstill on this project.
Related
I'm using the following code.
%matplotlib notebook
plt.style.use('seaborn-bright')
fig,ax = plt.subplots(figsize=(8,6), facecolor='w', edgecolor='blue')
ax.spines['top'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
plt.title('Clubs of Severn Valley Petanque League - performance in years 2014-2019',color='purple',size=14.5,y=1.09)
ax.set_ylabel('Year', color='indigo',size=10)
ax.set_ylabel('Points won', color='darkblue', fontdict={'fontsize': 12, 'fontweight': 'medium'})
years=[2014,2015, 2016,2017,2018, 2019]
plt.xticks(np.arange(5), (years))
plt.margins(x=0.006,y=0.009)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
ax.yaxis.set_major_formatter(plt.FuncFormatter('{:.0f}'.format))
ax.grid(True, 'major', 'y', ls='--', lw=.6, c='darkgray', alpha=.5)
ax.tick_params(axis='y', which='both', labelsize=10,bottom=False, top=False, labelbottom=True,
left=False, right=False, labelleft=True)
plt.legend(loc=2)
# Now I call each column( one for each team) one by one like so :
plt.plot(final.Points_Kinver.values, marker='o', label='Kinver Con Club')
plt.plot(final.Points_Nomads.values, marker='o', label='Nomads')
plt.plot(final.Points_TopPub.values, marker='o', label='Top Pub')
plt.plot(final.Points_Plough.values, marker='o', label='Plough')
plt.plot(final.Points_AlveleyRoyals.values, '-o', c='violet', label='Alveley Royals')
plt.plot(final.Points_Footloose.values, marker='o', c='yellow', label='Footloose')
plt.plot(final.Points_Gaters.values, '-o', c='orange', label="Gaters 'A'")
plt.plot(final.Points_AlveleyOaks.values, '-o',c='darkgreen', label='Alveley Oaks')
plt.show()
It works. Here is my chart .
However, I wish I could write plt.plot as one line, and this does not work.
plt.plot(final.columns.values, marker='o')
This causes an error.
TypeError Traceback (most recent call last)
<ipython-input-277-f48c020019fc> in <module>()
44 #mychart=final.plot( marker='o',ax=ax)
45
---> 46 plt.plot(final.columns.values, marker='o')
47
48 #plt.plot(final.Points_Kinver.values, marker='o', label='Kinver Con Club')
~\Anaconda3\lib\site-packages\matplotlib\pyplot.py in plot(*args, **kwargs)
3356 mplDeprecation)
3357 try:
-> 3358 ret = ax.plot(*args, **kwargs)
3359 finally:
3360 ax._hold = washold
~\Anaconda3\lib\site-packages\matplotlib\__init__.py in inner(ax, *args, **kwargs)
1853 "the Matplotlib list!)" % (label_namer, func.__name__),
1854 RuntimeWarning, stacklevel=2)
-> 1855 return func(ax, *args, **kwargs)
1856
1857 inner.__doc__ = _add_data_doc(inner.__doc__,
~\Anaconda3\lib\site-packages\matplotlib\axes\_axes.py in plot(self, *args, **kwargs)
1525 kwargs = cbook.normalize_kwargs(kwargs, _alias_map)
1526
-> 1527 for line in self._get_lines(*args, **kwargs):
1528 self.add_line(line)
1529 lines.append(line)
~\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _grab_next_args(self, *args, **kwargs)
404 this += args[0],
405 args = args[1:]
--> 406 for seg in self._plot_args(this, kwargs):
407 yield seg
408
~\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _plot_args(self, tup, kwargs)
381 x, y = index_of(tup[-1])
382
--> 383 x, y = self._xy_from_xy(x, y)
384
385 if self.command == 'plot':
~\Anaconda3\lib\site-packages\matplotlib\axes\_base.py in _xy_from_xy(self, x, y)
214 if self.axes.xaxis is not None and self.axes.yaxis is not None:
215 bx = self.axes.xaxis.update_units(x)
--> 216 by = self.axes.yaxis.update_units(y)
217
218 if self.command != 'plot':
~\Anaconda3\lib\site-packages\matplotlib\axis.py in update_units(self, data)
1467 neednew = self.converter != converter
1468 self.converter = converter
-> 1469 default = self.converter.default_units(data, self)
1470 if default is not None and self.units is None:
1471 self.set_units(default)
~\Anaconda3\lib\site-packages\matplotlib\category.py in default_units(data, axis)
113 # default_units->axis_info->convert
114 if axis.units is None:
--> 115 axis.set_units(UnitData(data))
116 else:
117 axis.units.update(data)
~\Anaconda3\lib\site-packages\matplotlib\category.py in __init__(self, data)
180 self._counter = itertools.count(start=0)
181 if data is not None:
--> 182 self.update(data)
183
184 def update(self, data):
~\Anaconda3\lib\site-packages\matplotlib\category.py in update(self, data)
199 for val in OrderedDict.fromkeys(data):
200 if not isinstance(val, VALID_TYPES):
--> 201 raise TypeError("{val!r} is not a string".format(val=val))
202 if val not in self._mapping:
203 self._mapping[val] = next(self._counter)
TypeError: ('Points_Kinver',) is not a string
I have also tried
ax=final.plot( marker='o')
for line, name in zip(ax.lines, final.columns):
y = line.get_ydata()[-1]
ax.annotate(name, xy=(1,y), xytext=(6,0), color=line.get_color(),
xycoords = ax.get_yaxis_transform(), textcoords="offset points",
size=8, va="center")
It works. However, it creates 2 figures.
Figure 1: the empty customized figure with all of my carefully coded formatting, more specifically, ticks and gridlines ( ax.grid...) and the chart title.
Figure 2: it contains the lines with labels, but all customized formatting and the title are lost.
Can you explain what is happening here?
My next step is labelling the end of each line instead of the legend.
so I thought I need my plt.plot code in a single line..
As you can see, I'm not very experienced in matplotlib/python. I am trying to practice by creating a visualisation for my local petanque association. thank you for your help with my little project.
I'm trying this code for featuretools:
features, feature_names = ft.dfs(entityset = es, target_entity = 'demo',
agg_primitives = ['count', 'max', 'time_since_first', 'median', 'time_since_last', 'avg_time_between',
'sum', 'mean'],
trans_primitives = ['is_weekend', 'year', 'week', 'divide_by_feature', 'percentile'])
But I had this error
TypeError Traceback (most recent call last)
<ipython-input-17-89e925ff895d> in <module>
3 agg_primitives = ['count', 'max', 'time_since_first', 'median', 'time_since_last', 'avg_time_between',
4 'sum', 'mean'],
----> 5 trans_primitives = ['is_weekend', 'year', 'week', 'divide_by_feature', 'percentile'])
~/.local/lib/python3.6/site-packages/featuretools/utils/entry_point.py in function_wrapper(*args, **kwargs)
44 ep.on_error(error=e,
45 runtime=runtime)
---> 46 raise e
47
48 # send return value
~/.local/lib/python3.6/site-packages/featuretools/utils/entry_point.py in function_wrapper(*args, **kwargs)
36 # call function
37 start = time.time()
---> 38 return_value = func(*args, **kwargs)
39 runtime = time.time() - start
40 except Exception as e:
~/.local/lib/python3.6/site-packages/featuretools/synthesis/dfs.py in dfs(entities, relationships, entityset, target_entity, cutoff_time, instance_ids, agg_primitives, trans_primitives, groupby_trans_primitives, allowed_paths, max_depth, ignore_entities, ignore_variables, seed_features, drop_contains, drop_exact, where_primitives, max_features, cutoff_time_in_index, save_progress, features_only, training_window, approximate, chunk_size, n_jobs, dask_kwargs, verbose, return_variable_types)
226 n_jobs=n_jobs,
227 dask_kwargs=dask_kwargs,
--> 228 verbose=verbose)
229 return feature_matrix, features
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/calculate_feature_matrix.py in calculate_feature_matrix(features, entityset, cutoff_time, instance_ids, entities, relationships, cutoff_time_in_index, training_window, approximate, save_progress, verbose, chunk_size, n_jobs, dask_kwargs)
265 cutoff_df_time_var=cutoff_df_time_var,
266 target_time=target_time,
--> 267 pass_columns=pass_columns)
268
269 feature_matrix = pd.concat(feature_matrix)
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/calculate_feature_matrix.py in linear_calculate_chunks(chunks, feature_set, approximate, training_window, verbose, save_progress, entityset, no_unapproximated_aggs, cutoff_df_time_var, target_time, pass_columns)
496 no_unapproximated_aggs,
497 cutoff_df_time_var,
--> 498 target_time, pass_columns)
499 feature_matrix.append(_feature_matrix)
500 # Do a manual garbage collection in case objects from calculate_chunk
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/calculate_feature_matrix.py in calculate_chunk(chunk, feature_set, entityset, approximate, training_window, verbose, save_progress, no_unapproximated_aggs, cutoff_df_time_var, target_time, pass_columns)
341 ids,
342 precalculated_features=precalculated_features_trie,
--> 343 training_window=window)
344
345 id_name = _feature_matrix.index.name
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/utils.py in wrapped(*args, **kwargs)
35 def wrapped(*args, **kwargs):
36 if save_progress is None:
---> 37 r = method(*args, **kwargs)
38 else:
39 time = args[0].to_pydatetime()
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/calculate_feature_matrix.py in calc_results(time_last, ids, precalculated_features, training_window)
316 ignored=all_approx_feature_set)
317
--> 318 matrix = calculator.run(ids)
319 return matrix
320
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/feature_set_calculator.py in run(self, instance_ids)
100 precalculated_trie=self.precalculated_features,
101 filter_variable=target_entity.index,
--> 102 filter_values=instance_ids)
103
104 # The dataframe for the target entity should be stored at the root of
~/.local/lib/python3.6/site-packages/featuretools/computational_backends/feature_set_calculator.py in _calculate_features_for_entity(self, entity_id, feature_trie, df_trie, full_entity_df_trie, precalculated_trie, filter_variable, filter_values, parent_data)
187 columns=columns,
188 time_last=self.time_last,
--> 189 training_window=self.training_window)
190
191 # Step 2: Add variables to the dataframe linking it to all ancestors.
~/.local/lib/python3.6/site-packages/featuretools/entityset/entity.py in query_by_values(self, instance_vals, variable_id, columns, time_last, training_window)
271
272 if columns is not None:
--> 273 df = df[columns]
274
275 return df
~/.local/lib/python3.6/site-packages/pandas/core/frame.py in __getitem__(self, key)
2686 return self._getitem_multilevel(key)
2687 else:
-> 2688 return self._getitem_column(key)
2689
2690 def _getitem_column(self, key):
~/.local/lib/python3.6/site-packages/pandas/core/frame.py in _getitem_column(self, key)
2693 # get column
2694 if self.columns.is_unique:
-> 2695 return self._get_item_cache(key)
2696
2697 # duplicate columns & possible reduce dimensionality
~/.local/lib/python3.6/site-packages/pandas/core/generic.py in _get_item_cache(self, item)
2485 """Return the cached item, item represents a label indexer."""
2486 cache = self._item_cache
-> 2487 res = cache.get(item)
2488 if res is None:
2489 values = self._data.get(item)
TypeError: unhashable type: 'set'
I also tried the simplest code for deep feature synthesis (dfs) as shown below, but it still encountered the same error
features, feature_names = ft.dfs(entityset = es, target_entity = 'demo')
I'm not really sure why I encountered this error, any help or recommendations on how to go about from here is deeply appreciated.
Thanks in advance for your help!
I found a solution, my current version had bugs in it that was fixed by the FeatureTools team. Just run pip install directly from master,
pip install --upgrade https://github.com/featuretools/featuretools/zipball/master
This fixed and has been released in Featuretools 0.9.1. If you upgrade to the latest version of Featuretools, it will go away.
I am using a while loop to calculate a cost function for memory reasons. When calculating the gradient, tensorflow will store Nm tensors where Nm is the number of iterations in my while loop (this cuases the same memory issues I had with the original energy functions). I do not want that as I don't have enough memory. So I want to register a new op along with a gradient function that both use a while loop. However I am having issues with using function.defun and a while loop. To simplify things, I have a small test example below:
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import sparse_ops
from tensorflow.python.framework import function
def _run(tensor):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
res = sess.run(tensor)
return res
#function.Defun(tf.float32,tf.float32,func_name ='tf_test_log')#,grad_func=tf_test_logGrad)
def tf_test_log(t_x,t_y):
#N = t_x.shape[0].value
condition = lambda i,m1: i<N
def body(index,x):
#return[(index+1),tf.concat([x, tf.expand_dims(tf.exp( tf.add( t_x[:,index],t_y[:,index]) ),1) ],1 ) ]
return[(index+1),tf.add(x, tf.exp( tf.add( t_x[:,0],t_y[:,0]) ) ) ]
i0 = tf.constant(0,dtype=tf.int32)
m0 = tf.zeros([N,1],dType)
ijk_0 = [i0,m0]
L,t_log_x = tf.while_loop(condition,body,ijk_0,
shape_invariants=[i0.get_shape(),
tf.TensorShape([N,None])]
)
return t_log_x
dType = tf.float32
N = np.int32(100)
t_N = tf.constant(N,dtype = tf.int32)
t_x = tf.constant(np.random.randn(N,N),dtype = dType)
t_y = tf.constant(np.random.randn(N,N),dtype = dType)
ys = _run(tf_test_log(t_x,t_y))
I then try to test the new op:
I get a Value error: The shape for while/Merge_1:0 is not an invariant for the loop. It enters the loop with shape (100, ?), but has shape after one iteration. Provide shape invariants using either the shape_invariants argument of tf.while_loop or set_shape() on the loop variables.
Note that calling
If i use a concatenate operation (instead of the add operation that gets returned by my while loop), I do not get any issues.
However, If I do not set N as a global variable (i.e. I do N = t_x.shape[0]) inside the body of the tf_test_log function, I get a Value error.
ValueError: Cannot convert a partially known TensorShape to a Tensor: (?, 1)
What is wrong with my code? Any help is greatly appreciated!
I am using python 3.5 on ubuntu 16.04 and tensorflow 1.4
full output:
ValueError Traceback (most recent call last)
~/Documents/TheEffingPhDHatersGonnaHate/PAM/defun_while.py in <module>()
51 t_x = tf.constant(np.random.randn(N,N),dtype = dType)
52 t_y = tf.constant(np.random.randn(N,N),dtype = dType)
---> 53 ys = _run(tf_test_log(t_x,t_y))
54
55
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in __call__(self, *args, **kwargs)
503
504 def __call__(self, *args, **kwargs):
--> 505 self.add_to_graph(ops.get_default_graph())
506 args = [ops.convert_to_tensor(_) for _ in args] + self._extra_inputs
507 ret, op = _call(self._signature, *args, **kwargs)
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
484 def add_to_graph(self, g):
485 """Adds this function into the graph g."""
--> 486 self._create_definition_if_needed()
487
488 # Adds this function into 'g'.
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
319 """Creates the function definition if it's not created yet."""
320 with context.graph_mode():
--> 321 self._create_definition_if_needed_impl()
322
323 def _create_definition_if_needed_impl(self):
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
336 # Call func and gather the output tensors.
337 with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338 outputs = self._func(*inputs)
339
340 # There is no way of distinguishing between a function not returning
~/Documents/TheEffingPhDHatersGonnaHate/PAM/defun_while.py in tf_test_log(t_x, t_y)
39 L,t_log_x = tf.while_loop(condition,body,ijk_0,
40 shape_invariants=[i0.get_shape(),
---> 41 tf.TensorShape([N,None])]
42 )
43 return t_log_x
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in while_loop(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name)
2814 loop_context = WhileContext(parallel_iterations, back_prop, swap_memory) # pylint: disable=redefined-outer-name
2815 ops.add_to_collection(ops.GraphKeys.WHILE_CONTEXT, loop_context)
-> 2816 result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
2817 return result
2818
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in BuildLoop(self, pred, body, loop_vars, shape_invariants)
2638 self.Enter()
2639 original_body_result, exit_vars = self._BuildLoop(
-> 2640 pred, body, original_loop_vars, loop_vars, shape_invariants)
2641 finally:
2642 self.Exit()
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in _BuildLoop(self, pred, body, original_loop_vars, loop_vars, shape_invariants)
2619 for m_var, n_var in zip(merge_vars, next_vars):
2620 if isinstance(m_var, ops.Tensor):
-> 2621 _EnforceShapeInvariant(m_var, n_var)
2622
2623 # Exit the loop.
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py in _EnforceShapeInvariant(merge_var, next_var)
576 "Provide shape invariants using either the `shape_invariants` "
577 "argument of tf.while_loop or set_shape() on the loop variables."
--> 578 % (merge_var.name, m_shape, n_shape))
579 else:
580 if not isinstance(var, (ops.IndexedSlices, sparse_tensor.SparseTensor)):
ValueError: The shape for while/Merge_1:0 is not an invariant for the loop. It enters the loop with shape (100, ?), but has shape <unknown> after one iteration. Provide shape invariants using either the `shape_invariants` argument of tf.while_loop or set_shape() on the loop variables.
Thanks #Alexandre Passos for the suggestion in the comment above!
The following piece of code is a modification of the original with a set_shape function added inside the body.
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import sparse_ops
from tensorflow.python.framework import function
def _run(tensor):
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
res = sess.run(tensor)
return res
#function.Defun(tf.float32,tf.float32,tf.float32,func_name ='tf_test_logGrad')
def tf_test_logGrad(t_x,t_y,grad):
return grad
#function.Defun(tf.float32,tf.float32,func_name ='tf_test_log')#,grad_func=tf_test_logGrad)
def tf_test_log(t_x,t_y):
#N = t_x.shape[0].value
condition = lambda i,m1: i<N
def body(index,x):
#return[(index+1),tf.concat([x, tf.expand_dims(tf.exp( tf.add( t_x[:,index],t_y[:,index]) ),1) ],1 ) ]
x = tf.add(x, tf.exp( tf.add( t_x[:,0],t_y[:,0]) ) )
x.set_shape([N])
return[(index+1), x]
i0 = tf.constant(0,dtype=tf.int32)
m0 = tf.zeros([N],dType)
ijk_0 = [i0,m0]
L,t_log_x = tf.while_loop(condition,body,ijk_0,
shape_invariants=[i0.get_shape(),
tf.TensorShape([N])]
)
return t_log_x
dType = tf.float32
N = np.int32(100)
t_N = tf.constant(N,dtype = tf.int32)
t_x = tf.constant(np.random.randn(N,N),dtype = dType)
t_y = tf.constant(np.random.randn(N,N),dtype = dType)
ys = _run(tf_test_log(t_x,t_y))
The Issue of global N still persists.
You still need to set the shape of the loop tensors as a global variable outside of the defun decorator. If you try to get it from the shape of the inputs of the defun decorator, you get:
TypeError Traceback (most recent call last)
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py in zeros(shape, dtype, name)
1438 shape = tensor_shape.as_shape(shape)
-> 1439 output = constant(zero, shape=shape, dtype=dtype, name=name)
1440 except (TypeError, ValueError):
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name, verify_shape)
207 tensor_util.make_tensor_proto(
--> 208 value, dtype=dtype, shape=shape, verify_shape=verify_shape))
209 dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
379 # exception when dtype is set to np.int64
--> 380 if shape is not None and np.prod(shape, dtype=np.int64) == 0:
381 nparray = np.empty(shape, dtype=np_dt)
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/numpy/core/fromnumeric.py in prod(a, axis, dtype, out, keepdims)
2517 return _methods._prod(a, axis=axis, dtype=dtype,
-> 2518 out=out, **kwargs)
2519
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/numpy/core/_methods.py in _prod(a, axis, dtype, out, keepdims)
34 def _prod(a, axis=None, dtype=None, out=None, keepdims=False):
---> 35 return umr_prod(a, axis, dtype, out, keepdims)
36
TypeError: __int__ returned non-int (type NoneType)
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
~/Documents/TheEffingPhDHatersGonnaHate/PAM/defun_while.py in <module>()
52 t_x = tf.constant(np.random.randn(N,N),dtype = dType)
53 t_y = tf.constant(np.random.randn(N,N),dtype = dType)
---> 54 ys = _run(tf_test_log(t_x,t_y))
55
56
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in __call__(self, *args, **kwargs)
503
504 def __call__(self, *args, **kwargs):
--> 505 self.add_to_graph(ops.get_default_graph())
506 args = [ops.convert_to_tensor(_) for _ in args] + self._extra_inputs
507 ret, op = _call(self._signature, *args, **kwargs)
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in add_to_graph(self, g)
484 def add_to_graph(self, g):
485 """Adds this function into the graph g."""
--> 486 self._create_definition_if_needed()
487
488 # Adds this function into 'g'.
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed(self)
319 """Creates the function definition if it's not created yet."""
320 with context.graph_mode():
--> 321 self._create_definition_if_needed_impl()
322
323 def _create_definition_if_needed_impl(self):
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/function.py in _create_definition_if_needed_impl(self)
336 # Call func and gather the output tensors.
337 with vs.variable_scope("", custom_getter=temp_graph.getvar):
--> 338 outputs = self._func(*inputs)
339
340 # There is no way of distinguishing between a function not returning
~/Documents/TheEffingPhDHatersGonnaHate/PAM/defun_while.py in tf_test_log(t_x, t_y)
33
34 i0 = tf.constant(0,dtype=tf.int32)
---> 35 m0 = tf.zeros([N],dType)
36
37
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py in zeros(shape, dtype, name)
1439 output = constant(zero, shape=shape, dtype=dtype, name=name)
1440 except (TypeError, ValueError):
-> 1441 shape = ops.convert_to_tensor(shape, dtype=dtypes.int32, name="shape")
1442 output = fill(shape, constant(zero, dtype=dtype), name=name)
1443 assert output.dtype.base_dtype == dtype
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, preferred_dtype)
834 name=name,
835 preferred_dtype=preferred_dtype,
--> 836 as_ref=False)
837
838
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/ops.py in internal_convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, ctx)
924
925 if ret is None:
--> 926 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
927
928 if ret is NotImplemented:
~/environments/tf_1_4_gpu/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py in _tensor_shape_tensor_conversion_function(s, dtype, name, as_ref)
248 if not s.is_fully_defined():
249 raise ValueError(
--> 250 "Cannot convert a partially known TensorShape to a Tensor: %s" % s)
251 s_list = s.as_list()
252 int64_value = 0
ValueError: Cannot convert a partially known TensorShape to a Tensor: (?,)
I am trying to recreate this example of bayesian PK/PD modelling using pymc3.....
The video shows the WinBUGS code and I am trying to convert to pymc3
https://www.youtube.com/watch?v=AQDXRoBan6Y
model here....
http://imgur.com/ckoKPRF
WinBUGS code is here ....
http://imgur.com/TsViyBC
My code is ....
from pymc3 import Model, Normal, Lognormal, Uniform
import numpy as np
import pandas as pd
data = pd.read_csv('/Users/Home/Documents/pymc3/fxa.data.csv' )
cobs = np.array(data['cobs'])
fxa = np.array(data['fxa.inh.obs'])
pkpd_model = Model()
with pkpd_model:
# Priors for unknown model parameters
emax = Uniform ('emax', lower =0, upper =100)
ec50 = Lognormal('ec50', mu=0, tau = 100000)
gamma = Uniform('gamma', lower=0, upper =10)
sigma = Uniform('sigma', lower = 0, upper = 1000 )
# Expected value of outcome
fxaMean = emax*(np.power(cobs, gamma)) / (np.power(ec50, gamma) + np.power(cobs, gamma))
# Likelihood (sampling distribution) of observations
fxa = Normal('fxa', mu=fxaMean, sd=sigma, observed=fxa )
But when I run the code I get the following error, which seems to relate to the way theano is interpreting the np.power function.
I am not sure how to proceed as I am a noob to pymc3 and theano and PK/PD modelling too!
Thanks in advance
Applied interval-transform to emax and added transformed emax_interval to model.
Applied log-transform to ec50 and added transformed ec50_log to model.
Applied interval-transform to gamma and added transformed gamma_interval to model.
Applied interval-transform to sigma and added transformed sigma_interval to model.
---------------------------------------------------------------------------
AsTensorError Traceback (most recent call last)
<ipython-input-28-1fa311a15ed0> in <module>()
14
15 # Likelihood (sampling distribution) of observations
---> 16 fxa = Normal('fxa', mu=fxaMean, sd=sigma, observed=fxa )
//anaconda/lib/python2.7/site-packages/pymc3/distributions/distribution.pyc in __new__(cls, name, *args, **kwargs)
23 data = kwargs.pop('observed', None)
24 dist = cls.dist(*args, **kwargs)
---> 25 return model.Var(name, dist, data)
26 elif name is None:
27 return object.__new__(cls) # for pickle
//anaconda/lib/python2.7/site-packages/pymc3/model.pyc in Var(self, name, dist, data)
282 self.named_vars[v.name] = v
283 else:
--> 284 var = ObservedRV(name=name, data=data, distribution=dist, model=self)
285 self.observed_RVs.append(var)
286 if var.missing_values:
//anaconda/lib/python2.7/site-packages/pymc3/model.pyc in __init__(self, type, owner, index, name, data, distribution, model)
556 self.missing_values = data.missing_values
557
--> 558 self.logp_elemwiset = distribution.logp(data)
559 self.model = model
560 self.distribution = distribution
//anaconda/lib/python2.7/site-packages/pymc3/distributions/continuous.pyc in logp(self, value)
191 sd = self.sd
192 mu = self.mu
--> 193 return bound((-tau * (value - mu)**2 + T.log(tau / np.pi / 2.)) / 2.,
194 tau > 0, sd > 0)
195
//anaconda/lib/python2.7/site-packages/theano/tensor/var.pyc in __radd__(self, other)
232 # ARITHMETIC - RIGHT-OPERAND
233 def __radd__(self, other):
--> 234 return theano.tensor.basic.add(other, self)
235
236 def __rsub__(self, other):
//anaconda/lib/python2.7/site-packages/theano/gof/op.pyc in __call__(self, *inputs, **kwargs)
609 """
610 return_list = kwargs.pop('return_list', False)
--> 611 node = self.make_node(*inputs, **kwargs)
612
613 if config.compute_test_value != 'off':
//anaconda/lib/python2.7/site-packages/theano/tensor/elemwise.pyc in make_node(self, *inputs)
541 using DimShuffle.
542 """
--> 543 inputs = list(map(as_tensor_variable, inputs))
544 shadow = self.scalar_op.make_node(
545 *[get_scalar_type(dtype=i.type.dtype).make_variable()
//anaconda/lib/python2.7/site-packages/theano/tensor/basic.pyc in as_tensor_variable(x, name, ndim)
206 except Exception:
207 str_x = repr(x)
--> 208 raise AsTensorError("Cannot convert %s to TensorType" % str_x, type(x))
209
210 # this has a different name, because _as_tensor_variable is the
AsTensorError: ('Cannot convert [Elemwise{mul,no_inplace}.0 Elemwise{mul,no_inplace}.0\n Elemwise{mul,no_inplace}.0 ..., Elemwise{mul,no_inplace}.0\n Elemwise{mul,no_inplace}.0 Elemwise{mul,no_inplace}.0] to TensorType', <type 'numpy.ndarray'>)
Doh - replaced np.power with ** ! working fine!
I'm trying to do a weighted Ridge Regression with sklearn. However, the code breaks when I call the fit method. The exception I get is :
Exception: Data must be 1-dimensional
But I'm sure (by checking through print-statements) that the data I'm passing has the right shapes.
print temp1.shape #(781, 21)
print temp2.shape #(781,)
print weights.shape #(781,)
result=RidgeCV(normalize=True).fit(temp1,temp2,sample_weight=weights)
What could be going wrong ??
Here's the whole output :
---------------------------------------------------------------------------
Exception Traceback (most recent call last)
<ipython-input-65-a5b1eba5d9cf> in <module>()
22
23
---> 24 result=RidgeCV(normalize=True).fit(temp2,temp1, sample_weight=weights)
25
26
/usr/local/lib/python2.7/dist-packages/sklearn/linear_model/ridge.pyc in fit(self, X, y, sample_weight)
868 gcv_mode=self.gcv_mode,
869 store_cv_values=self.store_cv_values)
--> 870 estimator.fit(X, y, sample_weight=sample_weight)
871 self.alpha_ = estimator.alpha_
872 if self.store_cv_values:
/usr/local/lib/python2.7/dist-packages/sklearn/linear_model/ridge.pyc in fit(self, X, y, sample_weight)
793 else alpha)
794 if error:
--> 795 out, c = _errors(weighted_alpha, y, v, Q, QT_y)
796 else:
797 out, c = _values(weighted_alpha, y, v, Q, QT_y)
/usr/local/lib/python2.7/dist-packages/sklearn/linear_model/ridge.pyc in _errors(self, alpha, y, v, Q, QT_y)
685 w = 1.0 / (v + alpha)
686 c = np.dot(Q, self._diag_dot(w, QT_y))
--> 687 G_diag = self._decomp_diag(w, Q)
688 # handle case where y is 2-d
689 if len(y.shape) != 1:
/usr/local/lib/python2.7/dist-packages/sklearn/linear_model/ridge.pyc in _decomp_diag(self, v_prime, Q)
672 def _decomp_diag(self, v_prime, Q):
673 # compute diagonal of the matrix: dot(Q, dot(diag(v_prime), Q^T))
--> 674 return (v_prime * Q ** 2).sum(axis=-1)
675
676 def _diag_dot(self, D, B):
/usr/local/lib/python2.7/dist-packages/pandas/core/ops.pyc in wrapper(left, right, name)
531 return left._constructor(wrap_results(na_op(lvalues, rvalues)),
532 index=left.index, name=left.name,
--> 533 dtype=dtype)
534 return wrapper
535
/usr/local/lib/python2.7/dist-packages/pandas/core/series.pyc in __init__(self, data, index, dtype, name, copy, fastpath)
209 else:
210 data = _sanitize_array(data, index, dtype, copy,
--> 211 raise_cast_failure=True)
212
213 data = SingleBlockManager(data, index, fastpath=True)
/usr/local/lib/python2.7/dist-packages/pandas/core/series.pyc in _sanitize_array(data, index, dtype, copy, raise_cast_failure)
2683 elif subarr.ndim > 1:
2684 if isinstance(data, np.ndarray):
-> 2685 raise Exception('Data must be 1-dimensional')
2686 else:
2687 subarr = _asarray_tuplesafe(data, dtype=dtype)
Exception: Data must be 1-dimensional
The error seems to be due to sample_weights being a Pandas series rather than a numpy array:
from sklearn.linear_model import RidgeCV
temp1 = pd.DataFrame(np.random.rand(781, 21))
temp2 = pd.Series(temp1.sum(1))
weights = pd.Series(1 + 0.1 * np.random.rand(781))
result = RidgeCV(normalize=True).fit(temp1, temp2,
sample_weight=weights)
# Exception: Data must be 1-dimensional
If you use a numpy array instead, the error goes away:
result = RidgeCV(normalize=True).fit(temp1, temp2,
sample_weight=weights.values)
This seems to be a bug; I've opened a scikit-learn issue to report this.