I have data like the sample below, which has 4 continuous columns [x0 to x3] and a binary column y. y has two values 1.0 and 0.0. I’m trying to check for correlation between the binary column y and one of the continuous columns x0, using the CatConCor function below, but I’m getting the error message below. The function creates a linear regression model and calcs the p value for the residuals with and without the categorical variable. If anyone can please point out the issue or how to fix it, it would be very much appreciated.
Data:
x_r x0 x1 x2 x3 y
0 0 0.466726 0.030126 0.998330 0.892770 0.0
1 1 0.173168 0.525810 -0.079341 -0.112151 0.0
2 2 -0.854467 0.770712 0.929614 -0.224779 0.0
3 3 -0.370574 0.568183 -0.928269 0.843253 0.0
4 4 -0.659431 -0.948491 -0.091534 0.706157 0.0
Code:
import numpy as np
import pandas as pd
from time import time
import scipy.stats as stats
from IPython.display import display # Allows the use of display() for DataFrames
# Pretty display for notebooks
%matplotlib inline
###########################################
# Suppress matplotlib user warnings
# Necessary for newer version of matplotlib
import warnings
warnings.filterwarnings("ignore", category = UserWarning, module = "matplotlib")
#
# Display inline matplotlib plots with IPython
from IPython import get_ipython
get_ipython().run_line_magic('matplotlib', 'inline')
###########################################
import matplotlib.pyplot as plt
import matplotlib.cm as cm
# correlation between categorical variable and continuous variable
def CatConCor(df,catVar,conVar):
import statsmodels.api as sm
from statsmodels.formula.api import ols
# subsetting data for one categorical column and one continuous column
data2=df.copy()[[catVar,conVar]]
data2[catVar]=data2[catVar].astype('category')
mod = ols(conVar+'~'+catVar,
data=data2).fit()
aov_table = sm.stats.anova_lm(mod, typ=2)
if aov_table['PR(>F)'][0] < 0.05:
print('Correlated p='+str(aov_table['PR(>F)'][0]))
else:
print('Uncorrelated p='+str(aov_table['PR(>F)'][0]))
# checking for correlation between categorical and continuous variables
CatConCor(df=train_df,catVar='y',conVar='x0')
Error:
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
<ipython-input-6-80f83b8c8e14> in <module>()
1 # checking for correlation between categorical and continuous variables
2
----> 3 CatConCor(df=train_df,catVar='y',conVar='x0')
<ipython-input-2-35404ba1d697> in CatConCor(df, catVar, conVar)
103
104 mod = ols(conVar+'~'+catVar,
--> 105 data=data2).fit()
106
107 aov_table = sm.stats.anova_lm(mod, typ=2)
~/anaconda2/envs/py36/lib/python3.6/site-packages/statsmodels/base/model.py in from_formula(cls, formula, data, subset, drop_cols, *args, **kwargs)
153
154 tmp = handle_formula_data(data, None, formula, depth=eval_env,
--> 155 missing=missing)
156 ((endog, exog), missing_idx, design_info) = tmp
157
~/anaconda2/envs/py36/lib/python3.6/site-packages/statsmodels/formula/formulatools.py in handle_formula_data(Y, X, formula, depth, missing)
63 if data_util._is_using_pandas(Y, None):
64 result = dmatrices(formula, Y, depth, return_type='dataframe',
---> 65 NA_action=na_action)
66 else:
67 result = dmatrices(formula, Y, depth, return_type='dataframe',
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/highlevel.py in dmatrices(formula_like, data, eval_env, NA_action, return_type)
308 eval_env = EvalEnvironment.capture(eval_env, reference=1)
309 (lhs, rhs) = _do_highlevel_design(formula_like, data, eval_env,
--> 310 NA_action, return_type)
311 if lhs.shape[1] == 0:
312 raise PatsyError("model is missing required outcome variables")
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/highlevel.py in _do_highlevel_design(formula_like, data, eval_env, NA_action, return_type)
163 return iter([data])
164 design_infos = _try_incr_builders(formula_like, data_iter_maker, eval_env,
--> 165 NA_action)
166 if design_infos is not None:
167 return build_design_matrices(design_infos, data,
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/highlevel.py in _try_incr_builders(formula_like, data_iter_maker, eval_env, NA_action)
60 "ascii-only, or else upgrade to Python 3.")
61 if isinstance(formula_like, str):
---> 62 formula_like = ModelDesc.from_formula(formula_like)
63 # fallthrough
64 if isinstance(formula_like, ModelDesc):
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/desc.py in from_formula(cls, tree_or_string)
162 tree = tree_or_string
163 else:
--> 164 tree = parse_formula(tree_or_string)
165 value = Evaluator().eval(tree, require_evalexpr=False)
166 assert isinstance(value, cls)
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/parse_formula.py in parse_formula(code, extra_operators)
146 tree = infix_parse(_tokenize_formula(code, operator_strings),
147 operators,
--> 148 _atomic_token_types)
149 if not isinstance(tree, ParseNode) or tree.type != "~":
150 tree = ParseNode("~", None, [tree], tree.origin)
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/infix_parser.py in infix_parse(tokens, operators, atomic_types, trace)
208
209 want_noun = True
--> 210 for token in token_source:
211 if c.trace:
212 print("Reading next token (want_noun=%r)" % (want_noun,))
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/parse_formula.py in _tokenize_formula(code, operator_strings)
92 else:
93 it.push_back((pytype, token_string, origin))
---> 94 yield _read_python_expr(it, end_tokens)
95
96 def test__tokenize_formula():
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/parse_formula.py in _read_python_expr(it, end_tokens)
42 origins = []
43 bracket_level = 0
---> 44 for pytype, token_string, origin in it:
45 assert bracket_level >= 0
46 if bracket_level == 0 and token_string in end_tokens:
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/util.py in next(self)
330 else:
331 # May raise StopIteration
--> 332 return six.advance_iterator(self._it)
333 __next__ = next
334
~/anaconda2/envs/py36/lib/python3.6/site-packages/patsy/tokens.py in python_tokenize(code)
33 break
34 origin = Origin(code, start, end)
---> 35 assert pytype not in (tokenize.NL, tokenize.NEWLINE)
36 if pytype == tokenize.ERRORTOKEN:
37 raise PatsyError("error tokenizing input "
AssertionError:
Upgrading patsy to 0.5.1 fixed the issue. I found the tip here:
https://github.com/statsmodels/statsmodels/issues/5343
Related
I'm uploading my dataset, and I'm copying my dataset, but an error is appearing.
import numpy as np
import pandas as pd
import mathplotlib.pyplot as plt
house_data=pd.read_csv("/home/houseprice.csv")
#we evaluate the price of a house for those cases where the information is missing, for each variable
def analyse_na_value(df, var):
df - df.copy()
# we indicate as a variable as 1 where the observation is missing
# we indicate as 0 where the observation has a real value
df[var] = np.where(df[var].isnull(), 1 , 0)
#print(df[var].isnull())
# we calculate the mean saleprice where the information is missing or present
df.groupby(var)['SalePrice'].median().plot.bar()
plt.title(var)
plt.show()
for var in vars_with_na:
analyse_na_value(house_data, var)
error,when I comment this code line, I don't get an error
df - df.copy()
TypeError Traceback (most recent call last)
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/array_ops.py in na_arithmetic_op(left, right, op, is_cmp)
142 try:
--> 143 result = expressions.evaluate(op, left, right)
144 except TypeError:
~/anaconda3/lib/python3.8/site-packages/pandas/core/computation/expressions.py in evaluate(op, a, b, use_numexpr)
232 if use_numexpr:
--> 233 return _evaluate(op, op_str, a, b) # type: ignore
234 return _evaluate_standard(op, op_str, a, b)
~/anaconda3/lib/python3.8/site-packages/pandas/core/computation/expressions.py in _evaluate_numexpr(op, op_str, a, b)
118 if result is None:
--> 119 result = _evaluate_standard(op, op_str, a, b)
120
~/anaconda3/lib/python3.8/site-packages/pandas/core/computation/expressions.py in _evaluate_standard(op, op_str, a, b)
67 with np.errstate(all="ignore"):
---> 68 return op(a, b)
69
TypeError: unsupported operand type(s) for -: 'str' and 'str'
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-31-25d58bc46c86> in <module>
15
16 for var in vars_with_na:
---> 17 analyse_na_value(house_data, var)
<ipython-input-31-25d58bc46c86> in analyse_na_value(df, var)
1 #we evaluate the price of a house for those cases where the information is missing, for each variable
2 def analyse_na_value(df, var):
----> 3 df - df.copy()
4
5 # we indicate as a variable as 1 where the observation is missing
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/__init__.py in f(self, other, axis, level, fill_value)
649 if isinstance(other, ABCDataFrame):
650 # Another DataFrame
--> 651 new_data = self._combine_frame(other, na_op, fill_value)
652
653 elif isinstance(other, ABCSeries):
~/anaconda3/lib/python3.8/site-packages/pandas/core/frame.py in _combine_frame(self, other, func, fill_value)
5864 return func(left, right)
5865
-> 5866 new_data = ops.dispatch_to_series(self, other, _arith_op)
5867 return new_data
5868
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/__init__.py in dispatch_to_series(left, right, func, axis)
273 # _frame_arith_method_with_reindex
274
--> 275 bm = left._mgr.operate_blockwise(right._mgr, array_op)
276 return type(left)(bm)
277
~/anaconda3/lib/python3.8/site-packages/pandas/core/internals/managers.py in operate_blockwise(self, other, array_op)
362 Apply array_op blockwise with another (aligned) BlockManager.
363 """
--> 364 return operate_blockwise(self, other, array_op)
365
366 def apply(self: T, f, align_keys=None, **kwargs) -> T:
~/anaconda3/lib/python3.8/site-packages/pandas/core/internals/ops.py in operate_blockwise(left, right, array_op)
36 lvals, rvals = _get_same_shape_values(blk, rblk, left_ea, right_ea)
37
---> 38 res_values = array_op(lvals, rvals)
39 if left_ea and not right_ea and hasattr(res_values, "reshape"):
40 res_values = res_values.reshape(1, -1)
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/array_ops.py in arithmetic_op(left, right, op)
188 else:
189 with np.errstate(all="ignore"):
--> 190 res_values = na_arithmetic_op(lvalues, rvalues, op)
191
192 return res_values
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/array_ops.py in na_arithmetic_op(left, right, op, is_cmp)
148 # will handle complex numbers incorrectly, see GH#32047
149 raise
--> 150 result = masked_arith_op(left, right, op)
151
152 if is_cmp and (is_scalar(result) or result is NotImplemented):
~/anaconda3/lib/python3.8/site-packages/pandas/core/ops/array_ops.py in masked_arith_op(x, y, op)
90 if mask.any():
91 with np.errstate(all="ignore"):
---> 92 result[mask] = op(xrav[mask], yrav[mask])
93
94 else:
TypeError: unsupported operand type(s) for -: 'str' and 'str'
1
As far to what I know the copy() function works with python3,
but in pandas,
and python3 does it work I don't know.
How can I get rid of this error without commenting that code line?
I think you are supposed to do df = df.copy(). I would recommend changing the variable though. Here is an official Pandas documentation on this function. What you are doing is subtracting the data frame from itself...
I have several parquet files (dataframes), which I load as one dask dataframe graph and sample.
Afterwards I perform some computations based on the original data in the dataframe and append the new columns to my dask dataframe.
Finally, I want to compute the mean() and std() for all columns in and do get a ValueError that I am not sure where it comes from or what I'm doing wrong.
import pandas as pd
import numpy as np
import tensorflow as tf
import os
from os.path import join
import dask
import dask.dataframe as dd
import dask.array as da
# read in the data
data_pq = dd.read_parquet(join(path_to_data,'filter_width_*_DNN_train.parquet'),chunksize='4GB')
print('Convert to single precission and sample')
data_pq = data_pq.astype(np.float32).sample(frac=0.1)
# ## compute the additional quantites (tensors)
# compute tensors R, S mag(U) etc.
mag_U = da.sqrt(data_pq['U_bar'].values**2 + data_pq['V_bar'].values**2 +data_pq['W_bar'].values**2)
mag_grad_c = da.sqrt(data_pq['grad_c_x_LES'].values**2 + data_pq['grad_c_y_LES'].values**2 +data_pq['grad_c_z_LES'].values**2)
sum_U = data_pq['U_bar'].values + data_pq['V_bar']+data_pq['W_bar'].values
sum_c = da.absolute(data_pq['grad_c_x_LES'].values) + da.absolute(data_pq['grad_c_y_LES'].values) +da.absolute(data_pq['grad_c_z_LES'].values)
grad_U = da.sqrt(data_pq['grad_U_x_LES'].values**2 + data_pq['grad_U_y_LES'].values**2 +data_pq['grad_U_z_LES'].values**2)
grad_V = da.sqrt(data_pq['grad_V_x_LES'].values**2 + data_pq['grad_V_y_LES'].values**2 +data_pq['grad_V_z_LES'].values**2)
grad_W = da.sqrt(data_pq['grad_W_x_LES'].values**2 + data_pq['grad_W_y_LES'].values**2 +data_pq['grad_W_z_LES'].values**2)
mag_grad_U = da.sqrt(grad_U**2 + grad_V**2 +grad_W**2)
sum_grad_U = da.absolute(grad_U) + da.absolute(grad_V) +da.absolute(grad_W)
print('Computing gradient_tensor')
gradient_tensor = da.array([
[data_pq['grad_U_x_LES'],data_pq['grad_V_x_LES'],data_pq['grad_W_x_LES']],
[data_pq['grad_U_y_LES'],data_pq['grad_V_y_LES'],data_pq['grad_W_y_LES']],
[data_pq['grad_U_z_LES'],data_pq['grad_V_z_LES'],data_pq['grad_W_z_LES']]
])
print('Computing S and R')
# symetric strain
Strain = 0.5*(gradient_tensor + da.transpose(gradient_tensor,(1,0,2)))
#anti symetric strain
Anti = 0.5*(gradient_tensor - da.transpose(gradient_tensor,(1,0,2)))
print('Computing lambdas')
lambda_1 = da.trace(Strain**2)
lambda_2 = da.trace(Anti**2)
lambda_3 = da.trace(Strain**3)
lambda_4 = da.trace(Anti**2 * Strain)
lambda_5 = da.trace(Anti**2 * Strain**2)
# Add to the dask dataframe
data_pq['mag_grad_c'] = mag_grad_c
data_pq['mag_U'] = mag_U
data_pq['sum_c'] = sum_c
data_pq['sum_U'] = sum_U
data_pq['sum_grad_U'] = sum_grad_U
data_pq['mag_grad_U'] = mag_grad_U
# REPARTITON
data_pq = data_pq.repartition(npartitions=lambda_1.npartitions)
data_pq['lambda_1'] = lambda_1
data_pq['lambda_2'] = lambda_2
data_pq['lambda_3'] = lambda_3
data_pq['lambda_4'] = lambda_4
data_pq['lambda_5'] = lambda_5
print('Done with feature computation')
# reindex and compute mean and std
data_pq = data_pq.reset_index().drop('index',axis=1)
# compute the mean and std
data_mean, data_std = dask.compute(data_pq.mean(),data_pq.std())
Not sure where it comes from. It says the indexes do not match.
This is the error message I get:
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) ~/Python/Data_driven_models/DASK_processing/Dask_parquet.py in <module>
119 data_pq = data_pq.reset_index().drop('index',axis=1)
120
--> 121 data_mean, data_std = dask.compute(data_pq.mean(),data_pq.std())
122
~/.local/lib/python3.6/site-packages/dask/base.py in compute(*args,
**kwargs)
450 postcomputes.append(x.__dask_postcompute__())
451
--> 452 results = schedule(dsk, keys, **kwargs)
453 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
454
~/.local/lib/python3.6/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
82 get_id=_thread_get_id,
83 pack_exception=pack_exception,
---> 84 **kwargs
85 )
86
~/.local/lib/python3.6/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
484 _execute_task(task, data) # Re-execute locally
485 else:
--> 486 raise_exception(exc, tb)
487 res, worker_id = loads(res_info)
488 state["cache"][key] = res
~/.local/lib/python3.6/site-packages/dask/local.py in reraise(exc, tb)
314 if exc.__traceback__ is not tb:
315 raise exc.with_traceback(tb)
--> 316 raise exc
317
318
~/.local/lib/python3.6/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
220 try:
221 task, data = loads(task_info)
--> 222 result = _execute_task(task, data)
223 id = get_id()
224 result = dumps((result, id))
~/.local/lib/python3.6/site-packages/dask/core.py in
_execute_task(arg, cache, dsk)
119 # temporaries by their reference count and can execute certain
120 # operations in-place.
--> 121 return func(*(_execute_task(a, cache) for a in args))
122 elif not ishashable(arg):
123 return arg
~/.local/lib/python3.6/site-packages/pandas/core/series.py in
__init__(self, data, index, dtype, name, copy, fastpath)
312 if len(index) != len(data):
313 raise ValueError(
--> 314 f"Length of passed values is {len(data)}, "
315 f"index implies {len(index)}."
316 )
ValueError: Length of passed values is 3728270, index implies 2135992.
I get a strange error when running the Tukey test. I hope somebody is able to help me with this as I tried a lot. This is my dataframe:
Name Score
1 A 2.29
2 B 2.19
This is my Tukey Test code:
#TUKEY HSD TEST
tukey = pairwise_tukeyhsd(endog=df['Score'].astype('float'),
groups=df['Name'],
alpha=0.05)
tukey.plot_simultaneous()
plt.vlines(x=49.57,ymin=-0.5,ymax=4.5, color="red")
tukey.summary()
This is the error:
<ipython-input-12-3e12e78a002f> in <module>()
2 tukey = pairwise_tukeyhsd(endog=df['Score'].astype('float'),
3 groups=df['Name'],
----> 4 alpha=0.05)
5
6 tukey.plot_simultaneous()
/usr/local/lib/python3.6/dist-packages/statsmodels/stats/multicomp.py in pairwise_tukeyhsd(endog, groups, alpha)
36 '''
37
---> 38 return MultiComparison(endog, groups).tukeyhsd(alpha=alpha)
/usr/local/lib/python3.6/dist-packages/statsmodels/sandbox/stats/multicomp.py in __init__(self, data, groups, group_order)
794 if group_order is None:
795 self.groupsunique, self.groupintlab = np.unique(groups,
--> 796 return_inverse=True)
797 else:
798 #check if group_order has any names not in groups
/usr/local/lib/python3.6/dist-packages/numpy/lib/arraysetops.py in unique(ar, return_index, return_inverse, return_counts, axis)
221 ar = np.asanyarray(ar)
222 if axis is None:
--> 223 return _unique1d(ar, return_index, return_inverse, return_counts)
224 if not (-ar.ndim <= axis < ar.ndim):
225 raise ValueError('Invalid axis kwarg specified for unique')
/usr/local/lib/python3.6/dist-packages/numpy/lib/arraysetops.py in _unique1d(ar, return_index, return_inverse, return_counts)
278
279 if optional_indices:
--> 280 perm = ar.argsort(kind='mergesort' if return_index else 'quicksort')
281 aux = ar[perm]
282 else:
**TypeError: '<' not supported between instances of 'float' and 'str'**
How can this error be resolved? Thanks in advance!
You have the problem because df['Name'] contains both floats and strings AND df['Name'] is of type pandas.core.series.Series. This combination leads to an error with numpy.unique() as seen from traceback. You can fix the problem with 2 ways.
tukey = pairwise_tukeyhsd(endog=df['Score'].astype('float'),
groups=list(df['Name']), # list instead of a Series
alpha=0.05)
OR
Make sure df['Name'] contains only numbers or only strings.
I’m new to the cvxpy package. I’m trying to use it to work through an example from the following blog:
https://towardsdatascience.com/integer-programming-in-python-1cbdfa240df2
Where we’re trying to optimize the combination of marketing channels sent to a customer.
There’s been some recent changes to the cvxpy package and I’m getting the error below when I try to run the sum_entries step, (which has in the latest version been changed to cvxpy.sum)
I think the problem is coming from the dimensions of “selection” and “TRANSFORMER” being incompatible, but I’m not familiar enough with the cvxpy package to know. Any tips are greatly appreciated.
Code:
test_probs.shape
(200, 8)
Code:
# selection = cvxpy.Bool(*test_probs.shape) # syntax changed in latest version
selection = cvxpy.Variable(*test_probs.shape, boolean=True)
# constraints
# Constant matrix that counts how many of each
# material we sent to each customer
TRANSFORMER = np.array([[1,0,0],
[0,1,0],
[0,0,1],
[1,1,0],
[1,0,1],
[0,1,1],
[1,1,1],
[0,0,0]])
# can't send customer more promotion than there is supply
# note: sum_entries changed to sum in latest cvxpy version
supply_constraint = cvxpy.sum(selection * TRANSFORMER, axis=0) <= supply
Error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-47-f2ebf41a00af> in <module>()
18 # note: sum_entries changed to sum in latest cvxpy version
19
---> 20 supply_constraint = cvxpy.sum(selection * TRANSFORMER, axis=0) <= supply
21
22
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in cast_op(self, other)
47 """
48 other = self.cast_to_const(other)
---> 49 return binary_op(self, other)
50 return cast_op
51
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in __mul__(self, other)
385 return cvxtypes.multiply_expr()(self, other)
386 elif self.is_constant() or other.is_constant():
--> 387 return cvxtypes.mul_expr()(self, other)
388 else:
389 warnings.warn("Forming a nonconvex expression.")
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/affine/binary_operators.py in __init__(self, lh_exp, rh_exp)
41
42 def __init__(self, lh_exp, rh_exp):
---> 43 super(BinaryOperator, self).__init__(lh_exp, rh_exp)
44
45 def name(self):
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/atom.py in __init__(self, *args)
42 self.args = [Atom.cast_to_const(arg) for arg in args]
43 self.validate_arguments()
---> 44 self._shape = self.shape_from_args()
45 if len(self._shape) > 2:
46 raise ValueError("Atoms must be at most 2D.")
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/affine/binary_operators.py in shape_from_args(self)
107 """Returns the (row, col) shape of the expression.
108 """
--> 109 return u.shape.mul_shapes(self.args[0].shape, self.args[1].shape)
110
111 def is_atom_convex(self):
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/utilities/shape.py in mul_shapes(lh_shape, rh_shape)
140 lh_old = lh_shape
141 rh_old = rh_shape
--> 142 lh_shape, rh_shape, shape = mul_shapes_promote(lh_shape, rh_shape)
143 if lh_shape != lh_old:
144 shape = shape[1:]
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/utilities/shape.py in mul_shapes_promote(lh_shape, rh_shape)
107 if lh_mat_shape[1] != rh_mat_shape[0]:
108 raise ValueError("Incompatible dimensions %s %s" % (
--> 109 lh_shape, rh_shape))
110 if lh_shape[:-2] != rh_shape[:-2]:
111 raise ValueError("Incompatible dimensions %s %s" % (
ValueError: Incompatible dimensions (1, 200) (8, 3)
Update:
I tried changing the selection shape as suggested in the comment below.
code:
selection = cvxpy.Variable(test_probs.shape, boolean=True)
and now I get the new error when I run the supply_constraint part of the code below.
code:
# constraints
# Constant matrix that counts how many of each
# material we sent to each customer
TRANSFORMER = np.array([[1,0,0],
[0,1,0],
[0,0,1],
[1,1,0],
[1,0,1],
[0,1,1],
[1,1,1],
[0,0,0]])
# can't send customer more promotion than there is supply
# note: sum_entries changed to sum in latest cvxpy version
supply_constraint = cvxpy.sum(selection * TRANSFORMER, axis=0) <= supply
Error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-10-6eb7a55ea896> in <module>()
18 # note: sum_entries changed to sum in latest cvxpy version
19
---> 20 supply_constraint = cvxpy.sum(selection * TRANSFORMER, axis=0) <= supply
21
22
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in cast_op(self, other)
47 """
48 other = self.cast_to_const(other)
---> 49 return binary_op(self, other)
50 return cast_op
51
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in __le__(self, other)
482 """NonPos : Creates an inequality constraint.
483 """
--> 484 return NonPos(self - other)
485
486 def __lt__(self, other):
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in cast_op(self, other)
47 """
48 other = self.cast_to_const(other)
---> 49 return binary_op(self, other)
50 return cast_op
51
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in __sub__(self, other)
370 """Expression : The difference of two expressions.
371 """
--> 372 return self + -other
373
374 #_cast_other
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in cast_op(self, other)
47 """
48 other = self.cast_to_const(other)
---> 49 return binary_op(self, other)
50 return cast_op
51
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/expressions/expression.py in __add__(self, other)
358 """Expression : Sum two expressions.
359 """
--> 360 return cvxtypes.add_expr()([self, other])
361
362 #_cast_other
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/affine/add_expr.py in __init__(self, arg_groups)
34 # For efficiency group args as sums.
35 self._arg_groups = arg_groups
---> 36 super(AddExpression, self).__init__(*arg_groups)
37 self.args = []
38 for group in arg_groups:
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/atom.py in __init__(self, *args)
42 self.args = [Atom.cast_to_const(arg) for arg in args]
43 self.validate_arguments()
---> 44 self._shape = self.shape_from_args()
45 if len(self._shape) > 2:
46 raise ValueError("Atoms must be at most 2D.")
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/atoms/affine/add_expr.py in shape_from_args(self)
42 """Returns the (row, col) shape of the expression.
43 """
---> 44 return u.shape.sum_shapes([arg.shape for arg in self.args])
45
46 def expand_args(self, expr):
~/anaconda2/envs/py36/lib/python3.6/site-packages/cvxpy/utilities/shape.py in sum_shapes(shapes)
50 raise ValueError(
51 "Cannot broadcast dimensions " +
---> 52 len(shapes)*" %s" % tuple(shapes))
53
54 longer = shape if len(shape) >= len(t) else t
ValueError: Cannot broadcast dimensions (3,) (1, 3)
Your issue is happening when you create the selection variable. You are unpacking the shape tuple into multiple arguments. The first argument to Variable should be a shape. So the correct construction is:
selection = cvxpy.Variable(test_probs.shape, boolean=True)
You can verify this is correct by inspecting the shape attribute:
selection.shape
Which should now give:
(200, 8)
I am trying to recreate this example of bayesian PK/PD modelling using pymc3.....
The video shows the WinBUGS code and I am trying to convert to pymc3
https://www.youtube.com/watch?v=AQDXRoBan6Y
model here....
http://imgur.com/ckoKPRF
WinBUGS code is here ....
http://imgur.com/TsViyBC
My code is ....
from pymc3 import Model, Normal, Lognormal, Uniform
import numpy as np
import pandas as pd
data = pd.read_csv('/Users/Home/Documents/pymc3/fxa.data.csv' )
cobs = np.array(data['cobs'])
fxa = np.array(data['fxa.inh.obs'])
pkpd_model = Model()
with pkpd_model:
# Priors for unknown model parameters
emax = Uniform ('emax', lower =0, upper =100)
ec50 = Lognormal('ec50', mu=0, tau = 100000)
gamma = Uniform('gamma', lower=0, upper =10)
sigma = Uniform('sigma', lower = 0, upper = 1000 )
# Expected value of outcome
fxaMean = emax*(np.power(cobs, gamma)) / (np.power(ec50, gamma) + np.power(cobs, gamma))
# Likelihood (sampling distribution) of observations
fxa = Normal('fxa', mu=fxaMean, sd=sigma, observed=fxa )
But when I run the code I get the following error, which seems to relate to the way theano is interpreting the np.power function.
I am not sure how to proceed as I am a noob to pymc3 and theano and PK/PD modelling too!
Thanks in advance
Applied interval-transform to emax and added transformed emax_interval to model.
Applied log-transform to ec50 and added transformed ec50_log to model.
Applied interval-transform to gamma and added transformed gamma_interval to model.
Applied interval-transform to sigma and added transformed sigma_interval to model.
---------------------------------------------------------------------------
AsTensorError Traceback (most recent call last)
<ipython-input-28-1fa311a15ed0> in <module>()
14
15 # Likelihood (sampling distribution) of observations
---> 16 fxa = Normal('fxa', mu=fxaMean, sd=sigma, observed=fxa )
//anaconda/lib/python2.7/site-packages/pymc3/distributions/distribution.pyc in __new__(cls, name, *args, **kwargs)
23 data = kwargs.pop('observed', None)
24 dist = cls.dist(*args, **kwargs)
---> 25 return model.Var(name, dist, data)
26 elif name is None:
27 return object.__new__(cls) # for pickle
//anaconda/lib/python2.7/site-packages/pymc3/model.pyc in Var(self, name, dist, data)
282 self.named_vars[v.name] = v
283 else:
--> 284 var = ObservedRV(name=name, data=data, distribution=dist, model=self)
285 self.observed_RVs.append(var)
286 if var.missing_values:
//anaconda/lib/python2.7/site-packages/pymc3/model.pyc in __init__(self, type, owner, index, name, data, distribution, model)
556 self.missing_values = data.missing_values
557
--> 558 self.logp_elemwiset = distribution.logp(data)
559 self.model = model
560 self.distribution = distribution
//anaconda/lib/python2.7/site-packages/pymc3/distributions/continuous.pyc in logp(self, value)
191 sd = self.sd
192 mu = self.mu
--> 193 return bound((-tau * (value - mu)**2 + T.log(tau / np.pi / 2.)) / 2.,
194 tau > 0, sd > 0)
195
//anaconda/lib/python2.7/site-packages/theano/tensor/var.pyc in __radd__(self, other)
232 # ARITHMETIC - RIGHT-OPERAND
233 def __radd__(self, other):
--> 234 return theano.tensor.basic.add(other, self)
235
236 def __rsub__(self, other):
//anaconda/lib/python2.7/site-packages/theano/gof/op.pyc in __call__(self, *inputs, **kwargs)
609 """
610 return_list = kwargs.pop('return_list', False)
--> 611 node = self.make_node(*inputs, **kwargs)
612
613 if config.compute_test_value != 'off':
//anaconda/lib/python2.7/site-packages/theano/tensor/elemwise.pyc in make_node(self, *inputs)
541 using DimShuffle.
542 """
--> 543 inputs = list(map(as_tensor_variable, inputs))
544 shadow = self.scalar_op.make_node(
545 *[get_scalar_type(dtype=i.type.dtype).make_variable()
//anaconda/lib/python2.7/site-packages/theano/tensor/basic.pyc in as_tensor_variable(x, name, ndim)
206 except Exception:
207 str_x = repr(x)
--> 208 raise AsTensorError("Cannot convert %s to TensorType" % str_x, type(x))
209
210 # this has a different name, because _as_tensor_variable is the
AsTensorError: ('Cannot convert [Elemwise{mul,no_inplace}.0 Elemwise{mul,no_inplace}.0\n Elemwise{mul,no_inplace}.0 ..., Elemwise{mul,no_inplace}.0\n Elemwise{mul,no_inplace}.0 Elemwise{mul,no_inplace}.0] to TensorType', <type 'numpy.ndarray'>)
Doh - replaced np.power with ** ! working fine!