Using the same example as from this previous question (code pasted below), we can get the 95% CI with the summary_table function from statsmodels outliers_influence. But now, how would it be possible to only subset the data points (x and y) that are outside the confidence interval?
import numpy as np
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import summary_table
#measurements genre
n = 100
x = np.linspace(0, 10, n)
e = np.random.normal(size=n)
y = 1 + 0.5*x + 2*e
X = sm.add_constant(x)
re = sm.OLS(y, X).fit()
st, data, ss2 = summary_table(re, alpha=0.05)
predict_ci_low, predict_ci_upp = data[:, 6:8].T
It might be a bit late for this, but you could put it in a pandas.DataFrame and filter depending on a list of booleans. Assuming I got your question:
import numpy as np
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import summary_table
import matplotlib.pyplot as plot
## Import pandas
import pandas as pd
#measurements genre
n = 100
x = np.linspace(0, 10, n)
e = np.random.normal(size=n)
y = 1 + 0.5*x + 2*e
X = sm.add_constant(x)
re = sm.OLS(y, X).fit()
st, data, ss2 = summary_table(re, alpha=0.05)
# Make prediction
prediction = re.predict(X)
predict_ci_low, predict_ci_upp = data[:, 6:8].T
# Put y and x in a pd.DataFrame
df = pd.DataFrame(y).set_index(x)
# Get the y values that are out of the ci intervals. This could be done directly in the df indexer
out_up = y > predict_ci_upp
out_down = y < predict_ci_low
# Plot everything
plot.plot(x, y, label = 'train')
plot.plot(df[out_up], marker = 'o', linewidth = 0)
plot.plot(df[out_down], marker = 'o', linewidth = 0)
plot.plot(x, predictionTrain, label = 'prediction')
plot.plot(x, predict_ci_upp, label = 'ci_up')
plot.plot(x, predict_ci_low, label = 'ci_low')
plot.legend(loc='best')
Here is the resulting plot:
Related
I am trying to plot the put/call option using python but I am having some errors when obtaining my values and plot looks weird. I think there is something wrong with my loop of the matrices. My put and call prices should be 0.37 & 1.03. But I just get a printed out matrix. Some help would be appreciated.
import matplotlib.pyplot as plt
import numpy as np
S = 8.5
K = 8
r = 0.02
sigma = 0.2
T = 1
h = 0.0005
N = int(T/h)
stock_price = np.zeros((N+1,N+1))
option_price_call = np.zeros((N+1,N+1))
option_price_put = np.zeros((N+1,N+1))
stock_price[0,0] = S
for j in range(1, N+1):
stock_price[0,j]= stock_price[0,j-1] *np.exp(sigma*np.sqrt(h)*np.random.normal())
for j in range(0, N+1):
option_price_call[N,j] = max(stock_price[N,j]-K,0)
option_price_put[N,j] = max(K-stock_price[N,j],0)
for i in range(N-1, -1, -1):
for j in range(0, i+1):
stock_price[i,j] = stock_price[i+1,j]*np.exp(-r*h)
option_price_call[i,j] = (option_price_call[i+1,j+1]+option_price_call[i+1,j])/2
option_price_put[i,j] = (option_price_put[i+1,j+1]+option_price_put[i+1,j])/2
print(option_price_call)
print(option_price_put)
plt.figure(1)
plt.plot(stock_price[0,:],option_price_call[0,:], 'r', label = "Call option")
plt.plot(stock_price[0,:],option_price_put[0,:], 'b', label = "Put option")
plt.xlabel("Stock")
plt.ylabel("Price")
plt.legend()
plt.show()
Here is an example.
import numpy as np
import matplotlib.pyplot as plt
import seaborn
# Fortis stock price
spot_price = 138.90
# Long put
strike_price_long_put = 135
premium_long_put = 4
# Long call
strike_price_long_call = 145
premium_long_call = 3.50
# Stock price range at expiration of the put
sT = np.arange(0.7*spot_price,1.3*spot_price,1)
def call_payoff(sT, strike_price, premium):
return np.where(sT > strike_price, sT - strike_price, 0) - premium
payoff_long_call = call_payoff(sT, strike_price_long_call, premium_long_call)
# Plot
fig, ax = plt.subplots()
ax.spines['bottom'].set_position('zero')
ax.plot(sT,payoff_long_call,label='Long Call',color='r')
plt.xlabel('Stock Price')
plt.ylabel('Profit and loss')
plt.legend()
plt.show()
I am trying to plot two true positive rate and false positive rate values for two different cases. While plotting, the second plot cancels out the display of the first sort of.
import math
import numpy as np
import random
import os
import sys
from scipy.spatial import distance
from matplotlib import pyplot as plt
import pandas as pd
def fextract(fname1, fname2):
file1 = open(fname1,'r')
file2 = open(fname2,'r')
cnt = 122
data1 = file1.readlines()
data2 = file2.readlines()
print(len(data1))
tpr = np.zeros((1,cnt))
fpr = np.zeros((1,cnt))
cnt2 = 0
for x in data1:
if(cnt2 == 120):
break
tpr[0,cnt2] = float(x)
cnt2 = cnt2 + 1
cnt2 = 0
for x in data2:
if(cnt2 == 120):
break
fpr[0,cnt2] = float(x)
cnt2 = cnt2 + 1
return tpr, fpr
def plotfig(x1,y1,x2,y2):
f1 = plt.figure()
lg = plt.plot(x2[0,:], y2[0,:], label="Skin Color (With Chrominance)")
lg = plt.plot(x1[0,:], y1[0,:], label="Skin Color")
plt.legend(loc='best')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.show()
tpr,fpr = fextract('tp.txt','fp.txt')
tpr2,fpr2 = fextract('tp2.txt','fp2.txt')
tpr3,fpr3 = fextract('tp3.txt','fp3.txt')
tpr4,fpr4 = fextract('tp7.txt','fp7.txt')
tpr5,fpr5 = fextract('tp8.txt','fp8.txt')
tpr6,fpr6 = fextract('tp9.txt','fp9.txt')
plotfig(fpr,tpr,fpr4,tpr4)
plotfig(fpr2,tpr2,fpr5,tpr5)
plotfig(fpr3,tpr3,fpr6,tpr6)
As seen above, the the orange line stops midway and the blue gets plotted completely.
I wrote a program to plot oscilloscope data and make a KDE scatter plot with a colorbar. Unfortunately it requires a third party lib (readTrc) as well as the oscilloscope binary file which size is 200MB. The lib can be found on github.
import pandas as pd
import readTrc
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import collections
from scipy.stats import gaussian_kde
trcpath = 'filename.trc' #Binary Oscilloscope File (200 MB)
datX, datY, m = readTrc.readTrc(trcpath)
srx, sry = pd.Series(datX * 1000), pd.Series(datY * 1000)
df = pd.concat([srx, sry], axis = 1)
df.set_index(0, inplace = True)
df = df.abs() #Build Dataframe from above file
fig = plt.figure()
#Eliminate Noise
df[df < 3] = None
df = df.dropna()
#x and y axes data to plot
q1 = np.array(df[1].tolist()[:-2])
q2 = np.array(df[1].tolist()[1:-1])
q3 = np.array(df[1].tolist()[2:])
dq1 = q2 - q1
dq2 = q3 - q2
#Create first Dataset
qqstack = []
xy = np.vstack([dq1,dq2])
#Determine max value for colorbar (highest repeating x/y combination)
df_d = pd.DataFrame([dq1,dq2]).T
for idx, row in df_d.iterrows():
if row[0] == row[1]:
qqstack.append((row[0], row[1]))
cbar_max = collections.Counter(qqstack).most_common(1)[0][-1]
#sort to show most present values last
z = gaussian_kde(xy)(xy)
idx = z.argsort()
x, y, z = dq1[idx], dq2[idx], z[idx]
#plot graph
plt.scatter(x, y,
c=z,
s=20,
cmap = plt.cm.get_cmap('jet'))
#create colormap variable
sm = plt.cm.ScalarMappable(cmap = plt.cm.get_cmap('jet'),
norm = matplotlib.colors.PowerNorm(vmin = -0.1, vmax = cbar_max, gamma = 1))
sm._A = []
fig.colorbar(sm, ticks = range(0, cbar_max, 250))
plt.grid(zorder = 0, alpha = 0.3)
plt.xlabel('dq1 / mV')
plt.ylabel('dq2 / mV')
plt.show()
How can I adjust the color allocation in the plot? I want there to be less blue space so the transition is visible more, like on this graph:
I am trying to achieve a plot similar to this one:
The color shows the clustering of the datapoints.
My code so far:
import pandas as pd
import readTrc
import matplotlib.pyplot as plt
import numpy as np
import os
import gc
trcpath = 'filename.trc'
datX, datY, m = readTrc.readTrc(trcpath)
srx, sry = pd.Series(datX * 1000), pd.Series(datY * 1000)
df_plot = pd.concat([srx, sry], axis = 1)
df_plot.set_index(0, inplace = True)
fig, ax = plt.subplots()
#Eliminate Noise
df_plot[df_plot < 3] = 0
df = df_plot[df_plot > 3]
df[df < 3] = None
df = df.dropna()
#Plot Parameters
p = np.array(df[1].tolist()[:-1])
p_nach = np.array(df[1].tolist()[1:])
d_t = np.array(pd.Series(df.index).diff().tolist()[1:])
#Graph Limit
graphlim = 101
#Plot
plt.scatter(p, p_nach,
edgecolors = 'none',
c = p,
s = 20,
cmap=plt.cm.get_cmap('jet'))
plt.xlim(0,graphlim)
plt.ylim(0,graphlim)
plt.xticks(range(0,graphlim,int(graphlim/10)))
plt.yticks(range(0,graphlim,int(graphlim/10)))
plt.colorbar()
plt.grid(zorder = 0, alpha = 0.3)
ax.set_xlabel('p / mV')
ax.set_ylabel('p_nach / mV')
##plt.savefig(dpi = 300)
plt.show()
##plt.close()
##fig.clear()
##gc.collect()
print('Progress... done!')
As you can see, the colorbar does not represent the clustering and instead the place on the x-axis. How do I configure my colorbar to represent the amount of datapoints in an area?
Folder with files: Link
import pandas as pd
import readTrc
import matplotlib.pyplot as plt
import numpy as np
import os
import gc
trcpath = 'filename.trc'
datX, datY, m = readTrc.readTrc(trcpath)
df = pd.DataFrame({'time': datX * 1000, 'volts': datY * 1000})
reduce_noise_df = df[df.volts >= 3.0]
d_t = reduce_noise_df.time.diff()[1:]
p = reduce_noise_df.volts[:-1]
p_nach = reduce_noise_df.volts[1:]
#Graph Limit
graphlim = 41
#Plot
fig, ax = plt.subplots(figsize=(6,6))
plt.scatter(p, p_nach,
edgecolors = 'none',
c = d_t,
s = 20,
cmap=plt.cm.get_cmap('jet'))
plt.xlim(0, graphlim)
plt.ylim(0, graphlim)
plt.xticks(range(0, graphlim, int(graphlim/10)))
plt.yticks(range(0, graphlim, int(graphlim/10)))
plt.colorbar()
plt.grid(zorder = 0, alpha = 0.3)
ax.set_xlabel('p / mV')
ax.set_ylabel('p_nach / mV')
plt.show()
I began be removing unnecessary code
The main issue was c = p instead of c = d_t.
Plot of waveform from your Le Croy WR640Zi colored by data density
import pandas as pd
import readTrc
import matplotlib.pyplot as plt
import numpy as np
import os
import gc
from scipy.stats import gaussian_kde
trcpath = 'filename.trc'
datX, datY, m = readTrc.readTrc(trcpath)
df = pd.DataFrame({'time': datX * 1000, 'volts': datY * 1000})
reduce_noise_df = df[df.volts >= 3.0]
y = np.array(reduce_noise_df.volts.tolist())
x = np.array(reduce_noise_df.time.tolist())
# Calculate point density
xy = np.vstack([x, y])
z = gaussian_kde(xy)(xy)
# Sort points by density
idx = z.argsort()
x, y, z = x[idx], y[idx], z[idx]
#Plot
fig, ax = plt.subplots(figsize=(6,6))
plt.scatter(x, y,
edgecolors = 'none',
c = z,
s = 20,
cmap=plt.cm.get_cmap('jet'))
plt.colorbar()
plt.grid(zorder = 0, alpha = 0.3)
ax.set_xlabel('Time (ms)')
ax.set_ylabel('Voltage (mV)')
plt.show()
I am working on a simple q learning code in python. After running several iterations the program suggest a valid path, but not always the shortest -which is the point of the program. I am not sure what I am overlooking. I am using a jupyter notebook.
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
iterations = 200
goalState = 5
gamma = 0.8
qValues = []
#actions
R = np.array([[-1, 0,-1,-1, 0,-1],
[ 0,-1,-1,-1,-1,100],
[-1,-1,-1,-1,-1,100],
[-1,-1,-1,-1, 0,100],
[ 0,-1,-1, 0,-1,-1],
[-1,-1,-1,-1,-1,100]])
#inital Q matrix
Q = np.zeros(R.shape)
for i in range(iterations):
state = np.random.randint(goalState + 1)
while state != goalState:
possibleActions = np.where(R[state] >= 0)[0]
action = possibleActions[np.random.randint(len(possibleActions+1))]
nextPossibleActions = np.where(R[action] >= 0)[0]
for k in nextPossibleActions:
qValues.append(Q[action][k])
qMax = max(qValues)
Q[state][action] += R[state][action] + gamma * qMax
state = action
Q = Q/Q.max() #normalising the matrix to percentage values
sns.set()
f, ax = plt.subplots(figsize=(8, 6))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
sns.heatmap(Q, cmap = cmap, annot=True, linewidths=.5, ax=ax)