This question already has answers here:
How to clear memory completely of all matplotlib plots
(5 answers)
Closed 3 years ago.
I will first describe the script and later will publish the code and a way to reproduce the problem, in my machine might not be the case for every machine.
So I have a script that build objects, clouds, with specific properties and export the data to files that is the input to a software called SHDOM, this tool analyze radiation properties and can be further read in the following link.
The script is its a bit long, sorry in advance:
import numpy as np
from itertools import product
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from mpl_toolkits.mplot3d import Axes3D
import gc
def writeDomainFile(type, file_name, temp_vec,
lwc, r_effective, dx, dy,
height_vec):
if type == 2:
assert list(lwc.shape) == list(r_effective.shape), "when the file type is 2 the shapes of lwc and r_effective shold be the same"
x_lwc, y_lwc, z_lwc = lwc.shape
x_r_effetive, y_r_effetive, z_r_effetive = r_effective.shape
fp = open(file_name, 'w')
# write the file format
fp.write("{:}\n".format(type))
# write the number of x, y, z points
fp.write("{:} {:} {:}\n".format(x_lwc, y_lwc, z_lwc))
# write the spacing resolution in x and y
fp.write("{:} {:}\n".format(dx, dy))
# write the height vector
fp.write(" ".join(map(str,height_vec)) + "\n")
# write the temprature vector
fp.write(" ".join(map(str,temp_vec)) + "\n")
indices = product(range(x_lwc), range(y_lwc), range(z_lwc))
print_indices = product(range(1, x_lwc + 1), range(1, y_lwc + 1), range(1, z_lwc + 1))
if type == 1: # only lwc
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(1 + list(triplet))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:}\n".format(*print_triplets, 0))
else:
fp.write("{:} {:} {:} {:6.4f}\n".format(*print_triplets, lwc[triplet]))
elif type == 2:
for triplet, print_triplets in zip(indices, print_indices):
# t = tuple(map(lambda x: x + 1, list(triplet)))
if not lwc[triplet]:
fp.write("{:} {:} {:} {:} {:}\n".format(*print_triplets, 0, 0))
else:
fp.write("{:} {:} {:} {:6.4f} {:4.2f}\n".format(*print_triplets, lwc[triplet], r_effective[triplet]))
fp.close()
return
def lapseRateTemp(base_temp, position):
temp = np.zeros(position.shape)
temp[0] = base_temp
temp[1:] = base_temp - 6.5 * position[1:]
return np.round(temp, 4)
def createCubicCloudDomain(cloud_tick, x_size, y_size,
dx, dy, dz, r_effective_option, lwc_option,
cloud_x_size, cloud_y_size):
# const
temp0 = 300 # kelvin
N_x = x_size / dx # number of x points
N_y = y_size / dy # number of y points
cld_base = 1.0 # the cloud always start from 1[km] above ground
x_center = N_x / 2
y_center = N_y / 2
prog_x = (cloud_x_size / 2) / dx
prog_y = (cloud_y_size / 2) / dy
cloud_x_west = int(x_center - prog_x)
cloud_x_east = int(x_center + prog_x)
cloud_y_south = int(y_center - prog_y)
cloud_y_north = int(y_center + prog_y)
cloud_x_vec = np.arange(cloud_x_west, cloud_x_east, 1)
cloud_y_vec = np.arange(cloud_y_south, cloud_y_north, 1)
for tick in cloud_tick: # cloud_tick might be a vector
cld_top = cld_base + tick
N_z = tick / dz # number of z points
cloud_z_vec = np.round(np.arange(cld_base, cld_top, dz), 4)
# temprature
temp_base = temp0 - 9.8 * cld_base
temp_vec = lapseRateTemp(temp_base, cloud_z_vec - cld_base,)
temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1))
temp_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud
plotTemperatureGradient(temp_domain, 'test.png')
# del temp_cloud
# del temp_domain
# gc.collect()
for r in r_effective_option:
for l in lwc_option:
r_effective_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
r)
lwc_cloud = np.full((cloud_x_vec.shape[0],
cloud_y_vec.shape[0],
cloud_z_vec.shape[0]),
l)
r_effective_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
lwc_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
# the positions to enter the cloud data
lwc_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = lwc_cloud
r_effective_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = r_effective_cloud
writeDomainFile(2, "test.txt", temp_vec, lwc_domain, r_effective_domain, dx, dy, cloud_z_vec)
plotGeneratedCloud(lwc_domain, r_effective_domain)
return
def plotTemperatureGradient(temp_mat, file_name):
xs, ys, zs = temp_mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = temp_mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
# map scalars to colors
minima = np.min(faltten_data[np.nonzero(faltten_data)])
maxima = np.max(faltten_data[np.nonzero(faltten_data)])
norm = matplotlib.colors.Normalize(vmin=minima, vmax=maxima, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap='jet')
rgba = mapper.to_rgba(faltten_data)
color_map[:,0:3] = rgba[:, 0:3]
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
p = ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
fig.colorbar(p)
# plt.title(title)
plt.savefig(file_name)
return
def plotGeneratedCloud(lwc, r_effective):
light_blue = np.array([0, 1, 0.7])
matrixPlotter(lwc, 'lwc.png',
'Liquid water content of the cloud',
light_blue)
# gc.collect()
light_pink = np.array([0.9, 0.6, 0.9])
matrixPlotter(r_effective, 'r_effective.png',
'Effective radius of the cloud',
light_pink)
return
def matrixPlotter(mat, file_name, title, c=np.array([0.2, 0.6, 0])):
xs, ys, zs = mat.shape
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
X, Y, Z = np.mgrid[:xs, :ys, :zs]
faltten_data = mat.ravel().astype(np.float16)
color_map = np.zeros((faltten_data.shape[0], 4))
color_map[:,0:3] = c
color_map[:,3] = np.where(faltten_data > 0, 0.07, 0)
ax.scatter(X, Y, Z, c=color_map.astype(np.float16))
ax.set_xlabel('X position [Arb.]')
ax.set_ylabel('Y position [Arb.]')
ax.set_zlabel('Z position [Arb.]')
plt.title(title)
plt.savefig(file_name)
return
# def main():
# return
if __name__ == '__main__':
# main()
createCubicCloudDomain([0.5], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
print("Done")
So the problem occur when:
I try to export all 3 plots, calling the following blocks:
plotGeneratedCloud(lwc_domain, r_effective_domain) or
temp_cloud = np.tile(temp_vec,(int(temp_vec.shape[0]), int(temp_vec.shape[0]), 1))
temp_domain = np.zeros((int(N_x), int(N_y), int(N_z)))
temp_domain[cloud_x_west:cloud_x_east, cloud_y_south: cloud_y_north, :] = temp_cloud
plotTemperatureGradient(temp_domain, 'test.png')
When I try to create more then 1 object (cloud), so calling the function with:
createCubicCloudDomain([0.5, 0.6], 2.02, 2.02, 0.01, 0.01, 0.01, [0.5, 0.4], [1, 0.3], 0.5, 0.5)
increasing the domain size like:
createCubicCloudDomain([0.5], 4.02, 4.02, 0.01, 0.01, 0.01, [0.5], [1], 0.5, 0.5)
The type of error I'm getting is the following:
File
"C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\backends\backend_agg.py",
line 396, in draw
self.figure.draw(self.renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py",
line 38, in draw_wrapper
return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\figure.py",
line 1735, in draw
mimage._draw_list_compositing_images( File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\image.py",
line 137, in _draw_list_compositing_images
a.draw(renderer) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\matplotlib\artist.py",
line 38, in draw_wrapper
return draw(artist, renderer, *args, **kwargs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py",
line 291, in draw
sorted(self.collections, File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\axes3d.py",
line 292, in
key=lambda col: col.do_3d_projection(renderer), File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\art3d.py",
line 538, in do_3d_projection
vxs, vys, vzs, vis = proj3d.proj_transform_clip(xs, ys, zs, renderer.M) File
"C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py",
line 214, in proj_transform_clip
vec = _vec_pad_ones(xs, ys, zs) File "C:\Users\davidsr\AppData\Local\Programs\Python\Python38-32\lib\site-packages\mpl_toolkits\mplot3d\proj3d.py",
line 189, in _vec_pad_ones
return np.array([xs, ys, zs, np.ones_like(xs)]) MemoryError: Unable to allocate array with shape (4, 2040200) and data type float64
I understand the problem is memory issues, so I thought that deleting the arrays once finish with them might solve the problem, so i used del function or even using the garbage collector solution used in the following question: How can I explicitly free memory in Python?
Would appreciate some help
The figure object persists in memory if you don't .close() it first, and it includes all the data it's plotting.
Try putting a plt.close(fig) before the return statement in matrixPlotter. Then your gc.collect() statements should be able to catch it if it's not collected by matplotlib (and I don't think it is).
Related
Is there anyway to get the rectangle points from the HoughLines Transformation results and apply the crop point to original image to get the cropped image. I have copied the code form the documentation. The idea is to extract the document from an image. Below is the result from the HoughLines Transformation and I required the intersection point to crop the image.
"""
#file hough_lines.py
#brief This program demonstrates line finding with the Hough transform
"""
import sys
import math
import cv2 as cv
import numpy as np
import matplotlib.pyplot as plt
def main(argv=[]):
default_file = "/Users/apple/Downloads/Unknown-4"
filename = argv[0] if len(argv) > 0 else default_file
# Loads an image
src = cv.imread(cv.samples.findFile(filename), cv.IMREAD_GRAYSCALE)
# Check if image is loaded fine
if src is None:
print ('Error opening image!')
print ('Usage: hough_lines.py [image_name -- default ' + default_file + '] \n')
return -1
dst = cv.Canny(src, 50, 200, None, 3)
# Copy edges to the images that will display the results in BGR
cdst = cv.cvtColor(dst, cv.COLOR_GRAY2BGR)
cdstP = np.copy(cdst)
lines = cv.HoughLines(dst, 1, np.pi / 180, 150, None, 0, 0)
if lines is not None:
for i in range(0, len(lines)):
rho = lines[i][0][0]
theta = lines[i][0][1]
a = math.cos(theta)
b = math.sin(theta)
x0 = a * rho
y0 = b * rho
pt1 = (int(x0 + 1000*(-b)), int(y0 + 1000*(a)))
pt2 = (int(x0 - 1000*(-b)), int(y0 - 1000*(a)))
cv.line(cdst, pt1, pt2, (0,0,255), 3, cv.LINE_AA)
linesP = cv.HoughLinesP(dst, 1, np.pi / 180, 50, None, 50, 10)
if linesP is not None:
for i in range(0, len(linesP)):
l = linesP[i][0]
cv.line(cdstP, (l[0], l[1]), (l[2], l[3]), (0,0,255), 3, cv.LINE_AA)
#cv.imshow("Source", src)
#plt.imshow(src)
plt.imshow(cdstP)
#plt.imshow(cdstP)
if __name__ == "__main__":
main()
I have a pickle file which contains 300 coordinates of my subject's location in time. There are some missing values in the middle of it for which I am using a particle filter to estimate those missing values. At the end, I am getting some predictions (not completely accurate) but in a bit drifted form.
So the position of my subject is, in fact, the position of my subject's nose. I take a total of 300 frames and each frame consists of a coordinate for nose in it. There are some frames which have the value of (0,0) meaning the values are missing. So in order to find them, I am implementing the particle filter. I am a newbie for particle filter so there are possibilities that I may have messed up the code. The results that I get, gives me the prediction for 300 frames with drifted values. You can get a clear idea form the image.
My measurement value is distance from four landmarks and I provide orientation angle to next point and distance to next point as additional measurements.
from filterpy.monte_carlo import systematic_resample
import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import norm
from numpy.random import randn
import scipy.stats
from numpy.random import uniform
import pickle
from math import *
#####################################################
def create_uniform_particles(x_range, y_range, hdg_range, N):
particles = np.empty((N, 3))
particles[:, 0] = uniform(x_range[0], x_range[1], size=N)
particles[:, 1] = uniform(y_range[0], y_range[1], size=N)
particles[:, 2] = uniform(hdg_range[0], hdg_range[1], size=N)
particles[:, 2] %= 2 * np.pi
return particles
def create_gaussian_particles(mean, std, N):
particles = np.empty((N, 3))
particles[:, 0] = mean[0] + (randn(N) * std[0])
particles[:, 1] = mean[1] + (randn(N) * std[1])
particles[:, 2] = mean[2] + (randn(N) * std[2])
particles[:, 2] %= 2 * np.pi
return particles
#####################################################
def predict(particles, u, std):
# move according to control input u (heading change, velocity)
#with noise Q (std heading change, std velocity)`
N = len(particles)
# update heading
#particles[:, 2] += u[0] + (randn(N) * std[0])
#particles[:, 2] %= 2 * np.pi
#u[0] += (randn(N) * std[0])
u[0] %= 2 * np.pi
# move in the (noisy) commanded direction
dist = (u[1]) #+ (randn(N) * std[1])
particles[:, 0] += np.cos(u[0]) * dist
particles[:, 1] += np.sin(u[0]) * dist
#####################################################
def update(particles, weights, z, R, landmarks):
for i, landmark in enumerate(landmarks):
distance = np.linalg.norm(particles[:, 0:2] - landmark, axis=1)
weights *= scipy.stats.norm(distance, R).pdf(z[i])
weights += 1.e-300 # avoid round-off to zero
weights /= sum(weights) # normalize
#####################################################
def estimate(particles, weights):
#returns mean and variance of the weighted particles
pos = particles[:, 0:2]
mean = np.average(pos, weights=weights, axis=0)
var = np.average((pos - mean)**2, weights=weights, axis=0)
return mean, var
#####################################################
def simple_resample(particles, weights):
N = len(particles)
cumulative_sum = np.cumsum(weights)
cumulative_sum[-1] = 1. # avoid round-off error
indexes = np.searchsorted(cumulative_sum, random(N))
# resample according to indexes
particles[:] = particles[indexes]
weights.fill(1.0 / N)
#####################################################
def neff(weights):
return 1. / np.sum(np.square(weights))
#####################################################
def resample_from_index(particles, weights, indexes):
particles[:] = particles[indexes]
weights[:] = weights[indexes]
weights.fill(1.0 / len(weights))
#####################################################
def read_pickle(pkl_file, f,j):
with open(pkl_file, 'rb') as res:
dets = pickle.load(res, encoding = 'latin1')
all_keyps = dets['all_keyps']
keyps_t = np.array(all_keyps[1])
keyps = np.zeros((keyps_t.shape[0], 4, 17))
for k in range(keyps.shape[0]):
if keyps_t[k]!=[]:
keyps[k] = keyps_t[k][0]
keyps = keyps[:,:2,:]
for i in range(keyps.shape[0]):
keyps[i][0] = keyps[i][0]/480*256
keyps[i][1] = keyps[i][1]/640*256
x0=keyps[f][0][j]
y0=keyps[f][1][j]
x1=keyps[f+1][0][j]
y1=keyps[f+1][1][j]
cord = np.array([x0,y0])
orientation = atan2((y1 - y0),(x1 - x0))
dist= sqrt((x1-x0) ** 2 + (y1-y0) ** 2)
u = np.array([orientation,dist])
return (cord, u)
#####################################################
def run_pf1(N, iters=298, sensor_std_err=.1,
do_plot=True, plot_particles=False,
xlim=(-256, 256), ylim=(-256, 256),
initial_x=None):
landmarks = np.array([[0, 0], [0, 256], [256,0], [256,256]])
NL = len(landmarks)
plt.figure()
# create particles and weights
if initial_x is not None:
particles = create_gaussian_particles(
mean=initial_x, std=(5, 5, np.pi/4), N=N)
else:
particles = create_uniform_particles((0,20), (0,20), (0, 6.28), N)
weights = np.ones(N) / N
if plot_particles:
alpha = .20
if N > 5000:
alpha *= np.sqrt(5000)/np.sqrt(N)
plt.scatter(particles[:, 0], particles[:, 1],
alpha=alpha, color='g')
xs = []
#robot_pos, u = read_pickle('.pkl',1,0)
for x in range(iters):
robot_pos, uv = read_pickle('.pkl',x,0)
print("orignal: ", robot_pos,)
# distance from robot to each landmark
zs = (norm(landmarks - robot_pos, axis=1) +
(randn(NL) * sensor_std_err))
# move diagonally forward to (x+1, x+1)
predict(particles, u=uv, std=(0, .0))
# incorporate measurements
update(particles, weights, z=zs, R=sensor_std_err,
landmarks=landmarks)
# resample if too few effective particles
if neff(weights) < N/2:
indexes = systematic_resample(weights)
resample_from_index(particles, weights, indexes)
assert np.allclose(weights, 1/N)
mu, var = estimate(particles, weights)
#mu +=(120,10)
xs.append(mu)
print("expected: ",mu)
if plot_particles:
plt.scatter(particles[:, 0], particles[:, 1],
color='k', marker=',', s=1)
p1 = plt.scatter(robot_pos[0], robot_pos[1], marker='+',
color='k', s=180, lw=3)
p2 = plt.scatter(mu[0], mu[1], marker='s', color='r')
print(p2)
xs = np.array(xs)
#plt.plot(xs[:, 0], xs[:, 1])
plt.legend([p1, p2], ['Actual', 'PF'], loc=4, numpoints=1)
plt.xlim(*xlim)
plt.ylim(*ylim)
print('final position error, variance:\n\t', mu - np.array([iters, iters]), var)
plt.show()
return(p2)
###############################
run_pf1(N=5000)
I expect a set of 300 coordinate values (estimated) as a result of the particle filter so I can replace my missing values in original files with this predicted ones.
Please see the following code:
class PickCursor(object):
def __init__(self, collection, alpha_other=0.3, tolerance=5):
self.collection = collection
self.alpha_other = alpha_other
self.pts= collection.get_offsets()
self.num_pts = len(self.pts)
# Need to set a tolerance to make it take effect
self.collection.set_picker(tolerance)
# Ensure that we have separate colors for each object
self.fc = collection.get_facecolors()
if len(self.fc) == 0:
raise ValueError('Collection must have a facecolor')
elif len(self.fc) == 1:
self.fc = np.tile(self.fc, (self.num_pts, 1))
# self.fc is a 2d array every row follows [r, g, b, a]
self.ind = None
self.point_selected = None
self.canvas = ax.figure.canvas
self.canvas.mpl_connect('pick_event', self.onselect)
def onselect(self, event):
self.ind = event.ind[0]
self.point_selected = self.pts[self.ind]
# Change alpha of other points
self.fc[:, -1] = self.alpha_other
self.fc[self.ind, -1] = 1
self.collection.set_facecolors(self.fc)
self.canvas.draw_idle()
fig, ax = plt.subplots(1, 1, figsize=(8, 8))
x = np.arange(0, 10)
y = np.arange(0, 10)
scat = ax.scatter(x, y, s=15)
PickCursor(scat)
The above code works! Basically, it will make points unselected transparent. However, if I wrap the code into a function like this:
def func():
fig, ax = plt.subplots(1, 1, figsize=(8, 8))
x = np.arange(0, 10)
y = np.arange(0, 10)
scat = ax.scatter(x, y, s=15)
PickCursor(scat)
func() ## this line does not work!!!
Anyone please shred some light on this? Thanks!
Your class is being garbage collected because you don't retain a reference to it.
def func():
# ...
return PickCursor(scat)
pc = func()
I am trying to make an radar chart to look at the characteristics of the clusters formed by KMeans. The chart was plotting, but when I changed the variables I want to plot, it throws a run time error with incomplete radarchart. I am using the following code:
def _scale_data(data, ranges):
(x1, x2) = ranges[0]
d = data[0]
return [(d - y1) / (y2 - y1) * (x2 - x1) + x1 for d, (y1, y2) in zip(data, ranges)]
class RadarChart():
def __init__(self, fig, location, sizes, variables, ranges, n_ordinate_levels = 6):
angles = np.arange(0, 360, 360./len(variables))
ix, iy = location[:] ; size_x, size_y = sizes[:]
axes = [fig.add_axes([ix, iy, size_x, size_y], polar = True,
label = "axes{}".format(i)) for i in range(len(variables))]
_, text = axes[0].set_thetagrids(angles, labels = variables)
for txt, angle in zip(text, angles):
if angle > -1 and angle < 181:
txt.set_rotation(angle - 90)
else:
txt.set_rotation(angle - 270)
for ax in axes[1:]:
ax.patch.set_visible(False)
ax.xaxis.set_visible(False)
ax.grid("off")
for i, ax in enumerate(axes):
grid = np.linspace(*ranges[i],num = n_ordinate_levels)
grid_label = [""]+["{:.0f}".format(x) for x in grid[1:-1]]
ax.set_rgrids(grid, labels = grid_label, angle = angles[i])
ax.set_ylim(*ranges[i])
self.angle = np.deg2rad(np.r_[angles, angles[0]])
self.ranges = ranges
self.ax = axes[0]
def plot(self, data, *args, **kw):
sdata = _scale_data(data, self.ranges)
self.ax.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kw)
def fill(self, data, *args, **kw):
sdata = _scale_data(data, self.ranges)
self.ax.fill(self.angle, np.r_[sdata, sdata[0]], *args, **kw)
def legend(self, *args, **kw):
self.ax.legend(*args, **kw)
def title(self, title, *args, **kw):
self.ax.text(0.9, 1, title, transform = self.ax.transAxes, *args, **kw)
#Plotting
fig = plt.figure(figsize=(14,16))
attributes = ['total_disabled', 'average_payment_amount', 'avg_days_btwn_payments', 'days_since_1st_transaction','days_since_last_transaction','amount_pending']
ranges = [[0.01, 1000], [0.01, 10000], [0.01, 50], [0.01, 1000], [0.01, 500], [0.01, 10000]]
index = [0, 1, 2, 3, 4, 5,6,7]
n_groups = n_clusters ; i_cols = 3
i_rows = n_groups//i_cols
size_x, size_y = (1/i_cols), (1/i_rows)
for ind in range(n_clusters):
ix = ind%3 ; iy = i_rows - ind//3
pos_x = ix*(size_x + 0.05) ; pos_y = iy*(size_y + 0.05)
location = [pos_x, pos_y] ; sizes = [size_x, size_y]
#______________________________________________________
data = np.array(clustered_df.loc[index[ind], attributes])
radar = RadarChart(fig, location, sizes, attributes, ranges)
radar.plot(data, color = 'b', linewidth=2.0)
radar.fill(data, alpha = 0.2, color = 'b')
radar.title(title = 'cluster nº{}'.format(index[ind]), color = 'r')
ind += 1
And it gives the following error:
C:\Users\arindam\Anaconda3\lib\site-packages\pandas\core\indexing.py:1020: FutureWarning:
Passing list-likes to .loc or [] with any missing label will raise
KeyError in the future, you can use .reindex() as an alternative.
See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike
C:\Users\arindam\Anaconda3\lib\site-packages\matplotlib\projections\polar.py:65: RuntimeWarning:
invalid value encountered in less
I hope someone can help me out with this. Thanks in advance.
I'm trying to do binary classification on two spirals. For testing, I am feeding my neural network the exact spiral data with no noise, and the model seems to work as the losses near 0 during SGD. However, after using my model to infer the exact same data points after SGD has completed, I get completely different losses than what was printed during the last epoch of SGD.
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
np.set_printoptions(threshold=np.nan)
# get the spiral points
t_p = np.linspace(0, 4, 1000)
x1_p = t_p * np.cos(t_p*2*np.pi)
y1_p = t_p * np.sin(t_p*2*np.pi)
x2_p = t_p * np.cos(t_p*2*np.pi + np.pi)
y2_p = t_p * np.sin(t_p*2*np.pi + np.pi)
plt.plot(x1_p, y1_p, x2_p, y2_p)
# generate data points
x1_dat = x1_p
y1_dat = y1_p
x2_dat = x2_p
y2_dat = y2_p
def model_variable(shape, name, initializer):
variable = tf.get_variable(name=name,
dtype=tf.float32,
shape=shape,
initializer=initializer
)
tf.add_to_collection('model_variables', variable)
return variable
class Model():
#layer specifications includes bias nodes
def __init__(self, sess, data, nEpochs, learning_rate, layer_specifications):
self.sess = sess
self.data = data
self.nEpochs = nEpochs
self.learning_rate = learning_rate
if layer_specifications[0] != 2 or layer_specifications[-1] != 1:
raise ValueError('First layer only two nodes, last layer only 1 node')
else:
self.layer_specifications = layer_specifications
self.build_model()
def build_model(self):
# x is the two nodes that will be layer one, will input an x, y coordinate
# and need to classify which spiral is it on, the non phase shifted or the phase
# shifted one.
# y is the output of the model
self.x = tf.placeholder(tf.float32, shape=[2, 1])
self.y = tf.placeholder(tf.float32, shape=[])
self.thetas = []
self.biases = []
for i in range(1, len(self.layer_specifications)):
self.thetas.append(model_variable([self.layer_specifications[i], self.layer_specifications[i-1]], 'theta'+str(i), tf.random_normal_initializer(stddev=0.1)))
self.biases.append(model_variable([self.layer_specifications[i], 1], 'bias'+str(i), tf.constant_initializer()))
#forward propagation
intermediate = self.x
for i in range(0, len(self.layer_specifications)-1):
if i != (len(self.layer_specifications) - 2):
intermediate = tf.nn.elu(tf.add(tf.matmul(self.thetas[i], intermediate), self.biases[i]))
else:
intermediate = tf.add(tf.matmul(self.thetas[i], intermediate), self.biases[i])
self.yhat = tf.squeeze(intermediate)
self.loss = tf.nn.sigmoid_cross_entropy_with_logits(self.yhat, self.y);
def train_init(self):
model_variables = tf.get_collection('model_variables')
self.optim = (
tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
.minimize(self.loss, var_list=model_variables)
)
self.check = tf.add_check_numerics_ops()
self.sess.run(tf.initialize_all_variables())
# here is where x and y combine to get just x in tf with shape [2, 1] and where label becomes y in tf
def train_iter(self, x, y):
loss, _, _ = sess.run([self.loss, self.optim, self.check],
feed_dict = {self.x: x, self.y: y})
print('loss: {0} on:{1}'.format(loss, x))
# here x and y are still x and y coordinates, label is separate
def train(self):
for _ in range(self.nEpochs):
for x, y, label in self.data():
print(label)
self.train_iter([[x], [y]], label)
print("NEW ONE:\n")
# here x and y are still x and y coordinates, label is separate
def infer(self, x, y, label):
return self.sess.run((tf.sigmoid(self.yhat), self.loss), feed_dict={self.x : [[x], [y]], self.y : label})
def data():
#so first spiral is label 0, second is label 1
for _ in range(len(x1_dat)-1, -1, -1):
for dat in range(2):
if dat == 0:
yield x1_dat[_], y1_dat[_], 0
else:
yield x2_dat[_], y2_dat[_], 1
layer_specifications = [2, 100, 100, 100, 1]
sess = tf.Session()
model = Model(sess, data, nEpochs=10, learning_rate=1.1e-2, layer_specifications=layer_specifications)
model.train_init()
model.train()
inferrences_1 = []
inferrences_2 = []
losses = 0
for i in range(len(t_p)-1, -1, -1):
infer, loss = model.infer(x1_p[i], y1_p[i], 0)
if infer >= 0.5:
print('loss: {0} on point {1}, {2}'.format(loss, x1_p[i], y1_p[i]))
losses = losses + 1
inferrences_1.append('r')
else:
inferrences_1.append('g')
for i in range(len(t_p)-1, -1, -1):
infer, loss = model.infer(x2_p[i], y2_p[i], 1)
if infer >= 0.5:
inferrences_2.append('r')
else:
print('loss: {0} on point {1}, {2}'.format(loss, x2_p[i], y2_p[i]))
losses = losses + 1
inferrences_2.append('g')
print('total losses: {}'.format(losses))
plt.scatter(x1_p, y1_p, c=inferrences_1)
plt.scatter(x2_p, y2_p, c=inferrences_2)
plt.show()