I am wondering why my code does not speed up when I increase number of threads. My app is generating output image from two input images. if I run only one thread, operation takes 28 seconds and if I run 2 to 8 threads operation takes time between 52 and 48 seconds.
This is fragment of my code:
def StartDuboisThreading():
global imageLeft
global imageRight
global anaglyphPixelArray
anaglyphPixelArray = None
numberOfThreads = threadSlider.get()
width, height = imageLeft.size
rgb_imgLeft = imageLeft.convert('RGB')
rgb_imgRight = imageRight.convert('RGB')
rowsPerThread = int(height / numberOfThreads)
remainder = height % numberOfThreads
threads = list()
start = time.time()
for i in range(0, numberOfThreads):
copyImgLeft = copy.deepcopy(rgb_imgLeft)
copyImgRight = copy.deepcopy(rgb_imgRight)
if (i != numberOfThreads - 1):
threads.append(ThreadWithReturnValue(target=ThreadFunctionWithArrays, args=(
i * rowsPerThread, (i + 1) * rowsPerThread, width, copyImgLeft, copyImgRight)))
threads[i].start()
else:
threads.append(ThreadWithReturnValue(target=ThreadFunctionWithArrays, args=(
i * rowsPerThread, height, width, copyImgLeft, copyImgRight)))
threads[i].start()
for i in range(0, numberOfThreads):
pixelsArray = threads[i].join()
if anaglyphPixelArray is None:
anaglyphPixelArray = pixelsArray
else:
anaglyphPixelArray = np.append(anaglyphPixelArray, pixelsArray, axis=0)
end = time.time()
timeElapsed = end - start
operationTimeText.configure(text='Time of operation: ' + timeElapsed.__str__())
CreateImage(anaglyphPixelArray)
I made a deepcopy of my input images objects because I thought that maybe my threads are reffering to the same memory at the same time and this can make a slight delay but this didn't work. Now I am in lack of ideas why this is only slowing down with more threads.
This is my thread function:
def ThreadFunctionWithArrays(startPointY, endPointY, pixelsWidth, pixelsArray1, pixelsArray2):
numberOfRows = endPointY - startPointY
pixelArray = GenerateEmptyPartMatrix(pixelsWidth, numberOfRows)
y = 0
print(str(startPointY) + " " + str(endPointY))
for j in range(startPointY, endPointY):
for i in range(0, pixelsWidth):
r1, g1, b1 = pixelsArray1.getpixel((i, j))
r2, g2, b2 = pixelsArray2.getpixel((i, j))
pixelArray[y][i] = (
r1 * 0.4561 + g1 * 0.500484 + b1 * 0.176381 - r2 * 0.0434706 - g2 * 0.0879388 - b2 * 0.00155529,
- r1 * 0.0400822 - g1 * 0.0378246 - b1 * 0.0157589 + r2 * 0.378476 + g2 * 0.73364 - b2 * 0.0184503,
- r1 * 0.0152161 - g1 * 0.0205971 - b1 * 0.00546856 - r2 * 0.0721527 - g2 * 0.112961 + b2 * 1.2264)
y +=1
return pixelArray
And here is definition of my extended Thread object:
class ThreadWithReturnValue(Thread):
def __init__(self, *args, **kwargs):
super(ThreadWithReturnValue, self).__init__(*args, **kwargs)
self._return = None
def run(self):
if self._target is not None:
self._return = self._target(*self._args, **self._kwargs)
def join(self, *args, **kwargs):
super(ThreadWithReturnValue, self).join(*args, **kwargs)
return self._return
Related
I want to perform Monte Carlo simulation to the particles which are interacting via Lennard-Jones potential + FENE potential. I'm getting negative values in the FENE potential which have the log value in it. The error is "RuntimeWarning: invalid value encountered in log return (-0.5 * K * R**2 * np.log(1-((np.sqrt(rij2) - r0) / R)**2))" The FENE potential is given by:
import numpy as np
def gen_chain(N, R0):
x = np.linspace(1, (N-1)*0.8*R0, num=N)
y = np.zeros(N)
z = np.zeros(N)
return np.column_stack((x, y, z))
def lj(rij2):
sig_by_r6 = np.power(sigma/rij2, 3)
sig_by_r12 = np.power(sig_by_r6, 2)
lje = 4.0 * epsilon * (sig_by_r12 - sig_by_r6)
return lje
def fene(rij2):
return (-0.5 * K * R**2 * np.log(1-((np.sqrt(rij2) - r0) / R)**2))
def total_energy(coord):
# Non-bonded
e_nb = 0
for i in range(N):
for j in range(i-1):
ri = coord[i]
rj = coord[j]
rij = ri - rj
rij2 = np.dot(rij, rij)
if (np.sqrt(rij2) < rcutoff):
e_nb += lj(rij2)
# Bonded
e_bond = 0
for i in range(1, N):
ri = coord[i]
rj = coord[i-1]
rij = ri - rj
rij2 = np.dot(rij, rij)
e_bond += fene(rij2)
return e_nb + e_bond
def move(coord):
trial = np.ndarray.copy(coord)
for i in range(N):
delta = (2.0 * np.random.rand(3) - 1) * max_delta
trial[i] += delta
return trial
def accept(delta_e):
beta = 1.0/T
if delta_e <= 0.0:
return True
random_number = np.random.rand(1)
p_acc = np.exp(-beta*delta_e)
if random_number < p_acc:
return True
return False
if __name__ == "__main__":
# FENE parameters
K = 40
R = 0.3
r0 = 0.7
# LJ parameters
sigma = r0/0.33
epsilon = 1.0
# MC parameters
N = 50 # number of particles
rcutoff = 2.5*sigma
max_delta = 0.01
n_steps = 10000000
T = 0.5
coord = gen_chain(N, R)
energy_current = total_energy(coord)
traj = open('traj.xyz', 'w')
for step in range(n_steps):
if step % 1000 == 0:
traj.write(str(N) + '\n\n')
for i in range(N):
traj.write("C %10.5f %10.5f %10.5f\n" % (coord[i][0], coord[i][1], coord[i][2]))
print(step, energy_current)
coord_trial = move(coord)
energy_trial = total_energy(coord_trial)
delta_e = energy_trial - energy_current
if accept(delta_e):
coord = coord_trial
energy_current = energy_trial
traj.close()
The problem is that calculating rij2 = np.dot(rij, rij) in total energy with the constant values you use is always a very small number. Looking at the expression inside the log used to calculate FENE, np.log(1-((np.sqrt(rij2) - r0) / R)**2), I first noticed that you're taking the square root of rij2 which is not consistent with the formula you provided.
Secondly, notice that ((rij2 - r0) / R)**2 is the same as ((r0 - rij2) / R)**2, since the sign gets lost when squaring. Because rij2 is very small (already in the first iteration -- I checked by printing the values), this will be more or less equal to ((r0 - 0.05)/R)**2 which will be a number bigger than 1. Once you subtract this value from 1 in the log expression, 1-((np.sqrt(rij2) - r0) / R)**2 will be equal to np.nan (standing for "Not A Number"). This will propagate through all the function calls (for example, calling energy_trial = total_energy(coord_trial) will effectively set energy_trial to np.nan), until an error will be raised by some function.
Maybe you could do something with np.isnan() call, documented here. Moreover, you should check how you iterate through the coord (there's some inconsistencies throughout the code) -- I suggest you check the code review community as well.
I am trying to implement self attention in Pytorch.
I need to calculate the following expressions.
Similarity function S (2 dimensional), P(2 dimensional), C'
S[i][j] = W1 * inp[i] + W2 * inp[j] + W3 * x1[i] * inp[j]
P[i][j] = e^(S[i][j]) / Sum for all j( e ^ (S[i]))
basically, P is a softmax function
C'[i] = Sum (for all j) P[i][j] * x1[j]
I tried the following code using for loops
for i in range(self.dim):
for j in range(self.dim):
S[i][j] = self.W1 * x1[i] + self.W2 * x1[j] + self.W3 * x1[i] * x1[j]
for i in range(self.dim):
for j in range(self.dim):
P[i][j] = torch.exp(S[i][j]) / torch.sum( torch.exp(S[i]))
# attend
for i in range(self.dim):
out[i] = 0
for j in range(self.dim):
out[i] += P[i][j] * x1[j]
Is there any faster way to implement this in Pytorch?
Here is an example of Self Attention I had implemented in Dual Attention for HSI Imagery
class PAM_Module(Module):
""" Position attention module https://github.com/junfu1115/DANet/blob/master/encoding/nn/attention.py"""
#Ref from SAGAN
def __init__(self, in_dim):
super(PAM_Module, self).__init__()
self.chanel_in = in_dim
self.query_conv = Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
self.key_conv = Conv2d(in_channels=in_dim, out_channels=in_dim//8, kernel_size=1)
self.value_conv = Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)
self.gamma = Parameter(torch.zeros(1))
self.softmax = Softmax(dim=-1)
def forward(self, x):
"""
inputs :
x : input feature maps( B X C X H X W)
returns :
out : attention value + input feature
attention: B X (HxW) X (HxW)
"""
m_batchsize, C, height, width = x.size()
proj_query = self.query_conv(x).view(m_batchsize, -1, width*height).permute(0, 2, 1)
proj_key = self.key_conv(x).view(m_batchsize, -1, width*height)
energy = torch.bmm(proj_query, proj_key)
attention = self.softmax(energy)
proj_value = self.value_conv(x).view(m_batchsize, -1, width*height)
out = torch.bmm(proj_value, attention.permute(0, 2, 1))
out = out.view(m_batchsize, C, height, width)
out = self.gamma*out + x
#out = F.avg_pool2d(out, out.size()[2:4])
return out
I would like to run these two parts of the code in parallel. Is this possible in python? How would i have to modify the code to accommodate this?
def smo(self, X, y):
iterations = 0
n_samples = X.shape[0]
# Initial coefficients
alpha = numpy.zeros(n_samples)
# Initial gradient
g = numpy.ones(n_samples)
while True:
yg = g * y
# KKT Conditions
y_pos_ind = (y == 1)
y_neg_ind = (numpy.ones(n_samples) - y_pos_ind).astype(bool)
alpha_pos_ind = (alpha >= self.C)
alpha_neg_ind = (alpha <= 0)
indices_violating_Bi_1 = y_pos_ind * alpha_pos_ind
indices_violating_Bi_2 = y_neg_ind * alpha_neg_ind
indices_violating_Bi = indices_violating_Bi_1 + indices_violating_Bi_2
yg_i = yg.copy()
yg_i[indices_violating_Bi] = float('-inf')
# First of the maximum violating pair
i = numpy.argmax(yg_i)
Kik = self.kernel_matrix(X, i)
indices_violating_Ai_1 = y_pos_ind * alpha_neg_ind
indices_violating_Ai_2 = y_neg_ind * alpha_pos_ind
indices_violating_Ai = indices_violating_Ai_1 + indices_violating_Ai_2
yg_j = yg.copy()
yg_j[indices_violating_Ai] = float('+inf')
# Second of the maximum violating pair
j = numpy.argmin(yg_j)
Kjk = self.kernel_matrix(X, j)
# Optimality criterion
if(yg_i[i] - yg_j[j]) < self.tol or (iterations >= self.max_iter):
break
min_term_1 = (y[i] == 1) * self.C - y[i] * alpha[i]
min_term_2 = y[j] * alpha[j] + (y[j] == -1) * self.C
min_term_3 = (yg_i[i] - yg_j[j]) / (Kik[i] + Kjk[j] - 2 * Kik[j])
# Direction search
lamda = numpy.min([min_term_1, min_term_2, min_term_3])
# Gradient update
g += lamda * y * (Kjk - Kik)
# Update coefficients
alpha[i] = alpha[i] + y[i] * lamda
alpha[j] = alpha[j] - y[j] * lamda
iterations += 1
print('{} iterations to arrive at the minimum'.format(iterations))
return alpha
I would like to run this line
Kik = self.kernel_matrix(X, i)
and this line
Kjk = self.kernel_matrix(X, j)
in parallel. How do i change the code to accommodate this?
Giving you a response with just the finished multi threading code probably wouldn't be that helpful to you and is hard given I don't know what the functions themselves do but check out this link: https://realpython.com/intro-to-python-threading/
The general idea is you will have to start a thread for each task you want to run in parallel like this:
thread1 = threading.Thread(target=kernel_matrix,args=(X,j))
thread1.start()
If you want to wait for a thread to finish you call thread.join()
You'll need to watch out for race conditions too good thread on that here: What is a race condition?
I've just started to play around with Reinforcement Learning these days and I found the Natural Evolution Strategy, I kind of understand how it works, but I'm very new with Python and I found this code which basically implements the NES algorithm
https://github.com/huseinzol05/Stock-Prediction-Models/blob/master/agent/updated-NES-google.ipynb
import numpy as np
import pandas as pd
import time
import matplotlib.pyplot as plt
import seaborn as sns
import random
sns.set()
# CSV containing the TSLA stock predictions in the form of
# [Date, Open, High, Low, Close, Adj Close, Volume] from
# Yahoo! Finance
df = pd.read_csv('TSLA.csv')
df.head()
def get_state(data, t, n):
d = t - n + 1
block = data[d : t + 1] if d >= 0 else -d * [data[0]] + data[0 : t + 1]
res = []
for i in range(n - 1):
res.append(block[i + 1] - block[i])
return np.array([res])
close = df.Close.values.tolist()
window_size = 30
skip = 1
l = len(close) - 1
class Deep_Evolution_Strategy:
inputs = None
def __init__(
self, weights, reward_function, population_size, sigma, learning_rate
):
self.weights = weights
self.reward_function = reward_function
self.population_size = population_size
self.sigma = sigma
self.learning_rate = learning_rate
def _get_weight_from_population(self, weights, population):
weights_population = []
for index, i in enumerate(population):
jittered = self.sigma * i
weights_population.append(weights[index] + jittered)
return weights_population
def get_weights(self):
return self.weights
def train(self, epoch = 100, print_every = 1):
lasttime = time.time()
for i in range(epoch):
population = []
rewards = np.zeros(self.population_size)
for k in range(self.population_size):
x = []
for w in self.weights:
x.append(np.random.randn(*w.shape))
population.append(x)
for k in range(self.population_size):
weights_population = self._get_weight_from_population(self.weights, population[k])
rewards[k] = self.reward_function(weights_population)
rewards = (rewards - np.mean(rewards)) / np.std(rewards)
for index, w in enumerate(self.weights):
A = np.array([p[index] for p in population])
self.weights[index] = (
w
+ self.learning_rate
/ (self.population_size * self.sigma)
* np.dot(A.T, rewards).T
)
class Model:
def __init__(self, input_size, layer_size, output_size):
self.weights = [
np.random.randn(input_size, layer_size),
np.random.randn(layer_size, output_size),
np.random.randn(layer_size, 1),
np.random.randn(1, layer_size),
]
def predict(self, inputs):
feed = np.dot(inputs, self.weights[0]) + self.weights[-1]
decision = np.dot(feed, self.weights[1])
buy = np.dot(feed, self.weights[2])
return decision, buy
def get_weights(self):
return self.weights
def set_weights(self, weights):
self.weights = weights
class Agent:
POPULATION_SIZE = 15
SIGMA = 0.1
LEARNING_RATE = 0.03
def __init__(self, model, money, max_buy, max_sell):
self.model = model
self.initial_money = money
self.max_buy = max_buy
self.max_sell = max_sell
self.es = Deep_Evolution_Strategy(
self.model.get_weights(),
self.get_reward,
self.POPULATION_SIZE,
self.SIGMA,
self.LEARNING_RATE,
)
def act(self, sequence):
decision, buy = self.model.predict(np.array(sequence))
return np.argmax(decision[0]), int(buy[0])
def get_reward(self, weights):
initial_money = self.initial_money
starting_money = initial_money
self.model.weights = weights
state = get_state(close, 0, window_size + 1)
inventory = []
quantity = 0
for t in range(0, l, skip):
action, buy = self.act(state)
next_state = get_state(close, t + 1, window_size + 1)
if action == 1 and initial_money >= close[t]:
if buy < 0:
buy = 1
if buy > self.max_buy:
buy_units = self.max_buy
else:
buy_units = buy
total_buy = buy_units * close[t]
initial_money -= total_buy
inventory.append(total_buy)
quantity += buy_units
elif action == 2 and len(inventory) > 0:
if quantity > self.max_sell:
sell_units = self.max_sell
else:
sell_units = quantity
quantity -= sell_units
total_sell = sell_units * close[t]
initial_money += total_sell
state = next_state
return ((initial_money - starting_money) / starting_money) * 100
def fit(self, iterations, checkpoint):
self.es.train(iterations, print_every = checkpoint)
def buy(self):
initial_money = self.initial_money
state = get_state(close, 0, window_size + 1)
starting_money = initial_money
states_sell = []
states_buy = []
inventory = []
quantity = 0
for t in range(0, l, skip):
action, buy = self.act(state)
next_state = get_state(close, t + 1, window_size + 1)
if action == 1 and initial_money >= close[t]:
if buy < 0:
buy = 1
if buy > self.max_buy:
buy_units = self.max_buy
else:
buy_units = buy
total_buy = buy_units * close[t]
initial_money -= total_buy
inventory.append(total_buy)
quantity += buy_units
states_buy.append(t)
elif action == 2 and len(inventory) > 0:
bought_price = inventory.pop(0)
if quantity > self.max_sell:
sell_units = self.max_sell
else:
sell_units = quantity
if sell_units < 1:
continue
quantity -= sell_units
total_sell = sell_units * close[t]
initial_money += total_sell
states_sell.append(t)
try:
invest = ((total_sell - bought_price) / bought_price) * 100
except:
invest = 0
state = next_state
invest = ((initial_money - starting_money) / starting_money) * 100
model = Model(window_size, 500, 3)
agent = Agent(model, 10000, 5, 5)
agent.fit(500, 10)
agent.buy()
As you can see, it is being used for stock prediction and it only uses the Close column, but I would like to try it with more parameters, let's say High and Low.
I'm struggling when I need to change it to use this 2 dimensional list. I've tried a simple change:
close = df.loc[:,['Close','Open']].values.tolist()
Which adds one more property at every row of the list. But when I run the code I start to see errors when I execute the agent.fit() call:
agent.fit(iterations = 500, checkpoint = 10)
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-225-d97697984016> in <module>()
----> 1 agent.fit(iterations = 500, checkpoint = 10)
<ipython-input-223-35d9fbba5756> in fit(self, iterations, checkpoint)
66
67 def fit(self, iterations, checkpoint):
---> 68 self.es.train(iterations, print_every = checkpoint)
69
70 def buy(self):
<ipython-input-220-84ca345091f4> in train(self, epoch, print_every)
33 self.weights, population[k]
34 )
---> 35 rewards[k] = self.reward_function(weights_population)
36 rewards = (rewards - np.mean(rewards)) / np.std(rewards)
37
<ipython-input-223-35d9fbba5756> in get_reward(self, weights)
36
37 self.model.weights = weights
---> 38 state = get_state(self.close, 0, self.window_size + 1)
39 inventory = []
40 quantity = 0
<ipython-input-219-0df8d8be24a9> in get_state(data, t, n)
4 res = []
5 for i in range(n - 1):
----> 6 res.append(block[i + 1] - block[i])
7 return np.array([res])
TypeError: unsupported operand type(s) for -: 'list' and 'list'
I assume that the first step is that I need to update my Model class to use a different input_size parameter right?
Any help would be appreciated! Thanks
I'm having a problem debugging the following code, for some reason The perceptron stops updating itself after a couple of steps with random values as the weights. I have tried not using a class for my work and edited everything to the bare minimum, but still had the same problem. I have also checked the Perceptron.train(), and it works just fine. So, I'm guessing the main problem is with the train function itself. I am kind of new to python programming so any help would be apreciated guys.
import random
import Plot as plt
import numpy as np
#-----Function Of the line that seperates the two different Data Types-----$
def f(x):
return x
#-----Activation Function-----#
def act(x):
if x >= 0:
return 1.0
return 0.0
class Point:
def __init__(self, x, y):
self.X = x
self.Y = y
if y > f(x):
self.Target = 1.0
else:
self.Target = 0.0
class Perceptron:
def __init__(self, n, actFunc = act, lr = 0.2):
self.Weights = [0 for i in range(n)]
self.ActFunc = actFunc
self.LR = lr
def guess(self, inputs):
valSum = 0
for i in range(len(inputs)):
valSum += self.Weights[i] * inputs[i]
return self.ActFunc(valSum)
def train(self, inputs, target):
cal = self.guess(inputs)
err = target - cal
for i in range(0, len(self.Weights)):
self.Weights[i] += self.LR * err * inputs[i]
def printWeights(self):
for i in range(len(self.Weights)):
print("WEIGHT[" + str(i) + "] = " + str(self.Weights[i]))
print("")
def lineFunc(self):
# y = w0 + w1x + w2y
# (1 - w2)y = w0 + w1x
# y = w0/(1-w2) + w1/(1 - w2)x
w0 = self.Weights[0]
w1 = self.Weights[1]
w2 = self.Weights[2]
return (str(w0/(1 - w2)) + " + " + str(w1/(1 - w2)) + " * x")
#-----INITIALISING DATA------#
brain = Perceptron(3)
n = 20
points = [Point(random.uniform(-10, 10), random.uniform(-10, 10)) for x in range(n)]
t = 1000
#-----Training-----#
for i in range(t):
point = points[random.randrange(0, n)]
brain.train([1, point.X, point.Y], point.Target)
brain.printWeights()
print(brain.lineFunc())
I did find the problem myself. There was a bug in the LineFunc() method. The return value was wrong and it should have been:
return (str(-w0/w2) + " + " + str(-w1/w2) + " * x")