I'm trying to automate data collection from an SR245 Boxcar using Python 3.6 and the PyVisa library (version 1.11.1). 9/10 times, it works great. However, three times over the course of two days it has caused the entire computer to crash and reboot (running on Windows 10). This has resulted in a lot of data loss, and I'm trying to figure out what I'm doing wrong that is leading to the whole system crashing. Code is below (it is part of a larger program, but I also run this piece of code by itself, and it has caused crashes). The data_processing file is not shown, but the functions there are simple calculations (e.g. divide the values in a list by the values in another list, return the average value from a list of integers, etc.)
import pyvisa
from pyvisa.constants import SerialTermination
import time
import numpy as np
from data_processing import *
def connect_boxcar(pNum):
rm = pyvisa.ResourceManager()
port = "COM"+pNum
sr = rm.open_resource(port)
return sr
def config_boxcar(boxcar):
#Configure the boxcar settings
boxcar.write_termination = '\r'
boxcar.read_termination='\r'
boxcar.baud_rate=19200
boxcar.end_output = SerialTermination.termination_char
def preset_scan(boxcar):
#Reset boxcar settings
boxcar.write('MR')
boxcar.write('MS;ET;T1;I2;W0')
def scan(boxcar, num):
#Send the SCAN command to the boxcar, set to the specified number of data points
command = 'SC1,2:' + str(num)
boxcar.write(command)
def read_data(boxcar, num):
#Read the stored scan data and return it as a value list
data_list = []
for x in range(num * 2):
data_list.append(float(boxcar.query('N')))
return data_list
def collect_baseline(boxcar, n):
#Get a baseline signal for later processing
config_boxcar(boxcar)
preset_scan(boxcar)
scan(boxcar, n)
raw_data = read_data(boxcar, n)
chan1 = raw_data[::2]
chan2 = raw_data[1::2]
normal_data = normalize(chan1, chan2, n)
return average_list(normal_data)
def main():
rm = pyvisa.ResourceManager()
n = 10
sleep_timer = 0.1 * n + 0.5
sr245 = rm.open_resource('COM5')
#Configure/preset
config_boxcar(sr245)
preset_scan(sr245)
#Set a timer to measure scanning time
t0 = time.time()
scan(sr245, n)
time.sleep(sleep_timer)
raw_data = read_data(sr245, n)
t1 = time.time()
#Breakdown data by channel and normalize
chan1 = raw_data[::2]
chan2 = raw_data[1::2]
normal_data = normalize(chan1, chan2, n)
elapsed_time = t1 - t0
print('Elapsed time: ', elapsed_time)
print('Channel 1: ', chan1)
print('Channel 2: ', chan2)
print('Normalized Data: ', normal_data)
print('Average Normalized Data: ', average_list(normal_data))
print('Standard Deviation: ', np.std(normal_data))
if __name__ == '__main__':
main()
Related
I had a dataset including about a million of rows. Before, I read the rows, preprocessed data and created a list of rows to be trained. Then I defined a Dataloader over this data like:
train_dataloader = torch.utils.data.DataLoader(mydata['train'],
batch_size=node_batch_size,shuffle=shuffle,collate_fn=data_collator)
Preprocessing could be time consuming, so I thought to define an IterableDataSet with __iter__ function. Then I could define my Dataloader like:
train_dataloader = torch.utils.data.DataLoader(myds['train'],
batch_size=node_batch_size,shuffle=shuffle,collate_fn=data_collator)
However, still to begin training it seems that it calls my preprocessing function and creates an Iteration over it. So, it seems I didn't gain much speed up.
Please guide me how could I use speed up in this case?
Here is my part of my class:
def __iter__(self):
iter_start = self.start
iter_end = self.num_samples
worker_info = torch.utils.data.get_worker_info()
if worker_info is None: # single-process data loading, return the full iterator
iter_start = self.start
iter_end = self.num_samples
else: # in a worker process
# split workload
per_worker = int(math.ceil((self.num_samples - self.start) / float(worker_info.num_workers)))
worker_id = worker_info.id
iter_start = self.start + worker_id * per_worker
iter_end = min(iter_start + per_worker, self.num_samples)
if self.flat_data:
return iter(self.flat_data)
else:
return iter(self.fill_data(iter_start, iter_end))
def fill_data(self, iter_start, iter_end, show_progress=False):
flat_data = []
if iter_end < 0:
iter_end = self.num_samples
kk = 0
dlog.info("========================== SPLIT: %s", self.split_name)
dlog.info("get data from %s to %s", iter_start, iter_end)
dlog.info("total rows: %s", len(self.split_df))
if show_progress:
pbar = tqdm(total = self.num_samples)
for index, d in self.split_df.iterrows():
if kk < iter_start:
dlog.info("!!!!!!!!! before start %s", iter_start)
kk += 1
continue
rel = d["prefix"]
...
# preprocessing and adding to returned list
I did preprosessing in the fill_data or __iter__ body. However, I can use a map for preprocessing. Then the preprocessing is called during training and for every batch and not before training.
import pandas as pd
import torch
class MyDataset(torch.utils.data.IterableDataset):
def __init__(self, fname, until=10):
self.df = pd.read_table("atomic/" + fname)
self.until = until
def preproc(self, t):
prefix, data = t
text = "Preproc: " + prefix + "|" + data
print(text) # to check when it is called
return text
def __iter__(self):
_iter = self.df_iter()
return map(self.preproc, _iter)
def df_iter(self):
ret = []
for idx, row in self.df.iterrows():
ret.append((row["prefix"],row["input_text"]))
return iter(ret)
I was doing a benchmark for myself that I encountered this interesting thing. I am trying to get the first 30 keys of a dictionary, and I have written three ways to get it as follows:
import time
dic = {str(i): i for i in range(10 ** 6)}
start_time = time.time()
x = list(dic.keys())[0:30]
print(time.time() - start_time)
start_time = time.time()
y = list(dic.keys())
x = y[0:30]
print(time.time() - start_time)
start_time = time.time()
z = dic.keys()
y = list(z)
x = y[0:30]
print(time.time() - start_time)
The results are:
0.015970945358276367
0.010970354080200195
0.01691460609436035
Surprisingly, the second method is much faster! Any thoughts on this?
Using Python's timeit module to measure various alternatives. I added mine which doesn't convert the keys to list:
from timeit import timeit
dic = {str(i): i for i in range(10 ** 6)}
def f1():
x = list(dic.keys())[0:30]
return x
def f2():
y = list(dic.keys())
x = y[0:30]
return x
def f3():
z = dic.keys()
y = list(z)
x = y[0:30]
return x
def f4():
x = [k for _, k in zip(range(30), dic.keys())]
return x
t1 = timeit(lambda: f1(), number=10)
t2 = timeit(lambda: f2(), number=10)
t3 = timeit(lambda: f3(), number=10)
t4 = timeit(lambda: f4(), number=10)
print(t1)
print(t2)
print(t3)
print(t4)
Prints:
0.1911074290110264
0.20418328599771485
0.18727918600779958
3.5186996683478355e-05
Maybe this is due to inaccuracies in your measure of time. You can use timeit for doing this kind of things:
import timeit
dic = {str(i): i for i in range(10 ** 6)}
# 27.5125/29.0836/26.8525
timeit.timeit("x = list(dic.keys())[0:30]", number=1000, globals={"dic": dic})
# 28.6648/26.4684/30.9534
timeit.timeit("y = list(dic.keys());x=y[0:30]", number=1000)
# 31.7345/29.5301/30.7541
timeit.timeit("z=dic.keys();y=list(z);x=y[0:30]", number=1000, globals={'dic': dic})
The comments show the times I got when running the same code 3 different times. As you can see, even by performing a large number of repetitions, it is possible to obtain quite large variations in time measured. This can be due to several different things:
An item can be in the cache of your processor or not.
Your processor can be occupied doing several other things.
Etc...
As stated by #Andrej Kesely, your bottleneck is due to the fact that you cast your dictionary keys into a list. By doing so, Python goes through the entire dictionary keys, because that's how it converts something to a list generally. Hence, by avoiding this, you can get much better results.
I've been trying to implement the fast powering algorithm into python code, but it turned out to be slower than the native approach, where's the problem?
import timeit
test = """
def Power(base, power, mod):
base_to_the_powers = [base]
temp = list(bin(power)[2:])
temp.reverse()
for i in range(len(temp)-1):
base_to_the_powers.append(base_to_the_powers[i]**2%mod)
values_to_multiply = [base_to_the_powers[index] for index,i in enumerate(temp) if i == '1' ]
x=1
for value in values_to_multiply:
x = (x* value) % 1000
return x
Power(3,218,1000)
"""
test2 ="""
result = (3**218)%1000
"""
print(timeit.timeit(test),"\n", timeit.timeit(test2))
I've been working in Reaction-Diffusion cellular automata with the cellpylib library for a course in my university (I wrote it all in one script so you don't have to install/download anything). I'd like to save the evolution of the automata data to a csv file to run some statistics. That is, I'd like to save the data in columns where the first column is 'number of "1"' and the second column: 'time steps'.
Thus, I need help in:
(1) Creating a variable that saves the amount of '1' per time step (I think so).
(2) I need to export all that data to a csv file (number of "1" and the corresponding iteration, from 1 to time_steps in the code below).
The code is the following.
#Libraries
import matplotlib
matplotlib.matplotlib_fname()
import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib.animation as animation
import numpy as np
import csv
# Conditions
#############################
theta = 1 # this is the condition for Moore neighbourhood
Int = 100 # this is the iteration speed (just for visualization)
time_steps = 100 # Iterations
size = 8 # this is the size of the matrix (8x8)
#############################
# Definitions
def plot2d_animate(ca, title=''):
c = mpl.colors.ListedColormap(['green', 'red', 'black', 'gray'])
n = mpl.colors.Normalize(vmin=0,vmax=3)
fig = plt.figure()
plt.title(title)
im = plt.imshow(ca[0], animated=True, cmap=c, norm=n)
i = {'index': 0}
def updatefig(*args):
i['index'] += 1
if i['index'] == len(ca):
i['index'] = 0
im.set_array(ca[i['index']])
return im,
ani = animation.FuncAnimation(fig, updatefig, interval=Int, blit=True)
plt.show()
def init_simple2d(rows, cols, val=1, dtype=np.int):
x = np.zeros((rows, cols), dtype=dtype)
x[x.shape[0]//2][x.shape[1]//2] = val
return np.array([x])
def evolve2d(cellular_automaton, timesteps, apply_rule, r=1, neighbourhood='Moore'):
_, rows, cols = cellular_automaton.shape
array = np.zeros((timesteps, rows, cols), dtype=cellular_automaton.dtype)
array[0] = cellular_automaton
von_neumann_mask = np.zeros((2*r + 1, 2*r + 1), dtype=bool)
for i in range(len(von_neumann_mask)):
mask_size = np.absolute(r - i)
von_neumann_mask[i][:mask_size] = 1
if mask_size != 0:
von_neumann_mask[i][-mask_size:] = 1
def get_neighbourhood(cell_layer, row, col):
row_indices = [0]*(2*r+1)
for i in range(-r,r+1):
row_indices[i+r]=(i+row) % cell_layer.shape[0]
col_indices = [0]*(2*r+1)
for i in range(-r,r+1):
col_indices[i+r]=(i+col) % cell_layer.shape[1]
n = cell_layer[np.ix_(row_indices, col_indices)]
if neighbourhood == 'Moore':
return n
elif neighbourhood == 'von Neumann':
return np.ma.masked_array(n, von_neumann_mask)
else:
raise Exception("unknown neighbourhood type: %s" % neighbourhood)
for t in range(1, timesteps):
cell_layer = array[t - 1]
for row, cell_row in enumerate(cell_layer):
for col, cell in enumerate(cell_row):
n = get_neighbourhood(cell_layer, row, col)
array[t][row][col] = apply_rule(n, (row, col), t)
return array
def ca_reaction_diffusion(neighbourhood, c, t):
center_cell = neighbourhood[1][1]
total = np.sum(neighbourhood==1)
if total >= theta and center_cell==0:
return 1
elif center_cell == 1:
return 2
elif center_cell == 2:
return 3
elif center_cell == 3:
return 0
else:
return 0
# Initial condition
cellular_automaton = init_simple2d(size, size, val=0, dtype=int)
# Excitable initial cells
cellular_automaton[:, [1,2], [1,1]] = 1
# The evolution
cellular_automaton = evolve2d(cellular_automaton,
timesteps=time_steps,
neighbourhood='Moore',
apply_rule=ca_reaction_diffusion)
animation=plot2d_animate(cellular_automaton)
Explanation of the code:
As you can see, there are 4 states: 0 (green), 1 (red), 2 (black) and 3 (gray). The way the automata evolves is with the cellular_automaton conditions. That is, for example, if a center cell has a value of 0 (excitable cell) and at least one cell (theta value) on its Moore neighbourhood is in state 1, in the following time step the same cell will be at state 1 (excited).
To notice:
The configuration of this matrix is toroidal, and the definitions are taken from the cellpylib library.
I've been stuck with this for over a week, so I'd really appreciate some help. Thanks in advance!
I am not well-experienced in this subject matter (and I was not fully clear on what you intended for me to do). I went through and implemented the counting of the specific "0", "1", "2" and "3" value cells in "evolve2d" function. This code should be viewed as "starter code"; whatever specifically you are trying to do should piggyback off of what I have given you. Additionally, this task could have been accomplished through some better code design and definitely, better planning of your function locations (as part of better coding practice and overall cleaner code that is easy to debug). Please peruse and UNDERSTAND the changes that I made.
#Libraries
import matplotlib
matplotlib.matplotlib_fname()
import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib.animation as animation
import numpy as np
import csv
# Conditions
#############################
theta = 1 # this is the condition for Moore neighbourhood
iter_speed = 100 # this is the iteration speed (just for visualization)
time_steps = 100 # Iterations
size = 8 # this is the size of the matrix (8x8)
#############################
# Definitions
def plot2d_animate(ca, title=''):
c = mpl.colors.ListedColormap(['green', 'red', 'black', 'gray'])
n = mpl.colors.Normalize(vmin=0,vmax=3)
fig = plt.figure()
plt.title(title)
im = plt.imshow(ca[0], animated=True, cmap=c, norm=n)
i = {'index': 0}
def updatefig(*args):
i['index'] += 1
if i['index'] == len(ca):
i['index'] = 0
im.set_array(ca[i['index']])
return im,
ani = animation.FuncAnimation(fig, updatefig, interval=iter_speed, blit=True)
plt.show()
#############I ADDED EXTRA ARGUMENTs FOR THE FUNCTION BELOW
def get_neighbourhood(cell_layer, row, col, r = 1, neighbourhood = "Moore"):
row_indices = [0]*(2*r+1)
for i in range(-r,r+1):
row_indices[i+r]=(i+row) % cell_layer.shape[0]
col_indices = [0]*(2*r+1)
for i in range(-r,r+1):
col_indices[i+r]=(i+col) % cell_layer.shape[1]
n = cell_layer[np.ix_(row_indices, col_indices)]
if neighbourhood == 'Moore':
return n
elif neighbourhood == 'von Neumann':
return np.ma.masked_array(n, von_neumann_mask)
else:
raise Exception("unknown neighbourhood type: %s" % neighbourhood)
def init_simple2d(rows, cols, val=1, dtype=np.int):
x = np.zeros((rows, cols), dtype=dtype)
x[x.shape[0]//2][x.shape[1]//2] = val
return np.array([x])
#Inner functions was moved due to bad coding practice. Arguments were also changed. Make sure you understand what I did.
def evolve2d(cellular_automaton, timesteps, apply_rule, r=1, neighbourhood='Moore'):
_, rows, cols = cellular_automaton.shape
array = np.zeros((timesteps, rows, cols), dtype=cellular_automaton.dtype)
array[0] = cellular_automaton
von_neumann_mask = np.zeros((2*r + 1, 2*r + 1), dtype=bool)
for i in range(len(von_neumann_mask)):
mask_size = np.absolute(r - i)
von_neumann_mask[i][:mask_size] = 1
if mask_size != 0:
von_neumann_mask[i][-mask_size:] = 1
#################################################
#These lists keep track of values over the course of the function:
Result_0 = ["Number of 0"]
Result_1 = ["Number of 1"]
Result_2 = ["Number of 2"]
Result_3 = ["Number of 3"]
#################################################
for t in range(1, timesteps):
#################################################
#This dictionary keeps track of values per timestep
value_iter_tracker = {0: 0, 1: 0, 2: 0, 3: 0 }
#################################################
cell_layer = array[t - 1]
for row, cell_row in enumerate(cell_layer):
for col, cell in enumerate(cell_row):
n = get_neighbourhood(cell_layer, row, col)
################################################
res = apply_rule(n, (row, col), t)
value_iter_tracker[res]+=1
array[t][row][col] = res
################################################
print(value_iter_tracker)
########################################################
#Now we need to add the results of the iteration dictionary to the corresponding
#lists in order to eventually export to the csv
Result_0.append(value_iter_tracker[0])
Result_1.append(value_iter_tracker[1])
Result_2.append(value_iter_tracker[2])
Result_3.append(value_iter_tracker[3])
########################################################
############################################################
#function call to export lists to a csv:
timesteps_result = list(range(1, timesteps))
timesteps_result = ["Time Step"] + timesteps_result
#If you don't understand what is going on here, put print statement and/or read docs
vals = zip(timesteps_result, Result_0, Result_1, Result_2, Result_3)
write_to_csv_file(list(vals))
############################################################
return array
################################################################################
#THIS CODE IS FROM:
#https://stackoverflow.com/questions/14037540/writing-a-python-list-of-lists-to-a-csv-file
import pandas as pd
def write_to_csv_file(data):
data = [list(x) for x in data]
my_df = pd.DataFrame(data)
my_df.to_csv('output1.csv', index=False, header=False)
################################################################################
def ca_reaction_diffusion(neighbourhood, c, t):
center_cell = neighbourhood[1][1]
total = np.sum(neighbourhood==1)
if total >= theta and center_cell==0:
return 1
elif center_cell == 1:
return 2
elif center_cell == 2:
return 3
elif center_cell == 3:
return 0
else:
return 0
# Initial condition
cellular_automaton = init_simple2d(size, size, val=0, dtype=int)
# Excitable initial cells
cellular_automaton[:, [1,2], [1,1]] = 1
# The evolution
cellular_automaton = evolve2d(cellular_automaton,
timesteps=time_steps,
neighbourhood='Moore',
apply_rule=ca_reaction_diffusion)
animation=plot2d_animate(cellular_automaton)
I have left comments that should clarify the changes that I made. Essentially, when you call the evolve2d function, a csv file called "output1.csv" is created with the timestep results. I used the pandas package to write the data into a csv but other methods could have been used as well. I will leave it to you to take advantage of the changes that I made for your use. Hope this helps.
I've never used incremental PCA which exists in sklearn and I'm a bit confused about it's parameters and not able to find a good explanation of them.
I see that there is batch_size in the constructor, but also, when using partial_fit method you can again pass only a part of your data, I've found the following way:
n = df.shape[0]
chunk_size = 100000
iterations = n//chunk_size
ipca = IncrementalPCA(n_components=40, batch_size=1000)
for i in range(0, iterations):
ipca.partial_fit(df[i*chunk_size : (i+1)*chunk_size].values)
ipca.partial_fit(df[iterations*chunk_size : n].values)
Now, what I don't understand is the following - when using partial fit, does the batch_size play any role at all, or not? And how are they related?
Moreover, if both are considered, how should I change their values properly, when wanting to increase the precision while increasing memory footprint (and the other way around, decrease the memory consumption for the price of decreased accuracy)?
The docs say:
batch_size : int or None, (default=None)
The number of samples to use for each batch. Only used when calling fit...
This param is not used within partial_fit, where the batch-size is controlled by the user.
Bigger batches will increase memory-consumption, smaller ones will decrease it.
This is also written in the docs:
This algorithm has constant memory complexity, on the order of batch_size, enabling use of np.memmap files without loading the entire file into memory.
Despite some checks and parameter-heuristics, the whole fit-function looks like this:
for batch in gen_batches(n_samples, self.batch_size_):
self.partial_fit(X[batch], check_input=False)
Here is some an incremental PCA code based on https://github.com/kevinhughes27/pyIPCA which is an implementation of CCIPCA method.
import scipy.sparse as sp
import numpy as np
from scipy import linalg as la
import scipy.sparse as sps
from sklearn import datasets
class CCIPCA:
def __init__(self, n_components, n_features, amnesic=2.0, copy=True):
self.n_components = n_components
self.n_features = n_features
self.copy = copy
self.amnesic = amnesic
self.iteration = 0
self.mean_ = None
self.components_ = None
self.mean_ = np.zeros([self.n_features], np.float)
self.components_ = np.ones((self.n_components,self.n_features)) / \
(self.n_features*self.n_components)
def partial_fit(self, u):
n = float(self.iteration)
V = self.components_
# amnesic learning params
if n <= int(self.amnesic):
w1 = float(n+2-1)/float(n+2)
w2 = float(1)/float(n+2)
else:
w1 = float(n+2-self.amnesic)/float(n+2)
w2 = float(1+self.amnesic)/float(n+2)
# update mean
self.mean_ = w1*self.mean_ + w2*u
# mean center u
u = u - self.mean_
# update components
for j in range(0,self.n_components):
if j > n: pass
elif j == n: V[j,:] = u
else:
# update the components
V[j,:] = w1*V[j,:] + w2*np.dot(u,V[j,:])*u / la.norm(V[j,:])
normedV = V[j,:] / la.norm(V[j,:])
normedV = normedV.reshape((self.n_features, 1))
u = u - np.dot(np.dot(u,normedV),normedV.T)
self.iteration += 1
self.components_ = V / la.norm(V)
return
def post_process(self):
self.explained_variance_ratio_ = np.sqrt(np.sum(self.components_**2,axis=1))
idx = np.argsort(-self.explained_variance_ratio_)
self.explained_variance_ratio_ = self.explained_variance_ratio_[idx]
self.components_ = self.components_[idx,:]
self.explained_variance_ratio_ = (self.explained_variance_ratio_ / \
self.explained_variance_ratio_.sum())
for r in range(0,self.components_.shape[0]):
d = np.sqrt(np.dot(self.components_[r,:],self.components_[r,:]))
self.components_[r,:] /= d
You can test it with
import pandas as pd, ccipca
df = pd.read_csv('iris.csv')
df = np.array(df)[:,:4].astype(float)
pca = ccipca.CCIPCA(n_components=2,n_features=4)
S = 10
print df[0, :]
for i in range(150): pca.partial_fit(df[i, :])
pca.post_process()
The resulting eigenvectors / values will not exaactly be the same as the batch PCA. Results are approximate, but they are useful.