I am working on a 2D matrix and finding sum of elements, below is my logic:
def calculateSum(a, x, y):
s = 0;
for i in range(0,x+1):
for j in range(0,y+1):
s = s + a[i][j];
print(s)
return s
def check(a):
arr = []
x = 0
y = 0
for i in range(len(a)):
row = []
y = 0
for j in range(len(a[i])):
row.append(calculateSum(a, x, y))
y = y + 1
x = x + 1
print(row)
check([[1, 2], [3, 4]])
calculateSum is the function that calculates sum of elements.
Now my question is, if the matrix size is huge then is there is a way to improve performance of the above program?
Update:
import numpy as np
def calculateSum(a, x, y):
return np.sum(a[x:,y:])
After using numpy I am getting error as TypeError: list indices must be integers or slices, not tuple if I use numpy
As the matrix dimensions increases, Efficiency will fall, the efficient way to deal with this is to parallelize the task of summing the values, this is possible because addition follows Associative property.
Luckily for you this parallelization is already implemented in a library known as numpy.
To get started with numpy, use pip install numpy To get an overview of the library visit: https://www.geeksforgeeks.org/numpy-in-python-set-1-introduction/
And for your question you will need to use function numpy.sum()
Edit:
Also as #Mad Physicist pointed out Numpy also has packed memory layout and the routines are implemented in C which boost its speed even further.
Related
I tried using Numpy, Scipy and Scikitlearn, but couldn't find what I need in any of them, basically I need to fit a curve to a dataset, but restricting some of the coefficients to known values, I found how to do it in MATLAB, using fittype, but couldn't do it in python.
In my case I have a dataset of X and Y and I need to find the best fitting curve, I know it's a polynomial of second degree (ax^2 + bx + c) and I know it's values of b and c, so I just needed it to find the value of a.
The solution I found in MATLAB was https://www.mathworks.com/matlabcentral/answers/216688-constraining-polyfit-with-known-coefficients which is the same problem as mine, but with the difference that their polynomial was of degree 5th, how could I do something similar in python?
To add some info: I need to fit a curve to a dataset, so things like scipy.optimize.curve_fit that expects a function won't work (at least as far as I tried).
The tools you have available usually expect functions only inputting their parameters (a being the only unknown in your case), or inputting their parameters and some data (a, x, and y in your case).
Scipy's curve-fit handles that use-case just fine, so long as we hand it a function that it understands. It expects x first and all your parameters as the remaining arguments:
from scipy.optimize import curve_fit
import numpy as np
b = 0
c = 0
def f(x, a):
return c+x*(b+x*a)
x = np.linspace(-5, 5)
y = x**2
# params == [1.]
params, _ = curve_fit(f, x, y)
Alternatively you can reach for your favorite minimization routine. The difference here is that you manually construct the error function so that it only inputs the parameters you care about, and then you don't need to provide that data to scipy.
from scipy.optimize import minimize
import numpy as np
b = 0
c = 0
x = np.linspace(-5, 5)
y = x**2
def error(a):
prediction = c+x*(b+x*a)
return np.linalg.norm(prediction-y)/len(prediction)**.5
result = minimize(error, np.array([42.]))
assert result.success
# params == [1.]
params = result.x
I don't think scipy has a partially applied polynomial fit function built-in, but you could use either of the above ideas to easily build one yourself if you do that kind of thing a lot.
from scipy.optimize import curve_fit
import numpy as np
def polyfit(coefs, x, y):
# build a mapping from null coefficient locations to locations in the function
# coefficients we're passing to curve_fit
#
# idx[j]==i means that unknown_coefs[i] belongs in coefs[j]
_tmp = [i for i,c in enumerate(coefs) if c is None]
idx = {j:i for i,j in enumerate(_tmp)}
def f(x, *unknown_coefs):
# create the entire polynomial's coefficients by filling in the unknown
# values in the right places, using the aforementioned mapping
p = [(unknown_coefs[idx[i]] if c is None else c) for i,c in enumerate(coefs)]
return np.polyval(p, x)
# we're passing an initial value just so that scipy knows how many parameters
# to use
params, _ = curve_fit(f, x, y, np.zeros((sum(c is None for c in coefs),)))
# return all the polynomial's coefficients, not just the few we just discovered
return np.array([(params[idx[i]] if c is None else c) for i,c in enumerate(coefs)])
x = np.linspace(-5, 5)
y = x**2
# (unknown)x^2 + 1x + 0
# params == [1, 0, 0.]
params = fit([None, 0, 0], x, y)
Similar features exist in nearly every mainstream scientific library; you just might need to reshape your problem a bit to frame it in terms of the available primitives.
I'm trying to write a script that computes numerical derivatives using the forward, backward, and centered approximations, and plots the results. I've made a linspace from 0 to 2pi with 100 points. I've made many arrays and linspaces in the past, but I've never seen this error: "ValueError: sequence too large; cannot be greater than 32"
I don't understand what the problem is. Here is my script:
import numpy as np
import matplotlib.pyplot as plt
def f(x):
return np.cos(x) + np.sin(x)
def f_diff(x):
return np.cos(x) - np.sin(x)
def forward(x,h): #forward approximation
return (f(x+h)-f(x))/h
def backward(x,h): #backward approximation
return (f(x)-f(x-h))/h
def center(x,h): #center approximation
return (f(x+h)-f(x-h))/(2*h)
x0 = 0
x = np.linspace(0,2*np.pi,100)
forward_result = np.zeros(x)
backward_result = np.zeros(x)
center_result = np.zeros(x)
true_result = np.zeros(x)
for i in range(x):
forward_result[i] = forward[x0,i]
true_result[i] = f_diff[x0]
print('Forward (x0={}) = {}'.format(x0,forward(x0,x)))
#print('Backward (x0={}) = {}'.format(x0,backward(x0,dx)))
#print('Center (x0={}) = {}'.format(x0,center(x0,dx)))
plt.figure()
plt.plot(x, f)
plt.plot(x,f_diff)
plt.plot(x, abs(forward_result-true_result),label='Forward difference')
I did try setting the linspace points to 32, but that gave me another error: "TypeError: 'numpy.float64' object cannot be interpreted as an integer"
I don't understand that one either. What am I doing wrong?
The issue starts at forward_result = np.zeros(x) because x is a numpy array not a dimension. Since x has 100 entries, np.zeros wants to create object in R^x[0] times R^x[1] times R^x[3] etc. The maximum dimension is 32.
You need a flat np array.
UPDATE: On request, I add corrected lines from code above:
forward_result = np.zeros(x.size) creates the array of dimension 1.
Corrected evaluation of the function is done via circular brackets. Also fixed the loop:
for i, h in enumerate(x):
forward_result[i] = forward(x0,h)
true_result[i] = f_diff(x0)
Finally, in the figure, you want to plot numpy array vs function. Fixed version:
plt.plot(x, [f(val) for val in x])
plt.plot(x, [f_diff(val) for val in x])
I have a system of equations to solve which goes like this:
https://imgur.com/oxVdM10
the left side of the equation is the sum, and the right side is the constraints. I have a system of N nonlinear equations with N variables.
I tried solving it with scipy fsolve but it won't converge, sympy also has a solver, doesn't converge either. I came across pyscipopt which seem to work but behaves inconsistently and breaks down.
import numpy as np
import networkx as nx
''' set up the problem with n nodes, and the constraint array c with the degree sequence'''
''' example : power law degree distribution'''
n=8
c = np.round(nx.utils.random_sequence.powerlaw_sequence(n))
''' other examples '''
#n=8
#c=[1, 2, 2, 1, 2, 1, 1, 2]
#n=3
#c=[2,1,1]
from pyscipopt import Model, quicksum
m = Model()
''' tolerance for convergence'''
tol = 1e-6
print('create the lagrange multipliers')
X = dict(zip(range(n), [m.addVar(vtype="C", lb=0) for i in range(n)]))
#X = dict(zip(range(n), [m.addVar(vtype="C") for i in range(n)]))
''' create the variable for the objective function because it's nonlinear
and must be linearized'''
Y = m.addVar(vtype='C')
print('set up the constraints')
''' the equations are essentially sum_i - degree_i <= tolerance'''
''' set up the constraints as sums of the lagrange multipliers as written in the papers'''
system =[]
for i in range(n):
idx = np.arange(n)
idx = idx[idx!=i]
k= quicksum(X[i]*X[j]/(1+X[i]*X[j]) for j in idx) -c[i]
''' the equations are essentially sum_i - degree_i <= tolerance'''
m.addCons(k<=tol)
system.append(k)
m.addCons( Y <= quicksum(system[j] for j in range(n)) )
m.setObjective(Y, 'minimize')
print('all constraints added, now optimization')
m.optimize()
So I have a system array where I store all the constraints to be optimized, the k values are the constraints, I perform a double loop over the entire data set.
1) are there better, or faster ways of achieving this? I'm guessing that for large N it wouldn't be efficient.
one example which works fast:
n=8
c=[1, 2, 2, 1, 2, 1, 1, 2]
but other examples, especially once I increase N (I use little n in the code), get stuck somehow in limbo.
edit: regarding the examples I have to say that any example should work. The hard constraint in designing an example is simply k_i <= N.
I have a 2d numpy matrix and want to calculate the following test statistic.
I have brute-force code to do it, but it seems like there should be a more general numpy solution that works for any 2D matrix, using things like np.diag(). I can't figure it out though.
def bruteforce(m):
s = 0.0
for (i,j) in itertools.product(range(0,m.shape[0]),range(0,m.shape[0])):
if i<j:
n = (m[i,j]-m[j,i])**2
d = m[i,j]+m[j,i]
if float(d) != 0.:
s = s+(float(n)/float(d))
else:
return('NA')
return(s)
Where in this case m is an NxN matrix of integers. Is there a way to do it vectorised in numpy, avoiding brute force loops like this?
If m is a square matrix, this will do the job:
import numpy as np
np.sum((m-m.T)**2/(m+m.T))/2
Here is a function that covers the case in which there is 0 in the denominator:
def find_s(m):
d=(m+m.T)
off_diag_indices=np.triu_indices(len(d),1)
if 0 in d[off_diag_indices]:
return 'NA'
else:
numerator=(m-m.T)**2
denominator=m+m.T
return np.sum(numerator[off_diag_indices]/denominator[off_diag_indices])
The reason that I used off_diag_indices is because we actually do allow to have 0 on the diagonal of m+m.T, because we never sum the elements on the diagonal.
According to MKL BLAS documentation
"All matrix-matrix operations (level 3) are threaded for both dense and sparse BLAS."
http://software.intel.com/en-us/articles/parallelism-in-the-intel-math-kernel-library
I have built Scipy with MKL BLAS. Using the test code below, I see the expected multithreaded speedup for dense, but not sparse, matrix multiplication. Are there any changes to Scipy to enable multithreaded sparse operations?
# test dense matrix multiplication
from numpy import *
import time
x = random.random((10000,10000))
t1 = time.time()
foo = dot(x.T, x)
print time.time() - t1
# test sparse matrix multiplication
from scipy import sparse
x = sparse.rand(10000,10000)
t1 = time.time()
foo = dot(x.T, x)
print time.time() - t1
As far as I know, the answer is no. But, you can build your own wrapper around the MKL sparse multiply routines. You asked about the multiplying two sparse matrices. Below is some a wrapper code I used for multiplying one sparse matrix times a dense vector, so it shouldn't be hard to adapt (look at the Intel MKL reference for mkl_cspblas_dcsrgemm). Also, be aware of how your scipy arrays are stored: default is coo, but csr (or csc) may be better choices. I chose csr, but MKL supports most types (just call the appropriate routine).
From what I could tell, both scipy's default and MKL are multithreaded. By changing OMP_NUM_THREADS I could see a difference in performance.
To use the function below, if you havea a recent version of MKL, just make sure you have LD_LIBRARY_PATHS set to include the relevant MKL directories. For older versions, you need to build some specific libraries. I got my information from IntelMKL in python
def SpMV_viaMKL( A, x ):
"""
Wrapper to Intel's SpMV
(Sparse Matrix-Vector multiply)
For medium-sized matrices, this is 4x faster
than scipy's default implementation
Stephen Becker, April 24 2014
stephen.beckr#gmail.com
"""
import numpy as np
import scipy.sparse as sparse
from ctypes import POINTER,c_void_p,c_int,c_char,c_double,byref,cdll
mkl = cdll.LoadLibrary("libmkl_rt.so")
SpMV = mkl.mkl_cspblas_dcsrgemv
# Dissecting the "cspblas_dcsrgemv" name:
# "c" - for "c-blas" like interface (as opposed to fortran)
# Also means expects sparse arrays to use 0-based indexing, which python does
# "sp" for sparse
# "d" for double-precision
# "csr" for compressed row format
# "ge" for "general", e.g., the matrix has no special structure such as symmetry
# "mv" for "matrix-vector" multiply
if not sparse.isspmatrix_csr(A):
raise Exception("Matrix must be in csr format")
(m,n) = A.shape
# The data of the matrix
data = A.data.ctypes.data_as(POINTER(c_double))
indptr = A.indptr.ctypes.data_as(POINTER(c_int))
indices = A.indices.ctypes.data_as(POINTER(c_int))
# Allocate output, using same conventions as input
nVectors = 1
if x.ndim is 1:
y = np.empty(m,dtype=np.double,order='F')
if x.size != n:
raise Exception("x must have n entries. x.size is %d, n is %d" % (x.size,n))
elif x.shape[1] is 1:
y = np.empty((m,1),dtype=np.double,order='F')
if x.shape[0] != n:
raise Exception("x must have n entries. x.size is %d, n is %d" % (x.size,n))
else:
nVectors = x.shape[1]
y = np.empty((m,nVectors),dtype=np.double,order='F')
if x.shape[0] != n:
raise Exception("x must have n entries. x.size is %d, n is %d" % (x.size,n))
# Check input
if x.dtype.type is not np.double:
x = x.astype(np.double,copy=True)
# Put it in column-major order, otherwise for nVectors > 1 this FAILS completely
if x.flags['F_CONTIGUOUS'] is not True:
x = x.copy(order='F')
if nVectors == 1:
np_x = x.ctypes.data_as(POINTER(c_double))
np_y = y.ctypes.data_as(POINTER(c_double))
# now call MKL. This returns the answer in np_y, which links to y
SpMV(byref(c_char("N")), byref(c_int(m)),data ,indptr, indices, np_x, np_y )
else:
for columns in xrange(nVectors):
xx = x[:,columns]
yy = y[:,columns]
np_x = xx.ctypes.data_as(POINTER(c_double))
np_y = yy.ctypes.data_as(POINTER(c_double))
SpMV(byref(c_char("N")), byref(c_int(m)),data,indptr, indices, np_x, np_y )
return y