Hi guys im not too into python but need to do some research. The problem mainly consists of a file that calculates a large number of non linear equations which takes quite some time. The idea is to implement Multiprocessing in some way. I was wondering if there is a "correct" way to do this, since the main file calls the "computational" script, should i focus on the main or the computational file for multiprocessing? There are more files involved but this should be a start.
Main file:
import numpy as np
import pandas as pd
from properties_basic import *
import sys
sys.path.append('../physics_sup/')
import os, glob, shutil
for filename in glob.glob("obl_point_data_*"):
os.remove(filename)
for filename in glob.glob("restart.pkl"):
os.remove(filename)
for filename in glob.glob("data.pvd"):
os.remove(filename)
for filename in glob.glob("__pycache__/conversio*"):
os.remove(filename)
for filename in glob.glob("__pycache__/mode*"):
os.remove(filename)
for filename in glob.glob("__pycache__/operato*"):
os.remove(filename)
for filename in glob.glob("__pycache__/physi*"):
os.remove(filename)
for filename in glob.glob("__pycache__/prope*"):
os.remove(filename)
from model_benchmark import Model
from darts.engines import value_vector, redirect_darts_output
import matplotlib.pyplot as plt
grid_1D = False
redirect_darts_output('run.log')
n = Model()
n.init()
injectionsrate = np.genfromtxt('injectionrate.txt')[0:].astype(float) #np.genfromtxt('InjectionMonthly.txt')[0:].astype(float)#
injectionsrate = injectionsrate / 20
#mu_w = CP.PropsSI('V', 'T', 22, 'P|liquid', bar2pa(130), 'Water') * 1000
#n.property_container.viscosity_ev = dict([('wat', ViscosityConst(mu_w))])
NT = 16 # 16
runtime = 50 # 365
#increase = np.repeat(0.000005,37)
#print(increase)
for i in range(NT):
n.inj_rate = injectionsrate[i]
n.injection_temperature = 273.15 + 22
n.set_boundary_conditions(injectionrate=injectionsrate[i], tempinj=273.15+22)
#n.property_container.kinetic_rate_ev = kinetic_advanced(comp_min=1e-11, rsa=int(2e-05 + increase[NT]))
n.run_python(runtime)
time_data = pd.DataFrame.from_dict(n.physics.engine.time_data)
time_data.to_pickle("darts_time_data.pkl")
writer = pd.ExcelWriter('time_data.xlsx')
time_data.to_excel(writer, 'Sheet1')
writer.save()
writer.close()
n.export_vtk()
n.save_restart_data()
n.load_restart_data()
injectionsrate2 = np.genfromtxt('injectionrate.txt')[15:].astype(float) #np.genfromtxt('InjectionMonthly.txt')[191:].astype(float)#
injectionsrate2 = injectionsrate2 / 20 #*2
#mu_w2 = CP.PropsSI('V', 'T', 10, 'P|liquid', bar2pa(130), 'Water') * 1000
#n.property_container.viscosity_ev = dict([('wat', ViscosityConst(1.3))])
n.property_container.kinetic_rate_ev = kinetic_advanced(comp_min=1e-11, rsa=2e-03)
days = 200
NT2 = 21 #21 # 252
runtime2 = 50 # 30
for i in range(NT2):
n.inj_rate = injectionsrate2[i]
n.injection_temperature = 273.15 + 10
n.set_boundary_conditions(injectionrate=injectionsrate2[i], tempinj=273.15 + 10)
n.run_python(runtime2)
time_data2 = pd.DataFrame.from_dict(n.physics.engine.time_data)
time_data2.to_pickle("darts_time_data2.pkl")
writer = pd.ExcelWriter('time_data2.xlsx')
time_data2.to_excel(writer, 'Sheet1')
writer.save()
writer.close()
n.export_vtk()
n.print_timers()
n.print_stat()
import darts.tools.plot_darts
from darts.tools.plot_darts import *
p_w = 'I1'
#ax = plot_water_rate_darts(p_w, time_data)
time_dataInjection = pd.read_pickle("darts_time_data.pkl")
time_dataInjection2= pd.read_pickle("darts_time_data2.pkl")
#ax = darts.tools.plot_darts.plot_water_rate_darts(p_w, time_dataInjection)
ax2 = darts.tools.plot_darts.plot_water_rate_darts(p_w, time_dataInjection2)
p_w2 = 'P1'
#ax3 = darts.tools.plot_darts.plot_water_rate_darts(p_w2, time_dataInjection)
ax4 = darts.tools.plot_darts.plot_water_rate_darts(p_w2, time_dataInjection2)
ax5 = darts.tools.plot_darts.plot_bhp_darts(p_w, time_dataInjection2)
plt.show()
The Non linear calculator:
from math import fabs
import pickle
import os
import numpy as np
from darts.engines import *
from darts.engines import print_build_info as engines_pbi
from darts.physics import print_build_info as physics_pbi
from darts.print_build_info import print_build_info as package_pbi
class DartsModel:
def __init__(self):
# print out build information
engines_pbi()
physics_pbi()
package_pbi()
self.timer = timer_node() # Create time_node object for time record
self.timer.start() # Start time record
self.timer.node["simulation"] = timer_node() # Create timer.node called "simulation" to record simulation time
self.timer.node["newton update"] = timer_node()
self.timer.node[
"initialization"] = timer_node() # Create timer.node called "initialization" to record initialization time
self.timer.node["initialization"].start() # Start recording "initialization" time
self.params = sim_params() # Create sim_params object to set simulation parameters
self.timer.node["initialization"].stop() # Stop recording "initialization" time
def init(self):
self.reservoir.init_wells()
self.physics.init_wells(self.reservoir.wells)
self.set_initial_conditions()
self.set_boundary_conditions()
self.set_op_list()
self.reset()
def reset(self)
self.physics.engine.init(self.reservoir.mesh, ms_well_vector(self.reservoir.wells),
op_vector(self.op_list),
self.params, self.timer.node["simulation"])
def set_initial_conditions(self):
pass
def set_boundary_conditions(self):
pass
def set_op_list(self):
self.op_list = [self.physics.acc_flux_itor]
def run(self, days=0):
if days:
runtime = days
else:
runtime = self.runtime
self.physics.engine.run(runtime)
def run_python(self, days=0, restart_dt=0, log_3d_body_path=0, timestep_python=False):
if days:
runtime = days
else:
runtime = self.runtime
mult_dt = self.params.mult_ts
max_dt = self.params.max_ts
self.e = self.physics.engine
t = self.e.t
if fabs(t) < 1e-15:
dt = self.params.first_ts
elif restart_dt > 0:
dt = restart_dt
else:
dt = self.params.max_ts
runtime += t
ts = 0
if log_3d_body_path and self.physics.n_vars == 3:
self.body_path_start()
while t < runtime:
if timestep_python:
converged = self.e.run_timestep(dt, t)
else:
converged = self.run_timestep_python(dt, t)
if converged:
t += dt
ts = ts + 1
print("# %d \tT = %3g\tDT = %2g\tNI = %d\tLI=%d"
% (ts, t, dt, self.e.n_newton_last_dt, self.e.n_linear_last_dt))
dt *= mult_dt
if dt > max_dt:
dt = max_dt
if t + dt > runtime:
dt = runtime - t
if log_3d_body_path and self.physics.n_vars == 3:
self.body_path_add_bodys(t)
nb_begin = self.reservoir.nx * self.reservoir.ny * (self.body_path_map_layer - 1) * 3
nb_end = self.reservoir.nx * self.reservoir.ny * (self.body_path_map_layer) * 3
self.save_matlab_map(self.body_path_axes[0] + '_ts_' + str(ts), self.e.X[nb_begin:nb_end:3])
self.save_matlab_map(self.body_path_axes[1] + '_ts_' + str(ts), self.e.X[nb_begin + 1:nb_end:3])
self.save_matlab_map(self.body_path_axes[2] + '_ts_' + str(ts), self.e.X[nb_begin + 2:nb_end:3])
else:
dt /= mult_dt
print("Cut timestep to %2.3f" % dt)
if dt < 1e-8:
break
self.e.t = runtime
print("TS = %d(%d), NI = %d(%d), LI = %d(%d)" % (self.e.stat.n_timesteps_total, self.e.stat.n_timesteps_wasted,
self.e.stat.n_newton_total, self.e.stat.n_newton_wasted,
self.e.stat.n_linear_total, self.e.stat.n_linear_wasted))
def load_restart_data(self, filename='restart.pkl'):
if os.path.exists(filename):
with open(filename, "rb") as fp:
data = pickle.load(fp)
days, X, arr_n = data
self.physics.engine.t = days
self.physics.engine.X = value_vector(X)
self.physics.engine.Xn = value_vector(X)
self.physics.engine.op_vals_arr_n = value_vector(arr_n)
def save_restart_data(self, filename='restart.pkl'):
"""
Function to save the simulation data for restart usage.
:param filename: Name of the file where restart_data stores.
"""
t = np.copy(self.physics.engine.t)
X = np.copy(self.physics.engine.X)
arr_n = np.copy(self.physics.engine.op_vals_arr_n)
data = [t, X, arr_n]
with open(filename, "wb") as fp:
pickle.dump(data, fp, 4)
def check_performance(self, overwrite=0, diff_norm_normalized_tol=1e-10, diff_abs_max_normalized_tol=1e-7,
rel_diff_tol=1, perf_file=''):
fail = 0
data_et = self.load_performance_data(perf_file)
if data_et and not overwrite:
data = self.get_performance_data()
nb = self.reservoir.mesh.n_res_blocks
nv = self.physics.n_vars
for v in range(nv):
sol_et = data_et['solution'][v:nb * nv:nv]
diff = data['solution'][v:nb * nv:nv] - sol_et
sol_range = np.max(sol_et) - np.min(sol_et)
diff_abs = np.abs(diff)
diff_norm = np.linalg.norm(diff)
diff_norm_normalized = diff_norm / len(sol_et) / sol_range
diff_abs_max_normalized = np.max(diff_abs) / sol_range
if diff_norm_normalized > diff_norm_normalized_tol or diff_abs_max_normalized > diff_abs_max_normalized_tol:
fail += 1
print(
'#%d solution check failed for variable %s (range %f): L2(diff)/len(diff)/range = %.2E (tol %.2E), max(abs(diff))/range %.2E (tol %.2E), max(abs(diff)) = %.2E' \
% (fail, self.physics.vars[v], sol_range, diff_norm_normalized, diff_norm_normalized_tol,
diff_abs_max_normalized, diff_abs_max_normalized_tol, np.max(diff_abs)))
for key, value in sorted(data.items()):
if key == 'solution' or type(value) != int:
continue
reference = data_et[key]
if reference == 0:
if value != 0:
print('#%d parameter %s is %d (was 0)' % (fail, key, value))
fail += 1
else:
rel_diff = (value - data_et[key]) / reference * 100
if abs(rel_diff) > rel_diff_tol:
print('#%d parameter %s is %d (was %d, %+.2f%%)' % (fail, key, value, reference, rel_diff))
fail += 1
if not fail:
print('OK, \t%.2f s' % self.timer.node['simulation'].get_timer())
return 0
else:
print('FAIL, \t%.2f s' % self.timer.node['simulation'].get_timer())
return 1
else:
self.save_performance_data(perf_file)
print('SAVED')
return 0
def get_performance_data(self):
perf_data = dict()
perf_data['solution'] = np.copy(self.physics.engine.X)
perf_data['reservoir blocks'] = self.reservoir.mesh.n_res_blocks
perf_data['variables'] = self.physics.n_vars
perf_data['OBL resolution'] = self.physics.n_points
perf_data['operators'] = self.physics.n_ops
perf_data['timesteps'] = self.physics.engine.stat.n_timesteps_total
perf_data['wasted timesteps'] = self.physics.engine.stat.n_timesteps_wasted
perf_data['newton iterations'] = self.physics.engine.stat.n_newton_total
perf_data['wasted newton iterations'] = self.physics.engine.stat.n_newton_wasted
perf_data['linear iterations'] = self.physics.engine.stat.n_linear_total
perf_data['wasted linear iterations'] = self.physics.engine.stat.n_linear_wasted
sim = self.timer.node['simulation']
jac = sim.node['jacobian assembly']
perf_data['simulation time'] = sim.get_timer()
perf_data['linearization time'] = jac.get_timer()
perf_data['linear solver time'] = sim.node['linear solver solve'].get_timer() + sim.node[
'linear solver setup'].get_timer()
interp = jac.node['interpolation']
perf_data['interpolation incl. generation time'] = interp.get_timer()
return perf_data
def save_performance_data(self, file_name=''):
import platform
if file_name == '':
file_name = 'perf_' + platform.system().lower()[:3] + '.pkl'
data = self.get_performance_data()
with open(file_name, "wb") as fp:
pickle.dump(data, fp, 4)
#staticmethod
def load_performance_data(file_name=''):
import platform
if file_name == '':
file_name = 'perf_' + platform.system().lower()[:3] + '.pkl'
if os.path.exists(file_name):
with open(file_name, "rb") as fp:
return pickle.load(fp)
return 0
def print_timers(self):
print(self.timer.print("", ""))
def print_stat(self):
self.physics.engine.print_stat()
def plot_layer_map(self, map_data, k, name, transpose=0):
import plotly
import plotly.graph_objs as go
nxny = self.reservoir.nx * self.reservoir.ny
layer_indexes = np.arange(nxny * (k - 1), nxny * k)
layer_data = np.zeros(nxny)
# for correct vizualization of inactive cells
layer_data.fill(np.nan)
active_mask = np.where(self.reservoir.discretizer.global_to_local[layer_indexes] > -1)
layer_data[active_mask] = map_data[self.reservoir.discretizer.global_to_local[layer_indexes][active_mask]]
layer_data = layer_data.reshape(self.reservoir.ny, self.reservoir.nx)
if transpose:
layer_data = layer_data.transpose()
y_axis = dict(scaleratio=1, scaleanchor='x', title='X, block')
x_axis = dict(title='Y, block')
else:
x_axis = dict(scaleratio=1, scaleanchor='x', title='X, block')
y_axis = dict(title='Y, block')
data = [go.Heatmap(
z=layer_data)]
layout = go.Layout(title='%s, layer %d' % (name, k),
xaxis=x_axis,
yaxis=y_axis)
fig = go.Figure(data=data, layout=layout)
plotly.offline.plot(fig, filename='%s_%d_map.html' % (name, k))
def plot_layer_map_offline(self, map_data, k, name, transpose=0):
import plotly
plotly.offline.init_notebook_mode()
self.plot_layer_map(map_data, k, name, transpose)
def plot_layer_surface(self, map_data, k, name, transpose=0):
import plotly
import plotly.graph_objs as go
nxny = self.reservoir.nx * self.reservoir.ny
layer_indexes = np.arange(nxny * (k - 1), nxny * k)
layer_data = np.zeros(nxny)
# for correct vizualization of inactive cells
layer_data.fill(np.nan)
active_mask = np.where(self.reservoir.discretizer.global_to_local[layer_indexes] > -1)
layer_data[active_mask] = map_data[self.reservoir.discretizer.global_to_local[layer_indexes][active_mask]]
layer_data = layer_data.reshape(self.reservoir.ny, self.reservoir.nx)
if transpose:
layer_data = layer_data.transpose()
data = [go.Surface(z=layer_data)]
plotly.offline.plot(data, filename='%s_%d_surf.html' % (name, k))
def plot_geothermal_temp_layer_map(self, X, k, name, transpose=0):
import plotly
import plotly.graph_objs as go
import numpy as np
from darts.models.physics.iapws.iapws_property import iapws_temperature_evaluator
nxny = self.reservoir.nx * self.reservoir.ny
temperature = iapws_temperature_evaluator()
layer_pres_data = np.zeros(nxny)
layer_enth_data = np.zeros(nxny)
layer_indexes = np.arange(nxny * (k - 1), nxny * k)
active_mask = np.where(self.reservoir.discretizer.global_to_local[layer_indexes] > -1)
layer_pres_data[active_mask] = X[2 * self.reservoir.discretizer.global_to_local[layer_indexes][active_mask]]
layer_enth_data[active_mask] = X[2 * self.reservoir.discretizer.global_to_local[layer_indexes][active_mask] + 1]
# used_data = map_data[2 * nxny * (k-1): 2 * nxny * k]
T = np.zeros(nxny)
T.fill(np.nan)
for i in range(0, nxny):
if self.reservoir.discretizer.global_to_local[nxny * (k - 1) + i] > -1:
T[i] = temperature.evaluate([layer_pres_data[i], layer_enth_data[i]])
layer_data = T.reshape(self.reservoir.ny, self.reservoir.nx)
if transpose:
layer_data = layer_data.transpose()
y_axis = dict(scaleratio=1, scaleanchor='x', title='X, block')
x_axis = dict(title='Y, block')
else:
x_axis = dict(scaleratio=1, scaleanchor='x', title='X, block')
y_axis = dict(title='Y, block')
data = [go.Heatmap(
z=layer_data)]
layout = go.Layout(title='%s, layer %d' % (name, k),
xaxis=x_axis,
yaxis=y_axis)
fig = go.Figure(data=data, layout=layout)
plotly.offline.plot(fig, filename='%s_%d_map.html' % (name, k))
def plot_1d(self, map_data, name):
import plotly
import plotly.graph_objs as go
import numpy as np
nx = self.reservoir.nx
data = [go.Scatter(x=np.linspace(0, 1, nx), y=map_data[1:nx])]
plotly.offline.plot(data, filename='%s_surf.html' % name)
def plot_1d_all(self, map_data):
import plotly
import plotly.graph_objs as go
import numpy as np
nx = self.reservoir.nx
nc = self.physics.n_components
data = []
for i in range(nc - 1):
data.append(go.Scatter(x=np.linspace(0, 1, nx), y=map_data[i + 1::nc][1:nx], dash='dash'))
plotly.offline.plot(data, filename='Compositions.html')
def plot_cumulative_totals_mass(self):
import plotly.offline as po
import plotly.graph_objs as go
import numpy as np
import pandas as pd
nc = self.physics.n_components
darts_df = pd.DataFrame(self.physics.engine.time_data)
total_df = pd.DataFrame()
total_df['time'] = darts_df['time']
time_diff = darts_df['time'].diff()
time_diff[0] = darts_df['time'][0]
for c in range(nc):
total_df['Total injection c %d' % c] = 0
total_df['Total production c %d' % c] = 0
search_str = ' : c %d rate (Kmol/day)' % c
for col in darts_df.columns:
if search_str in col:
inj_mass = darts_df[col] * time_diff
prod_mass = darts_df[col] * time_diff
# assuming that any well can inject and produce over the whole time
inj_mass[inj_mass < 0] = 0
prod_mass[prod_mass > 0] = 0
total_df['Total injection c %d' % c] += inj_mass
total_df['Total production c %d' % c] -= prod_mass
data = []
for c in range(nc):
data.append(go.Scatter(x=total_df['time'], y=total_df['Total injection c %d' % c].cumsum(),
name='%s injection' % self.physics.components[c]))
data.append(go.Scatter(x=total_df['time'], y=total_df['Total production c %d' % c].cumsum(),
name='%s production' % self.physics.components[c]))
layout = go.Layout(title='Cumulative total masses (kmol)', xaxis=dict(title='Time (days)'),
yaxis=dict(title='Mass (kmols)'))
fig = go.Figure(data=data, layout=layout)
po.plot(fig, filename='Cumulative_totals_mass.html')
def plot_mass_balance_error(self):
import plotly.offline as po
import plotly.graph_objs as go
import numpy as np
import pandas as pd
nc = self.physics.n_components
darts_df = pd.DataFrame(self.physics.engine.time_data)
total_df = pd.DataFrame()
total_df['time'] = darts_df['time']
time_diff = darts_df['time'].diff()
time_diff[0] = darts_df['time'][0]
for c in range(nc):
total_df['Total source-sink c %d' % c] = 0
search_str = ' : c %d rate (Kmol/day)' % c
for col in darts_df.columns:
if search_str in col:
mass = darts_df[col] * time_diff
total_df['Total source-sink c %d' % c] += mass
data = []
for c in range(nc):
total_df['Total mass balance error c %d' % c] = darts_df['FIPS c %d (kmol)' % c] - total_df[
'Total source-sink c %d' % c].cumsum()
total_df['Total mass balance error c %d' % c] -= darts_df['FIPS c %d (kmol)' % c][0] - \
total_df['Total source-sink c %d' % c][0]
data.append(go.Scatter(x=total_df['time'], y=total_df['Total mass balance error c %d' % c],
name='%s' % self.physics.components[c]))
layout = go.Layout(title='Mass balance error (kmol)', xaxis=dict(title='Time (days)'),
yaxis=dict(title='Mass (kmols)'))
fig = go.Figure(data=data, layout=layout)
po.plot(fig, filename='Mass_balance_error.html')
def plot_FIPS(self):
import plotly.offline as po
import plotly.graph_objs as go
import numpy as np
import pandas as pd
nc = self.physics.n_components
darts_df = pd.DataFrame(self.physics.engine.time_data)
data = []
for c in range(nc):
data.append(go.Scatter(x=darts_df['time'], y=darts_df['FIPS c %d (kmol)' % c],
name='%s' % self.physics.components[c]))
layout = go.Layout(title='FIPS (kmol)', xaxis=dict(title='Time (days)'),
yaxis=dict(title='Mass (kmols)'))
fig = go.Figure(data=data, layout=layout)
po.plot(fig, filename='FIPS.html')
def plot_totals_mass(self):
import plotly.offline as po
import plotly.graph_objs as go
import numpy as np
import pandas as pd
nc = self.physics.n_components
darts_df = pd.DataFrame(self.physics.engine.time_data)
total_df = pd.DataFrame()
total_df['time'] = darts_df['time']
for c in range(nc):
total_df['Total injection c %d' % c] = 0
total_df['Total production c %d' % c] = 0
search_str = ' : c %d rate (Kmol/day)' % c
for col in darts_df.columns:
if search_str in col:
inj_mass = darts_df[col].copy()
prod_mass = darts_df[col].copy()
# assuming that any well can inject and produce over the whole time
inj_mass[inj_mass < 0] = 0
prod_mass[prod_mass > 0] = 0
total_df['Total injection c %d' % c] += inj_mass
total_df['Total production c %d' % c] -= prod_mass
data = []
for c in range(nc):
data.append(go.Scatter(x=total_df['time'], y=total_df['Total injection c %d' % c],
name='%s injection' % self.physics.components[c]))
data.append(go.Scatter(x=total_df['time'], y=total_df['Total production c %d' % c],
name='%s production' % self.physics.components[c]))
layout = go.Layout(title='Total mass rates (kmols/day)', xaxis=dict(title='Time (days)'),
yaxis=dict(title='Rate (kmols/day)'))
fig = go.Figure(data=data, layout=layout)
po.plot(fig, filename='Totals_mass_rates.html')
def plot_1d_compare(self, map_data1, map_data2):
import plotly
import plotly.graph_objs as go
import numpy as np
nx = self.reservoir.nx
nc = self.physics.n_components
data = []
for i in range(nc - 1):
data.append(go.Scatter(x=np.linspace(0, 1, nx), y=map_data1[i + 1::nc][1:nx],
name="Comp = %d, dt = 5 days" % (i + 1)))
for i in range(nc - 1):
data.append(go.Scatter(x=np.linspace(0, 1, nx), y=map_data2[i + 1::nc][1:nx],
name="Comp = %d, dt = 50 days" % (i + 1), line=dict(dash='dot')))
plotly.offline.plot(data, filename='Compositions.html')
def body_path_start(self):
with open('body_path.txt', "w") as fp:
itor = self.physics.acc_flux_itor
self.processed_body_idxs = set()
for i, p in enumerate(itor.axis_points):
fp.write('%d %lf %lf %s\n' % (p, itor.axis_min[i], itor.axis_max[i], self.body_path_axes[i]))
fp.write('Body Index Data\n')
def body_path_add_bodys(self, time):
with open('body_path.txt', "a") as fp:
fp.write('T=%lf\n' % time)
itor = self.physics.acc_flux_itor
all_idxs = set(itor.body_data.keys())
new_idxs = all_idxs - self.processed_body_idxs
for i in new_idxs:
fp.write('%d\n' % i)
self.processed_body_idxs = all_idxs
def save_matlab_map(self, name, np_arr):
import scipy.io
scipy.io.savemat(name + '.mat', dict(x=np_arr))
def export_vtk(self, file_name='data', local_cell_data={}, global_cell_data={}, vars_data_dtype=np.float32,
export_grid_data=True):
# get current engine time
t = self.physics.engine.t
nb = self.reservoir.mesh.n_res_blocks
nv = self.physics.n_vars
X = np.array(self.physics.engine.X, copy=False)
for v in range(nv):
local_cell_data[self.physics.vars[v]] = X[v:nb * nv:nv].astype(vars_data_dtype)
self.reservoir.export_vtk(file_name, t, local_cell_data, global_cell_data, export_grid_data)
# destructor to force to destroy all created C objects and free memory
def __del__(self):
for name in list(vars(self).keys()):
delattr(self, name)
def run_timestep_python(self, dt, t):
max_newt = self.params.max_i_newton
max_residual = np.zeros(max_newt + 1)
self.e.n_linear_last_dt = 0
well_tolerance_coefficient = 1e2
self.timer.node['simulation'].start()
for i in range(max_newt+1):
self.e.run_single_newton_iteration(dt)
self.e.newton_residual_last_dt = self.e.calc_newton_residual()
max_residual[i] = self.e.newton_residual_last_dt
counter = 0
for j in range(i):
if abs(max_residual[i] - max_residual[j])/max_residual[i] < 1e-3:
counter += 1
if counter > 2:
print("Stationary point detected!")
break
self.e.well_residual_last_dt = self.e.calc_well_residual()
self.e.n_newton_last_dt = i
# check tolerance if it converges
if ((self.e.newton_residual_last_dt < self.params.tolerance_newton and self.e.well_residual_last_dt < well_tolerance_coefficient * self.params.tolerance_newton )
or self.e.n_newton_last_dt == self.params.max_i_newton):
if (i > 0): # min_i_newton
break
r_code = self.e.solve_linear_equation()
self.timer.node["newton update"].start()
self.e.apply_newton_update(dt)
self.timer.node["newton update"].stop()
# End of newton loop
converged = self.e.post_newtonloop(dt, t)
self.timer.node['simulation'].stop()
return converged
When I use vtkPolyDataWriter to create a vtk legacy file, I obtain the new version of the data file (5.1) with connectivities and offsets. Is it possible to change that and get the 'legacy' old format?
It seems gmsh is not able to read vtk file with the new format version..
(I work with python 3.8 and vtk package version 9.0.1)
Nope, you cannot programmatically choose the version. The only way to write with an old format is to downgrade vtk to the desired version.
For those interested, I post the code I used to convert new to old format. It works for polydata and unstructured grid vtk meshes.
import math
import re
def convert_to_old_format(mesh_fname: str, save_fname: str, copy_lines: bool = False):
conv = Converter(mesh_fname, save_fname, copy_lines)
conv.read()
conv.replace()
conv.write()
def change_vtk_version(filename: str, v: float = 5.1):
with open(filename, "r") as f:
x = f.read()
x = re.sub(r"(# vtk DataFile Version) (.+)", f"\\1 {v}", x)
with open(filename, "w") as f:
f.write(x)
class Converter:
def __init__(self, inp, out=None, copy_lines=False):
if out is None:
out = inp
self.inp = inp
self.out = out
self.copy_lines = copy_lines # if line cells should be copied
self.original = None
self.lines = None
self.polys = None
self.cells = None
def read(self):
with open(self.inp, "r") as f:
self.original = f.read().split("\n")
lines_original = list(map(lambda x: x.strip().split(), self.original))
for i, l in enumerate(self.original):
if "LINES" in l:
self.lines = NewContent("LINES", lines_original, i)
elif "POLYGONS" in l:
self.polys = NewContent("POLYGONS", lines_original, i)
elif "CELLS" in l:
self.cells = NewContent("CELLS", lines_original, i)
def replace(self):
if self.polys is not None:
self.original = self.polys.replace(self.original)
if self.cells is not None:
self.original = self.cells.replace(self.original)
if self.lines is not None:
self.original = self.lines.replace(self.original, replace=self.copy_lines)
def write(self):
with open(self.out, "w") as f:
f.write("\n".join(self.original))
change_vtk_version(self.out, 4.2)
class NewContent:
def __init__(self, kw, content, ln):
self.kw = kw
self.ln = ln
self.name = content[ln][0]
self.no = int(content[ln][1])
self.nc = int(content[ln][2])
flat_list = [item for line in content[ln + 2 :] for item in line]
flat_list = list(filter("".__ne__, flat_list))
self.offsets = list(map(int, flat_list[0 : self.no]))
self.connectivity = list(
map(int, flat_list[self.no + 2 : self.no + 2 + self.nc])
)
#property
def remove(self):
return self.ln, self.ln + math.ceil(self.no / 9) + math.ceil(self.nc / 9) + 3
def replace(self, lines, replace=True):
nb_cells = self.no - 1
new_content = []
if replace:
new_content = [f"{self.kw} {nb_cells} {nb_cells + self.nc}"]
for i in range(nb_cells):
nb_points = self.offsets[i + 1] - self.offsets[i]
ids = self.connectivity[self.offsets[i] : self.offsets[i + 1]]
new_content.append(f"{nb_points} {' '.join(map(str, ids))}")
lines_to_keep = lines
a, b = self.remove
del lines_to_keep[a:b]
lines_to_keep[a:a] = new_content
lines_to_keep = list(filter("".__ne__, lines_to_keep))
return lines_to_keep
if __name__ == "__main__":
convert_to_old_format("mesh.vtk", "mesh_old_format.vtk")
# change_vtk_version("mesh.vtk", 8.6)
An instance of the Rabbit class is created in a for loop and given a datetime object as init variable
but all instances get the same datetime value.
I expected a time difference for each instance; what am I missing in understanding here?
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = datetime.now(),
tob = datetime.now()
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = str(dob.strftime("%x"))
self.tob = str(tob.strftime("%X"))
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
dob_time = self.dob+" at "+self.tob
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
print()
print('Number of rabbits is ',Rabbit_birth.num_of_rabbits)
rabbits=[]
def create_rabbit():
lr=[]
rabbit = ('name', 'fur colour', 'eye colour', 'sex', 'breed')
for item in rabbit:
r = input(item+' >')
lr.append(r)
r0,r1,r2,r3,r4=lr
name = Rabbit_birth(r0,r1,r2,r3,r4)
return name
for i in range(3):
name=create_rabbit()
rabbits.append(name)
print(rabbits[i].display_rabbit())
I see the mistake I have made, it's a matter of passing the datetime object in the wrong scope.
Here is the relevant corrected code:
## <- class experiment
from datetime import datetime
class Rabbit_birth:
num_eyes = 2 # <- Class variable
num_legs = 4
num_ears = 2
num_tails = 1
num_of_rabbits = 0
def __init__(self, name = 'Rabbit',
fur_colour = 'unknown',
eye_colour = 'unknown',
sex = 'unsexed',
breed = 'unknown',
dob = None, # This line changed
tob = None # This line changed
):
self.name = name # <- Instance variable
self.fur_colour = fur_colour
self.eye_colour = eye_colour
self.sex = sex
self.breed = breed
self.dob = datetime.now() # This line changed
self.tob = datetime.now() # This line changed
Rabbit_birth.num_of_rabbits += 1
def display_rabbit(self):
# This line below changed
dob_time = str(self.dob.strftime("%x"))+" at "+str(self.tob.strftime("%X"))
return ("{0.name} is a {0.sex} rabbit"
" and has {0.fur_colour} fur and "
"{0.eye_colour} eyes, {0.name} is "
"of the {0.breed} breed of rabbits.\n"
"{0.name}'s DOB is ").format(self) + dob_time
################################################
When I run a crawler in anaconda prompt I got error:
ModuleNotFoundError: no module named 'ihs'
there is part in Pyhon file like following:
from ihs.items import IhsItem
from ihs.library import Library
which library should I install to fix this error ?
full code:
import scrapy
import re
import logging
import os
import sys
import smtplib
import urlparse
from bs4 import BeautifulSoup
from scrapy.spider import BaseSpider
from datetime import datetime
from scrapy.exceptions import CloseSpider
from ihs.items import IhsItem
from ihs.library import Library
class McewingpartnersSpider(scrapy.Spider):
name = "mcewingpartners"
lib = Library()
allowed_domains = ["mcewingpartners.com"]
start_urls = (
'http://www.mcewingpartners.com/?/rent/residential/',
)
def __init__(self):
self.log_dir = os.getcwd()
formatter = logging.Formatter('%(asctime)s %(message)s')
self.log = logging.getLogger('log-mcewingpartners')
self.log.setLevel(logging.INFO)
log_file_name = datetime.now().strftime('logs/mcewingpartners%H:%M_%d-%m-%Y.log')
ch_file = logging.FileHandler(log_file_name, 'w')
ch_file.setLevel(logging.ERROR)
ch_file.setFormatter(formatter)
self.log.addHandler(ch_file)
# add formatter to ch
ch_stream = logging.StreamHandler()
ch_stream.setFormatter(formatter)
# add ch to logger
self.log.addHandler(ch_stream)
self.log.info("mcewingpartners Ready.")
def parse(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
# Property Urls
property_urls = []
_property_urls = response.xpath("//a[#class='listing-item']/#href").extract()
for _property_url in _property_urls:
token = 'self.location'
if token in _property_url[0:len(token) + 1]:
_property_url = _property_url[len(token) + 2:len(_property_url)-1]
_property_url = urlparse.urljoin(site, _property_url)
if _property_url not in property_urls:
property_urls.append(_property_url)
# check last page
if response.meta.get('prev_urls') == property_urls:
return # reached the last page
for property_url in property_urls:
yield scrapy.Request(property_url, callback = self.parse_property_urls, dont_filter = True)
#next_page_url = response.xpath("//*[#id='pageNext2']/#href").extract()
#if len(next_page_url) > 0:
# next_page_url = next_page_url[0].strip()
# if domain not in next_page_url:
# next_page_url = urlparse.urljoin(site, next_page_url)
# yield scrapy.Request(next_page_url, callback = self.parse)
except Exception as e:
err = "Error in parse: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_property_urls(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if urlparse.urlparse(site).scheme == '':
if site.startswith('//'):
site = 'http:' + site
else:
site = 'http://' + site
item = IhsItem()
item['spider_identifier'] = self.name
item['property_url'] = response.url
#Static Value
item['property_type'] = 'Other Residential'
#Static Value
item['listing_type'] = 'For Rent'
#Default value for short_description
item['short_description'] = 'Please Contact Agent'
short_description = response.xpath("//div[#class='col-md-24 property-description']//h3/text()").extract()
if len(short_description) > 0:
short_description = short_description[0].strip().encode('ascii','ignore')
item['short_description'] = short_description.strip()
#Default value for description
item['description'] = 'Please Contact Agent'
# Aggregation of multiple values from multiple xpaths
description = []
_description = response.xpath("//div[#class='col-md-24 property-description']//div[#class='inner border-bot']/p/text()").extract()
for _pd in _description:
if _pd not in description:
description.append(_pd)
if len(description) > 0:
item['description'] = ' '.join(description).encode('ascii','ignore').strip()
#Default value for price
item['price'] = 'Please Contact Agent'
price = response.xpath("//div[#class='row property-header']//div[#class='inner border-bot']//h4/text()").extract()
if len(price) > 0:
price = price[0].strip().encode('ascii','ignore')
item['price'] = price.strip()
#Default value for bedroom
item['bedroom'] = '0'
bedroom = response.xpath("(//div[#class='bbc-icon filter-bed']/text())[2]").extract()
if len(bedroom) > 0:
bedroom = bedroom[0].strip().encode('ascii','ignore')
item['bedroom'] = bedroom.strip()
#Default value for bathroom
item['bathroom'] = '0'
bathroom = response.xpath("(//div[#class='bbc-icon filter-bath']/text())[2]").extract()
if len(bathroom) > 0:
bathroom = bathroom[0].strip().encode('ascii','ignore')
item['bathroom'] = bathroom.strip()
#Default value for parking
item['parking'] = '0'
parking = response.xpath("(//div[#class='bbc-icon filter-car']/text())[2]").extract()
if len(parking) > 0:
parking = parking[0].strip().encode('ascii','ignore')
item['parking'] = parking.strip()
#Default value for photo
item['photo'] = []
# Aggregation of multiple values from multiple xpaths
photo = []
_photo = response.xpath("//div[#class='carousel-inner']//img/#src").extract()
for _pd in _photo:
if _pd not in photo:
photo.append(_pd)
if len(photo) > 0:
item['photo'] = photo
#Default value for land_area
item['land_area'] = '0'
#Default value for building_area
item['building_area'] = '0'
#Default value for inspection_date
#item['inspection_date'] = ''
#inspection_date = response.xpath("//div[#class='font13 colorthreefontcolor left valueInsp']/text()").extract()
#if len(inspection_date) > 0:
# inspection_date = inspection_date[0].strip().encode('ascii','ignore')
# item['inspection_date'] = inspection_date.strip()
#self.parse_agent_info_url(self, response)
#agent_info_url
#agent_info_url = response.xpath("//div[#id='property_staffMember']/div[#class='left staffImageContainer']/a/#href").extract()
#if len(agent_info_url) > 0:
# agent_info_url = agent_info_url[0].strip()
# agent_info_url = urlparse.urljoin(site, agent_info_url)
scrapy.Request(site, callback=self.parse_agent_info_url, dont_filter=True, meta={'item': item})
#else:
# # Default Value for items in the above branch url
# #Default value for agent_name
# item['agent_name'] = self.name.split('_')[0]
#
#Default value for agency_name
# item['agency_name'] = ''
#Default value for agent_proprietor_name
# item['agent_proprietor_name'] = self.name.split('_')[0]
#Default value for agent_licencee_number
# item['agent_licencee_number'] = self.name.split('_')[0]
#Default value for agent_licencee_name
# item['agent_licencee_name'] = self.name.split('_')[0]
#Default value for agency_logo
# item['agency_logo'] = ''
#Default value for agency_email
# item['agency_email'] = ''
#Default value for agent_email
# item['agent_email'] = 'ihomeseek#outlook.com'
#Default value for agent_mobile
# item['agent_mobile'] = ''
#Default value for agency_phone
# item['agency_phone'] = ''
#Default value for agent_fax
# item['agent_fax'] = ''
#Default value for agent_color
# item['agent_color'] = ''
yield item
except Exception as e:
err = "Error in parse_property_urls: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
def parse_agent_info_url(self, response):
try:
site = response.url
domain = "mcewingpartners.com"
if site.startswith('//') or site.startswith('\\'):
site = 'http:' + site
else:
site = 'http://' + site
item = response.meta['item']
#Default value for agent_name
item['agent_name'] = self.name.split('_')[0]
agent_name = response.xpath("//a[#class='agent-logo-inner']//img/#src").extract()
if len(agent_name) > 0:
agent_name = agent_name[0].strip().encode('ascii','ignore')
item['agent_name'] = agent_name.strip()
#Static Value
item['agency_name'] = 'McEwing Partners'
#Static Value
item['agent_proprietor_name'] = 'McEwing Partners'
#Static Value
item['agent_licencee_number'] = 'xxxxxxx'
#Static Value
item['agent_licencee_name'] = 'xxxxxxx'
#Default value for agency_logo
item['agency_logo'] = ''
agency_logo = response.xpath("//ul[#class='images']/li[#class='images']/img/#src").extract()
if len(agency_logo) > 0:
agency_logo = agency_logo[0].strip().encode('ascii','ignore')
item['agency_logo'] = agency_logo.strip()
#Default value for agency_email
item['agency_email'] = ''
agency_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agency_email) > 0:
agency_email = agency_email[0].strip().encode('ascii','ignore')
item['agency_email'] = agency_email.strip()
#Default value for agent_email
item['agent_email'] = '** no mail **'
agent_email = response.xpath("//*[#id='staffMember']/div[2]/span/a/text()").extract()
if len(agent_email) > 0:
agent_email = agent_email[0].strip().encode('ascii','ignore')
item['agent_email'] = agent_email.strip()
#Default value for agent_mobile
item['agent_mobile'] = ''
agent_mobile = response.xpath("//a[#class='property-staff-link']//span/text()").extract()
if len(agent_mobile) > 0:
agent_mobile = agent_mobile[0].strip().encode('ascii','ignore')
item['agent_mobile'] = agent_mobile.strip()
#Static Value
item['agency_phone'] = '03 5975 4555'
#Static Value
item['agent_fax'] = '03 5975 6444'
#Static Value
item['agent_color'] = '#3F3F3F'
yield item
except Exception as e:
err = "Error in parse_agent_info_url: %s, Message: %s, Traceback: %s " % (response.url, str(e), sys.exc_info())
self.log.error(err)
self.lib.PrintException('mcewingpartners')
#self.lib.send_email(err, self.name)
pass
SPIDER = McewingpartnersSpider()
when l run the following program l got this error :
originDataset = dataset.lmdbDataset(originPath, 'abc', *args)
TypeError: __init__() takes from 1 to 4 positional arguments but 9 were given
This error is relate to the second code source l presented below. it's strange because l don't have 9 argument. what's wrong with my code ?
import sys
origin_path = sys.path
sys.path.append("..")
import dataset
sys.path = origin_path
import lmdb
def writeCache(env, cache):
with env.begin(write=True) as txn:
for k, v in cache.iteritems():
txn.put(k, v)
def convert(originPath, outputPath):
args = [0] * 6
originDataset = dataset.lmdbDataset(originPath, 'abc', *args)
print('Origin dataset has %d samples' % len(originDataset))
labelStrList = []
for i in range(len(originDataset)):
label = originDataset.getLabel(i + 1)
labelStrList.append(label)
if i % 10000 == 0:
print(i)
lengthList = [len(s) for s in labelStrList]
items = zip(lengthList, range(len(labelStrList)))
items.sort(key=lambda item: item[0])
env = lmdb.open(outputPath, map_size=1099511627776)
cnt = 1
cache = {}
nSamples = len(items)
for i in range(nSamples):
imageKey = 'image-%09d' % cnt
labelKey = 'label-%09d' % cnt
origin_i = items[i][1]
img, label = originDataset[origin_i + 1]
cache[labelKey] = label
cache[imageKey] = img
if cnt % 1000 == 0 or cnt == nSamples:
writeCache(env, cache)
cache = {}
print('Written %d / %d' % (cnt, nSamples))
cnt += 1
nSamples = cnt - 1
cache['num-samples'] = str(nSamples)
writeCache(env, cache)
print('Convert dataset with %d samples' % nSamples)
if __name__ == "__main__":
convert('/share/datasets/scene_text/Synth90k/synth90k-val-lmdb', '/share/datasets/scene_text/Synth90k/synth90k-val-ordered-lmdb')
convert('/share/datasets/scene_text/Synth90k/synth90k-train-lmdb', '/share/datasets/scene_text/Synth90k/synth90k-train-ordered-lmdb')
which calls the following program :
#!/usr/bin/python
# encoding: utf-8
import random
import torch
from torch.utils.data import Dataset
from torch.utils.data import sampler
import torchvision.transforms as transforms
import lmdb
import six
import sys
from PIL import Image
import numpy as np
class lmdbDataset(Dataset):
def __init__(self, root=None, transform=None, target_transform=None):
self.env = lmdb.open(
root,
max_readers=1,
readonly=True,
lock=False,
readahead=False,
meminit=False)
if not self.env:
print('cannot creat lmdb from %s' % (root))
sys.exit(0)
with self.env.begin(write=False) as txn:
nSamples = int(txn.get('num-samples'))
self.nSamples = nSamples
self.transform = transform
self.target_transform = target_transform
def __len__(self):
return self.nSamples
def __getitem__(self, index):
assert index <= len(self), 'index range error'
index += 1
with self.env.begin(write=False) as txn:
img_key = 'image-%09d' % index
imgbuf = txn.get(img_key)
buf = six.BytesIO()
buf.write(imgbuf)
buf.seek(0)
try:
img = Image.open(buf).convert('L')
except IOError:
print('Corrupted image for %d' % index)
return self[index + 1]
if self.transform is not None:
img = self.transform(img)
label_key = 'label-%09d' % index
label = str(txn.get(label_key))
if self.target_transform is not None:
label = self.target_transform(label)
return (img, label)
class resizeNormalize(object):
def __init__(self, size, interpolation=Image.BILINEAR):
self.size = size
self.interpolation = interpolation
self.toTensor = transforms.ToTensor()
def __call__(self, img):
img = img.resize(self.size, self.interpolation)
img = self.toTensor(img)
img.sub_(0.5).div_(0.5)
return img
class randomSequentialSampler(sampler.Sampler):
def __init__(self, data_source, batch_size):
self.num_samples = len(data_source)
self.batch_size = batch_size
def __iter__(self):
n_batch = len(self) // self.batch_size
tail = len(self) % self.batch_size
index = torch.LongTensor(len(self)).fill_(0)
for i in range(n_batch):
random_start = random.randint(0, len(self) - self.batch_size)
batch_index = random_start + torch.range(0, self.batch_size - 1)
index[i * self.batch_size:(i + 1) * self.batch_size] = batch_index
# deal with tail
if tail:
random_start = random.randint(0, len(self) - self.batch_size)
tail_index = random_start + torch.range(0, tail - 1)
index[(i + 1) * self.batch_size:] = tail_index
return iter(index)
def __len__(self):
return self.num_samples
class alignCollate(object):
def __init__(self, imgH=32, imgW=128, keep_ratio=False, min_ratio=1):
self.imgH = imgH
self.imgW = imgW
self.keep_ratio = keep_ratio
self.min_ratio = min_ratio
def __call__(self, batch):
images, labels = zip(*batch)
imgH = self.imgH
imgW = self.imgW
if self.keep_ratio:
ratios = []
for image in images:
w, h = image.size
ratios.append(w / float(h))
ratios.sort()
max_ratio = ratios[-1]
imgW = int(np.floor(max_ratio * imgH))
imgW = max(imgH * self.min_ratio, imgW) # assure imgH >= imgW
transform = resizeNormalize((imgW, imgH))
images = [transform(image) for image in images]
images = torch.cat([t.unsqueeze(0) for t in images], 0)
return images, labels