i use the pytorch DistributedDataParallel to train my network,but i want to get Confusion Matrix,this is my code
def drawMatrix(self,root_name,epoch):
self.matrix = self.getMatrix(self.normalize)
plt.imshow(self.matrix, cmap=plt.cm.Blues) #
plt.title("Normalized confusion matrix") # title
plt.xlabel("Predict label")
plt.ylabel("Truth label")
plt.yticks(range(self.num_classes), self.labels_name) #
plt.xticks(range(self.num_classes), self.labels_name, rotation=45) #
for x in range(self.num_classes):
for y in range(self.num_classes):
value = float(format('%.2f' % self.matrix[y, x])) #
plt.text(x, y, value, verticalalignment='center', horizontalalignment='center') #
plt.tight_layout() #
plt.colorbar() #
print('绘图3')
plt.savefig('./checkpoint/'+root_name+'/ConfusionMatrix'+str(epoch)+'.png', bbox_inches='tight') #
plt.show()
when i use the shell like
python -m torch.distributed.launch --master_port 12347 --nproc_per_node=1 main.py violence5 RGB --div_type none --arch resnet50 --num_segments 8 --gd 20 --lr 0.01 --lr_scheduler step --lr_steps 15 30 45 --epochs 60 --batch-size 16 --wd 5e-4 --dropout 0.5 --consensus_type=avg --eval-freq=1 -j 4 --npb
it throw a error occured in plt.imshow(self.matrix, cmap=plt.cm.Blues)
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -11) local_rank: 0
i use the gui it can run
--master_port 12347 --nproc_per_node=1 main.py violence5 RGB --div_type none --arch resnet50 --num_segments 8 --gd 20 --lr 0.01 --lr_scheduler step --lr_steps 15 30 45 --epochs 60 --batch-size 16 --wd 5e-4 --dropout 0.5 --consensus_type=avg --eval-freq=1 -j 4 --npb
how can i run the code in the shell?
Related
I would like to plot a line plot (source: pandas dataframe) over a hvplot (source: xarray/ NetCDF).
The xarray looks like this:
dataDIR = 'ceilodata.nc'
DS = xr.open_dataset(dataDIR)
DS = DS.transpose()
print(DS)
<xarray.Dataset>
Dimensions: (range_hr: 32, range: 1024, layer: 3, time: 5760)
Coordinates:
* range_hr (range_hr) float32 0.001 4.995 9.99 ... 144.9 149.9 154.8
* range (range) float32 14.98 29.97 44.96 ... 1.533e+04 1.534e+04
* layer (layer) int32 1 2 3
* time (time) datetime64[ns] 2022-03-18 ... 2022-03-18T23:59:46
Data variables: (12/41)
zenith float32 ...
wavelength float32 ...
scaling float32 ...
range_gate_hr float32 ...
range_gate float32 ...
longitude float32 ...
... ...
cbe (layer, time) int16 ...
beta_raw_hr (range_hr, time) float32 ...
beta_raw (range, time) float32 ...
bcc (time) int8 ...
base (time) float32 ...
average_time (time) int32 ...
Attributes: (12/13)
comment:
software_version: 15.06.1 2.13 1.040 1
title: CHM15k Nimbus
wmo_id: 10865
month: 3
source: CHM160138
... ...
serlom: TUB160038
location: muenchen
year: 2022
device_name: CHM160138
institution: DWD
day: 18
The pandas dataframe source looks like this:
df = pd.read_csv('PTU.csv')
print(df)
Unnamed: 0 PTU
0 2022-03-18 07:38:56 451.839
1 2022-03-18 07:38:57 468.826
2 2022-03-18 07:38:58 469.093
3 2022-03-18 07:38:59 469.356
4 2022-03-18 07:39:00 469.623
... ... ...
6140 2022-03-18 09:21:16 31690.600
6141 2022-03-18 09:21:17 31694.700
6142 2022-03-18 09:21:18 31692.900
6143 2022-03-18 09:21:19 31712.000
6144 2022-03-18 09:21:20 31711.500
[6145 rows x 2 columns]
Both are time dependend datasets but have different time stamps and frequencies. Time is index in each data set.
I tried to plot them together with additional imports of holoviews. While each single plot is no problem, plotting them together seems not to work the way I tried it:
import hvplot.pandas
import holoviews as hv
# cmap of the xarray:
ceilo = (DS.b_r.hvplot(cmap="viridis_r", width = 850, height = 600, title = 'title', clim = (5, 80))
# line plot of the data frame
p = df.hvplot.line()
# add pressure line plot to pcolormeshplot using * which overlays the line on the plot
ceilo * p
but this ended in an error message with the following complete traceback:
---------------------------------------------------------------------------
NotImplementedError Traceback (most recent call last)
<ipython-input-10-2b1c6baca339> in <module>
24 p = df.hvplot.line()
25 # add pressure line plot to pcolormeshplot using * which overlays the line on the plot
---> 26 ceilo * df
c:\python38\lib\site-packages\pandas\core\ops\common.py in new_method(self, other)
68 other = item_from_zerodim(other)
69
---> 70 return method(self, other)
71
72 return new_method
c:\python38\lib\site-packages\pandas\core\arraylike.py in __rmul__(self, other)
118 #unpack_zerodim_and_defer("__rmul__")
119 def __rmul__(self, other):
--> 120 return self._arith_method(other, roperator.rmul)
121
122 #unpack_zerodim_and_defer("__truediv__")
c:\python38\lib\site-packages\pandas\core\frame.py in _arith_method(self, other, op)
6936 other = ops.maybe_prepare_scalar_for_op(other, (self.shape[axis],))
6937
-> 6938 self, other = ops.align_method_FRAME(self, other, axis, flex=True, level=None)
6939
6940 new_data = self._dispatch_frame_op(other, op, axis=axis)
c:\python38\lib\site-packages\pandas\core\ops\__init__.py in align_method_FRAME(left, right, axis, flex, level)
275 elif is_list_like(right) and not isinstance(right, (ABCSeries, ABCDataFrame)):
276 # GH 36702. Raise when attempting arithmetic with list of array-like.
--> 277 if any(is_array_like(el) for el in right):
278 raise ValueError(
279 f"Unable to coerce list of {type(right[0])} to Series/DataFrame"
c:\python38\lib\site-packages\holoviews\core\element.py in __iter__(self)
94 def __iter__(self):
95 "Disable iterator interface."
---> 96 raise NotImplementedError('Iteration on Elements is not supported.')
97
98
NotImplementedError: Iteration on Elements is not supported.
Is the different time frequency a problem here? The line plot should be orientated along the x- and the y-axis considering the right time stamp and altitude of the underlying cmap-(matplotlib)-plot.
To illustrate what I am aiming for, here is a picture of my goal:
Thanks for reading / helping.
I found a solution for this case:
Both dataset time columns have to have the same format. In my case it's: datetime64[ns] (to adopt to the NetCDF xarray). That is why I converted the dataframe time column to datetime64[ns]:
df.Datetime = df.Datetime.astype('datetime64')
Also I found the data to be type "object". So I transformed it to "float":
df.PTU = df.PTU.astype(float) # convert to correct data type
The last step was choosing hvplot as this helps in plotting xarray data
import hvplot.xarray
hvplot.quadmesh
And here is my final solution:
title = ('Ceilo data + '\ndate: '+ str(DS.year) + '-' + str(DS.month) + '-' + str(DS.day))
ceilo = (DS.br.hvplot.quadmesh(cmap="viridis_r", width = 850, height = 600, title = title,
clim = (1000, 10000), # set colorbar limits
cnorm = ('log'), # choose log scale
clabel = ('colorbar title'),
rot = 0 # degree rotation of ticks
)
)
# from: https://justinbois.github.io/bootcamp/2020/lessons/l27_holoviews.html
# take care! may take 2...3 minutes to be ploted:
p = hv.Points(data=df,
kdims=['Datetime', 'PTU'],
).opts(#alpha=0.7,
color='red',
size=1,
ylim=(0, 5000))
# add PTU line plot to quadmesh plot using * which overlays the line on the plot
ceilo * p
I wrote a class to rescale images, but the RGB value became ranging from 0 to 1 after preocessing. What happened to the RGB which intuitively should be ranging from 0-255 ? Following are the Rescale class and the RGB values after rescaling.
Question:
Do I still need a Min-Max Normalization, map the RGB value to 0-1?
How do I apply transforms.Normalization, where do I put the Normalization, before or after the Rescale, how do I calculate the mean and variance, use the RGB value ranging from 0-255 or 0-1？
Thanks for your time!
class Rescale(object):
def __init__(self, output_size):
assert isinstance(output_size, (int, tuple))
self.output_size = output_size
def __call__(self, sample):
image, anno = sample['image'], sample['anno']
# get orginal width and height of image
h, w = image.shape[0:2]
# if output_size is an integer
if isinstance(self.output_size, int):
if h > w:
new_h, new_w = h * self.output_size / w, self.output_size
else:
new_h, new_w = self.output_size / h, w * self.output_size / h
# if output size is a tuple (a, b)
else:
new_h, new_w = self.output_size
new_h, new_w = int(new_h), int(new_w)
image = transform.resize(image, (new_h, new_w))
return {'image': image, 'anno': anno}
[[[0.67264216 0.50980392 0.34503034]
[0.67243905 0.51208121 0.34528431]
[0.66719145 0.51817184 0.3459951 ]
...
[0.23645098 0.2654311 0.3759458 ]
[0.24476471 0.28003857 0.38963938]
[0.24885877 0.28807445 0.40935877]]
[[0.67465196 0.50994608 0.3452402 ]
[0.68067157 0.52031373 0.3531848 ]
[0.67603922 0.52732436 0.35839216]
...
[0.23458333 0.25195098 0.36822142]
[0.2461343 0.26886127 0.38314558]
[0.2454384 0.27233056 0.39977664]]
[[0.67707843 0.51237255 0.34766667]
[0.68235294 0.5219951 0.35553024]
[0.67772059 0.52747687 0.35659176]
...
[0.24485294 0.24514568 0.36592999]
[0.25407436 0.26205475 0.38063318]
[0.2597007 0.27202914 0.40214216]]
...
[[[172 130 88]
[172 130 88]
[172 130 88]
...
[ 63 74 102]
[ 65 76 106]
[ 67 77 112]]
[[173 131 89]
[173 131 89]
[173 131 89]
...
[ 65 74 103]
[ 64 75 105]
[ 63 73 108]]
[[173 131 89]
[174 132 90]
[174 132 90]
...
[ 63 72 101]
[ 62 71 102]
[ 61 69 105]]
...
You can use torchvision to accomplish this.
transform = transforms.Compose([
transforms.Resize(output_size),
transforms.ToTensor(),
])
This requires a PIL image as input. It will return the tensor in [0, 1] range.You may also add mean-standard normalization as below
transform = transforms.Compose([
transforms.Resize(output_size),
transforms.ToTensor(),
transforms.Normalize(mean, std),
])
Here mean and std are per channel mean and standard deviation of all pixels of all images in the training set. You need to calculate them after resizing all images and converting to torch Tensor. One way to do this would be to apply first two transformation (resize and ToTensor) and then calculate mean and std over all training images like this
x = torch.concatenate([train_data[i] for i in range(len(train_data))])
mean = torch.mean(x, dim=(0, 1))
std = torch.std(x, dim=(0, 1))
Then you use this mean and std value with Normalize transorm above.
I have pre-trained weights for maskrcnn in caffe2 in .pkl extension and it's config file as yaml. If I try to load it directly it throws Improper config format: . Is there a way to use it without installing caffe2.
Config.py
MODEL:
TYPE: generalized_rcnn
CONV_BODY: FPN.add_fpn_ResNet101_conv5_body
NUM_CLASSES: 6
FASTER_RCNN: True
MASK_ON: True
NUM_GPUS: 8
SOLVER:
WEIGHT_DECAY: 0.0001
LR_POLICY: steps_with_decay
# 1x schedule (note TRAIN.IMS_PER_BATCH: 1)
BASE_LR: 0.01
GAMMA: 0.1
MAX_ITER: 180000
STEPS: [0, 120000, 160000]
FPN:
FPN_ON: True
MULTILEVEL_ROIS: True
MULTILEVEL_RPN: True
MRCNN:
ROI_MASK_HEAD: mask_rcnn_heads.mask_rcnn_fcn_head_v1up4convs
RESOLUTION: 28 # (output mask resolution) default 14
ROI_XFORM_METHOD: RoIAlign
ROI_XFORM_RESOLUTION: 14 # default 7
ROI_XFORM_SAMPLING_RATIO: 2 # default 0
DILATION: 1 # default 2
CONV_INIT: MSRAFill # default GaussianFill
TRAIN:
# md5sum of weights pkl file: aa14062280226e48f569ef1c7212e7c7
DATASETS: ('medline_train',)
SCALES: (400,)
MAX_SIZE: 512
IMS_PER_BATCH: 1
BATCH_SIZE_PER_IM: 512
RPN_PRE_NMS_TOP_N: 2000 # Per FPN level
USE_FLIPPED: False
TEST:
DATASETS: ('medline_val',)
SCALE: 400
MAX_SIZE: 512
NMS: 0.5
RPN_PRE_NMS_TOP_N: 1000 # Per FPN level
RPN_POST_NMS_TOP_N: 1000
FORCE_JSON_DATASET_EVAL: True
OUTPUT_DIR: .
I have a csv file which looks like below
date mse
2018-02-11 14.34
2018-02-12 7.24
2018-02-13 4.5
2018-02-14 3.5
2018-02-16 12.67
2018-02-21 45.66
2018-02-22 15.33
2018-02-24 98.44
2018-02-26 23.55
2018-02-27 45.12
2018-02-28 78.44
2018-03-01 34.11
2018-03-05 23.33
2018-03-06 7.45
... ...
Now I want to get two clusters for the mse values so that I know what values lies to which cluster and their mean.
Now since I do not have any other set of values apart from mse (I have to provide X and Y), I would like to use just mse values to get a k means cluster.For now for the other set of values, I pass it as range which is of same size as no of mse values.This is what I did
from sklearn.cluster import KMeans
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
df = pd.read_csv("generate_csv/all_data_device.csv", parse_dates=["date"])
f1 = df['mse'].values
# generate another list
f2 = list(range(0, len(f1)))
X = np.array(list(zip(f1, f2)))
kmeans = KMeans(n_clusters=2).fit(X)
labels = kmeans.predict(X)
# Centroid values
centroids = kmeans.cluster_centers_
#print(centroids)
fig = plt.figure()
ax = Axes3D(fig)
ax.scatter(X[:, 0], X[:, 1], c=labels)
ax.scatter(centroids[:, 0], centroids[:, 1], marker='*', c='#050505', s=1000)
plt.title('K Mean Classification')
plt.show()
How can I just use the mse values to get the k means cluster?I am aware of the function 'reshape()' but not quite sure how to use it?
Demo:
In [29]: kmeans = KMeans(n_clusters=2)
In [30]: df['label'] = kmeans.fit_predict(df[['mse']])
# NOTE: ----> ^ ^
In [31]: df
Out[31]:
date mse label
0 2018-02-11 14.34 0
1 2018-02-12 7.24 0
2 2018-02-13 4.50 0
3 2018-02-14 3.50 0
4 2018-02-16 12.67 0
5 2018-02-21 45.66 0
6 2018-02-22 15.33 0
7 2018-02-24 98.44 1
8 2018-02-26 23.55 0
9 2018-02-27 45.12 0
10 2018-02-28 78.44 1
11 2018-03-01 34.11 0
12 2018-03-05 23.33 0
13 2018-03-06 7.45 0
plotting:
In [64]: ax = df[df['label']==0].plot.scatter(x='mse', y='label', s=50, color='white', edgecolor='black')
In [65]: df[df['label']==1].plot.scatter(x='mse', y='label', s=50, color='white', ax=ax, edgecolor='red')
Out[65]: <matplotlib.axes._subplots.AxesSubplot at 0xfa42be0>
In [66]: plt.scatter(kmeans.cluster_centers_.ravel(), [0.5]*len(kmeans.cluster_centers_), s=100, color='green', marker='*')
Out[66]: <matplotlib.collections.PathCollection at 0xfabf208>
I'm trying to learn how to use a random forest generator in Python by adapting code found here: http://blog.yhathq.com/posts/random-forests-in-python.html to a fake data set
I'm trying to predict whether a person is male (0) or female (1) based on their weight and height
Data looks like so:
Weight Height Gender
150 60 1
250 85 0
175 75 0
100 62 1
90 58 1
200 80 0
... ... ...
165 66 0
Now I'm trying to classify the test set into male and female
Here is the code:
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import numpy as np
xl = pd.ExcelFile(fakedata.xlsx')
df = xl.parse()
df.head()
df['is_train'] = np.random.uniform(0, 1, len(df)) <= .75
train, test = df[df['is_train']==True], df[df['is_train']==False]
features = df.columns[:2]
clf = RandomForestClassifier(n_jobs=2)
y, _ = pd.factorize(train['Gender'])
clf.fit(train[features], y)
I understand what this code accomplishes up to here but I run into problems after:
preds = train['Gender'][clf.predict(test[features])]
print(pd.crosstab(test['Gender'], preds, rownames=['actual'], colnames=['preds']))
gives me the error
ValueError: cannot reindex from a duplicate axis
What exactly am I missing?
You shouldn't index by the predictions in your line preds = train['Gender'][clf.predict(test[features])]. Your predictions should simply be
preds = clf.predict(test[features])