I'm trying to make a simple style copy net, combining two images. I'm a newbie and doing it from the example to get some experience in programming. The main idea is to copy style on the target image. Here's the code I wrote:
def preprocess(image_path):
img = load_img(image_path, target_size = (img_height, img_width))
img = img_to_array(img)
img = np.expand_dims(img, axis = 0)
img = vgg19.preprocess_input(img)
return img
def deprocess(x):
x[:, :, 0] += 103.939
x[:, :, 1] += 116.779
x[:, :, 2] += 123.68
x = x[:, :, ::-1]
x = np.clip(x, 0, 255).astype('unit8')
return x
target_image = backend.variable(preprocess(target_image_path))
sr_image = backend.variable(preprocess(sr_image_path))
if backend.image_data_format() == 'channels_first':
combination_image = backend.placeholder((1,3,img_height, img_width))
else:
combination_image = backend.placeholder((1,img_height, img_width,3))
input_tensor = backend.concatenate([target_image, sr_image, combination_image], axis = 0)
model = vgg19.VGG19(input_tensor = input_tensor, weights = 'imagenet', include_top = False)
print('Model loaded successfully')
def content_loss(base, combination):
return backend.sum(backend.square(combination - base))
def gram_matrix(x):
features = backend.batch_flatten(backend.permute_dimensions(x, (2, 0, 1)))
gram = backend.dot(features, backend.transpose(features))
return gram
def style_loss(style, combination):
S = gram_matrix(style)
C = gram_matrix(combination)
channels = 3
size = img_height * img_width
return backend.sum(backend.square(S - C)) / (4.0 * (channels ** 2) * (size ** 2))
def total_variation_loss(x):
a = backend.square(
x[:, :img_height - 1, :img_width - 1, :] -
x[:, 1:, :img_width - 1, :])
b = backend.square(
x[:, :img_height - 1, :img_width - 1, :] -
x[:, :img_height - 1, 1:, :])
return backend.sum(backend.pow(a + b, 1.25))
outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
content_layer = 'block5_conv2'
style_layers = ['block1_conv1',
'block2_conv1',
'block3_conv1',
'block4_conv1',
'block5_conv1']
total_variation_weight = 1e-4
style_weight = 1.
content_weight = 0.025
loss = backend.variable(0.0)
layer_features = outputs_dict[content_layer]
target_image_features = layer_features[0, :, :, :]
combination_features = layer_features[2, :, :, :]
loss = loss + content_weight * content_loss(target_image_features, combination_features)
for layer_name in style_layers:
layer_features = outputs_dict[layer_name]
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[2, :, :, :]
sl = style_loss(style_reference_features, combination_features)
loss = loss + (style_weight / len(style_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)
grads = backend.gradients(loss, combination_image)
outputs = [loss]
if isinstance(grads, (list,tuple)):
outputs += grads
else:
outputs.append(grads)
f_outputs = backend.function([combination_image], outputs)
def eval_loss_and_grads(x):
if backend.image_data_format() == 'channels_first':
x = x.reshape((1, 3, img_height, img_width))
else:
x = x.reshape((1, img_height, img_width, 3))
outs = f_outputs([x])
loss_value = outs[0]
if len(outs[1:]) == 1:
grad_values = outs[1].flatten().astype('float64')
else:
grad_values = np.array(outs[1:]).flatten().astype('float64')
return loss_value, grad_values
class Evaluator(object):
def _unit_(self):
self.loss_value = None
self.grads_values = None
def loss(self, x):
assert self.loss_value is None
loss_value, grad_values = eval_loss_and_grads(x)
self.loss_value = loss_value
self.grads_values = grad_values
return self.loss_value
def grads(self, x):
assert self.loss_value is not None
grad_values = np.copy(self.grad_values)
self.loss_value = None
self.grad_values = None
return grad_values
evaluator = Evaluator()
result_prefix = 'result'
iterations = 20
x = preprocess(target_image_path)
x = x.flatten()
for i in range(iterations):
print('Start of iterations', i)
start_time = time.time()
x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x,
fprime = evaluator.grads, maxfun = 20)
print('Current loss value:', min_val)
img = x.copy().reshape((img_height, img_width, 3))
img = deprocess(img)
fname = result_prefix + '_at_iteration_%d.png' % i
save_img('D:\study\Stylecopy\fmane', img)
print('image saved as:', fname)
end_time = time.time()
print(' Iteration %d completed in %ds' % (i , end_time - start_time))
Here's the error I got:
AttributeError Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_12264/556996678.py in <module>
7 print('Start of iterations', i)
8 start_time = time.time()
----> 9 x, min_val, info = fmin_l_bfgs_b(evaluator.loss, x,
10 fprime = evaluator.grads, maxfun = 20)
11 print('Current loss value:', min_val)
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\lbfgsb.py in fmin_l_bfgs_b(func, x0, fprime, args, approx_grad, bounds, m, factr, pgtol, epsilon, iprint, maxfun, maxiter, disp, callback, maxls)
195 'maxls': maxls}
196
--> 197 res = _minimize_lbfgsb(fun, x0, args=args, jac=jac, bounds=bounds,
198 **opts)
199 d = {'grad': res['jac'],
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\lbfgsb.py in _minimize_lbfgsb(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, finite_diff_rel_step, **unknown_options)
304 iprint = disp
305
--> 306 sf = _prepare_scalar_function(fun, x0, jac=jac, args=args, epsilon=eps,
307 bounds=new_bounds,
308 finite_diff_rel_step=finite_diff_rel_step)
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\optimize.py in _prepare_scalar_function(fun, x0, jac, args, bounds, epsilon, finite_diff_rel_step, hess)
259 # ScalarFunction caches. Reuse of fun(x) during grad
260 # calculation reduces overall function evaluations.
--> 261 sf = ScalarFunction(fun, x0, args, grad, hess,
262 finite_diff_rel_step, bounds, epsilon=epsilon)
263
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\_differentiable_functions.py in __init__(self, fun, x0, args, grad, hess, finite_diff_rel_step, finite_diff_bounds, epsilon)
138
139 self._update_fun_impl = update_fun
--> 140 self._update_fun()
141
142 # Gradient evaluation
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\_differentiable_functions.py in _update_fun(self)
231 def _update_fun(self):
232 if not self.f_updated:
--> 233 self._update_fun_impl()
234 self.f_updated = True
235
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\_differentiable_functions.py in update_fun()
135
136 def update_fun():
--> 137 self.f = fun_wrapped(self.x)
138
139 self._update_fun_impl = update_fun
~\anaconda3\envs\CNN_base\lib\site-packages\scipy\optimize\_differentiable_functions.py in fun_wrapped(x)
132 # Overwriting results in undefined behaviour because
133 # fun(self.x) will change self.x, with the two no longer linked.
--> 134 return fun(np.copy(x), *args)
135
136 def update_fun():
~\AppData\Local\Temp/ipykernel_12264/3220866978.py in loss(self, x)
29
30 def loss(self, x):
---> 31 assert self.loss_value is None
32 loss_value, grad_values = eval_loss_and_grads(x)
33 self.loss_value = loss_value
AttributeError: 'Evaluator' object has no attribute 'loss_value'
I'm dealing with this problem and don't know how to solve it. I've double checked the code with the example (https://www.kaggle.com/code/gabrieltangzy/p2p-gan-newver-slice-cezanne), but haven't found any mistakes. I suppose it may occur because of difference in python versions. I use 3.9.7
Related
Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 9 months ago.
Improve this question
RuntimeError Traceback (most recent call last)
/tmp/ipykernel_33/2056227650.py in <module>
----> 1 learn.fit_one_cycle(n, max_learning_rate)
2 learn.recorder.plot_losses()
/opt/conda/lib/python3.7/site-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)
21 callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,
22 final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))
---> 23 learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
24
25 def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr, moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,
/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
198 else: self.opt.lr,self.opt.wd = lr,wd
199 callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
--> 200 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
201
202 def create_opt(self, lr:Floats, wd:Floats=0.)->None:
/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
104 if not cb_handler.skip_validate and not learn.data.empty_val:
105 val_loss = validate(learn.model, learn.data.valid_dl, loss_func=learn.loss_func,
--> 106 cb_handler=cb_handler, pbar=pbar)
107 else: val_loss=None
108 if cb_handler.on_epoch_end(val_loss): break
/opt/conda/lib/python3.7/site-packages/fastai/basic_train.py in validate(model, dl, loss_func, cb_handler, pbar, average, n_batch)
61 if not is_listy(yb): yb = [yb]
62 nums.append(first_el(yb).shape[0])
---> 63 if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
64 if n_batch and (len(nums)>=n_batch): break
65 nums = np.array(nums, dtype=np.float32)
/opt/conda/lib/python3.7/site-packages/fastai/callback.py in on_batch_end(self, loss)
306 "Handle end of processing one batch with `loss`."
307 self.state_dict['last_loss'] = loss
--> 308 self('batch_end', call_mets = not self.state_dict['train'])
309 if self.state_dict['train']:
310 self.state_dict['iteration'] += 1
/opt/conda/lib/python3.7/site-packages/fastai/callback.py in __call__(self, cb_name, call_mets, **kwargs)
248 "Call through to all of the `CallbakHandler` functions."
249 if call_mets:
--> 250 for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
251 for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
252
/opt/conda/lib/python3.7/site-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
239 def _call_and_update(self, cb, cb_name, **kwargs)->None:
240 "Call `cb_name` on `cb` and update the inner state."
--> 241 new = ifnone(getattr(cb, f'on_{cb_name}')(**self.state_dict, **kwargs), dict())
242 for k,v in new.items():
243 if k not in self.state_dict:
/opt/conda/lib/python3.7/site-packages/object_detection_fastai/callbacks/callbacks.py in on_batch_end(self, last_output, last_target, **kwargs)
125 scores = scores[:total_nms_examples]
126 preds = preds[:total_nms_examples]
--> 127 to_keep = nms(bbox_pred, scores, self.nms_thresh)
128 bbox_pred, preds, scores = bbox_pred[to_keep].cpu(), preds[to_keep].cpu(), scores[to_keep].cpu()
129
/opt/conda/lib/python3.7/site-packages/object_detection_fastai/helper/object_detection_helper.py in nms(boxes, scores, thresh)
156 mask_keep = iou_vals <= thresh
157 if len(mask_keep.nonzero()) == 0: break
--> 158 idx_first = mask_keep.nonzero().min().item()
159 boxes, scores, indexes = boxes[mask_keep], scores[mask_keep], indexes[mask_keep]
160 return LongTensor(to_keep)
RuntimeError: min(): Expected reduction dim to be specified for input.numel() == 0. Specify the reduction dim with the 'dim' argument.
# %% [code] {"id":"asY9Gl86zcYl","outputId":"71cad981-691c-489e-f203-66ee5df9e25a","execution":{"iopub.status.busy":"2022-05-02T07:47:26.313181Z","iopub.execute_input":"2022-05-02T07:47:26.313767Z","iopub.status.idle":"2022-05-02T07:47:36.173129Z","shell.execute_reply.started":"2022-05-02T07:47:26.313727Z","shell.execute_reply":"2022-05-02T07:47:36.172282Z"}}
%reload_ext autoreload
%autoreload 2
%matplotlib inline
!pip install -U plotly
import json
from pathlib import Path
import plotly
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots
from tqdm import tqdm
import pandas as pd
import random
import cv2
# %% [code] {"id":"NvOPXHfk2PzM","outputId":"4849c9ec-32d2-4aeb-b8f6-5a881423392e","execution":{"iopub.status.busy":"2022-05-02T07:47:36.175278Z","iopub.execute_input":"2022-05-02T07:47:36.175539Z","iopub.status.idle":"2022-05-02T07:47:36.278379Z","shell.execute_reply.started":"2022-05-02T07:47:36.175505Z","shell.execute_reply":"2022-05-02T07:47:36.277469Z"}}
folder = "midog-challenge"
midog_folder = Path("../input") / Path(folder)
print(list(midog_folder.glob("*.*")))
# %% [code] {"id":"3hzo-Io-zsRk","outputId":"5a4b2a8b-e79a-4371-dd41-c819c44026a9","execution":{"iopub.status.busy":"2022-05-02T07:47:36.280155Z","iopub.execute_input":"2022-05-02T07:47:36.280431Z","iopub.status.idle":"2022-05-02T07:47:45.692203Z","shell.execute_reply.started":"2022-05-02T07:47:36.280394Z","shell.execute_reply":"2022-05-02T07:47:45.691328Z"}}
!pip install -U object-detection-fastai
from object_detection_fastai.helper.wsi_loader import *
from object_detection_fastai.loss.RetinaNetFocalLoss import RetinaNetFocalLoss
from object_detection_fastai.models.RetinaNet import RetinaNet
from object_detection_fastai.callbacks.callbacks import BBMetrics, PascalVOCMetricByDistance, PascalVOCMetric, PascalVOCMetricByDistance
# %% [code] {"id":"jiZJLWqD5Rpr","execution":{"iopub.status.busy":"2022-05-02T07:47:45.694445Z","iopub.execute_input":"2022-05-02T07:47:45.694685Z","iopub.status.idle":"2022-05-02T07:47:45.801985Z","shell.execute_reply.started":"2022-05-02T07:47:45.694652Z","shell.execute_reply":"2022-05-02T07:47:45.800941Z"}}
image_folder = midog_folder / "images"
hamamatsu_rx_ids = list(range(0, 51))
hamamatsu_360_ids = list(range(51, 101))
aperio_ids = list(range(101, 151))
leica_ids = list(range(151, 201))
# %% [code] {"id":"tNwJXJufaVt-","outputId":"0e710552-9a03-4825-f1a3-97444a2be238","execution":{"iopub.status.busy":"2022-05-02T07:47:45.803635Z","iopub.execute_input":"2022-05-02T07:47:45.804129Z","iopub.status.idle":"2022-05-02T07:47:46.032320Z","shell.execute_reply.started":"2022-05-02T07:47:45.804090Z","shell.execute_reply":"2022-05-02T07:47:46.031354Z"}}
annotation_file = midog_folder / "MIDOG.json"
print(annotation_file," ",image_folder)
rows = []
with open(annotation_file) as f:
data = json.load(f)
categories = {1: 'mitotic figure', 2: 'hard negative'}
for row in data["images"]:
file_name = row["file_name"]
image_id = row["id"]
width = row["width"]
height = row["height"]
scanner = "Hamamatsu XR"
if image_id in hamamatsu_360_ids:
scanner = "Hamamatsu S360"
if image_id in aperio_ids:
scanner = "Aperio CS"
if image_id in leica_ids:
scanner = "Leica GT450"
for annotation in [anno for anno in data['annotations'] if anno["image_id"] == image_id]:
box = annotation["bbox"]
cat = categories[annotation["category_id"]]
rows.append([file_name, image_id, width, height, box, cat, scanner])
df = pd.DataFrame(rows, columns=["file_name", "image_id", "width", "height", "box", "cat", "scanner"])
df.head()
# %% [markdown] {"id":"r2Tm_N5PqbMJ"}
# ### Visual Examples
# %% [code] {"id":"KIALOeDIuCKo","execution":{"iopub.status.busy":"2022-05-02T07:47:53.807583Z","iopub.execute_input":"2022-05-02T07:47:53.807862Z","iopub.status.idle":"2022-05-02T07:47:53.904670Z","shell.execute_reply.started":"2022-05-02T07:47:53.807827Z","shell.execute_reply":"2022-05-02T07:47:53.903659Z"}}
def sample_function(y, classes, size, level_dimensions, level):
width, height = level_dimensions[level]
if len(y[0]) == 0:
return randint(0, width - size[0]), randint(0, height -size[1])
else:
#if randint(0, 5) < 2:
if True:
class_id = np.random.choice(classes, 1)[0] # select a random class
ids = np.array(y[1]) == class_id # filter the annotations according to the selected class
xmin, ymin, _, _ = np.array(y[0])[ids][randint(0, np.count_nonzero(ids) - 1)] # randomly select one of the filtered annotatons as seed for the training patch
# To have the selected annotation not in the center of the patch and an random offset.
xmin += random.randint(-size[0]/2, size[0]/2)
ymin += random.randint(-size[1]/2, size[1]/2)
xmin, ymin = max(0, int(xmin - size[0] / 2)), max(0, int(ymin -size[1] / 2))
xmin, ymin = min(xmin, width - size[0]), min(ymin, height - size[1])
return xmin, ymin
else:
return randint(0, width - size[0]), randint(0, height -size[1])
# %% [code] {"id":"HH_0sG8w4TfA","outputId":"d9de5667-10c5-46b1-c0e0-28a99e7f95d7","execution":{"iopub.status.busy":"2022-05-02T07:47:58.157246Z","iopub.execute_input":"2022-05-02T07:47:58.157508Z","iopub.status.idle":"2022-05-02T07:48:00.797900Z","shell.execute_reply.started":"2022-05-02T07:47:58.157480Z","shell.execute_reply":"2022-05-02T07:48:00.797151Z"}}
def create_wsi_container(annotations_df: pd.DataFrame):
container = []
for image_name in tqdm(annotations_df["file_name"].unique()):
image_annos = annotations_df[annotations_df["file_name"] == image_name]
bboxes = [box for box in image_annos["box"]]
labels = [label for label in image_annos["cat"]]
container.append(SlideContainer(image_folder/image_name, y=[bboxes, labels], level=res_level,width=patch_size, height=patch_size, sample_func=sample_function))
return container
train_scanner = "Aperio CS" #["Hamamatsu XR", "Hamamatsu S360", "Aperio CS"]
val_scanner = "Hamamatsu XR" #["Hamamatsu XR", "Hamamatsu S360", "Aperio CS"]
patch_size = 256
res_level = 0
train_annos = df[df["scanner"].isin(train_scanner.split(","))]
train_container = create_wsi_container(train_annos)
val_annos = df[df["scanner"].isin(val_scanner.split(","))]
valid_container = create_wsi_container(val_annos)
f"Created: {len(train_container)} training WSI container and {len(valid_container)} validation WSI container"
# %% [code] {"cellView":"form","id":"Mei_iD1sxJCA","outputId":"2c87f1b1-e9fd-4de1-bd16-6fdc5224d05a","execution":{"iopub.status.busy":"2022-05-02T07:48:18.070337Z","iopub.execute_input":"2022-05-02T07:48:18.070756Z","iopub.status.idle":"2022-05-02T07:48:18.213409Z","shell.execute_reply.started":"2022-05-02T07:48:18.070722Z","shell.execute_reply":"2022-05-02T07:48:18.212544Z"}}
import numpy as np
train_samples_per_scanner = 1500
val_samples_per_scanner = 500
train_images = list(np.random.choice(train_container, train_samples_per_scanner))
print('training_images =',len(train_images))
valid_images = list(np.random.choice(valid_container, val_samples_per_scanner))
print('validation_images =',len(valid_images))
# %% [code] {"id":"UZKoHWjR1mUG","outputId":"5cd9552e-8a9a-4aeb-f898-614e30e17eca","execution":{"iopub.status.busy":"2022-05-02T07:48:20.127048Z","iopub.execute_input":"2022-05-02T07:48:20.129167Z","iopub.status.idle":"2022-05-02T07:48:48.039604Z","shell.execute_reply.started":"2022-05-02T07:48:20.129128Z","shell.execute_reply":"2022-05-02T07:48:48.038869Z"}}
batch_size = 64
do_flip = True
flip_vert = True
max_rotate = 90
max_zoom = 1.1
max_lighting = 0.2
max_warp = 0.2
p_affine = 0.75
p_lighting = 0.75
tfms = get_transforms(do_flip=do_flip,
flip_vert=flip_vert,
max_rotate=max_rotate,
max_zoom=max_zoom,
max_lighting=max_lighting,
max_warp=max_warp,
p_affine=p_affine,
p_lighting=p_lighting)
train, valid = ObjectItemListSlide(train_images), ObjectItemListSlide(valid_images)
item_list = ItemLists(".", train, valid)
lls = item_list.label_from_func(lambda x: x.y, label_cls=SlideObjectCategoryList)
lls = lls.transform(tfms, tfm_y=True, size=patch_size)
data = lls.databunch(bs=batch_size, collate_fn=bb_pad_collate,num_workers=0).normalize()
# %% [code] {"id":"g5PZ9e-c2k1S","outputId":"317422c4-b572-4898-dd82-84c054f98735","execution":{"iopub.status.busy":"2022-05-02T07:48:54.016278Z","iopub.execute_input":"2022-05-02T07:48:54.016534Z","iopub.status.idle":"2022-05-02T07:48:54.455317Z","shell.execute_reply.started":"2022-05-02T07:48:54.016506Z","shell.execute_reply":"2022-05-02T07:48:54.454672Z"}}
scales = [2]
ratios=[1]
#The feature map sizes. [(64,64), (32,32) , (16,16), (8,8), (4,4)]
sizes=[(32,32)]
anchors = create_anchors(sizes=sizes, ratios=ratios, scales=scales)
fig,ax = plt.subplots(figsize=(15,15))
ax.imshow(image2np(data.valid_ds[0][0].data))
for i, bbox in enumerate(anchors[:len(scales)*len(ratios)*len(sizes)]):
bb = bbox.numpy()
x = (bb[0] + 1) * patch_size / 2
y = (bb[1] + 1) * patch_size / 2
w = bb[2] * patch_size / 2
h = bb[3] * patch_size / 2
rect = [x,y,w,h]
draw_rect(ax,rect)
# %% [code] {"id":"Mfrl8VJ94edJ","outputId":"3793f0b0-fa4c-4c87-a4ed-14840d18b4c2","execution":{"iopub.status.busy":"2022-05-02T07:49:03.926600Z","iopub.execute_input":"2022-05-02T07:49:03.926887Z","iopub.status.idle":"2022-05-02T07:49:39.263251Z","shell.execute_reply.started":"2022-05-02T07:49:03.926856Z","shell.execute_reply":"2022-05-02T07:49:39.262482Z"}}
all_boxes, all_labels = show_anchors_on_images(data, anchors, figsize=(12, 12))
# %% [code] {"id":"J-qEw_bN41cG","outputId":"eaf1c350-1c1f-4d50-a919-311ee538e612","execution":{"iopub.status.busy":"2022-05-02T07:49:39.264920Z","iopub.execute_input":"2022-05-02T07:49:39.265265Z","iopub.status.idle":"2022-05-02T07:49:39.546091Z","shell.execute_reply.started":"2022-05-02T07:49:39.265226Z","shell.execute_reply":"2022-05-02T07:49:39.545301Z"}}
from fastai.utils.collect_env import show_install
show_install()
# %% [code] {"id":"te6ci-Z35_5I","outputId":"67f2f18b-041b-44fd-fdc6-1a53121c445a","execution":{"iopub.status.busy":"2022-05-02T07:49:39.548856Z","iopub.execute_input":"2022-05-02T07:49:39.549088Z","iopub.status.idle":"2022-05-02T07:49:47.657381Z","shell.execute_reply.started":"2022-05-02T07:49:39.549058Z","shell.execute_reply":"2022-05-02T07:49:47.656648Z"}}
backbone = "ResNet101" #["ResNet18", "ResNet34", "ResNet50", "ResNet101", "ResNet150"]
backbone_model = models.resnet18
if backbone == "ResNet34":
backbone_model = models.resnet34
if backbone == "ResNet50":
backbone_model = models.resnet50
if backbone == "ResNet101":
backbone_model = models.resnet101
if backbone == "ResNet150":
backbone_model = models.resnet150
pre_trained_on_imagenet = True
encoder = create_body(models.resnet101, pre_trained_on_imagenet, -2)
loss_function = "FocalLoss"
if loss_function == "FocalLoss":
crit = RetinaNetFocalLoss(anchors)
channels = 128
final_bias = -4
n_conv = 3
model = RetinaNet(encoder, n_classes=data.train_ds.c,
n_anchors=len(scales) * len(ratios),
sizes=[size[0] for size in sizes],
chs=channels, # number of hidden layers for the classification head
final_bias=final_bias,
n_conv=n_conv # Number of hidden layers
)
# %% [code] {"id":"waS3UZNaNz4p","outputId":"9b49bd00-7b21-4cad-8588-a46248953370","execution":{"iopub.status.busy":"2022-05-02T07:49:47.659276Z","iopub.execute_input":"2022-05-02T07:49:47.659604Z","iopub.status.idle":"2022-05-02T07:49:47.758263Z","shell.execute_reply.started":"2022-05-02T07:49:47.659566Z","shell.execute_reply":"2022-05-02T07:49:47.757182Z"}}
voc = PascalVOCMetric(anchors, patch_size, [str(i) for i in data.train_ds.y.classes[1:]])
voc
# %% [code] {"id":"nwgGmvsPZ3bN","execution":{"iopub.status.busy":"2022-05-02T07:49:47.759735Z","iopub.execute_input":"2022-05-02T07:49:47.760020Z","iopub.status.idle":"2022-05-02T07:49:47.973584Z","shell.execute_reply.started":"2022-05-02T07:49:47.759982Z","shell.execute_reply":"2022-05-02T07:49:47.972749Z"}}
learn = Learner(data, model, loss_func=crit,
callback_fns=[BBMetrics, ShowGraph],
metrics=[voc]
)
learn.split([model.encoder[6], model.c5top5])
learn.freeze_to(-2)
# %% [code] {"id":"NWwQRvzw6vnr","outputId":"e9c8b89b-0a90-468b-de6d-351fd09bf467","execution":{"iopub.status.busy":"2022-05-02T07:47:46.267766Z","iopub.status.idle":"2022-05-02T07:47:46.268297Z","shell.execute_reply.started":"2022-05-02T07:47:46.268040Z","shell.execute_reply":"2022-05-02T07:47:46.268066Z"}}
learn.lr_find()
learn.recorder.plot(suggestion=True)
# %% [code] {"id":"gK1s61jSJtie","outputId":"aa0f9d65-3d19-42d6-b17e-7a29fe5fca8f","execution":{"iopub.status.busy":"2022-05-01T17:20:57.239658Z","iopub.execute_input":"2022-05-01T17:20:57.239975Z","iopub.status.idle":"2022-05-01T19:08:05.944655Z","shell.execute_reply.started":"2022-05-01T17:20:57.239931Z","shell.execute_reply":"2022-05-01T19:08:05.939947Z"}}
max_learning_rate = 1e-3
n=250
learn.fit_one_cycle(n, max_learning_rate)
learn.recorder.plot_sched()
learn.recorder.plot_losses()
This issue arises when the tensor on which you take max/min becomes empty.
I'm trying to measure the latent space clustering but the error raised.
class AutoEncoder(nn.Module):
def __init__(self, input_dim1, input_dim2, hidden_dims, agg, sep_decode):
super(AutoEncoder, self).__init__()
self.agg = agg
self.sep_decode = sep_decode
print("hidden_dims:", hidden_dims)
self.encoder_layers = []
self.encoder2_layers = []
dims = [[input_dim1, input_dim2]] + hidden_dims
for i in range(len(dims) - 1):
if i == 0:
layer = nn.Sequential(nn.Linear(dims[i][0], dims[i+1]), nn.ReLU())
layer2 = nn.Sequential(nn.Linear(dims[i][1], dims[i+1]), nn.ReLU())
elif i != 0 and i < len(dims) - 2:
layer = nn.Sequential(nn.Linear(dims[i], dims[i+1]), nn.ReLU())
layer2 = nn.Sequential(nn.Linear(dims[i], dims[i+1]), nn.ReLU())
else:
layer = nn.Linear(dims[i], dims[i+1])
layer2 = nn.Linear(dims[i], dims[i+1])
self.encoder_layers.append(layer)
self.encoder2_layers.append(layer2)
self.encoder = nn.Sequential(*self.encoder_layers)
self.encoder2 = nn.Sequential(*self.encoder2_layers)
self.decoder_layers = []
self.decoder2_layers = []
hidden_dims.reverse()
dims = hidden_dims + [[input_dim1, input_dim2]]
if self.agg == "concat" and not self.sep_decode:
dims[0] = 2 * dims[0]
for i in range(len(dims) - 1):
if i < len(dims) - 2:
layer = nn.Sequential(nn.Linear(dims[i], dims[i+1]), nn.ReLU())
layer2 = nn.Sequential(nn.Linear(dims[i], dims[i+1]), nn.ReLU())
else:
layer = nn.Linear(dims[i], dims[i+1][0])
layer2 = nn.Linear(dims[i], dims[i+1][1])
self.decoder_layers.append(layer)
self.decoder2_layers.append(layer2)
self.decoder = nn.Sequential(*self.decoder_layers)
self.decoder2 = nn.Sequential(*self.decoder2_layers)
def forward(self, x1, x2):
z1 = self.encoder(x1)
z2 = self.encoder2(x2)
if self.agg == "max":
z = torch.max(z1, z2)
elif self.agg == "multi":
z = z1 * z2
elif self.agg == "sum":
z = z1 + z2
elif self.agg == "concat":
z = torch.cat([z1, z2], dim=1)
if self.sep_decode:
x_bar1 = self.decoder(z1)
x_bar1 = F.normalize(x_bar1, dim=-1)
x_bar2 = self.decoder2(z2)
x_bar2 = F.normalize(x_bar2, dim=-1)
else:
x_bar1 = self.decoder(z)
x_bar1 = F.normalize(x_bar1, dim=-1)
x_bar2 = self.decoder2(z)
x_bar2 = F.normalize(x_bar2, dim=-1)
return x_bar1, x_bar2, z
class TopicCluster(nn.Module):
def __init__(self, args):
super(TopicCluster, self).__init__()
self.alpha = 1.0
self.dataset_path = args.dataset_path
self.args = args
self.device = args.device
self.temperature = args.temperature
self.distribution = args.distribution
self.agg_method = args.agg_method
self.sep_decode = (args.sep_decode == 1)
input_dim1 = args.input_dim1
input_dim2 = args.input_dim2
hidden_dims = eval(args.hidden_dims)
self.model = AutoEncoder(input_dim1, input_dim2, hidden_dims, self.agg_method, self.sep_decode)
if self.agg_method == "concat":
self.topic_emb = Parameter(torch.Tensor(args.n_clusters, 2*hidden_dims[-1]))
else:
self.topic_emb = Parameter(torch.Tensor(args.n_clusters, hidden_dims[-1]))
torch.nn.init.xavier_normal_(self.topic_emb.data)
def pretrain(self, input_data, pretrain_epoch=200):
pretrained_path = os.path.join(self.dataset_path, f"pretrained_{args.suffix}.pt")
if os.path.exists(pretrained_path) and self.args.load_pretrain:
# load pretrain weights
print(f"loading pretrained model from {pretrained_path}")
self.model.load_state_dict(torch.load(pretrained_path))
else:
train_loader = DataLoader(input_data, batch_size=self.args.batch_size, shuffle=True)
optimizer = Adam(self.model.parameters(), lr=self.args.lr)
for epoch in range(pretrain_epoch):
total_loss = 0
for batch_idx, (x1, x2, _, weight) in enumerate(train_loader):
x1 = x1.to(self.device)
x2 = x2.to(self.device)
weight = weight.to(self.device)
optimizer.zero_grad()
x_bar1, x_bar2, z = self.model(x1, x2)
loss = cosine_dist(x_bar1, x1) + cosine_dist(x_bar2, x2) #, weight)
total_loss += loss.item()
loss.backward()
optimizer.step()
print(f"epoch {epoch}: loss = {total_loss / (batch_idx+1):.4f}")
torch.save(self.model.state_dict(), pretrained_path)
print(f"model saved to {pretrained_path}")
def cluster_assign(self, z):
if self.distribution == 'student':
p = 1.0 / (1.0 + torch.sum(
torch.pow(z.unsqueeze(1) - self.topic_emb, 2), 2) / self.alpha)
p = p.pow((self.alpha + 1.0) / 2.0)
p = (p.t() / torch.sum(p, 1)).t()
else:
self.topic_emb.data = F.normalize(self.topic_emb.data, dim=-1)
z = F.normalize(z, dim=-1)
sim = torch.matmul(z, self.topic_emb.t()) / self.temperature
p = F.softmax(sim, dim=-1)
return p
def forward(self, x1, x2):
x_bar1, x_bar2, z = self.model(x1, x2)
p = self.cluster_assign(z)
return x_bar1, x_bar2, z, p
def target_distribution(self, x1, x2, freq, method='all', top_num=0):
_, _, z = self.model(x1, x2)
p = self.cluster_assign(z).detach()
if method == 'all':
q = p**2 / (p * freq.unsqueeze(-1)).sum(dim=0)
q = (q.t() / q.sum(dim=1)).t()
elif method == 'top':
assert top_num > 0
q = p.clone()
sim = torch.matmul(self.topic_emb, z.t())
_, selected_idx = sim.topk(k=top_num, dim=-1)
for i, topic_idx in enumerate(selected_idx):
q[topic_idx] = 0
q[topic_idx, i] = 1
return p, q
def cosine_dist(x_bar, x, weight=None):
if weight is None:
weight = torch.ones(x.size(0), device=x.device)
cos_sim = (x_bar * x).sum(-1)
cos_dist = 1 - cos_sim
cos_dist = (cos_dist * weight).sum() / weight.sum()
return cos_dist
def train(args, emb_dict):
# ipdb.set_trace()
inv_vocab = {k: " ".join(v) for k, v in emb_dict["inv_vocab"].items()}
vocab = {" ".join(k):v for k, v in emb_dict["vocab"].items()}
print(f"Vocab size: {len(vocab)}")
embs = F.normalize(torch.tensor(emb_dict["vs_emb"]), dim=-1)
embs2 = F.normalize(torch.tensor(emb_dict["oh_emb"]), dim=-1)
freq = np.array(emb_dict["tuple_freq"])
if not args.use_freq:
freq = np.ones_like(freq)
input_data = TensorDataset(embs, embs2, torch.arange(embs.size(0)), torch.tensor(freq))
topic_cluster = TopicCluster(args).to(args.device)
topic_cluster.pretrain(input_data, args.pretrain_epoch)
train_loader = DataLoader(input_data, batch_size=args.batch_size, shuffle=False)
optimizer = Adam(topic_cluster.parameters(), lr=args.lr)
# topic embedding initialization
embs = embs.to(args.device)
embs2 = embs2.to(args.device)
x_bar1, x_bar2, z = topic_cluster.model(embs, embs2)
z = F.normalize(z, dim=-1)
print(f"Running K-Means for initialization")
kmeans = KMeans(n_clusters=args.n_clusters, n_init=5)
if args.use_freq:
y_pred = kmeans.fit_predict(z.data.cpu().numpy(), sample_weight=freq)
else:
y_pred = kmeans.fit_predict(z.data.cpu().numpy())
print(f"Finish K-Means")
freq = torch.tensor(freq).to(args.device)
y_pred_last = y_pred
topic_cluster.topic_emb.data = torch.tensor(kmeans.cluster_centers_).to(args.device)
topic_cluster.train()
i = 0
for epoch in range(50):
if epoch % 5 == 0:
_, _, z, p = topic_cluster(embs, embs2)
z = F.normalize(z, dim=-1)
topic_cluster.topic_emb.data = F.normalize(topic_cluster.topic_emb.data, dim=-1)
if not os.path.exists(os.path.join(args.dataset_path, f"clusters_{args.suffix}")):
os.makedirs(os.path.join(args.dataset_path, f"clusters_{args.suffix}"))
embed_save_path = os.path.join(args.dataset_path, f"clusters_{args.suffix}/embed_{epoch}.pt")
torch.save({
"inv_vocab": emb_dict['inv_vocab'],
"embed": z.detach().cpu().numpy(),
"topic_embed": topic_cluster.topic_emb.detach().cpu().numpy(),
}, embed_save_path)
f = open(os.path.join(args.dataset_path, f"clusters_{args.suffix}/{epoch}.txt"), 'w')
pred_cluster = p.argmax(-1)
result_strings = []
for j in range(args.n_clusters):
if args.sort_method == 'discriminative':
word_idx = torch.arange(embs.size(0))[pred_cluster == j]
sorted_idx = torch.argsort(p[pred_cluster == j][:, j], descending=True)
word_idx = word_idx[sorted_idx]
else:
sim = torch.matmul(topic_cluster.topic_emb[j], z.t())
_, word_idx = sim.topk(k=30, dim=-1)
word_cluster = []
freq_sum = 0
for idx in word_idx:
freq_sum += freq[idx].item()
if inv_vocab[idx.item()] not in word_cluster:
word_cluster.append(inv_vocab[idx.item()])
if len(word_cluster) >= 10:
break
result_strings.append((freq_sum, f"Topic {j} ({freq_sum}): " + ', '.join(word_cluster)+'\n'))
result_strings = sorted(result_strings, key=lambda x: x[0], reverse=True)
for result_string in result_strings:
f.write(result_string[1])
for x1, x2, idx, weight in train_loader:
if i % args.update_interval == 0:
p, q = topic_cluster.target_distribution(embs, embs2, freq.clone().fill_(1), method='all', top_num=epoch+1)
y_pred = p.cpu().numpy().argmax(1)
delta_label = np.sum(y_pred != y_pred_last).astype(np.float32) / y_pred.shape[0]
y_pred_last = y_pred
if i > 0 and delta_label < args.tol:
print(f'delta_label {delta_label:.4f} < tol ({args.tol})')
print('Reached tolerance threshold. Stopping training.')
return None
i += 1
x1 = x1.to(args.device)
x2 = x2.to(args.device)
idx = idx.to(args.device)
weight = weight.to(args.device)
x_bar1, x_bar2, _, p = topic_cluster(x1, x2)
reconstr_loss = cosine_dist(x_bar1, x1) + cosine_dist(x_bar2, x2) #, weight)
kl_loss = F.kl_div(p.log(), q[idx], reduction='none').sum(-1)
kl_loss = (kl_loss * weight).sum() / weight.sum()
loss = args.gamma * kl_loss + reconstr_loss
if i % args.update_interval == 0:
print(f"KL loss: {kl_loss}; Reconstruction loss: {reconstr_loss}")
optimizer.zero_grad()
loss.backward()
optimizer.step()
return None
if __name__ == "__main__":
# CUDA_VISIBLE_DEVICES=0 python3 latent_space_clustering.py --dataset_path ./pandemic --input_emb_name po_tuple_features_all_svos.pk
parser = argparse.ArgumentParser(
description='train',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--dataset_path', type=str)
parser.add_argument('--input_emb_name', type=str)
parser.add_argument('--lr', type=float, default=5e-4)
parser.add_argument('--n_clusters', default=30, type=int)
parser.add_argument('--input_dim1', default=1000, type=int)
parser.add_argument('--input_dim2', default=1000, type=int)
parser.add_argument('--agg_method', default="multi", choices=["sum", "multi", "concat", "attend"], type=str)
parser.add_argument('--sep_decode', default=0, choices=[0, 1], type=int)
parser.add_argument('--pretrain_epoch', default=100, type=int)
parser.add_argument('--load_pretrain', default=False, action='store_true')
parser.add_argument('--temperature', default=0.1, type=float)
parser.add_argument('--sort_method', default='generative', choices=['generative', 'discriminative'])
parser.add_argument('--distribution', default='softmax', choices=['softmax', 'student'])
parser.add_argument('--batch_size', default=256, type=int)
parser.add_argument('--use_freq', default=False, action='store_true')
parser.add_argument('--hidden_dims', default='[1000, 2000, 1000, 100]', type=str)
parser.add_argument('--suffix', type=str, default='')
parser.add_argument('--gamma', default=5, type=float, help='weight of clustering loss')
parser.add_argument('--update_interval', default=100, type=int)
parser.add_argument('--tol', default=0.001, type=float)
args = parser.parse_args()
args.cuda = torch.cuda.is_available()
print("use cuda: {}".format(args.cuda))
args.device = torch.device("cuda" if args.cuda else "cpu")
print(args)
with open(os.path.join(args.dataset_path, args.input_emb_name), "rb") as fin:
emb_dict = pk.load(fin)
candidate_idx = train(args, emb_dict)
print(candidate_idx)
The error I'm getting is: RuntimeError: mat1 and mat2 shapes cannot be multiplied (256x726 and 1000x1000). I cannot figure out which part is the problem. Please help me.. Thank you so much
for the images runtime error like
enter image description here
I'm using Pytorch SSD that loads a pre-trained model on the COCO dataset from Torch HUB. Modified the code in an API format to fetch some images and detect the objects in it.
Trying to save each inference image output in the /output using matplotlib's .savefig() method but getting an error:
import torch
import matplotlib.patches as patches
from matplotlib import pyplot as plt
class ObjectDetector:
def __init__(self):
self.precision = 'fp32'
self.detect_model = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_ssd', model_math=self.precision)
self.utils = torch.hub.load('NVIDIA/DeepLearningExamples:torchhub', 'nvidia_ssd_processing_utils')
def process(self):
self.fetch_images()
self.create_model()
self.display_detections()
def fetch_images(self):
"""To get the images from a website"""
self.images = ['http://images.cocodataset.org/val2017/000000397133.jpg','http://images.cocodataset.org/val2017/000000037777.jpg','http://images.cocodataset.org/val2017/000000252219.jpg']
return self.images
def create_model(self):
self.detect_model.to('cuda')
self.detect_model.eval()
self.inputs = [self.utils.prepare_input(uri) for uri in self.images]
tensor = self.utils.prepare_tensor(self.inputs, self.precision == 'fp16')
with torch.no_grad():
detections_batch = self.detect_model(tensor)
results_per_input = self.utils.decode_results(detections_batch)
self.best_results_per_input = [self.utils.pick_best(results, 0.40) for results in results_per_input]
self.classes_to_labels = self.utils.get_coco_object_dictionary()
return self.best_results_per_input, self.classes_to_labels
def display_detections(self):
output_dir = "../data/vision/ssd/output"
for image_idx in range(len(self.best_results_per_input)):
fig, ax = plt.subplots(figsize=(20, 10))
# Show original, denormalized image...
image = self.inputs[image_idx] / 2 + 0.5
ax.imshow(image)
# ...with detections
bboxes, classes, confidences = self.best_results_per_input[image_idx]
for idx in range(len(bboxes)):
left, bot, right, top = bboxes[idx]
x, y, w, h = [val * 300 for val in [left, bot, right - left, top - bot]]
rect = patches.Rectangle((x, y), w, h, linewidth=1, edgecolor='r', facecolor='none')
ax.add_patch(rect)
ax.text(x, y, "{} {:.0f}%".format(self.classes_to_labels[classes[idx] - 1], confidences[idx] * 100), bbox=dict(facecolor='white', alpha=0.5))
plt.savefig(output_dir + str(image) + '.jpg')
plt.show()
if __name__== '__main__':
det = ObjectDetector()
det.process()
del det
The above code throws the following error:
---------------------------------------------------------------------------
FileNotFoundError Traceback (most recent call last)
<ipython-input-9-acbe775772c1> in <module>
63 if __name__== '__main__':
64 det = ObjectDetector()
---> 65 det.process()
66
67 del det
<ipython-input-9-acbe775772c1> in process(self)
14 self.fetch_images()
15 self.create_model()
---> 16 self.display_detections()
17
18
<ipython-input-9-acbe775772c1> in display_detections(self)
57 ax.text(x, y, "{} {:.0f}%".format(self.classes_to_labels[classes[idx] - 1], confidences[idx] * 100), bbox=dict(facecolor='white', alpha=0.5))
58
---> 59 plt.savefig(output_dir + str(image) + '.jpg')
60 plt.show()
61
~/venv38/lib/python3.8/site-packages/matplotlib/pyplot.py in savefig(*args, **kwargs)
721 def savefig(*args, **kwargs):
722 fig = gcf()
--> 723 res = fig.savefig(*args, **kwargs)
724 fig.canvas.draw_idle() # need this if 'transparent=True' to reset colors
725 return res
~/venv38/lib/python3.8/site-packages/matplotlib/figure.py in savefig(self, fname, transparent, **kwargs)
2201 self.patch.set_visible(frameon)
2202
-> 2203 self.canvas.print_figure(fname, **kwargs)
2204
2205 if frameon:
~/venv38/lib/python3.8/site-packages/matplotlib/backend_bases.py in print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, **kwargs)
2117
2118 try:
-> 2119 result = print_method(
2120 filename,
2121 dpi=dpi,
~/venv38/lib/python3.8/site-packages/matplotlib/cbook/deprecation.py in wrapper(*args, **kwargs)
356 f"%(removal)s. If any parameter follows {name!r}, they "
357 f"should be pass as keyword, not positionally.")
--> 358 return func(*args, **kwargs)
359
360 return wrapper
~/venv38/lib/python3.8/site-packages/matplotlib/backends/backend_agg.py in print_jpg(self, filename_or_obj, dryrun, pil_kwargs, *args, **kwargs)
597 pil_kwargs.setdefault("quality", rcParams["savefig.jpeg_quality"])
598 pil_kwargs.setdefault("dpi", (self.figure.dpi, self.figure.dpi))
--> 599 return background.save(
600 filename_or_obj, format='jpeg', **pil_kwargs)
601
~/venv38/lib/python3.8/site-packages/PIL/Image.py in save(self, fp, format, **params)
2153 fp = builtins.open(filename, "r+b")
2154 else:
-> 2155 fp = builtins.open(filename, "w+b")
2156
2157 try:
FileNotFoundError: [Errno 2] No such file or directory: '../data/vision/ssd/output[[[0.1050852 0.07895297 0.08367175]\n [0.31462591 0.31466424 0.32513717]\n [0.28277484 0.25506944 0.23508735]\n ...\n [0.42182888 0.27386384 0.07784647]\n [0.67421166 0.57844825 0.39889071]\n [0.554919 0.33316082 0.09618731]]\n\n [[0.05228582 0.03646781 0.0400054 ]\n [0.06949542 0.06235639 0.05692344]\n [0.25959795 0.18080175 0.18654409]\n ...\n [0.60428691 0.30419598 0.06168084]\n [0.62523846 0.37480789 0.15464491]\n [0.40595506 0.21335363 0.0789785 ]]\n\n [[0.10904118 0.11286539 0.09207947]\n [0.0804173 0.04945466 0.03713621]\n [0.24569849 0.12457102 0.1002835 ]\n ...\n [0.8473525 0.49805938 0.01584464]\n [0.62128949 0.34659926 0.04259144]\n [0.60784509 0.39757653 0.1146472 ]]\n\n ...\n\n [[0.54990582 0.37598903 0.20369267]\n [0.5526588 0.38010985 0.19625383]\n [0.56226779 0.38371096 0.20185737]\n ...\n [0.29863339 0.2165191 0.14226269]\n [0.30894688 0.23059896 0.16393229]\n [0.31879315 0.21973148 0.16671452]]\n\n [[0.54124921 0.37518263 0.19985079]\n [0.54947818 0.38385507 0.19607851]\n [0.54889008 0.37478852 0.18892228]\n ...\n [0.29478525 0.22002212 0.15326277]\n [0.31478406 0.23243116 0.16062237]\n [0.30818757 0.21890863 0.14786195]]\n\n [[0.53892612 0.37097071 0.1888549 ]\n [0.54983966 0.38421659 0.19571689]\n [0.55770917 0.38090676 0.18950984]\n ...\n [0.316164 0.24439232 0.16849774]\n [0.32127783 0.23892493 0.16441515]\n [0.30470566 0.21542674 0.14437993]]].jpg'
plt.savefig() can be saved like this. Also, to generate a random unique string, import uuid can be used.
def display_detections(self):
output_dir = "../data/vision/ssd/output"
for image_idx in range(len(self.best_results_per_input)):
fig, ax = plt.subplots(figsize=(20, 10))
# Show original, denormalized image...
image = self.inputs[image_idx] / 2 + 0.5
ax.imshow(image)
# ...with detections
bboxes, classes, confidences = self.best_results_per_input[image_idx]
for idx in range(len(bboxes)):
left, bot, right, top = bboxes[idx]
x, y, w, h = [val * 300 for val in [left, bot, right - left, top - bot]]
rect = patches.Rectangle((x, y), w, h, linewidth=1, edgecolor='r', facecolor='none')
ax.add_patch(rect)
ax.text(x, y, "{} {:.0f}%".format(self.classes_to_labels[classes[idx] - 1], confidences[idx] * 100), bbox=dict(facecolor='white', alpha=0.5))
plt.axis('off')
plt.savefig(output_dir + "/" + "Image" + "_" + str(uuid.uuid4()))
plt.show()
I am training a neural networks with three different output prediction. For computing the loss of one output I need one of the input that is passed into the network. I am not able to access it as the training data is feed into the network by a keras data generator object. Is there any workaround for this problem.
This is the Generator class that feds data into the model
class DataGenerator(tf.keras.utils.Sequence):
def __init__(self,list_ID,centers,sizes,batch_size=2,dims=(512,512),n_channels=3,n_classes=10,shuffle=True) -> None:
assert len(list_ID) == len(centers)
self.dims = dims
self.batch_size = batch_size
self.list_ID = list_ID
self.centers = centers
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.sizes = sizes
self.on_epoch_end()
self.mask = None
def __len__(self):
return int(np.floor(len(self.list_ID) / self.batch_size))
def on_epoch_end(self):
self.indexes = np.arange(len(self.list_ID))
if self.shuffle:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_ID_temp = [self.list_ID[k] for k in indexes]
centers_temp = [self.centers[k] for k in indexes]
sizes_temp = [self.sizes[k] for k in indexes]
X, y = self.__datageneration(list_ID_temp, centers_temp,sizes_temp)
return X, y
def __datageneration(self, list_ID_temp,centers_temp,sizes_temp):
X = np.empty((self.batch_size,*self.dims,self.n_channels))
Y_center = np.empty((self.batch_size,128,128,1))
Y_dimension = np.empty((self.batch_size,128,128,2))
Y_offset = np.empty((self.batch_size,128,128,2))
self.mask = np.empty((self.batch_size,128,128,1))
for i,ID in enumerate(list_ID_temp):
image = cv2.imread(path+'/'+ID) / 255.0
heat_center, self.mask[i,] = gaussian_2d(centers_temp[i],image.shape)
'''Here I tried to save mask which is what I need,
as an attribute to data generator but when accessed by loss function
the value is just None which is what I initialized it as in init method'''
heat_size,heat_off = size_off_heatmap(sizes_temp[i], centers_temp[i],image.shape)
image = cv2.resize(image,(512,512))
X[i,] = image
Y_center[i,] = heat_center
Y_dimension[i,] = heat_size
Y_offset[i,] = heat_off
return (X,{'center_output':Y_center,'size_output':Y_dimension,'offset_output':Y_offset})
This is the generator class I implemented and I needed the mask , which I tried to write as an attribute of data generator object(I have commented the code. For reference I will also include the function that will return the mask and the error function that requires the mask.
Function returning mask
def gaussian_2d(centers, img_shape):
heatmap = []
y_index = np.tile(np.arange(128), (128, 1))
mask = np.zeros((128,128,1))
width = img_shape[1]
height = img_shape[0]
for x_o, y_o in centers:
x = int(x_o / width * 128)
y = int(y_o / height * 128)
mask[y,x] = 1
gauss = np.exp(-((y_index.T - y) ** 2 + (y_index - x) ** 2) / 2 * 0.2 ** 2)
heatmap.append(gauss)
if len(heatmap) > 1:
heatmap = np.stack(heatmap)
heatmap = np.max(heatmap, axis=0)
else:
heatmap = np.array(heatmap)
heatmap = heatmap.reshape((128, 128,1))
return heatmap,mask
Loss function
def final_loss(mask):
def l1_loss(y_true, y_pred):
y_true = tf.cast(y_true, tf.float32)
y_pred = tf.cast(y_pred, tf.float32)
n = tf.reduce_sum(tf.cast(tf.equal(mask, 1.0),dtype=tf.float32))
tot_loss = tf.reduce_sum(tf.abs(y_pred - y_true))
if tf.greater(n,0):
loss = tot_loss / (n)
else:
loss = tot_loss
return loss
return l1_loss
The error show is as below
Epoch 1/10
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-27-74a28b075f52> in <module>()
----> 1 model.fit(gen,epochs=10,verbose=1,callbacks=Callback(patience=4))
9 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
ValueError: in user code:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
<ipython-input-24-c45fe131feb7>:5 l1_loss *
n = tf.reduce_sum(tf.cast(tf.equal(mask, 1.0),dtype=tf.float32))
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py:201 wrapper **
return target(*args, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:1679 equal
return gen_math_ops.equal(x, y, name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_math_ops.py:3179 equal
name=name)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:540 _apply_op_helper
(input_name, err))
ValueError: Tried to convert 'x' to a tensor and failed. Error: None values not supported.
'''
I have encountered the following error. This is a different rnn structure. I have implemented to use in graph convolution. The problem is that the hidden is updated in-place operation. However, I have to update its value in each forward call. How can I do that? Thanks in advance.
RuntimeError Traceback (most recent call last)
<ipython-input-110-b4425651d544> in <module>()
8 out = model(x[i])
9 loss = mael(out, x[i+1])
---> 10 loss.backward(retain_graph=True)
11 optimizer.step()
12 print(loss.item())
1 frames
/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
130 Variable._execution_engine.run_backward(
131 tensors, grad_tensors_, retain_graph, create_graph,
--> 132 allow_unreachable=True) # allow_unreachable flag
133
134
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [1, 100]], which is output 0 of SelectBackward, is at version 1; expected version 0 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
This is a different rnn structure. I have implemented to use in graph convolution. The problem is that the hidden is updated in-place operation. However, I have to update its value in each forward call. How can I do that? Thanks in advance.
class RNN(nn.Module):
def __init__(self, input_dim, hidden_dim):
super(RNN,self).__init__()
self.input_dim = input_dim
self.hidden_dim = hidden_dim
self.weight = Parameter(torch.rand(10,input_dim,hidden_dim, requires_grad=True))
self.weight_h = Parameter(torch.rand(10,input_dim,hidden_dim, requires_grad=True))
self.bias = Parameter(torch.rand(10,input_dim,hidden_dim, requires_grad=True))
self.hidden = torch.rand(10,input_dim, hidden_dim)
self.weight_2 = Parameter(torch.rand(10,input_dim,hidden_dim,requires_grad=True))
self.weight_h_2 = Parameter(torch.rand(10,hidden_dim,hidden_dim, requires_grad=True))
self.bias_2 = Parameter(torch.rand(10,input_dim,hidden_dim, requires_grad=True))
self.tanh = Tanh()
self.iteration = 0
self.next_layer = False
self.hidden_init = torch.rand(1,1)
def set_hidden(self,x):
y = self.tanh(mm(x, self.weight[self.iteration]) + mm(self.hidden_init, self.weight_h[self.iteration]) + self.bias[self.iteration])
return y
def set_hidden_state_layer_2(self, x, hidden):
y = self.tanh(mm(x, self.weight_2[self.iteration]) + mm(hidden, self.weight_h_2[self.iteration]) + self.bias_2[self.iteration])
return y
def forward(self, x):
try:
dim_1, dim_2, dim_3 = x.shape
except:
x = torch.unsqueeze(x,0)
if self.iteration == 10:
self.next_layer = True
self.iteration = 0
if self.next_layer:
self.hidden[self.iteration] = self.set_hidden_state_layer_2(x, self.hidden[self.iteration].clone())
self.iteration = self.iteration + 1
return self.hidden[self.iteration - 1]
else:
hidden_init = torch.rand(1,1)
self.hidden[self.iteration] = self.tanh(mm(x, self.weight[self.iteration]) + mm(self.hidden_init, self.weight_h[self.iteration]) + self.bias[self.iteration])
self.iteration = self.iteration + 1
return self.hidden[self.iteration - 1]
model = RNN(1,100)
mael = nn.L1Loss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
x = torch.rand(11,1)
x_2 = torch.rand(11,1)
for i in range(10):
optimizer.zero_grad()
out = model(x[i])
loss = mael(out, x[i+1])
loss.backward(retain_graph=True)
optimizer.step()
print(loss.item())