inside "PWCDCNet.py" I have defined the following ANN:
class Conv2D(tfk.layers.Layer):
def __init__(self, filters, kernel_size, strides, name=None, padding=1, dilation_rate=1):
super(Conv2D, self).__init__(name=name)
self.conv_out = tfk.layers.Conv2D(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
kernel_initializer='he_normal',
dilation_rate=dilation_rate,
activation=tfk.layers.LeakyReLU(0.1))
def call(self, inputs):
return self.conv_out(inputs)
class DeConv2D(tfk.layers.Layer):
def __init__(self, filters, kernel_size=4, strides=2, name=None):
super(DeConv2D, self).__init__(name=name)
self.deconv_out = tfk.layers.Conv2DTranspose(filters=filters,
kernel_size=kernel_size,
strides=strides,
padding='same',
name=name)
def call(self, inputs):
return self.deconv_out(inputs)
def CostVolumn(c1, warp, search_range, name='cost_volumn'):
padded_lvl = tf.pad(warp, [[0, 0], [search_range, search_range], [search_range, search_range], [0, 0]])
_, h, w, _ = tf.unstack(tf.shape(c1))
max_offset = search_range * 2 + 1
cost_vol = []
for y in range(0, max_offset):
for x in range(0, max_offset):
slice = tf.slice(padded_lvl, [0, y, x, 0], [-1, h, w, -1])
cost = tf.reduce_mean(c1 * slice, axis=3, keepdims=True)
cost_vol.append(cost)
cost_vol = tf.concat(cost_vol, axis=3)
cost_vol = tf.nn.leaky_relu(cost_vol, alpha=0.1, name=name)
return cost_vol
class PredictFlow(tfk.layers.Layer):
def __init__(self, name=None):
super(PredictFlow, self).__init__()
self.conv_out = tfk.layers.Conv2D(filters=2,
kernel_size=3,
strides=1,
name=name,
padding='same')
def call(self, inputs):
return self.conv_out(inputs)
class PWCDCNet(tf.keras.Model):
def __init__(self, max_displacement=4):
super(PWCDCNet, self).__init__()
self.conv1a = Conv2D( 16, kernel_size=3, strides=2, name='conv1a')
self.conv1aa = Conv2D( 16, kernel_size=3, strides=1, name='conv1aa')
self.conv1b = Conv2D( 16, kernel_size=3, strides=1, name='conv1b')
self.conv2a = Conv2D( 32, kernel_size=3, strides=2, name='conv2a')
self.conv2aa = Conv2D( 32, kernel_size=3, strides=1, name='conv2aa')
self.conv2b = Conv2D( 32, kernel_size=3, strides=1, name='conv2b')
self.conv3a = Conv2D( 64, kernel_size=3, strides=2, name='conv3a')
self.conv3aa = Conv2D( 64, kernel_size=3, strides=1, name='conv3aa')
self.conv3b = Conv2D( 64, kernel_size=3, strides=1, name='conv3b')
self.conv4a = Conv2D( 96, kernel_size=3, strides=2, name='conv4a')
self.conv4aa = Conv2D( 96, kernel_size=3, strides=1, name='conv4aa')
self.conv4b = Conv2D( 96, kernel_size=3, strides=1, name='conv4b')
self.conv5a = Conv2D(128, kernel_size=3, strides=2, name='conv5a')
self.conv5aa = Conv2D(128, kernel_size=3, strides=1, name='conv5aa')
self.conv5b = Conv2D(128, kernel_size=3, strides=1, name='conv5b')
self.conv6aa = Conv2D(196, kernel_size=3, strides=2, name='conv6aa')
self.conv6a = Conv2D(196, kernel_size=3, strides=1, name='conv6a')
self.conv6b = Conv2D(196, kernel_size=3, strides=1, name='conv6b')
self.LeakyReLU = tfk.layers.LeakyReLU(0.1, name='LeakyReLU')
self.conv6_0 = Conv2D(128, kernel_size=3, strides=1, name='conv6_0')
self.conv6_1 = Conv2D(128, kernel_size=3, strides=1, name='conv6_1')
self.conv6_2 = Conv2D(96, kernel_size=3, strides=1, name='conv6_2')
self.conv6_3 = Conv2D(64, kernel_size=3, strides=1, name='conv6_3')
self.conv6_4 = Conv2D(32, kernel_size=3, strides=1, name='conv6_4')
self.deconv6 = DeConv2D(2, kernel_size=4, strides=2, name='deconv_6')
self.upfeat6 = DeConv2D(2, kernel_size=4, strides=2, name='upfeat_6')
self.conv5_0 = Conv2D(128, kernel_size=3, strides=1, name='conv5_0')
self.conv5_1 = Conv2D(128, kernel_size=3, strides=1, name='conv5_1')
self.conv5_2 = Conv2D(96, kernel_size=3, strides=1, name='conv5_2')
self.conv5_3 = Conv2D(64, kernel_size=3, strides=1, name='conv5_3')
self.conv5_4 = Conv2D(32, kernel_size=3, strides=1, name='conv5_4')
self.deconv5 = DeConv2D(2, kernel_size=4, strides=2, name='deconv_5')
self.upfeat5 = DeConv2D(2, kernel_size=4, strides=2, name='upfeat_5')
self.conv4_0 = Conv2D(128, kernel_size=3, strides=1, name='conv4_0')
self.conv4_1 = Conv2D(128, kernel_size=3, strides=1, name='conv4_1')
self.conv4_2 = Conv2D(96, kernel_size=3, strides=1, name='conv4_2')
self.conv4_3 = Conv2D(64, kernel_size=3, strides=1, name='conv4_3')
self.conv4_4 = Conv2D(32, kernel_size=3, strides=1, name='conv4_4')
self.deconv4 = DeConv2D(2, kernel_size=4, strides=2, name='deconv4')
self.upfeat4 = DeConv2D(2, kernel_size=4, strides=2, name='upfeat4')
self.conv3_0 = Conv2D(128, kernel_size=3, strides=1, name='conv3_0')
self.conv3_1 = Conv2D(128, kernel_size=3, strides=1, name='conv3_1')
self.conv3_2 = Conv2D(96, kernel_size=3, strides=1, name='conv3_2')
self.conv3_3 = Conv2D(64, kernel_size=3, strides=1, name='conv3_3')
self.conv3_4 = Conv2D(32, kernel_size=3, strides=1, name='conv3_4')
self.deconv3 = DeConv2D(2, kernel_size=4, strides=2, name='deconv3')
self.upfeat3 = DeConv2D(2, kernel_size=4, strides=2, name='upfeat3')
self.conv2_0 = Conv2D(128, kernel_size=3, strides=1, name='conv2_0')
self.conv2_1 = Conv2D(128, kernel_size=3, strides=1, name='conv2_1')
self.conv2_2 = Conv2D(96, kernel_size=3, strides=1, name='conv2_2')
self.conv2_3 = Conv2D(64, kernel_size=3, strides=1, name='conv2_3')
self.conv2_4 = Conv2D(32, kernel_size=3, strides=1, name='conv2_4')
self.deconv2 = DeConv2D(2, kernel_size=4, strides=2, name='deconv2')
self.dc_conv1 = Conv2D(128, kernel_size=3, strides=1, padding=1, dilation_rate=1, name='dc_conv1')
self.dc_conv2 = Conv2D(128, kernel_size=3, strides=1, padding=2, dilation_rate=2, name='dc_conv2')
self.dc_conv3 = Conv2D(128, kernel_size=3, strides=1, padding=4, dilation_rate=4, name='dc_conv3')
self.dc_conv4 = Conv2D(96, kernel_size=3, strides=1, padding=8, dilation_rate=8, name='dc_conv4')
self.dc_conv5 = Conv2D(64, kernel_size=3, strides=1, padding=16, dilation_rate=16, name='dc_conv5')
self.dc_conv6 = Conv2D(32, kernel_size=3, strides=1, padding=1, dilation_rate=1, name='dc_conv6')
self.predict_flow6 = PredictFlow(name='predict_flow6')
self.predict_flow5 = PredictFlow(name='predict_flow5')
self.predict_flow4 = PredictFlow(name='predict_flow4')
self.predict_flow3 = PredictFlow(name='predict_flow3')
self.predict_flow2 = PredictFlow(name='predict_flow2')
self.dc_conv7 = PredictFlow(name='dc_conv7')
def call(self, inputs):
im1 = inputs[:, :, :, :3]
im2 = inputs[:, :, :, 3:]
c11 = self.conv1b(self.conv1aa(self.conv1a(im1)))
c21 = self.conv1b(self.conv1aa(self.conv1a(im2)))
c12 = self.conv2b(self.conv2aa(self.conv2a(c11)))
c22 = self.conv2b(self.conv2aa(self.conv2a(c21)))
c13 = self.conv3b(self.conv3aa(self.conv3a(c12)))
c23 = self.conv3b(self.conv3aa(self.conv3a(c22)))
c14 = self.conv4b(self.conv4aa(self.conv4a(c13)))
c24 = self.conv4b(self.conv4aa(self.conv4a(c23)))
c15 = self.conv5b(self.conv5aa(self.conv5a(c14)))
c25 = self.conv5b(self.conv5aa(self.conv5a(c24)))
c16 = self.conv6b(self.conv6a(self.conv6aa(c15)))
c26 = self.conv6b(self.conv6a(self.conv6aa(c25)))
### 6th flow
corr6 = CostVolumn(c1=c16, warp=c26, search_range=4)
x = tf.concat([self.conv6_0(corr6), corr6], 3)
x = tf.concat([self.conv6_1(x), x], 3)
x = tf.concat([self.conv6_2(x), x], 3)
x = tf.concat([self.conv6_3(x), x], 3)
x = tf.concat([self.conv6_4(x), x], 3)
flow6 = self.predict_flow6(x)
up_flow6 = self.deconv6(flow6)
up_feat6 = self.upfeat6(x)
### 5th flow
warp5 = bilinear_warp(c25, up_flow6*0.625)
corr5 = CostVolumn(c1=c15, warp=warp5, search_range=4)
x = tf.concat([corr5, c15, up_flow6, up_feat6], 3)
x = tf.concat([self.conv5_0(x), x], 3)
x = tf.concat([self.conv5_1(x), x], 3)
x = tf.concat([self.conv5_2(x), x], 3)
x = tf.concat([self.conv5_3(x), x], 3)
x = tf.concat([self.conv5_4(x), x], 3)
flow5 = self.predict_flow5(x)
up_flow5 = self.deconv5(flow5)
up_feat5 = self.upfeat5(x)
### 4th flow
warp4 = bilinear_warp(c24, up_flow5*1.25)
corr4 = CostVolumn(c1=c14, warp=warp4, search_range=4)
x = tf.concat([corr4, c14, up_flow5, up_feat5], 3)
x = tf.concat([self.conv4_0(x), x], 3)
x = tf.concat([self.conv4_1(x), x], 3)
x = tf.concat([self.conv4_2(x), x], 3)
x = tf.concat([self.conv4_3(x), x], 3)
x = tf.concat([self.conv4_4(x), x], 3)
flow4 = self.predict_flow4(x)
up_flow4 = self.deconv4(flow4)
up_feat4 = self.upfeat4(x)
### 3rd flow
warp3 = bilinear_warp(c23, up_flow4*2.5)
corr3 = CostVolumn(c1=c13, warp=warp3, search_range=4)
x = tf.concat([corr3, c13, up_flow4, up_feat4], 3)
x = tf.concat([self.conv3_0(x), x], 3)
x = tf.concat([self.conv3_1(x), x], 3)
x = tf.concat([self.conv3_2(x), x], 3)
x = tf.concat([self.conv3_3(x), x], 3)
x = tf.concat([self.conv3_4(x), x], 3)
flow3 = self.predict_flow3(x)
up_flow3 = self.deconv3(flow3)
up_feat3 = self.upfeat3(x)
# 2nd flow
warp2 = bilinear_warp(c22, up_flow3*5.0)
corr2 = CostVolumn(c1=c12, warp=warp2, search_range=4)
x = tf.concat([corr2, c12, up_flow3, up_feat3], 3)
x = tf.concat([self.conv2_0(x), x], 3)
x = tf.concat([self.conv2_1(x), x], 3)
x = tf.concat([self.conv2_2(x), x], 3)
x = tf.concat([self.conv2_3(x), x], 3)
x = tf.concat([self.conv2_4(x), x], 3)
flow2 = self.predict_flow2(x)
x = self.dc_conv4(self.dc_conv3(self.dc_conv2(self.dc_conv1(x))))
flow2 = flow2 + self.dc_conv7(self.dc_conv6(self.dc_conv5(x)))
return flow2
This ANN gets imported into another file:
from PWCNet_tf2.PWCDCNet import PWCDCNet
PWC_model = tf.keras.Model(PWCDCNet())
checkpoint = tf.train.Checkpoint(PWC_model)
save_path = checkpoint.save('/some_directories/PWCNet_tf2/checkpoints/0001/tf_ckpt')
checkpoint.restore(save_path)
But if I try to:
output = PWC_model(input)
I get the error:
NotImplementedError: When subclassing the `Model` class, you should implement a `call` method.
I am wondering, as a call method is implemented for every Layer, but also for the ANN as a whole. Is the error due to the way I import the model?
Thanks to all of you in advance!
The error was that I called:
PWC_model = tf.keras.Model(PWCDCNet())
instead of:
PWC_model = PWCDCNet()
Related
After resnet convolution, I want to further compress the 256 dimensions to 20 dimensions. I directly wrote a layer in the back, but after forward propagation, there is an error in this layer, I don't know why？
def forward(self, x):
x = self.conv1(x)
dif_residual1 = self.downsample1(x)
x = self.layer1_1(x)
x =x + dif_residual1
residual = x
x = self.layer1_2(x)
x = x + residual
residual = x
x = self.layer1_3(x)
x = x + residual
if self.out_channel != 256:
x = self.layer2
filters = torch.ones(self.batch_size, self.out_channel, 1, 1).detach().requires_grad_(False).to(self.device)
x = F.conv2d(x, weight=filters, padding=0)
The dimension of x before I do if is：
x = {Tensor:(1,256,117,240)}
But after the if statement is executed, it becomes what the picture shows。
The error I get is this：
x = F.conv2d(feature, weight=filters, padding=0)
TypeError: conv2d() received an invalid combination of arguments - got (Sequential, weight=Tensor, padding=int), but expected one of:
* (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)
* (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)
Encounter a new problem：
File "D:\software\Anaconda\envs\torch1.10\lib\site-packages\torch\autograd\__init__.py", line 173, in backward
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [1, 1, 117, 240]], which is output 0 of AddBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).
My code：
class VGG(nn.Module):
def __init__(self, in_channel, out_channel=None, init_weights=True, device='gpu',batch_size=1):
super(VGG, self).__init__()
self.batch_size = batch_size
self.out_channel = out_channel
if device == 'gpu':
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
else:
self.device = torch.device("cpu")
modes = 'reflect'
out_channel1 = 64
self.conv1_1 = nn.Sequential(
nn.Conv2d(in_channels=in_channel, out_channels=out_channel1, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel1),
nn.LeakyReLU()
)
self.conv1_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel1, out_channels=out_channel1, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel1),
nn.LeakyReLU()
)
out_channel2 = 128
self.conv2_1 = nn.Sequential(
nn.Conv2d(in_channels=out_channel1, out_channels=out_channel2, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel2),
nn.LeakyReLU()
)
self.conv2_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel2, out_channels=out_channel2, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel2),
nn.LeakyReLU()
)
out_channel3 = 256
self.conv3_1 = nn.Sequential(
nn.Conv2d(in_channels=out_channel2, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
self.conv3_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
if out_channel == None:
self.out_channel = 256
self.conv3_3 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1,
padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
else:
self.conv3_3 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU(),
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel, kernel_size=3, stride=1, padding=1, padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel),
nn.LeakyReLU()
)
if init_weights:
self._init_weight()
def forward(self, x):
x = self.conv1_1(x)
x = self.conv1_2(x)
x = self.conv2_1(x)
x = self.conv2_2(x)
x = self.conv3_1(x)
x = self.conv3_2(x)
x = self.conv3_3(x)
feature = x
filters = torch.ones(self.batch_size, self.out_channel, 1, 1).detach().requires_grad_(False).to(self.device)
x = F.conv2d(x, weight = filters, padding = 0)
return x,feature
out_channel = 20
model = VGG(in_channel=12, out_channel=out_channel, init_weights=True, batch_size=batch_size)
for epoch in range(start_epoch+1,epochs):
# train
model.train()
running_loss = 0.0
train_bar = tqdm(train_loader, file=sys.stdout)
for step, data in enumerate(train_bar):
images, labels = data
optimizer.zero_grad()
outputs,feature = model(images.to(device))
outputs = tonser_nolmal(outputs)
loss = loss_function(outputs, labels.to(device))
loss.backward()
optimizer.step()
running_loss += loss.item()
train_bar.desc = "train epoch[{}/{}] loss:{:.6f}".format(epoch + 1,
epochs,
loss)
checkpoint = {
"net": model.state_dict(),
"optimizer": optimizer.state_dict(),
"epoch": epoch
}
torch.save(checkpoint, save_path + "/model-{}.pth".format(epoch))
# validate
model.eval()
count_acc = 0.0
count_mae = 0.0
with torch.no_grad():
val_bar = tqdm(validate_loader, file=sys.stdout)
for val_data in val_bar:
val_images, val_labels = val_data
outputs,_ = model(val_images.to(device))
# outputs = F.normalize(outputs,dim=3)
outputs = tonser_nolmal(outputs)
loss = loss_function(outputs, val_labels.to(device))
count_acc = count_acc + loss.item()
mae = Evaluation().MAE(outputs, val_labels.to(device))
count_mae = count_mae + mae.item()
The error is likely to be caused by the following variable assignment:
if self.out_channel != 256:
x = self.layer2
which can be easily fixed by changing it to
x = self.layer2(x)
Update:
As OP updated his code, I did some test. There were several things which I found problematic:
self._init_weight was not provided, so I commented it out;
filters = torch.ones(self.batch_size, self.out_channel, 1, 1).detach().requires_grad_(False).to(self.device). The filter weight should have a shape of (c_out, c_in, kernel_size, kernel_size). However, batch_size appeared in the position of out_channels.
The role of filter in the forward was not clear to me. If you wanted to reduce the out_channels further from 256 to 20, then initializing your model with VGG(..., out_channel=20) is sufficient. Basically, self.conv3_3 would do the job.
On my end, I modified the code a little bit and it ran successfully:
import sys
import torch
import torch.nn as nn
from tqdm import tqdm
from torchvision.datasets import FakeData
from torch.utils.data import DataLoader
import torch.nn.functional as F
dataset = [torch.randn(12, 64, 64) for _ in range(1000)]
train_loader = DataLoader(dataset, batch_size=1, shuffle=True)
class VGG(nn.Module):
def __init__(self, in_channel, out_channel=None, init_weights=True, device='cpu', batch_size=1):
super(VGG, self).__init__()
self.batch_size = batch_size
self.out_channel = out_channel
self.device = device
modes = 'reflect'
out_channel1 = 64
self.conv1_1 = nn.Sequential(
nn.Conv2d(in_channels=in_channel, out_channels=out_channel1, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel1),
nn.LeakyReLU()
)
self.conv1_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel1, out_channels=out_channel1, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel1),
nn.LeakyReLU()
)
out_channel2 = 128
self.conv2_1 = nn.Sequential(
nn.Conv2d(in_channels=out_channel1, out_channels=out_channel2, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel2),
nn.LeakyReLU()
)
self.conv2_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel2, out_channels=out_channel2, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel2),
nn.LeakyReLU()
)
self.out_channel3 = out_channel3 = 256
self.conv3_1 = nn.Sequential(
nn.Conv2d(in_channels=out_channel2, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
self.conv3_2 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode = modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
self.out_channel = out_channel
if out_channel == None:
self.conv3_3 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1,
padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU()
)
else:
self.conv3_3 = nn.Sequential(
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel3, kernel_size=3, stride=1, padding=1, padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel3),
nn.LeakyReLU(),
nn.Conv2d(in_channels=out_channel3, out_channels=out_channel, kernel_size=3, stride=1, padding=1, padding_mode=modes, bias=False),
nn.BatchNorm2d(out_channel),
nn.LeakyReLU()
)
# The implementation of _init_weight is not found
# if init_weights:
# self._init_weight()
def forward(self, x):
x = self.conv1_1(x)
x = self.conv1_2(x)
x = self.conv2_1(x)
x = self.conv2_2(x)
x = self.conv3_1(x)
x = self.conv3_2(x)
x = self.conv3_3(x)
feature = x
if x.shape[1] == 256: # self.out_channel is None
filters = torch.ones(20, self.out_channel3, 1, 1).to(self.device)
x = F.conv2d(x, weight = filters, padding = 0)
return x, feature
out_channel = 20
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = VGG(in_channel=12, out_channel=None, init_weights=True, device=device, batch_size=1)
model.to(device)
print(model(next(iter(train_loader)).to(device))[0].shape)
model = VGG(in_channel=12, out_channel=20, init_weights=True, device=device, batch_size=1)
model.to(device)
print(model(next(iter(train_loader)).to(device))[0].shape)
Outputs:
torch.Size([1, 20, 64, 64])
torch.Size([1, 20, 64, 64])
Below code works fine and generate proper results.
import torch
import torch.nn as nn
import torch.nn.functional as F
from modules import ConvLSTMCell, Sign
class EncoderCell(nn.Module):
def __init__(self):
super(EncoderCell, self).__init__()
self.conv = nn.Conv2d(
3, 64, kernel_size=3, stride=2, padding=1, bias=False)
self.rnn1 = ConvLSTMCell(
64,
256,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn2 = ConvLSTMCell(
256,
512,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn3 = ConvLSTMCell(
512,
512,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
def forward(self, input, hidden1, hidden2, hidden3):
x = self.conv(input)
hidden1 = self.rnn1(x, hidden1)
x = hidden1[0]
hidden2 = self.rnn2(x, hidden2)
x = hidden2[0]
hidden3 = self.rnn3(x, hidden3)
x = hidden3[0]
return x, hidden1, hidden2, hidden3
class Binarizer(nn.Module):
def __init__(self):
super(Binarizer, self).__init__()
self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
self.sign = Sign()
def forward(self, input):
feat = self.conv(input)
x = F.tanh(feat)
return self.sign(x)
class DecoderCell(nn.Module):
def __init__(self):
super(DecoderCell, self).__init__()
self.conv1 = nn.Conv2d(
32, 512, kernel_size=1, stride=1, padding=0, bias=False)
self.rnn1 = ConvLSTMCell(
512,
512,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn2 = ConvLSTMCell(
128,
512,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn3 = ConvLSTMCell(
128,
256,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=3,
bias=False)
self.rnn4 = ConvLSTMCell(
64,
128,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=3,
bias=False)
self.conv2 = nn.Conv2d(
32, 3, kernel_size=1, stride=1, padding=0, bias=False)
def forward(self, input, hidden1, hidden2, hidden3, hidden4):
x = self.conv1(input)
hidden1 = self.rnn1(x, hidden1)
x = hidden1[0]
x = F.pixel_shuffle(x, 2)
hidden2 = self.rnn2(x, hidden2)
x = hidden2[0]
x = F.pixel_shuffle(x, 2)
hidden3 = self.rnn3(x, hidden3)
x = hidden3[0]
x = F.pixel_shuffle(x, 2)
hidden4 = self.rnn4(x, hidden4)
x = hidden4[0]
x = F.pixel_shuffle(x, 2)
x = F.tanh(self.conv2(x)) / 2
return x, hidden1, hidden2, hidden3, hidden4
Now i have changed in self.con and add pretrained resent with layer. Now it shows tensor mismatched error after training. All things are same just add this line in code. I put ** in those line
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models
from modules import ConvLSTMCell, Sign
class EncoderCell(nn.Module):
def __init__(self):
super(EncoderCell, self).__init__()
#self.conv = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
**resConv = models.resnet50(pretrained=True)
resConv.layer4 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
self.conv = resConv.layer4**
self.rnn1 = ConvLSTMCell(
64,
256,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn2 = ConvLSTMCell(
256,
512,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn3 = ConvLSTMCell(
512,
512,
kernel_size=3,
stride=2,
padding=1,
hidden_kernel_size=1,
bias=False)
def forward(self, input, hidden1, hidden2, hidden3):
x = self.conv(input)
hidden1 = self.rnn1(x, hidden1)
x = hidden1[0]
hidden2 = self.rnn2(x, hidden2)
x = hidden2[0]
hidden3 = self.rnn3(x, hidden3)
x = hidden3[0]
return x, hidden1, hidden2, hidden3
class Binarizer(nn.Module):
def __init__(self):
super(Binarizer, self).__init__()
self.conv = nn.Conv2d(512, 32, kernel_size=1, bias=False)
self.sign = Sign()
def forward(self, input):
feat = self.conv(input)
x = F.tanh(feat)
return self.sign(x)
class DecoderCell(nn.Module):
def __init__(self):
super(DecoderCell, self).__init__()
**resConv = models.resnet50(pretrained=True)
resConv.layer4 = nn.Conv2d(32, 512, kernel_size=3, stride=2, padding=1, bias=False)
self.conv1 = resConv.layer4**
self.rnn1 = ConvLSTMCell(
512,
512,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn2 = ConvLSTMCell(
128,
512,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=1,
bias=False)
self.rnn3 = ConvLSTMCell(
128,
256,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=3,
bias=False)
self.rnn4 = ConvLSTMCell(
64,
128,
kernel_size=3,
stride=1,
padding=1,
hidden_kernel_size=3,
bias=False)
**resConv2 = models.resnet50(pretrained=True)
resConv2.layer4 = nn.Conv2d(32, 3, kernel_size=1, stride=1, padding=0, bias=False)
self.conv2 = resConv2.layer4**
def forward(self, input, hidden1, hidden2, hidden3, hidden4):
x = self.conv1(input)
hidden1 = self.rnn1(x, hidden1)
x = hidden1[0]
x = F.pixel_shuffle(x, 2)
hidden2 = self.rnn2(x, hidden2)
x = hidden2[0]
x = F.pixel_shuffle(x, 2)
hidden3 = self.rnn3(x, hidden3)
x = hidden3[0]
x = F.pixel_shuffle(x, 2)
hidden4 = self.rnn4(x, hidden4)
x = hidden4[0]
x = F.pixel_shuffle(x, 2)
x = F.tanh(self.conv2(x)) / 2
return x, hidden1, hidden2, hidden3, hidden4
You are doing it a wrong way, some explanation is,
**resConv = models.resnet50(pretrained=True) # you are reading a model
now you are replacing the layer in that model with newly initialized layer. Secondly, layer4 in resnet50 is a sequential block containing multiple layers. Use print to see exact the layers in model.
resConv.layer4 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)
here you are using new layer.
self.conv = resConv.layer4**
As per your query regarding usage of pretrained layer, you should do it like this,
resConv = models.resnet50(pretrained=True)
print(resConv) #see the layer which you want to use
self.conv = resConv.conv1 # replace conv1 with that layer
# note: conv1 is the name of first conv layer in resnet
To add to this, I would also recommend acquiring and adding this layer (or the weights and biases) outside of the object initialization. Something like:
enc = EncoderCell()
resnet50 = models.resnet50(pretrained=True)
and then either
enc.conv = resnet50.conv1
or more ideally
enc.conv.load_state_dict(resnet50.layer1.state_dict())
The reason being, calling state_dict() on a nn.Module class creates a clone of the parameters (weights and biases in this case) which can be loaded via nn.Module.load_state_dict() method as long as the two instances of nn.Module share the same shape. So you get the pretrained weights and they are completely detached from the pretrained model. Then you can get rid of the pretrained model since it could be rather large in memory.
del resnet50
I submitted a potential improvement to the other answer, but to address the errors you are getting I am answering here also. If the code runs before your edits, and the layer you are trying to change is the same shape as the previous one, then my guess is that it may have to do with the computational graph that is formed from creating the resnet50 object. I would recommended the approach I mentioned in my edit to the other answer, but I will state it here again (note, this assumes you keep the code as it was originally):
# instantiate you encoder (repeat these steps with the decoder as well)
enc = EncoderCell()
# get the pretrained model
resnet = models.resnet50(pretrained=True)
# load the state dict into the regular conv layer
enc.conv.load_state_dict(resnet50.layer4.state_dict())
This should load the pretrained weights and biases from the resnet50 model into your conv layer, and this can be done to the decoder conv layer as well as long as they all share the same shape.
To do more testing with your mismatch error I would recommend either using a debugger or print statements in the forward() method of the models to see the shape of the tensor after each layer is applied, like so
def forward(self, input, hidden1, hidden2, hidden3, hidden4):
print(x.size())
x = self.conv1(input)
print(x.size())
hidden1 = self.rnn1(x, hidden1)
x = hidden1[0]
x = F.pixel_shuffle(x, 2)
hidden2 = self.rnn2(x, hidden2)
x = hidden2[0]
x = F.pixel_shuffle(x, 2)
hidden3 = self.rnn3(x, hidden3)
x = hidden3[0]
x = F.pixel_shuffle(x, 2)
hidden4 = self.rnn4(x, hidden4)
x = hidden4[0]
x = F.pixel_shuffle(x, 2)
x = F.tanh(self.conv2(x)) / 2
return x, hidden1, hidden2, hidden3, hidden4
and of course you can put the print statements where ever else in the forward method. I would also highly recommend a debugger; pycharm makes this quite easy, and also makes it easy to see the state of variables in scientific mode beside the python console it gives. It might be worth looking up ways to calculate size of variables after they pass through certain layers like convolutional layers. This is well understood and formulas exist to calculate the size of the dimensions based on the initial size, the filter size, stride width, and padding.
I want to design a model by tensorflow2.0,when Icompile the model,it report an error
'my_layer' object has no attribute '_dynamic'
the code is
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
class my_layer(layers.Layer):
def __init__(self,classes):
self.conv1 = layers.Conv2D(32,(3,3),strides=1,padding='same')
self.conv2 = layers.Conv2D(64,(3,3),strides=1,padding='same')
self.conv3 = layers.Conv2D(32, (3, 3), strides=1, padding='same')
self.conv4 = layers.Conv2D(classes, (3, 3), strides=1, padding='same')
self.bn = layers.BatchNormalization()
self.glbavgpool = layers.GlobalMaxPooling2D()
self.fc = layers.Dense(classes)
def call(self,inputs):
x = self.conv1(inputs)
x = self.bn(x)
x = keras.activations.relu(x)
x = self.conv2(x)
x = keras.activations.relu(x)
x = self.conv3(x)
x = self.conv4(x)
x = self.bn(x)
x = self.glbavgpool(x)
out = self.fc(x)
return out
class mymodel(keras.Model):
def __init__(self,classes):
super(mymodel,self).__init__()
self.ml = my_layer(classes=classes)
def call(self,inputs):
return self.ml(inputs)
then I put all the custom layers to my_model, it worked. I think it's probably the wrong way to use mylayer.
You're seeing this error because you forgot to call the superclass constructor in your my_layer class. Add this following line:
class my_layer(layers.Layer):
def __init__(self,classes):
super(my_layer, self).__init__() # <-- Remember to call superclass constructor!!
self.conv1 = layers.Conv2D(32,(3,3),strides=1,padding='same')
self.conv2 = layers.Conv2D(64,(3,3),strides=1,padding='same')
self.conv3 = layers.Conv2D(32, (3, 3), strides=1, padding='same')
self.conv4 = layers.Conv2D(classes, (3, 3), strides=1, padding='same')
self.bn = layers.BatchNormalization()
self.glbavgpool = layers.GlobalMaxPooling2D()
self.fc = layers.Dense(classes)
My environment
Ubuntu 18.04
Python 3.6.8
Tensorflow 1.12.0
The problem
I made UNet to segment images by using Tensorflow. I refered the original paper, and I implemented the same structure except for the output channel number (used 3 but not 2). My model seems to work well, but the loss does not converge, it is like electrocardiogram ...
I used MSE as loss function, but some websites said that MSE does not valid in UNet, so I change to use dice loss. But still, it does not work. I suspect that the network structure is bad.
The codes
import tensorflow as tf
tf.reset_default_graph()
with tf.name_scope('input'):
X = tf.placeholder(tf.float32, shape=[None, 572, 572, 3], name='X')
y = tf.placeholder(tf.float32, shape=[None, 388, 388, 3], name='y')
# Encoding
with tf.name_scope('layer1'):
conv1 = tf.layers.conv2d(X, filters=64, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv1')
conv2 = tf.layers.conv2d(conv1, filters=64, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv2')
with tf.name_scope('layer2'):
pool1 = tf.nn.max_pool(conv2, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID', name='pool1')
conv3 = tf.layers.conv2d(pool1, filters=128, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv3')
conv4 = tf.layers.conv2d(conv3, filters=128, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv4')
with tf.name_scope('layer3'):
pool2 = tf.nn.max_pool(conv4, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID', name='pool2')
conv5 = tf.layers.conv2d(pool2, filters=256, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv5')
conv6 = tf.layers.conv2d(conv5, filters=256, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv6')
with tf.name_scope('layer4'):
pool3 = tf.nn.max_pool(conv6, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID', name='pool3')
conv7 = tf.layers.conv2d(pool3, filters=512, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv7')
conv8 = tf.layers.conv2d(conv7, filters=512, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv8')
with tf.name_scope('layer5'):
pool4 = tf.nn.max_pool(conv8, ksize=[1,2,2,1], strides=[1,2,2,1], padding='VALID', name='pool4')
conv9 = tf.layers.conv2d(pool4, filters=1024, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv9')
conv10 = tf.layers.conv2d(conv9, filters=1024, kernel_size=3, strides=1, activation=tf.nn.relu, name='conv10')
#Decoding
with tf.name_scope('layer6'):
up_conv1 = tf.layers.conv2d_transpose(conv10, filters=512, kernel_size=2, strides=2)
croped_conv8 = tf.image.central_crop(conv8, 7/8)
concat1 = tf.concat([croped_conv8, up_conv1], axis=-1)
conv11 = tf.layers.conv2d(concat1, filters=512, kernel_size=3, activation=tf.nn.relu, name='conv11')
conv12 = tf.layers.conv2d(conv11, filters=512, kernel_size=3, activation=tf.nn.relu, name='conv12')
with tf.name_scope('layer7'):
up_conv2 = tf.layers.conv2d_transpose(conv12, filters=256, kernel_size=2, strides=2)
croped_conv6 = tf.image.central_crop(conv6, 13/17)
concat2 = tf.concat([croped_conv6, up_conv2], axis=-1)
conv13 = tf.layers.conv2d(concat2, filters=256, kernel_size=3, activation=tf.nn.relu, name='conv13')
conv14 = tf.layers.conv2d(conv13, filters=256, kernel_size=3, activation=tf.nn.relu, name='conv14')
with tf.name_scope('layer8'):
up_conv3 = tf.layers.conv2d_transpose(conv14, filters=128, kernel_size=2, strides=2)
croped_conv4 = tf.image.central_crop(conv4, 5/7)
concat3 = tf.concat([croped_conv4, up_conv3], axis=-1)
conv15 = tf.layers.conv2d(concat3, filters=128, kernel_size=3, activation=tf.nn.relu, name='conv15')
conv16 = tf.layers.conv2d(conv15, filters=128, kernel_size=3, activation=tf.nn.relu, name='conv16')
with tf.name_scope('layer8'):
up_conv4 = tf.layers.conv2d_transpose(conv16, filters=64, kernel_size=2, strides=2)
croped_conv2 = tf.image.central_crop(conv2, 49/71)
concat4 = tf.concat([croped_conv2, up_conv4], axis=-1)
conv17 = tf.layers.conv2d(concat4, filters=64, kernel_size=3, activation=tf.nn.relu, name='conv17')
conv18 = tf.layers.conv2d(conv17, filters=64, kernel_size=3, activation=tf.nn.relu, name='conv18')
output = tf.layers.conv2d(conv18, filters=3, kernel_size=1, name='output')
with tf.name_scope('train'):
dice = 2 * tf.math.reduce_sum(output*y) / (tf.math.reduce_sum(output) + tf.math.reduce_sum(y) + 1)
loss = 1 - dice
optimizer = tf.train.AdamOptimizer()
training_op = optimizer.minimize(loss)
with tf.name_scope('save'):
saver = tf.train.Saver()
loss_sumary = tf.summary.scalar('ls', loss)
filewriter = tf.summary.FileWriter('./', tf.get_default_graph())
I am new with keras and have been learning it for about 3 weeks now. I apologies if my question sounds a bit stupid.
I am currently doing semantic medical image segmentation of 512x512. I'm using UNet from this link https://github.com/zhixuhao/unet . Basically, I want to segment a brain from an image (so two-class segmentation, background vs foreground)
I have made a few modification of the network and I'm getting some results which i am happy with. But I think I can improve the segmentation results by imposing more weight on the foreground because the number of pixels of the brain is much smaller than the number of background pixels. In some cases the brain does not appear in the image especially those located in the bottom slices.
I don't know which part of the code I need to modify in https://github.com/zhixuhao/unet
I would really appreciate if anyone can help me with this. Thanks a lot in advance!
import numpy as np
import os
import skimage.io as io
import skimage.transform as trans
import numpy as np
from keras.models import *
from keras.layers import *
from keras.optimizers import *
from keras.callbacks import ModelCheckpoint, LearningRateScheduler
from keras import backend as keras
def unet(pretrained_weights=None, input_size=(256, 256, 1)):
inputs = Input(input_size)
conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3)
conv3 = BatchNormalization()(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4)
conv4 = BatchNormalization()(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4)
conv5 = BatchNormalization()(conv5)
conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5)
conv5 = BatchNormalization()(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv2D(512, 2, activation='relu', padding='same', kernel_initializer='he_normal')(
UpSampling2D(size=(2, 2))(drop5))
merge6 = concatenate([drop4, up6], axis=3)
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6)
conv6 = BatchNormalization()(conv6)
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)
conv6 = BatchNormalization()(conv6)
up7 = Conv2D(256, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv6))
merge7 = concatenate([conv3, up7], axis=3)
conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7)
conv7 = BatchNormalization()(conv7)
conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)
conv7 = BatchNormalization()(conv7)
up8 = Conv2D(128, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv7))
merge8 = concatenate([conv2, up8], axis=3)
conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8)
conv8 = BatchNormalization()(conv8)
conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8)
conv8 = BatchNormalization()(conv8)
up9 = Conv2D(64, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9)
conv9 = BatchNormalization()(conv9)
conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
conv9 = BatchNormalization()(conv9)
conv9 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
conv9 = BatchNormalization()(conv9)
conv10 = Conv2D(1, 1, activation='sigmoid')(conv9)
model = Model(input=inputs, output=conv10)
model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy', metrics=['accuracy'])
# model.summary()
if (pretrained_weights):
model.load_weights(pretrained_weights)
return model
Here's the main.py
from model2 import *
from data2 import *
from keras.models import load_model
class_weight= {0:0.10, 1:0.90}
myGene = trainGenerator(2,'data/brainTIF/trainNew','image','label',save_to_dir = None)
model = unet()
model_checkpoint = ModelCheckpoint('unet_brainTest_e10_s5.hdf5',
monitor='loss')
model.fit_generator(myGene,steps_per_epoch=5,epochs=10,callbacks = [model_checkpoint])
testGene = testGenerator("data/brainTIF/test3")
results = model.predict_generator(testGene,18,verbose=1)
saveResult("data/brainTIF/test_results3",results)
As an option for class_weight for binary classes, you can also handle imbalanced classes using Synthetic Oversampling Technique (SMOTE), increasing the size of the minority group:
from imblearn.over_sampling import SMOTE
sm = SMOTE()
x, y = sm.fit_sample(X_train, Y_train)