I am trying to rescale an image of size 255, 255 having 5 channels to something more manageable for my CNN, i.e 5, 40, 40
Here is a minimally reproducible example -
import torch
from torch import nn
import torchvision as TV
from torchvision import transforms
import numpy as np
x = np.random.rand(5,255,255)
transform = TV.transforms.Resize([transforms.Scale(48)])
transform(torch.from_numpy(x))
But I get the following error -
File "C:\Users\thoma\anaconda3\envs\tf-latest\lib\site-packages\torchvision\transforms\transforms.py", line 186, in __init__ assert isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2) AssertionError
Related
I'm trying to use the pre-trained model of detectron2. While running the following code, it shows NotImplementedError.
import torch
torch.__version__
import torchvision
#torchvision.__version__
!pip install detectron2 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html
import detectron2
from detectron2.utils.logger import setup_logger
setup_logger()
import numpy as np
import os, json, cv2, random
import matplotlib.pyplot as plt
%matplotlib inline
from detectron2 import model_zoo
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog, DatasetCatalog
from detectron2.structures import BoxMode
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCOInstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCOInstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
predictor = DefaultPredictor(cfg)
And it shows the following error:
---------------------------------------------------------------------------
NotImplementedError Traceback (most recent call last)
<ipython-input-27-699e754fc9df> in <module>
----> 1 predictor = DefaultPredictor(cfg)
4 frames
/usr/local/lib/python3.8/dist-packages/iopath/common/file_io.py in _isfile(self, path, **kwargs)
438 bool: true if the path is a file
439 """
--> 440 raise NotImplementedError()
441
442 def _isdir(self, path: str, **kwargs: Any) -> bool:
NotImplementedError:
I had the same issue and solved it by manually downloading .pkl file and giving path to cfg.MODEL.WEIGHTS variable
You can try this:
model_weights_url = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
print(f"Downloading model from {model_weights_url}...")
local_model_weights_path = Path("./temp/downloads/model.pkl")
os.makedirs(local_model_weights_path.parent, exist_ok=True)
urllib.request.urlretrieve(model_weights_url, local_model_weights_path)
cfg.MODEL.WEIGHTS = str(local_model_weights_path)
I had the same issue just today. I manually downloaded the R-50.pkl from https://dl.fbaipublicfiles.com/detectron2/ImageNetPretrained/MSRA/R-50.pkl and set cfg.MODEL.WEIGHTS = "R-50.pkl" #path to file
How to troubleshoot this? I've tried setting dtype=None in the image.img_to_array method.
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import matplotlib.pyplot as plt
from keras.preprocessing import image
image_size = (180, 180)
batch_size = 32
model = keras.models.load_model('best_model.h5')
img = keras.preprocessing.image.load_img(
"GarnetCreek_7-15-2019.jpeg", target_size=image_size
)
img_array = image.img_to_array(img)
img_array = tf.expand_dims(img_array, 0) # Create batch axis
predictions = model.predict(img_array)
score = predictions[0]
This raises the following error:
Traceback (most recent call last):
img_array = image.img_to_array(img, dtype=None)
return image.img_to_array(img, data_format=data_format, **kwargs)
x = np.asarray(img, dtype=dtype)
return array(a, dtype, copy=False, order=order)
TypeError: __array__() takes 1 positional argument but 2 were given
Has anyone seen this before? Many thanks!
This error sometimes is due to a bug in Pillow 8.3.0 as it is here. (You may not use import PIL directly in your code, however some libraries such as tf.keras.preprocessing.image.load_img use PIL internally)
So, downgrading from PIL 8.3.0 to 8.2.0 may work.
Check PIL version:
import PIL
print(PIL.__version__)
If it is 8.3.0, then you may downgrade to 8.2.0:
!pip install pillow==8.2.0
File "D:\Users\Watson Rockstar\Anaconda3\lib\site-packages\sklearn\utils\validation.py", line 205, in check_consistent_length
" samples: %r" % [int(l) for l in lengths])
ValueError:
Found input variables with inconsistent numbers of samples: [2883, 1236]
This dataset totally has 4119 data, and the Xtrain volum= (2883,18), Xtest volum = (1236,18)
I have tried to use LabelEncoder and OneHotEncoder to sovle the problems, but it is not helpful:
# Ignore the warnings
import warnings
warnings.filterwarnings('always')
warnings.filterwarnings('ignore')
# data visualisation and manipulation
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import style
import seaborn as sns
import missingno as msno
#configure
# sets matplotlib to inline and displays graphs below the corressponding cell.
#import the necessary modelling algos.
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
#preprocessing
from sklearn.preprocessing import LabelEncoder
telebanking = pd.read_csv('bank-additional.csv')
telebank = telebanking.drop(['duration','default'],axis =1)
def transform(feature):
le = LabelEncoder()
telebank[feature] = le.fit_transform(telebank[feature])
print(le.classes_)
cat_telebank=telebank.select_dtypes(include='object')
cat_telebank.columns
for col in cat_telebank.columns:
transform(col)
scaler=StandardScaler()
scaled_telebank=scaler.fit_transform(telebank.drop('y',axis=1))
X=scaled_telebank
Y=telebank['y'].as_matrix()
Xtrain,Xtest,Ytrain,Ytest = train_test_split(X,Y,test_size=0.3)
def compare(model):
clf = model
clf.fit(Xtrain,Ytrain)
pred = clf.predict(Xtrain)
acc.append(accuracy_score(pred,Ytest))
prec.append(precision_score(pred,Ytest))
rec.append(recall_score(pred,Ytest))
auroc.append(roc_auc_score(pred,Ytest))
acc=[]
prec=[]
rec=[]
auroc=[]
models=[RandomForestClassifier(),DecisionTreeClassifier()]
model_names=['RandomForestClassifier','DecisionTreeClassifier']
for model in range(len(models)):
compare(models[model])
d={'Modelling Algo':model_names,'Accuracy':acc,'Precision':prec,'Recall':rec,'Area Under ROC Curve':auroc}
met_telebank=pd.DataFrame(d)
met_telebank
It is the first warning's detail.
Xtrain,Xtest,Ytrain,Ytest = train_test_split(X,Y,test_size=0.3)
should be
Xtrain,Ytrain,Xtest,Ytest = train_test_split(X,Y,test_size=0.3)
This is causing the error, because it wants to use Xtest as the Ytrain values.
My system is configured with 16GB RAM. I have tried to train image similarity model on 20 millions images(total size 10GB) using VGG19 and KNN's nearest neighbor. When tried to read images i am getting Memory error. Even I have tried to train model on 200000(total size 770MB) but issue is same. How I can read millions of images to train ML models.
Ubuntu 18.04.2 LTS,Core™ i7,Intel® HD Graphics 5500 (Broadwell GT2), 64-bit, 16GB RAM
import os
import skimage.io
import tensorflow as tf
from skimage.transform import resize
import numpy as np
from sklearn.neighbors import NearestNeighbors
import matplotlib.pyplot as plt
from matplotlib import offsetbox
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from sklearn import manifold
import pickle
skimage.io.use_plugin('matplotlib')
dirPath = 'train_data'
args = [os.path.join(dirPath, filename) for filename in os.listdir(dirPath)]
imgs_train = [skimage.io.imread(arg, as_gray=False) for arg in args]
shape_img = (130, 130, 3)
model = tf.keras.applications.VGG19(weights='imagenet', include_top=False,
input_shape=shape_img)
model.summary()
shape_img_resize = tuple([int(x) for x in model.input.shape[1:]])
input_shape_model = tuple([int(x) for x in model.input.shape[1:]])
output_shape_model = tuple([int(x) for x in model.output.shape[1:]])
n_epochs = None
def resize_img(img, shape_resized):
img_resized = resize(img, shape_resized,
anti_aliasing=True,
preserve_range=True)
assert img_resized.shape == shape_resized
return img_resized
def normalize_img(img):
return img / 255.
def transform_img(img, shape_resize):
img_transformed = resize_img(img, shape_resize)
img_transformed = normalize_img(img_transformed)
return img_transformed
def apply_transformer(imgs, shape_resize):
imgs_transform = [transform_img(img, shape_resize) for img in imgs]
return imgs_transform
imgs_train_transformed = apply_transformer(imgs_train, shape_img_resize)
X_train = np.array(imgs_train_transformed).reshape((-1,) + input_shape_model)
E_train = model.predict(X_train)
E_train_flatten = E_train.reshape((-1, np.prod(output_shape_model)))
knn = NearestNeighbors(n_neighbors=5, metric="cosine")
knn.fit(E_train_flatten)
Knowing that keras is working well with generator, you should consider using one:
python generator tutorial,
using a generator with keras (example)
It allows you to load your image during your training, batch by batch.
For start in Tensorflow, I am triying to reproduce the basic example of the estimator with the IRIS data set, but with my own data.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
from six.moves.urllib.request import urlopen
import tensorflow as tf
from pandas import DataFrame, read_csv
import numpy as np
TESTFILE = "test.csv"
TRAINFILE = "train.csv"
# Load datasets
training_set = tf.contrib.learn.datasets.base.load_csv_with_header(filename=TRAINFILE, target_dtype=np.int, features_dtype=np.float32, target_column=0)
test_set = tf.contrib.learn.datasets.base.load_csv_with_header(filename=TESTFILE, target_dtype=np.int, features_dtype=np.float32, target_column=0)
# Specify that all features have real-value data
feature_columns = [tf.feature_column.numeric_column("x", shape=[4])]
# Build 3 layer DNN with 10, 20, 10 units respectively.
classifier = tf.estimator.DNNClassifier(feature_columns=feature_columns,
hidden_units=[10, 20, 10],
n_classes=3,
model_dir="/tmp/Goldman_model")
# Define the training inputs
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": np.array(training_set.data)},
y=np.array(training_set.target),
num_epochs=None,
shuffle=True)
# Train model.
classifier.train(input_fn=train_input_fn, steps=2000)
But, when I try to train the classifier, I receive the next error:
File "C:\Users\***\Anaconda3\lib\site-packages\tensorflow\python\framework\common_shapes.py", line 691, in _call_cpp_shape_fn_impl
raise ValueError(err.message)
ValueError: Dimensions must be equal, but are 160 and 128
for 'dnn/head/sparse_softmax_cross_entropy_loss/xentropy/xentropy'
(op: 'SparseSoftmaxCrossEntropyWithLogits') with input shapes: [160,3], [128].
And I have absolutely no idea what to do next.
Thank you very much for the answers,
JF Palomeque
Your labels and predictions have different dimensions (160 and 128).