This my first use of Pytorch, However I use tensorflow as below:
path ="./Train_set"
img_gen = tf.keras.preprocessing.image.ImageDataGenerator()
test_set = img_gen.flow_from_directory(path,(224, 224),'rgb')
So my dataset is an image dataset .
Now, I would like to download my dataset with Pytorch, with this manner, I don't find anything in train_set:
import torch
import torch.nn as nn
import torch.nn.functional as F
DS="./path"
from torch.utils.data import DataLoader
train_set= DataLoader(DS, batch_size=64, shuffle=True)
So, How can I change the code
Thanks
Related
The following code run in Colab and I got the following error:
NameError: name 'MINST' is not defined
What do I need to do?
import torch
import torchvision
from torchvision.datasets import MNIST
dataset = MINST(root='data/', download=True)
len(dataset)
test_dataset = MINST(root='data/', train=False)
len(test_dataset)
dataset[0]
It is what it say it is a NameError
You imported the MNIST dataset and try to access MINST which is not a valid name.
Your code should be:
import torch
import torchvision
from torchvision.datasets import MNIST
dataset = MNIST(root='data/', download=True)
len(dataset)
test_dataset = MINST(root='data/', train=False)
len(test_dataset)
dataset[0]
Here is an autoencoder I created from Pytorch tutorials :
epochs = 1000
from pylab import plt
plt.style.use('seaborn')
import torch.utils.data as data_utils
import torch
import torchvision
import torch.nn as nn
from torch.autograd import Variable
cuda = torch.cuda.is_available()
FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor
import numpy as np
import pandas as pd
import datetime as dt
features = torch.tensor(np.array([ [1,2,3],[1,2,3],[100,200,500] ]))
print(features)
batch = 10
data_loader = torch.utils.data.DataLoader(features, batch_size=2, shuffle=False)
encoder = nn.Sequential(nn.Linear(3,batch), nn.Sigmoid())
decoder = nn.Sequential(nn.Linear(batch,3), nn.Sigmoid())
autoencoder = nn.Sequential(encoder, decoder)
optimizer = torch.optim.Adam(params=autoencoder.parameters(), lr=0.001)
encoded_images = []
for i in range(epochs):
for j, images in enumerate(data_loader):
# images = images.view(images.size(0), -1)
images = Variable(images).type(FloatTensor)
optimizer.zero_grad()
reconstructions = autoencoder(images)
loss = torch.dist(images, reconstructions)
loss.backward()
optimizer.step()
# encoded_images.append(encoder(images))
# print(decoder(torch.tensor(np.array([1,2,3])).type(FloatTensor)))
encoded_images = []
for j, images in enumerate(data_loader):
images = images.view(images.size(0), -1)
images = Variable(images).type(FloatTensor)
encoded_images.append(encoder(images))
I can see the encoded images do have newly created dimension of 10. In order to understand the matrix operations going on under the hood I'm attempting to print the matrix dimensions of encoder and decoder but shape is not available on nn.Sequential
How to print the matrix dimensions of nn.Sequential ?
A nn.Sequential is not a "layer", but rather a "container". It can store several layers and manage their execution (and some other functionalities).
In your case, each nn.Sequential holds both the linear layer and the non-linear nn.Sigmoid activation. To get the shape of the weights of the first layer in a nn.Sequential you can simply do:
encoder[0].weight.shape
I currently have following script that helps to find the best model for a doc2vec model. It works like this: First train a few models based on given parameters and then test against a classifier. Finally, it outputs the best model and classifier (I hope).
Data
Example data (data.csv) can be downloaded here: https://pastebin.com/takYp6T8
Note that the data has a structure that should make an ideal classifier with 1.0 accuracy.
Script
import sys
import os
from time import time
from operator import itemgetter
import pickle
import pandas as pd
import numpy as np
from argparse import ArgumentParser
from gensim.models.doc2vec import Doc2Vec
from gensim.models import Doc2Vec
import gensim.models.doc2vec
from gensim.models import KeyedVectors
from gensim.models.doc2vec import TaggedDocument, Doc2Vec
from sklearn.base import BaseEstimator
from gensim import corpora
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
dataset = pd.read_csv("data.csv")
class Doc2VecModel(BaseEstimator):
def __init__(self, dm=1, size=1, window=1):
self.d2v_model = None
self.size = size
self.window = window
self.dm = dm
def fit(self, raw_documents, y=None):
# Initialize model
self.d2v_model = Doc2Vec(size=self.size, window=self.window, dm=self.dm, iter=5, alpha=0.025, min_alpha=0.001)
# Tag docs
tagged_documents = []
for index, row in raw_documents.iteritems():
tag = '{}_{}'.format("type", index)
tokens = row.split()
tagged_documents.append(TaggedDocument(words=tokens, tags=[tag]))
# Build vocabulary
self.d2v_model.build_vocab(tagged_documents)
# Train model
self.d2v_model.train(tagged_documents, total_examples=len(tagged_documents), epochs=self.d2v_model.iter)
return self
def transform(self, raw_documents):
X = []
for index, row in raw_documents.iteritems():
X.append(self.d2v_model.infer_vector(row))
X = pd.DataFrame(X, index=raw_documents.index)
return X
def fit_transform(self, raw_documents, y=None):
self.fit(raw_documents)
return self.transform(raw_documents)
param_grid = {'doc2vec__window': [2, 3],
'doc2vec__dm': [0,1],
'doc2vec__size': [100,200],
'logreg__C': [0.1, 1],
}
pipe_log = Pipeline([('doc2vec', Doc2VecModel()), ('log', LogisticRegression())])
log_grid = GridSearchCV(pipe_log,
param_grid=param_grid,
scoring="accuracy",
verbose=3,
n_jobs=1)
fitted = log_grid.fit(dataset["posts"], dataset["type"])
# Best parameters
print("Best Parameters: {}\n".format(log_grid.best_params_))
print("Best accuracy: {}\n".format(log_grid.best_score_))
print("Finished.")
I do have following questions regarding my script (I combine them here to avoid three posts with the same code snippet):
What's the purpose of def __init__(self, dm=1, size=1, window=1):? Can I possibly remove this part, somehow (tried unsuccessfully)?
How can I add a RandomForest classifier (or others) to the GridSearch workflow/pipeline?
How could a train/test data split added to the code above, as the current script only trains on the full dataset?
1) init() lets you define the parameters you would like your class to take at initialization (equivalent to contructor in java).
Please look at these questions for more details:
Python __init__ and self what do they do?
Python constructors and __init__
2) Why do you want to add the RandomForestClassifier and what will be its input?
Looking at your other two questions, do you want to compare the output of RandomForestClassifier with LogisticRegression here? If so, you are doing good in this question of yours.
3) You have imported the train_test_split, just use it.
X_train, X_test, y_train, y_test = train_test_split(dataset["posts"], dataset["type"])
fitted = log_grid.fit(X_train, y_train)
I am trying to use Pytorch to run classification on a dataset of images of cats and dogs. In my code I am so far downloading the data and going into the folder train which has two folders in it called "cats" and "dogs." I am then trying to load this data into a dataloader and iterate through batches, but it is giving me some error I don't understand in the iteration step.
Since it is Google Colabs I have code in there for downloading data and installing libraries. Any other advice on my code so far would be appreciated as well.
!pip install torch
!pip install torchvision
from __future__ import print_function, division
import os
import torch
import pandas as pd
import numpy as np
# For showing and formatting images
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# For importing datasets into pytorch
import torchvision.datasets as dataset
# Used for dataloaders
import torch.utils.data as data
# For pretrained resnet34 model
import torchvision.models as models
# For optimisation function
import torch.nn as nn
import torch.optim as optim
!wget http://files.fast.ai/data/dogscats.zip
!unzip dogscats.zip
batch_size = 256
train_raw = dataset.ImageFolder(PATH+"train", transform=transforms.ToTensor())
train_loader = data.DataLoader(train_raw, batch_size=batch_size, shuffle=True)
for batch_idx, (data, target) in enumerate(train_loader):
print("Data: ", batch_idx)
The error comes up on the last lines and is below:
RuntimeErrorTraceback (most recent call last)
<ipython-input-66-c32dd0c1b880> in <module>()
----> 1 for batch_idx, (data, target) in enumerate(train_loader):
2 print("Data: ", batch_idx)
3
/usr/local/lib/python2.7/dist-packages/torch/utils/data/dataloader.pyc in __next__(self)
257 if self.num_workers == 0: # same-process loading
258 indices = next(self.sample_iter) # may raise StopIteration
--> 259 batch = self.collate_fn([self.dataset[i] for i in indices])
260 if self.pin_memory:
261 batch = pin_memory_batch(batch)
/usr/local/lib/python2.7/dist-packages/torch/utils/data/dataloader.pyc in default_collate(batch)
133 elif isinstance(batch[0], collections.Sequence):
134 transposed = zip(*batch)
--> 135 return [default_collate(samples) for samples in transposed]
136
137 raise TypeError((error_msg.format(type(batch[0]))))
/usr/local/lib/python2.7/dist-packages/torch/utils/data/dataloader.pyc in default_collate(batch)
110 storage = batch[0].storage()._new_shared(numel)
111 out = batch[0].new(storage)
--> 112 return torch.stack(batch, 0, out=out)
113 elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
114 and elem_type.__name__ != 'string_':
/usr/local/lib/python2.7/dist-packages/torch/functional.pyc in stack(sequence, dim, out)
62 inputs = [t.unsqueeze(dim) for t in sequence]
63 if out is None:
---> 64 return torch.cat(inputs, dim)
65 else:
66 return torch.cat(inputs, dim, out=out)
RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 0. Got 400 and 487 in dimension 2 at /pytorch/torch/lib/TH/generic/THTensorMath.c:2897
Thanks
I think the main problem was images being of different size . I may have understood ImageFolder in other way but, i think you don't need labels for images if the directory structure is as specified in pytorch and pytorch will figure out the labels for you.
I would also add more things to your transform that automatically resizes every images from the folder such as:
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)
transform = transforms.Compose(
[transforms.ToTensor(),transforms.Resize((224,224)),
normalize])
Also you can use other tricks to make your DataLoader much faster such as adding batch_size and number of cpu workers such as:
testloader = DataLoader(testset, batch_size=16,
shuffle=False, num_workers=4)
I think this will make you pipeline much faster.
I see two problems in your code first you are importing import torch.utils.data as data and again replacing that in the data loader. Please keep the imported module and your variable name in separate namespace. I think this error could be because of different sizes of data returned by dataloder(images) and labels. As you can see there is an error in concatenation because the first dimension ie. the label size and number of images in folder do not match. Hope this helps.
I think I was wrong in my comment to Manoj Acharya, the problem was in the batch_size being put into the dataloader. I read the below source and it seems you can't batch images together with different sizes:
https://medium.com/#yvanscher/pytorch-tip-yielding-image-sizes-6a776eb4115b
So in my code after changing the data variable Manoj points out I changed the batch_size to 1 and the program stopped failing. I want to put it in batches though so I added a further transform CenterCrop() to resize all images to the same size. Below is my new code:
!pip install torch
!pip install torchvision
from __future__ import print_function, division
import os
import torch
import pandas as pd
import numpy as np
# For showing and formatting images
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# For importing datasets into pytorch
import torchvision.datasets as dataset
# Used for dataloaders
from torch.utils.data import DataLoader
# For pretrained resnet34 model
import torchvision.models as models
# For optimisation function
import torch.nn as nn
import torch.optim as optim
# For turning data into tensors
import torchvision.transforms as transforms
!wget http://files.fast.ai/data/dogscats.zip
!unzip dogscats.zip
batch_size = 256
sz = 224
train_raw = dataset.ImageFolder(PATH+"train", transform=transforms.Compose([transforms.CenterCrop(sz),transforms.ToTensor()]))
train_loader = DataLoader(train_raw,batch_size=batch_size, shuffle=True)
for batch_idx, (data, target) in enumerate(train_loader):
print("Data: ", batch_idx)
Thanks
I am trying to predict the result for a single image but it is giving an irrelevant result. I have trained the model on cifar 10 dataset
I have used keras and tensorflow to train this model. I suppose the input which I am providing is not of the correct size.
here is the gist of training code :https://github.com/09rohanchopra/cifar10/blob/master/cifar10-simple-cnn.ipynb
Code for predicting simgle image
#
from keras.preprocessing import image
from keras.models import load_model
from scipy.misc import imread,imresize
import numpy as np
model=load_model('augmented_best_model.h5')
im=imread('1.jpg')
im=im/255
im=im.resize(im,(32,32))
pr=model.predict(im.reshape(-1,3,32,32))
imread is now deprecated. Better alternative is to use keras functions.
x = image.load_img('test/2.jpg', target_size=(32,32))
x = image.img_to_array(x)
x = x.reshape((1,) + x.shape)
x = x/255.
I got the answer
x=imread('test/2.jpg',mode='RGB')
x=imresize(x,(32,32))
x=np.invert(x)
x=x.reshape(-1,32,32,3)