Generate CSV file for each classes based on nuisances dataset - pytorch

Question

Please help me out, I am very confused. The dataset contains three folders train, nuisances, and testing set. The train folder contains 10 classes like aeroplane, train, etc. However, the nuisances folder consists of 5 classes like nuisances = ['shape', 'pose', 'texture', 'context', 'weather', 'occlusion'] and each class contains 10 classes same as the training dataset.
Dataset Structure
- dataset
- train folder
- images
- aeroplane
- train
- etc
- labels.csv
- nuisances folder
-context
- Images
- labels.csv
-occlusion
- Images
- labels.csv
- test folder
- images
- labels.csv
When testing, the test script generates a single .csv file containing all classes img_name, preds.
Code
from torch.utils.data import Dataset, DataLoader
import pandas as pd
from torchvision import transforms
from PIL import Image
import torch
import torch.nn as nn
from glob import glob
from pathlib import PurePath
import numpy as np
import timm
import torchvision
import time
img_list = glob('/media/cvpr/CM_22/ROBINv1.1-cls-pose/iid_test/Images/*.jpg')
name_list = [
'aeroplane',
'bicycle',
'boat',
'bus',
'car',
'chair',
'diningtable',
'motorbike',
'sofa',
'train'
]
# conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -c pytorch
class PoseData(Dataset):
def __init__(self, transforms) -> None:
"""
the data folder should look like
- datafolder
- Images
- labels.csv
"""
super().__init__()
self.img_list = glob('/media/cvpr/CM_22/ROBINv1.1-cls-pose/iid_test/Images/*.jpg')
self.img_list = sorted(self.img_list, key=lambda x: PurePath(x).parts[-1][:-4])
self.trs = transforms
def __len__(self):
return len(self.img_list)
def __getitem__(self, index):
image_dir = self.img_list[index]
image_name = PurePath(image_dir).parts[-1]
image = Image.open(image_dir)
image = self.trs(image)
return image, image_name
if __name__ == "__main__":
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
tfs = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
normalize,
])
model = timm.models.resnet152(pretrained=True, num_classes=10)
#model = torch.nn.DataParallel(model)
model.load_state_dict(torch.load('resnet152_best.pth.tar')['state_dict'], strict=False)
model = model.cuda()
model.eval()
dataset = PoseData(tfs)
loader = DataLoader(dataset, batch_size=128, shuffle=False, drop_last=False, num_workers=4)
image_dir = []
preds = []
for image, pth in loader:
image_dir.append(list(pth))
image = image.cuda()
with torch.no_grad():
model.eval()
pred = model(image)
#pred = torch.softmax(pred[:, :10], dim=1)
pred = torch.argmax(pred[:, :10], dim=1)
p = []
for i in range(pred.size(0)):
p.append(name_list[pred[i].item()])
p = np.array(p)
preds.append(p)
print(len(np.concatenate(preds)))
image_dir = np.array(sum(image_dir, []))
preds = np.concatenate(preds)
csv = {'imgs': np.array(image_dir), 'pred': np.array(preds),
}
csv = pd.DataFrame(csv)
print(csv)
csv.to_csv('evaluation/cls_ref/res/iid.csv', index=False)
The iid.csv is completely fine but I also require the separate .csv file for nuisances = ['shape', 'pose', 'texture', 'context', 'weather', 'occlusion'] so that I can evaluate accuracy on each nuisances classes. I have no idea how to build logic for this task