3、 How to customize data sets?

---

Preface

MNIST Data, the most basic data set, has been destroyed by the students who are on the path of programming apes , So today I will teach you how to make a custom dataset .

---

One 、 Defined datasets , No pretreatment .

Here are some Inline code slice .

import  os
from  torch.utils.data import  Dataset ,DataLoader
from PIL import Image

#1. Create dataset class , Use torch.utils.data Medium Dataset Method .
class My_Dataset(Dataset):

    #2. Loop to find the file path , And add labels 
    def __init__(self,main_dir,data_type,transforms):
        self.dataset=[]# The empty list is the database with a new label 
        self.transforms=transforms
        if data_type==0:
            data_filename='train'
        elif data_type is 1:
            data_filename='val'
        else:
            data_filename='test'
        for i , cls_filename in enumerate(
        os.listdir(os.path.join(main_dir,data_filename))):
            for i ,img_data in enumerate(os.listdir(
            os.path.join(main_dir,data_filename,cls_filename))):
                self.dataset.append([os.path.join(main_dir,
                data_filename,cls_filename,img_data),int(img_data[0]) ])

    #3. Calculate picture length , Convenient for later iterations 
    def __len__(self):
        return len(self.dataset)# To get the length of the picture , Easy to iterate 

    #4、 Take out the image path , And open , Convenient for data preprocessing 
    def __getitem__(self, index):
        img,label=self.dataset[index]
        img_data=Image.open(img)
        img_data=self.transforms(img_data)
        return img_data,label

Example ：@TOC

Two 、 Define datasets , And do data preprocessing .

Including rotation 、 tailoring 、 Into a tensor 、 expand 、 Regularization and so on .

1. Pretreatment part

#4、 Take out the image path , And open , Convenient for data preprocessing 
def __getitem__(self, index):
    img,label=self.dataset[index]
    img_data=self.data_process(Image.open(img))
    return img_data,label

#5. Data processing , Data to enhance 、 Add noise, etc 
def data_process(self,x):
    return transforms.Compose([transforms.ToTensor(),
    transforms.Normalize(mean=(0.5,),std=(0.5,))])(x)

2. Define data process

The code is as follows （ Example ）：

import  os
from  torchvision import transforms
from  torch.utils.data import  Dataset ,DataLoader
from PIL import Image

#1. Create dataset class , Use torch.utils.data Medium Dataset Method .
class My_Dataset(Dataset):

    #2. Loop to find the file path , And add labels 
    def __init__(self,main_dir,data_type):
        self.dataset=[]# The empty list is the database with a new label 
        if data_type==0:
            data_filename='train'
        elif data_type is 1:
            data_filename='val'
        else:
            data_filename='test'
        for i , cls_filename in enumerate(
        os.listdir(os.path.join(main_dir,data_filename))):
            for i ,img_data in enumerate(os.listdir(
            os.path.join(main_dir,data_filename,cls_filename))):
                self.dataset.append([os.path.join(main_dir,
                data_filename,cls_filename,img_data),i ])

    #3. Calculate picture length , Convenient for later iterations 
    def __len__(self):
        return len(self.dataset)# To get the length of the picture , Easy to iterate 

    #4、 Take out the image path , And open , Convenient for data preprocessing 
    def __getitem__(self, index):
        img,label=self.dataset[index]
        img_data=self.data_process(Image.open(img))
        return img_data,label

    #5. Data processing , Data to enhance 、 Add noise, etc 
    def data_process(self,x):
        return transforms.Compose([transforms.ToTensor(),
        transforms.Normalize(mean=(0.5,),std=(0.5,))])(x)
``