1. #!/usr/bin/env python 
    2. # -*- coding:utf-8 -*-
    3. import os
    4. from PIL import Image
    5. import torch
    6. import torchvision
    7. import sys
    8. from efficientnet_pytorch import EfficientNet
    9. device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    10. from PIL import Image
    11. from torch import optim
    12. from torch import nn
    13. os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
    14. import d2lzh_pytorch as d2l
    15. from time import time
    16. import time
    17. import  csv
    18. from torchvision import models
    19. import torch.nn.functional as F
    20. device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    22. class MyDataset(torch.utils.data.Dataset):  # 创建自己的类：MyDataset,这个类是继承的torch.utils.data.Dataset
    23.     def __init__(self, is_train,root):  # 初始化一些需要传入的参数
    24.         super(MyDataset, self).__init__()
    25.         fh = open(root, 'r', encoding="utf-8")  # 按照传入的路径和txt文本参数，打开这个文本，并读取内容
    26.         imgs = []  # 创建一个名为img的空列表，一会儿用来装东西
    27.         for line in fh:  # 按行循环txt文本中的内容
    28.             line = line.rstrip()  # 删除 本行string 字符串末尾的指定字符，这个方法的详细介绍自己查询python
    29.             words = line.split(':')  # 是英文的:  通过指定分隔符对字符串进行切片，默认为所有的空字符，包括空格、换行、制表符等
    30.             imgs.append((words[0], int(words[-1])))  # 把txt里的内容读入imgs列表保存，具体是words几要看txt内容而定
    31.         self.imgs = imgs
    32.         self.is_train = is_train
    33.         if self.is_train:
    34.             self.train_tsf = torchvision.transforms.Compose([
    35.                 torchvision.transforms.RandomResizedCrop(524, scale=(0.1, 1), ratio=(0.5, 2)),
    36.                 torchvision.transforms.ToTensor()
    37.             ])
    38.         else:
    39.             self.test_tsf = torchvision.transforms.Compose([
    40.                 torchvision.transforms.Resize(size=524),
    41.                 torchvision.transforms.CenterCrop(size=500),
    42.                 torchvision.transforms.ToTensor()])
    44.     def __getitem__(self, index):  # 这个方法是必须要有的，用于按照索引读取每个元素的具体内容
    45.         feature, label = self.imgs[index]  # fn是图片path #fn和label分别获得imgs[index]也即是刚才每行中word[0]和word[1]的信息
    46.         feature = Image.open(feature).convert('RGB')  # 按照path读入图片from PIL import Image # 按照路径读取图片
    47.         if self.is_train:
    48.             feature = self.train_tsf(feature)
    49.         else:
    50.             feature = self.test_tsf(feature)
    51.         return feature, label
    53.     def __len__(self):  # 这个函数也必须要写，它返回的是数据集的长度，也就是多少张图片，要和loader的长度作区分
    54.         return len(self.imgs)
    56. def get_k_fold_data(k, k1, image_dir):#image_dir即为上图包含总的数据集信息的txt文本文件的具体路径
    57.     # 返回第i折交叉验证时所需要的训练和验证数据
    58.     assert k > 1
    59.     # if k1==0:#第一次需要打开文件
    61.     file = open(image_dir, 'r', encoding='utf-8')
    62.     imgs_ls = []
    63.     for line in file.readlines():
    64.         # if len(line):
    65.         imgs_ls.append(line)
    66.     file.close()
    67.     #print(len(imgs_ls))
    68.     avg = len(imgs_ls) // 10
    69.     #print(avg)
    70.     # print(    len(   imgs_ls[k*avg:avg*(k+1)]   )    )
    71.     f1 = open('.\\shallow\\test_k.txt', 'w')
    72.     f2 = open('.\\shallow\\train_k.txt', 'w')
    73.     for i, j in enumerate(imgs_ls): #这个for循环很关键
    74.         if (i // avg) == k1: #根据整除结果不同，测试集有10种划分方式
    75.             f2.write(j)
    76.         else:
    77.             f1.write(j)
    78.     f1.close()
    79.     f2.close()
    82. #训练函数部分
    83. def train(i,train_iter, test_iter, net, loss, optimizer, device, num_epochs):
    84.     net = net.to(device)
    85.     print("training on ", device)
    86.     start = time.time()
    87.     test_acc_max_l = []
    88.     train_acc_max_l = []
    89.     train_l_min_l=[]
    90.     for epoch in range(num_epochs):  #迭代100次
    91.         batch_count = 0
    92.         train_l_sum, train_acc_sum, test_acc_sum, n = 0.0, 0.0, 0.0, 0
    93.         for X, y in train_iter:
    94.             X = X.to(device)
    95.             y = y.to(device)
    96.             y_hat = net(X)
    97.             l = loss(y_hat, y)
    98.             optimizer.zero_grad()
    99.             l.backward()
    100.             optimizer.step()
    101.             train_l_sum += l.cpu().item()
    102.             train_acc_sum += (y_hat.argmax(dim=1) == y).sum().cpu().item()
    103.             n += y.shape[0]
    104.             batch_count += 1
    105.         #至此，一个epoches完成
    106.         test_acc_sum= d2l.evaluate_accuracy(test_iter, net)
    107.         train_l_min_l.append(train_l_sum/batch_count)
    108.         train_acc_max_l.append(train_acc_sum/n)
    109.         test_acc_max_l.append(test_acc_sum)
    110.         print('fold %d epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
    111.               % (i+1,epoch + 1, train_l_sum / batch_count, train_acc_sum / n, test_acc_sum))
    112.     #train_l_min_l.sort()
    113.     #￥train_acc_max_l.sort()
    114.     index_max=test_acc_max_l.index(max(test_acc_max_l))
    115.     f = open("./shallow/results.txt", "a")
    117.     if i==0:
    118.         f.write("%d fold"+"   "+"train_loss"+"       "+"train_acc"+"      "+"test_acc")
    119.     f.write('\n' +"fold"+str(i+1)+":"+str(train_l_min_l[index_max]) + " ;" + str(train_acc_max_l[index_max]) + " ;" + str(test_acc_max_l[index_max]))
    120.     f.close()
    121.     print('fold %d, train_loss_min %.4f, train acc max%.4f, test acc max %.4f, time %.1f sec'
    122.             % (i + 1, train_l_min_l[index_max], train_acc_max_l[index_max], test_acc_max_l[index_max], time.time() - start))
    123.     return train_l_min_l[index_max],train_acc_max_l[index_max],test_acc_max_l[index_max]
    125. #k折交叉验证部分
    126. def k_fold(k,image_dir,num_epochs,device,batch_size,optimizer,loss,net):
    127.     train_k = './/shallow//train_k.txt'
    128.     test_k = './/shallow//test_k.txt'
    129.     #loss_acc_sum,train_acc_sum, test_acc_sum = 0,0,0
    130.     Ktrain_min_l = []
    131.     Ktrain_acc_max_l = []
    132.     Ktest_acc_max_l = []
    133.     for i in range(k):
    134.         get_k_fold_data(k, i,image_dir)
    135.         #修改train函数，使其返回每一批次的准确率，tarin_ls用列表表示
    136.         train_data = MyDataset(is_train=True, root=train_k)
    137.         test_data = MyDataset(is_train=False, root=test_k)
    139.         train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=10, shuffle=True, num_workers=num_workers)
    140.         test_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=10, shuffle=True, num_workers=num_workers)
    142.         loss_min,train_acc_max,test_acc_max=train(i,train_loader,test_loader, net, loss, optimizer, device, num_epochs)
    143.         Ktrain_min_l.append(loss_min)
    144.         Ktrain_acc_max_l.append(train_acc_max)
    145.         Ktest_acc_max_l.append(test_acc_max)
    146.         #train_acc_sum += train_acc# train函数epoches（即第k个数据集被测试后）结束后，累加
    147.         #test_acc_sum += test_acc#
    148.         #loss_acc_sum+=loss_acc
    149.         #print('fold %d, lose_rmse_max %.4f, train_rmse_max %.4f, test_rmse_max %.4f ' %(i+1, loss_acc,train_acc, test_acc_max_l[i]))
    150.     return sum(Ktrain_min_l)/len(Ktrain_min_l),sum(Ktrain_acc_max_l)/len(Ktrain_acc_max_l),sum(Ktest_acc_max_l)/len(Ktest_acc_max_l)
    153. #from efficientnet_pytorch import EfficientNet
    154. #net = EfficientNet.from_pretrained('efficientnet-b0')
    155. #net._fc = nn.Linear(1280, 2)
    156. #output_params = list(map(id, net._fc.parameters()))
    157. #feature_params = filter(lambda p: id(p) not in output_params, net.parameters())
    158. #lr = 0.01
    159. #optimizer = optim.SGD([{'params': feature_params},
    160. #                       {'params': net._fc.parameters(), 'lr': lr * 10}],
    161. #                      lr=lr, weight_decay=0.001)
    162. net = models.resnet18(pretrained=True)
    163. class Net(nn.Module):
    164.     def __init__(self, model):  # 此处的model参数是已经加载了预训练参数的模型，方便继承预训练成果
    165.         super(Net, self).__init__()
    166.         # 取掉model的后两层
    167.         self.resnet18_layer = nn.Sequential(*list(model.children())[:-1])
    168.         self.fc1 = nn.Linear(512, 256)
    169.         self.fc2 = nn.Linear(256, 128)
    170.         self.fc3 = nn.Linear(128, 32)
    171.         self.fc4 = nn.Linear(32, 2)
    173.     def forward(self, x):
    174.         x = self.resnet18_layer(x)
    175.         x = x.view(x.size(0), -1)
    176.         x = F.relu(self.fc1(x))
    177.         x = F.relu(self.fc2(x))
    178.         x = F.relu(self.fc3(x))
    179.         x = self.fc4(x)
    181.         return x
    182. net = Net(net)
    185. batch_size=10
    186. lr = 0.01
    187. k=6
    188. image_dir='.\\shallow\\all_shuffle_datas.txt'
    189. num_epochs=20
    190. num_workers = 0
    192. optimizer = optim.Adam(net.parameters(),lr = lr,weight_decay=0.001)
    193. loss = torch.nn.CrossEntropyLoss()
    194. loss_k,train_k, valid_k=k_fold(k,image_dir,num_epochs,device,batch_size,optimizer,loss,net)
    195. f=open("./shallow/results.txt","a")
    196. f.write('\n'+"avg in k fold:"+"\n"+str(loss_k)+" ;"+str(train_k)+" ;"+str(valid_k))
    197. f.close()
    199. print('%d-fold validation: min loss rmse %.5f, max train rmse %.5f,max test rmse %.5f' % (k,loss_k,train_k, valid_k))
    200. print("Congratulations!!! hou bin")
    201. #torch.save(net.module.state_dict(), ".\\bird_model_k.pt")