比赛题

https://www.kaggle.com/c/nlp-getting-started
简单的说就是句子的二分类问题

基本思路

  1. 建立词典
  2. 输入句子序列（编号）
  3. 输入到embeding
  4. embeding再输入LSTM
  5. LSTM输入到Linear
  6. 最后利用输出的数和1，0计算损失
  7. 使用torch.nn.MSELoss()计算loss
  8. 优化算法optim.SGD(model.parameters(), lr=learning_rate)

总的架构：Embedding+LSTM+Linear

代码

  2. import pandas as pd
  3. import numpy as np
  4. from sklearn.preprocessing import MinMaxScaler
  5. import time
  6. import copy
  7. import torch.nn as nn
  8. import torch.nn.functional as F
  9. import torch.optim as optim
  10. import torch.autograd as autograd
  11. import torch.nn.functional
  12. from torch.utils.data import Dataset, DataLoader
  13. from torchvision import transforms
  14. import warnings
  15. import torch
  16. import time
  19. USE_CUDA = torch.cuda.is_available()
  21. base_path="D:\\New_desktop\\nlp-getting-started\\"
  22. read_train=pd.read_csv(base_path+'train.csv')
  23. train_data=read_train.iloc[0:read_train.shape[0],[1,2,3]]
  25. train_label=read_train.iloc[0:read_train.shape[0],[4]]
  26. train_label=torch.tensor(train_label[:read_train.shape[0]].values,dtype=torch.float)
  29. read_test = pd.read_csv(base_path + 'test.csv')
  30. test_data = read_test.iloc[0:read_train.shape[0], [1, 2, 3]]
  33. a=[]
  34. sentence=""
  35. for i in range(0, len(train_data)):
  36.     sentence=sentence+str(train_data.iloc[i]['keyword'])
  37.     sentence+=" "
  38.     sentence=sentence+str(train_data.iloc[i]['location'])
  39.     sentence += " "
  40.     sentence=sentence+str(train_data.iloc[i]['text'])
  41.     sentence += " "
  43. for i in range(0, len(test_data)):
  44.     sentence=sentence+str(test_data.iloc[i]['keyword'])
  45.     sentence+=" "
  46.     sentence=sentence+str(test_data.iloc[i]['location'])
  47.     sentence += " "
  48.     sentence=sentence+str(test_data.iloc[i]['text'])
  49.     sentence += " "
  51. dict = sentence.split()
  53. dict=set(dict)
  55. print(len(dict))
  56. w2i={}
  58. def word2index():
  59.     index=0
  60.     for i in dict:
  61.         w2i[i]=index
  62.         index+=1
  64. word2index()
  66. train_data['keyword'] = [[w2i[i] for i in str(x).split()] for x in train_data.keyword]
  67. train_data['location'] = [[w2i[i] for i in str(x).split()] for x in train_data.location]
  68. train_data['text'] = [[w2i[i] for i in str(x).split()] for x in train_data.text]
  69. #print(train_data[:read_train.shape[0]])
  71. test_data['keyword'] = [[w2i[i] for i in str(x).split()] for x in test_data.keyword]
  72. test_data['location'] = [[w2i[i] for i in str(x).split()] for x in test_data.location]
  73. test_data['text'] = [[w2i[i] for i in str(x).split()] for x in test_data.text]
  76. data_list=[]
  77. max_len=0
  78. for i in range(0, len(train_data)):
  79.     if len(train_data.iloc[i]['text'])>max_len:
  80.         max_len=len(train_data.iloc[i]['text'])
  81. print(max_len)
  82. for x in train_data.text:
  83.     while len(x)<max_len:
  84.         x.append(0)
  86. for i in range(0, len(train_data)):
  87.     data_list.append(train_data.iloc[i]['text'])
  88. #print(len(data_list))
  90. traindata_tensor = torch.Tensor(data_list)
  92. if USE_CUDA:
  93.     print("using GPU")
  94.     traindata_tensor =traindata_tensor.cuda()
  95.     train_label = train_label.cuda()
  96. #print(a.shape)
  97. #train_data=torch.tensor(train_data[:read_train.shape[0]].values,dtype=torch.float)
  98. #print(train_data)
  99. #print (len(train_data.iloc[0]))
  101. '''
  102. 输入数据格式：
  103. input(seq_len, batch, input_size)
  104. h0(num_layers * num_directions, batch, hidden_size)
  105. c0(num_layers * num_directions, batch, hidden_size)
  107. 输出数据格式：
  108. output(seq_len, batch, hidden_size * num_directions)
  109. hn(num_layers * num_directions, batch, hidden_size)
  110. cn(num_layers * num_directions, batch, hidden_size)
  111. '''
  112. class LSTMTagger(nn.Module):
  114.     def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size,batch_size,str_len):
  115.         super(LSTMTagger, self).__init__()
  117.         self.hidden_dim = hidden_dim
  118.         self.str_len=str_len
  119.         self.batch_size=batch_size
  121.         self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
  123.         self.lstm = nn.LSTM(embedding_dim, hidden_dim)
  125.         self.hidden2tag = nn.Linear(str_len*hidden_dim, tagset_size)
  126.         self.hidden = self.init_hidden()
  128.     def init_hidden(self):
  129.         return (torch.zeros(1, self.batch_size, self.hidden_dim).cuda(),
  130.                 torch.zeros(1, self.batch_size, self.hidden_dim).cuda())
  133.     def forward(self, sentence,state):
  134.         embeds = self.word_embeddings(sentence)
  135.         #print(embeds.shape)
  136.         self.hidden=state
  137.         lstm_out, self.hidden = self.lstm(embeds.view(self.str_len, len(sentence), -1), self.hidden)
  138.         #print("ls",lstm_out.shape)
  139.         tag_space = self.hidden2tag(lstm_out.view(self.batch_size,-1))
  140.         #print(tag_space.shape)
  141.         self.dropout = nn.Dropout(config.hidden_dropout_prob)
  142.         #tag_scores = F.log_softmax(tag_space,dim=1)
  143.         return tag_space ,self.hidden
  145. model = LSTMTagger(10, 100, len(dict),1,16,31)
  149. model=model.cuda()
  151. def train(net, train_data, train_label,num_epochs, learning_rate,  batch_size):
  152.     train_ls=[]
  153.     #loss =torch.nn.CrossEntropyLoss()
  154.     state=None
  155.     loss = torch.nn.MSELoss()
  156.     optimizer = optim.SGD(model.parameters(), lr=learning_rate)
  157.     dataset = torch.utils.data.TensorDataset(train_data, train_label)
  158.     train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True)
  160.     iter=0
  161.     for epoch in range(num_epochs):
  162.         correct = 0
  163.         total=0
  164.         start = time.time()
  165.         index=0
  166.         for X, y in train_iter:
  167.             iter+=1
  168.             index+=1
  169.             if index>=470:
  170.                 break
  171.             if state is not None:
  172.                 if isinstance (state, tuple): # LSTM, state:(h, c)
  173.                     state = (state[0].detach(), state[1].detach())
  174.                 else:
  175.                     state = state.detach()
  176.             X = X.long()
  177.             (output, state) = model(X, state)
  179.             #correct+=predicted.data.eq(label.data).cpu().sum()
  180.             l = loss(output.float(), y)
  181.             l.backward()
  182.             optimizer.step()
  183.             optimizer.zero_grad()
  184.             #print(output)
  186.             output = [1 if i > 0.5 else 0 for i in output]
  188.             for i in range(len(output)):
  189.                 total+=1
  190.                 if output[i]==y[i]:
  191.                     correct+=1
  193.         acc = correct / total
  194.         end = time.time()
  195.         print("epoch ", str(epoch), "time: ",end-start ," loss: ", l.item())
  196.         print("correct: ", correct, " total: ", total, "acc: ", acc)
  197.         print("\n")
  198.         train_ls.append(l.item())
  199.     return train_ls
  203. #torch.save(model_object, 'model.pkl')
  204. #model = torch.load('model.pkl')
  205. def out_put(model):
  206.     test_list = []
  207.     max_len = 31
  208.     for x in test_data.text:
  209.         while len(x) < max_len:
  210.             x.append(0)
  211.         if len(x)>max_len:
  212.             x =x[0:30]
  213.     test_num=len(test_data.text)
  214.     print("sda",test_num)
  215.     for i in range(0, len(test_data)):
  216.         data_list.append(test_data.iloc[i]['text'])
  218.     test_data_tensor = torch.Tensor(data_list)
  220.     if USE_CUDA:
  221.         print("using GPU")
  222.         test_data_tensor = test_data_tensor.cuda()
  224.     with torch.no_grad():
  225.         result_dataset = torch.utils.data.TensorDataset(test_data_tensor)
  226.         result_dataloader = torch.utils.data.DataLoader(result_dataset, batch_size=16, shuffle=False)
  227.         state=None
  228.         index=0
  229.         for X in result_dataloader:
  230.             X=X[0]
  231.             if index>=203:
  232.                 break
  233.             index+=1
  234.             print(X.shape)
  235.             if state is not None:
  236.                 if isinstance(state, tuple):  # LSTM, state:(h, c)
  237.                     state = (state[0].detach(), state[1].detach())
  238.                 else:
  239.                     state = state.detach()
  240.             X = X.long()
  241.             (output, state) = model(X, state)
  243.             # correct+=predicted.data.eq(label.data).cpu().sum()
  246.             output = [1 if i > 0.5 else 0 for i in output]
  248.             for i in range(len(output)):
  249.                 test_list.append(output[i])
  251.     print(test_list)
  252.     print(len(test_list))
  253.     while len(test_list)<3263:
  254.         test_list.append(0)
  255.     df_output = pd.DataFrame()
  256.     aux = pd.read_csv(base_path + 'test.csv')
  257.     df_output['id'] = aux['id']
  258.     df_output['target'] = test_list
  259.     df_output[['id', 'target']].to_csv(base_path + 's1mple.csv', index=False)
  261. train(model,traindata_tensor,train_label,250,0.1,16)
  262. out_put(model)

结果

本地训练集

image.png

在线跑分

只有5分多
排名93%
效果奇差

image.png

image.png

总结

  1. 由于编程水平的限制，没有用更好的损失函数和优化算法
  2. 由于模型的输入被限制为了定长的batch_size的句子链，导致了部分的训练数据不可用以及部分的测试数据只能随便填（输入batchsize=16，只能取到16的倍数）
  3. 由于数据较少以及全用于训练，以至于本地的测试出现了过拟合的现象（本地最高训练准确率达到了98%，但实际显然在在线跑分上差到家了）
  4. 由于第二点的限制，导致一开始出现了最后一次读取的数据不足16的现象，导致了输入的维度不对，导致了Bug，但是经验有限，一直没看出来，尤其是使用GPU进行运行的时候，报错都特别的诡异，建议之后遇到bug可以关闭GPU再进行分析
  5. 还有一个问题是没有充分的利用信息，我只用了text这个维度进行预测

    改进

    减小训练

    初步随手设置15epoch，减少过拟合
    得分增加
    image.png

Drop_out

加上dropout
成绩有所进步

  1. class LSTMTagger(nn.Module):
  3.     def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size,batch_size,str_len):
  4.         super(LSTMTagger, self).__init__()
  6.         self.hidden_dim = hidden_dim
  7.         self.str_len=str_len
  8.         self.batch_size=batch_size
  10.         self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
  12.         self.lstm = nn.LSTM(embedding_dim, hidden_dim)
  13.         self.dropout = nn.Dropout(0.5)
  14.         self.hidden2tag = nn.Linear(str_len*hidden_dim, tagset_size)
  15.         self.hidden = self.init_hidden()
  17.     def init_hidden(self):
  18.         return (torch.zeros(1, self.batch_size, self.hidden_dim).cuda(),
  19.                 torch.zeros(1, self.batch_size, self.hidden_dim).cuda())
  22.     def forward(self, sentence,state,train_flag):
  23.         embeds = self.word_embeddings(sentence)
  24.         #print(embeds.shape)
  25.         self.hidden=state
  26.         lstm_out, self.hidden = self.lstm(embeds.view(self.str_len, len(sentence), -1), self.hidden)
  27.         #print("ls",lstm_out.shape)
  28.         if train_flag:
  29.             lstm_out=self.dropout(lstm_out)
  30.         tag_space = self.hidden2tag(lstm_out.view(self.batch_size,-1))
  31.         #print(tag_space.shape)
  33.         #tag_scores = F.log_softmax(tag_space,dim=1)
  34.         return tag_space ,self.hidden

image.png

增加embedding维数和hidden_dim

懒的提交测试了。。。

  1. model = LSTMTagger(300, 256, len(dict),1,16,31)

最终代码（不改了）

image.png

  1. import pandas as pd
  2. import numpy as np
  3. from sklearn.preprocessing import MinMaxScaler
  4. import time
  5. import copy
  6. import torch.nn as nn
  7. import torch.nn.functional as F
  8. import torch.optim as optim
  9. import torch.autograd as autograd
  10. import torch.nn.functional
  11. from torch.utils.data import Dataset, DataLoader
  12. from torchvision import transforms
  13. import warnings
  14. import torch
  15. import time
  18. USE_CUDA = torch.cuda.is_available()
  20. base_path="D:\\New_desktop\\nlp-getting-started\\"
  21. read_train=pd.read_csv(base_path+'train.csv')
  22. train_data=read_train.iloc[0:read_train.shape[0],[1,2,3]]
  24. train_label=read_train.iloc[0:read_train.shape[0],[4]]
  25. train_label=torch.tensor(train_label[:read_train.shape[0]].values,dtype=torch.float)
  28. read_test = pd.read_csv(base_path + 'test.csv')
  29. test_data = read_test.iloc[0:read_train.shape[0], [1, 2, 3]]
  32. a=[]
  33. sentence=""
  34. for i in range(0, len(train_data)):
  35.     sentence=sentence+str(train_data.iloc[i]['keyword'])
  36.     sentence+=" "
  37.     sentence=sentence+str(train_data.iloc[i]['location'])
  38.     sentence += " "
  39.     sentence=sentence+str(train_data.iloc[i]['text'])
  40.     sentence += " "
  42. for i in range(0, len(test_data)):
  43.     sentence=sentence+str(test_data.iloc[i]['keyword'])
  44.     sentence+=" "
  45.     sentence=sentence+str(test_data.iloc[i]['location'])
  46.     sentence += " "
  47.     sentence=sentence+str(test_data.iloc[i]['text'])
  48.     sentence += " "
  50. dict = sentence.split()
  52. dict=set(dict)
  54. print(len(dict))
  55. w2i={}
  57. def word2index():
  58.     index=0
  59.     for i in dict:
  60.         w2i[i]=index
  61.         index+=1
  63. word2index()
  65. train_data['keyword'] = [[w2i[i] for i in str(x).split()] for x in train_data.keyword]
  66. train_data['location'] = [[w2i[i] for i in str(x).split()] for x in train_data.location]
  67. train_data['text'] = [[w2i[i] for i in str(x).split()] for x in train_data.text]
  68. #print(train_data[:read_train.shape[0]])
  70. test_data['keyword'] = [[w2i[i] for i in str(x).split()] for x in test_data.keyword]
  71. test_data['location'] = [[w2i[i] for i in str(x).split()] for x in test_data.location]
  72. test_data['text'] = [[w2i[i] for i in str(x).split()] for x in test_data.text]
  75. data_list=[]
  76. max_len=0
  77. for i in range(0, len(train_data)):
  78.     if len(train_data.iloc[i]['text'])>max_len:
  79.         max_len=len(train_data.iloc[i]['text'])
  80. print(max_len)
  81. for x in train_data.text:
  82.     while len(x)<max_len:
  83.         x.append(0)
  85. for i in range(0, len(train_data)):
  86.     data_list.append(train_data.iloc[i]['text'])
  87. #print(len(data_list))
  89. traindata_tensor = torch.Tensor(data_list)
  91. if USE_CUDA:
  92.     print("using GPU")
  93.     traindata_tensor =traindata_tensor.cuda()
  94.     train_label = train_label.cuda()
  95. #print(a.shape)
  96. #train_data=torch.tensor(train_data[:read_train.shape[0]].values,dtype=torch.float)
  97. #print(train_data)
  98. #print (len(train_data.iloc[0]))
  100. '''
  101. 输入数据格式：
  102. input(seq_len, batch, input_size)
  103. h0(num_layers * num_directions, batch, hidden_size)
  104. c0(num_layers * num_directions, batch, hidden_size)
  106. 输出数据格式：
  107. output(seq_len, batch, hidden_size * num_directions)
  108. hn(num_layers * num_directions, batch, hidden_size)
  109. cn(num_layers * num_directions, batch, hidden_size)
  110. '''
  111. class LSTMTagger(nn.Module):
  113.     def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size,batch_size,str_len):
  114.         super(LSTMTagger, self).__init__()
  116.         self.hidden_dim = hidden_dim
  117.         self.str_len=str_len
  118.         self.batch_size=batch_size
  120.         self.word_embeddings = nn.Embedding(vocab_size, embedding_dim)
  122.         self.lstm = nn.LSTM(embedding_dim, hidden_dim)
  123.         self.dropout = nn.Dropout(0.5)
  124.         self.hidden2tag = nn.Linear(str_len*hidden_dim, tagset_size)
  125.         self.hidden = self.init_hidden()
  127.     def init_hidden(self):
  128.         return (torch.zeros(1, self.batch_size, self.hidden_dim).cuda(),
  129.                 torch.zeros(1, self.batch_size, self.hidden_dim).cuda())
  132.     def forward(self, sentence,state,train_flag):
  133.         embeds = self.word_embeddings(sentence)
  134.         #print(embeds.shape)
  135.         self.hidden=state
  136.         lstm_out, self.hidden = self.lstm(embeds.view(self.str_len, len(sentence), -1), self.hidden)
  137.         #print("ls",lstm_out.shape)
  138.         if train_flag:
  139.             lstm_out=self.dropout(lstm_out)
  140.         tag_space = self.hidden2tag(lstm_out.view(self.batch_size,-1))
  141.         #print(tag_space.shape)
  143.         #tag_scores = F.log_softmax(tag_space,dim=1)
  144.         return tag_space ,self.hidden
  146. model = LSTMTagger(300, 256, len(dict),1,16,31)
  150. model=model.cuda()
  151. print(model)
  153. exit()
  154. def train(model, train_data, train_label,num_epochs, learning_rate,  batch_size):
  155.     train_ls=[]
  156.     #loss =torch.nn.CrossEntropyLoss()
  157.     state=None
  158.     loss = torch.nn.MSELoss()
  159.     optimizer = optim.SGD(model.parameters(), lr=learning_rate,weight_decay=0)
  160.     dataset = torch.utils.data.TensorDataset(train_data, train_label)
  161.     train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True)
  163.     iter=0
  164.     for epoch in range(num_epochs):
  165.         correct = 0
  166.         total=0
  167.         start = time.time()
  168.         index=0
  169.         for X, y in train_iter:
  170.             iter+=1
  171.             index+=1
  172.             if index>400:
  173.                 break
  174.             if state is not None:
  175.                 if isinstance (state, tuple): # LSTM, state:(h, c)
  176.                     state = (state[0].detach(), state[1].detach())
  177.                 else:
  178.                     state = state.detach()
  179.             X = X.long()
  180.             (output, state) = model(X, state,True)
  182.             #correct+=predicted.data.eq(label.data).cpu().sum()
  183.             l = loss(output.float(), y)
  184.             l.backward()
  185.             optimizer.step()
  186.             optimizer.zero_grad()
  187.             #print(output)
  189.             output = [1 if i > 0.5 else 0 for i in output]
  191.             for i in range(len(output)):
  192.                 total+=1
  193.                 if output[i]==y[i]:
  194.                     correct+=1
  196.         acc = correct / total
  197.         end = time.time()
  198.         print("epoch ", str(epoch), "time: ",end-start ," loss: ", l.item(),"correct: ", correct, " total: ", total, "acc: ", acc)
  199.         torch.save(model, 'model.pkl')
  200.         eval(model, train_data, train_label, batch_size)
  201.         train_ls.append(l.item())
  203.     return train_ls
  206. Train=train_data
  208. #model = torch.load('model.pkl')
  209. def eval(model,train_data, train_label,batch_size):
  210.     test_list=[]
  211.     dataset = torch.utils.data.TensorDataset(train_data, train_label)
  212.     train_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=False)
  213.     with torch.no_grad():
  214.         state = None
  215.         index = 0
  216.         for X, y in train_iter:
  217.             index += 1
  218.             if index <=400:
  219.                 continue
  220.             if index >= 470:
  221.                 break
  223.             if state is not None:
  224.                 if isinstance(state, tuple):  # LSTM, state:(h, c)
  225.                     state = (state[0].detach(), state[1].detach())
  226.                 else:
  227.                     state = state.detach()
  228.             X = X.long()
  229.             (output, state) = model(X, state,False)
  230.             output = [1 if i > 0.5 else 0 for i in output]
  231.             for i in range(len(output)):
  232.                 test_list.append(output[i])
  233.     correct=0
  234.     for i in range(0, len(test_list)):
  235.         if train_label[i]==test_list[i]:
  236.             correct+=1
  237.     print("eval: ",correct/len(test_list))
  239. def out_put(model):
  240.     test_list = []
  241.     max_len = 31
  242.     for x in test_data.text:
  243.         while len(x) < max_len:
  244.             x.append(0)
  245.         if len(x)>max_len:
  246.             x =x[0:30]
  247.     test_num=len(test_data.text)
  248.     #print("sda",test_num)
  249.     for i in range(0, len(test_data)):
  250.         data_list.append(test_data.iloc[i]['text'])
  252.     test_data_tensor = torch.Tensor(data_list)
  254.     if USE_CUDA:
  255.         print("using GPU")
  256.         test_data_tensor = test_data_tensor.cuda()
  258.     with torch.no_grad():
  259.         result_dataset = torch.utils.data.TensorDataset(test_data_tensor)
  260.         result_dataloader = torch.utils.data.DataLoader(result_dataset, batch_size=16, shuffle=False)
  261.         state=None
  262.         index=0
  263.         for X in result_dataloader:
  264.             X=X[0]
  265.             if index>=203:
  266.                 break
  267.             index+=1
  268.             if state is not None:
  269.                 if isinstance(state, tuple):  # LSTM, state:(h, c)
  270.                     state = (state[0].detach(), state[1].detach())
  271.                 else:
  272.                     state = state.detach()
  273.             X = X.long()
  274.             (output, state) = model(X, state,False)
  276.             # correct+=predicted.data.eq(label.data).cpu().sum()
  279.             output = [1 if i > 0.5 else 0 for i in output]
  281.             for i in range(len(output)):
  282.                 test_list.append(output[i])
  284.     print(len(test_list))
  285.     while len(test_list)<3263:
  286.         test_list.append(0)
  287.     df_output = pd.DataFrame()
  288.     aux = pd.read_csv(base_path + 'test.csv')
  289.     df_output['id'] = aux['id']
  290.     df_output['target'] = test_list
  291.     df_output[['id', 'target']].to_csv(base_path + 's1mple.csv', index=False)
  293. train(model,traindata_tensor,train_label,30,0.01,16)
  294. print("\n")
  296. eval(model,traindata_tensor, train_label,16)
  298. out_put(model)