试图实现Extractive Summarization as Text Matching中的模型

数据处理读取

基本思路

  1. 暂时使用lead3里的处理和读取
  2. 然后对应的候选摘要是对原Docment里的所有句子和原文档计算Rouge平均值，然后选择较高的5个。原论文使用的是BertExt，没弄明白，先用这个代替
  3. 然后对5个句子排列组合C(5,3)得到10个句子，这里用到了python里的from itertools import combinations。使用起来比较方便
  4. 对所有的候选和所有的数据用token编码，并转为tensor

    调用结果

    ```python from Dataloader import Loader document_path=”train.txt.src” label_path=”train.txt.tgt” base_path=”D:\New_desktop\summarization_papers\“

def Train(): loader=Loader(“bert”) train_data,train_label,train_candi=loader.read_data(base_path+document_path,base_path+label_path,50) print(“\n”) print(train_data.size()) print(train_label.size()) print(train_candi.size())

Train()

  1. ![image.png](https://cdn.nlark.com/yuque/0/2020/png/296244/1594915936844-f0640533-a931-4b80-9205-fbe691233e28.png#align=left&display=inline&height=245&margin=%5Bobject%20Object%5D&name=image.png&originHeight=245&originWidth=785&size=17815&status=done&style=none&width=785)
  2. ```python
  3. import re
  5. import numpy as np
  7. import torch.nn as nn
  8. import torch.nn.functional as F
  9. import torch.optim as optim
  10. import torch.autograd as autograd
  11. import torch.nn.functional
  12. from torch.utils.data import Dataset, DataLoader
  14. import torch
  15. import time
  16. import argparse
  18. import os
  19. from itertools import combinations
  21. from transformers import BertTokenizer
  22. from transformers import BertForSequenceClassification
  23. from transformers import BertConfig
  24. from transformers import BertModel
  26. USE_CUDA = torch.cuda.is_available()
  28. base_path="D:\\New_desktop\\summarization_papers\\"
  29. tokenizer = BertTokenizer.from_pretrained(base_path)
  30. print("----init tokenizer finished----")
  31. def normalizeString(s):
  32.     #s = re.sub(r"([.!?])", r" \1", s)
  33.     #s = re.sub(r"[^a-zA-Z.!?]+", r" ", s)
  34.     s=s.strip()
  35.     s = s.replace("\n", "")
  36.     return s
  38. def possess_sentence(s):
  39.     lines=s.split("##SENT##")
  40.     for i in lines:
  41.         i=normalizeString(i)
  42.     return lines
  44. from rouge import Rouge
  45. from util import ProgressBar
  47. class Loader:
  48.     def __init__(self, name):
  49.         self.name = name
  50.         self.train_data={}
  51.         self.train_data['text']=[]
  52.         self.train_data['label']=[]
  53.         self.train_data["candi"]=[]
  54.         self.rouge=Rouge()
  55.     def get_document(self,document):
  56.         sentences=possess_sentence(document)
  57.         return sentences
  59.     def get_labels(self,label):
  60.         sentences = possess_sentence(label)
  61.         return sentences
  63.     def get_score(self,sen1,sen2):
  64.         score=0
  65.         rouge_score = self.rouge.get_scores(sen1, sen2)
  66.         score+= rouge_score[0]["rouge-1"]['r']
  67.         score+= rouge_score[0]["rouge-2"]['r']
  68.         score+= rouge_score[0]["rouge-l"]['r']
  69.         return score/3
  70.     def read_data(self,path1,path2,pairs_num,max_len=40):
  71.         print("----start Read train data----")
  72.         fo = open(path1, "r", encoding='gb18030', errors='ignore')
  73.         fl = open(path2, "r", encoding='gb18030', errors='ignore')
  74.         data_list=[]
  75.         label_list=[]
  76.         candi_list=[]
  77.         pbar = ProgressBar(n_total=pairs_num, desc='Loading')
  78.         for i in range(pairs_num):
  79.             pbar(i, {'current': i})
  80.             line1 = fo.readline()
  81.             line2 = fl.readline()
  82.             #line1="A ##SENT## B ##SENT## C ##SENT## D ##SENT## E ##SENT## F"
  83.             do = self.get_document(line1)
  84.             la = self.get_labels(line2)
  85.             sentences={}
  86.             document = " ".join(do)
  87.             la = " ".join(la)
  88.             for i in do:
  89.                 if i != None:
  90.                  sentences[i]=self.get_score(i,document)
  92.             sentences = sorted(sentences.items(), key=lambda x: x[1],reverse = True)
  94.             candidata_sentence_set=sentences[:5]
  95.             sentences=[]
  96.             #print(candidata_sentence_set)
  97.             #print(type(candidata_sentence_set))
  98.             for i in candidata_sentence_set:
  99.                 sentences.append(i[0])
  100.             #indices = list(combinations(sentences, 2))
  101.             indices = list(combinations(sentences, 3))
  102.             candidata=[]
  103.             #print(sentences)
  104.             #print(indices)
  105.             for i in indices:
  106.                 #print(type(i))
  107.                 #print(len(i))
  108.                 candidata.append(" ".join(i))
  109.             #print(candidata[0])
  110.             #print(type(candidata[0]))
  111.             candidata_data=[]
  112.             for i in candidata:
  113.                 candidata_data.append(tokenizer.encode(i, add_special_tokens=False))
  114.             #print(len(candidata_data))
  115.             #print(candidata_data[0])
  117.             self.train_data['text'].append(tokenizer.encode(document, add_special_tokens=False))
  118.             self.train_data['label'].append(tokenizer.encode(la, add_special_tokens=False))
  119.             self.train_data['candi'].append(candidata_data)
  120.         for x in self.train_data['text']:
  121.             if len(x) >= max_len:
  122.                 x = x[0:max_len-1]
  123.             while len(x) < max_len:
  124.                 x.append(0)
  125.             x.append(102)
  126.             l = x
  127.             x = [101]
  128.             x.extend(l)
  129.             data_list.append(x)
  130.         for x in self.train_data['label']:
  131.             if len(x) >= max_len:
  132.                 x = x[0:max_len-1]
  133.             while len(x) < max_len:
  134.                 x.append(0)
  135.             x.append(102)
  136.             l = x
  137.             x = [101]
  138.             x.extend(l)
  139.             label_list.append(x)
  140.         for i in self.train_data['candi']:
  141.             temp=[]
  142.             for x in i:
  143.                 if len(x) >= max_len:
  144.                     x = x[0:max_len-1]
  145.                 while len(x) < max_len:
  146.                     x.append(0)
  147.                 x.append(102)
  148.                 l = x
  149.                 x = [101]
  150.                 x.extend(l)
  151.                 temp.append(x)
  152.             candi_list.append(temp)
  153.         train_data = torch.tensor(data_list)
  154.         train_label = torch.Tensor(label_list)
  155.         train_candi = torch.Tensor(candi_list)
  156.         return train_data,train_label,train_candi

构造损失函数

补充资料

原论文开源代码里用的损失函数我完全没有用过，看论文的时候只有几个公式觉得很简单来着。。。
开始补充资料
这个损失函数看起来很有意思？
似乎我之前做句子相似度匹配TextMatching的时候可以用的？

MarginRankingLoss 你们可能对这个损失函数比较陌生。在机器学习领域，了解一个概念最直观的最快速的方式即是从它的名字开始。前端

MarginRankingLoss也是如此，拆分一下，Margin，Ranking，Loss。机器学习

Margin：前端同窗对Margin是再熟悉不过了，它表示两个元素之间的间隔。在机器学习中其实Margin也有相似的意思，它能够理解为一个可变的加在loss上的一个偏移量。也就是代表这个方法能够手动调节偏移。固然Margin不是重点。函数

Ranking：它是该损失函数的重点和核心，也就是排序！若是排序的内容仅仅是两个元素而已，那么对于某一个元素，只有两个结果，那就是在第二个元素以前或者在第二个元素以前。其实这就是该损失函数的核心了。学习

咱们看一下它的loss funcion表达式。cdn

Match_Sum复现 - 图1

margin咱们能够先无论它，其实模型的含义不言而喻。blog

y只能有两个取值，也就是1或者-1。排序

  1. 当y=1的时候，表示咱们预期x1的排名要比x2高，也就是x1-x2>0
  2. 当y=-1的时候，表示咱们预期x1的排名要比x2高，也就是x1-x2<0

何时用？

  1. GAN
  2. 排名任务
  3. 开源实现和实例很是少