Bert魔改

阅读源码

初步对bert的态度是作为工具使用
暂时不关注内部的细节
看一看输出和输入即可

  1.     def __init__(self, config):
  2.         super(BertForSequenceClassification, self).__init__(config)
  3.         self.num_labels = config.num_labels
  5.         self.bert = BertModel(config)
  6.         self.dropout = nn.Dropout(config.hidden_dropout_prob)
  7.         self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
  9.         self.init_weights()
  11.     def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
  12.                 position_ids=None, head_mask=None, inputs_embeds=None, labels=None):
  14.         outputs = self.bert(input_ids,
  15.                             attention_mask=attention_mask,
  16.                             token_type_ids=token_type_ids,
  17.                             position_ids=position_ids,
  18.                             head_mask=head_mask,
  19.                             inputs_embeds=inputs_embeds)
  21.         pooled_output = outputs[1]
  22.         pooled_output = self.dropout(pooled_output)
  23.         logits = self.classifier(pooled_output)
  25.         outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here
  27.         if labels is not None:
  28.             if self.num_labels == 1:
  29.                 #  We are doing regression
  30.                 loss_fct = MSELoss()
  31.                 loss = loss_fct(logits.view(-1), labels.view(-1))
  32.             else:
  33.                 loss_fct = CrossEntropyLoss()
  34.                 loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))
  35.             outputs = (loss,) + outputs
  37.         return outputs  # (loss), logits, (hidden_states), (attentions)

不难发现本质就是bert的输出，加上了一个dropout层，最后用一个线性的层计算到label上

loss: (optional, returned when labels is provided) torch.FloatTensor of shape (1,): Classification (or regression if config.num_labels==1) loss.

logits: torch.FloatTensor of shape (batch_size, config.num_labels) Classification (or regression if config.num_labels==1) scores (before SoftMax).

_

原模型

  1. class BertModel(BertPreTrainedModel):
  2.     """
  4.     The model can behave as an encoder (with only self-attention) as well
  5.     as a decoder, in which case a layer of cross-attention is added between
  6.     the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,
  7.     Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
  9.     To behave as an decoder the model needs to be initialized with the
  10.     :obj:`is_decoder` argument of the configuration set to :obj:`True`; an
  11.     :obj:`encoder_hidden_states` is expected as an input to the forward pass.
  13.     .. _`Attention is all you need`:
  14.         https://arxiv.org/abs/1706.03762
  16.     """
  18.     def __init__(self, config):
  19.         super().__init__(config)
  20.         self.config = config
  22.         self.embeddings = BertEmbeddings(config)
  23.         self.encoder = BertEncoder(config)
  24.         self.pooler = BertPooler(config)
  26.         self.init_weights()
  28. [DOCS]    def get_input_embeddings(self):
  29.         return self.embeddings.word_embeddings
  32. [DOCS]    def set_input_embeddings(self, value):
  33.         self.embeddings.word_embeddings = value
  36.     def _prune_heads(self, heads_to_prune):
  37.         """ Prunes heads of the model.
  38.             heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
  39.             See base class PreTrainedModel
  40.         """
  41.         for layer, heads in heads_to_prune.items():
  42.             self.encoder.layer[layer].attention.prune_heads(heads)
  44. [DOCS]    @add_start_docstrings_to_callable(BERT_INPUTS_DOCSTRING)
  45.     def forward(
  46.         self,
  47.         input_ids=None,
  48.         attention_mask=None,
  49.         token_type_ids=None,
  50.         position_ids=None,
  51.         head_mask=None,
  52.         inputs_embeds=None,
  53.         encoder_hidden_states=None,
  54.         encoder_attention_mask=None,
  55.     ):
  56.         r"""
  57.     Return:
  58.         :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
  59.         last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
  60.             Sequence of hidden-states at the output of the last layer of the model.
  61.         pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
  62.             Last layer hidden-state of the first token of the sequence (classification token)
  63.             further processed by a Linear layer and a Tanh activation function. The Linear
  64.             layer weights are trained from the next sentence prediction (classification)
  65.             objective during pre-training.
  67.             This output is usually *not* a good summary
  68.             of the semantic content of the input, you're often better with averaging or pooling
  69.             the sequence of hidden-states for the whole input sequence.
  70.         hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_hidden_states=True``):
  71.             Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
  72.             of shape :obj:`(batch_size, sequence_length, hidden_size)`.
  74.             Hidden-states of the model at the output of each layer plus the initial embedding outputs.
  75.         attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``config.output_attentions=True``):
  76.             Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
  77.             :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.
  79.             Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
  80.             heads.
  82.     Examples::
  84.         from transformers import BertModel, BertTokenizer
  85.         import torch
  87.         tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
  88.         model = BertModel.from_pretrained('bert-base-uncased')
  90.         input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
  91.         outputs = model(input_ids)
  93.         last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
  95.         """
  97.         if input_ids is not None and inputs_embeds is not None:
  98.             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
  99.         elif input_ids is not None:
  100.             input_shape = input_ids.size()
  101.         elif inputs_embeds is not None:
  102.             input_shape = inputs_embeds.size()[:-1]
  103.         else:
  104.             raise ValueError("You have to specify either input_ids or inputs_embeds")
  106.         device = input_ids.device if input_ids is not None else inputs_embeds.device
  108.         if attention_mask is None:
  109.             attention_mask = torch.ones(input_shape, device=device)
  110.         if token_type_ids is None:
  111.             token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)
  113.         # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
  114.         # ourselves in which case we just need to make it broadcastable to all heads.
  115.         if attention_mask.dim() == 3:
  116.             extended_attention_mask = attention_mask[:, None, :, :]
  117.         elif attention_mask.dim() == 2:
  118.             # Provided a padding mask of dimensions [batch_size, seq_length]
  119.             # - if the model is a decoder, apply a causal mask in addition to the padding mask
  120.             # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length]
  121.             if self.config.is_decoder:
  122.                 batch_size, seq_length = input_shape
  123.                 seq_ids = torch.arange(seq_length, device=device)
  124.                 causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None]
  125.                 causal_mask = causal_mask.to(
  126.                     attention_mask.dtype
  127.                 )  # causal and attention masks must have same type with pytorch version < 1.3
  128.                 extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :]
  129.             else:
  130.                 extended_attention_mask = attention_mask[:, None, None, :]
  131.         else:
  132.             raise ValueError(
  133.                 "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(
  134.                     input_shape, attention_mask.shape
  135.                 )
  136.             )
  138.         # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
  139.         # masked positions, this operation will create a tensor which is 0.0 for
  140.         # positions we want to attend and -10000.0 for masked positions.
  141.         # Since we are adding it to the raw scores before the softmax, this is
  142.         # effectively the same as removing these entirely.
  143.         extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype)  # fp16 compatibility
  144.         extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0
  146.         # If a 2D ou 3D attention mask is provided for the cross-attention
  147.         # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
  148.         if self.config.is_decoder and encoder_hidden_states is not None:
  149.             encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
  150.             encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
  151.             if encoder_attention_mask is None:
  152.                 encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
  154.             if encoder_attention_mask.dim() == 3:
  155.                 encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :]
  156.             elif encoder_attention_mask.dim() == 2:
  157.                 encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :]
  158.             else:
  159.                 raise ValueError(
  160.                     "Wrong shape for encoder_hidden_shape (shape {}) or encoder_attention_mask (shape {})".format(
  161.                         encoder_hidden_shape, encoder_attention_mask.shape
  162.                     )
  163.                 )
  165.             encoder_extended_attention_mask = encoder_extended_attention_mask.to(
  166.                 dtype=next(self.parameters()).dtype
  167.             )  # fp16 compatibility
  168.             encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0
  169.         else:
  170.             encoder_extended_attention_mask = None
  172.         # Prepare head mask if needed
  173.         # 1.0 in head_mask indicate we keep the head
  174.         # attention_probs has shape bsz x n_heads x N x N
  175.         # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
  176.         # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
  177.         if head_mask is not None:
  178.             if head_mask.dim() == 1:
  179.                 head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1)
  180.                 head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1)
  181.             elif head_mask.dim() == 2:
  182.                 head_mask = (
  183.                     head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1)
  184.                 )  # We can specify head_mask for each layer
  185.             head_mask = head_mask.to(
  186.                 dtype=next(self.parameters()).dtype
  187.             )  # switch to fload if need + fp16 compatibility
  188.         else:
  189.             head_mask = [None] * self.config.num_hidden_layers
  191.         embedding_output = self.embeddings(
  192.             input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
  193.         )
  194.         encoder_outputs = self.encoder(
  195.             embedding_output,
  196.             attention_mask=extended_attention_mask,
  197.             head_mask=head_mask,
  198.             encoder_hidden_states=encoder_hidden_states,
  199.             encoder_attention_mask=encoder_extended_attention_mask,
  200.         )
  201.         sequence_output = encoder_outputs[0]
  202.         pooled_output = self.pooler(sequence_output)
  204.         outputs = (sequence_output, pooled_output,) + encoder_outputs[
  205.             1:
  206.         ]  # add hidden_states and attentions if they are here
  207.         return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)
  211. [DOCS]@add_start_docstrings(
  212.     """Bert Model with two heads on top as done during the pre-training: a `masked language modeling` head and
  213.     a `next sentence prediction (classification)` head. """,
  214.     BERT_START_DOCSTRING,
  215. )
  1. from transformers import BertModel, BertTokenizer
  2. import torch
  4. tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
  5. model = BertModel.from_pretrained('bert-base-chinese')
  7. input_ids = torch.tensor(tokenizer.encode("美国", add_special_tokens=True)).unsqueeze(0)  # Batch size 1
  8. print(input_ids)
  9. print(input_ids.shape)
  10. outputs = model(input_ids)
  11. #print(outputs)
  13. last_hidden_states = outputs[0]  # The last hidden-state is the first element of the output tuple
  14. print(last_hidden_states.shape)
  16. others = outputs[1]
  17. print(others.shape)

image.png

代码源头

https://github.com/huggingface/transformers/tree/master/examples/ner
利用pytorch实现的bert_ner
https://huggingface.co/transformers/_modules/transformers/modeling_bert.html#BertForSequenceClassification