关于抽取bert里面第几层的代码:
(
＃我们取零，因为据我了解，这就是[CLS]令牌...
＃想法是也要合并最后4层而不是最后一层，因为它太接近输出了
＃层，它可能没有那么有效，因为它受到o / p的更多控制。
)
https://www.kaggle.com/c/google-quest-challenge/discussion/123770
class BertForSequenceClassification_v2(BertPreTrainedModel):
r"""
**labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``:
Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs:
**loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``:
Classification (or regression if config.num_labels==1) loss.
**logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)``
Classification (or regression if config.num_labels==1) scores (before SoftMax).
**hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``)
list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings)
of shape ``(batch_size, sequence_length, hidden_size)``:
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
outputs = model(input_ids, labels=labels)
loss, logits = outputs[:2]
"""
def __init__(self, config):
super(BertForSequenceClassification_v2, self).__init__(config)
# config.output_hidden_states=True (make sure)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
self.init_weights()
def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,
position_ids=None, head_mask=None, inputs_embeds=None, labels=None,
extra_feats=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
# sequence_output = outputs[0]
# pooled_output = outputs[1]
hidden_states = outputs[2] #hidden_states: 12 layers tuples each is of (batch_size, sequence_length, hidden_size) + embedding``
# print(seq[-1].shape, seq[-1][:, 0].shape)
# we are taking zero because in my understanding that's the [CLS] token...
# idea is to pool last 4 layers as well instead of just the last one, since it's too close to the output
# layers, it might not be that efficient as it's more regulated by the o/p's..
h12 = hidden_states[-1][:, 0].reshape((-1, 1, 768))
h11 = hidden_states[-2][:, 0].reshape((-1, 1, 768))
h10 = hidden_states[-3][:, 0].reshape((-1, 1, 768))
h9 = hidden_states[-4][:, 0].reshape((-1, 1, 768))
all_h = torch.cat([h9, h10, h11, h12], 1) #Also don't forget to add the last CLS token seq_op/pooled_op as you wish..
mean_pool = torch.mean(all_h, 1)
pooled_output = self.dropout(mean_pool)
logits = self.classifier(pooled_output)
outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here
return outputs # (loss), logits, (hidden_states), (attentions)