Upload 3 files

config.json

@@ -1,12 +1,12 @@
 {
- "_name_or_path": "babylm/ltgbert-100m-2024",
+ "_name_or_path": "ltg/ltg-bert-babylm",
  "architectures": [
  "LtgBertForMaskedLM"
  ],
  "attention_probs_dropout_prob": 0.1,
  "auto_map": {
- "AutoConfig": "ltg/ltg-bert-babylm--configuration_ltgbert.LtgBertConfig",
- "AutoModelForMaskedLM": "ltg/ltg-bert-babylm--modeling_ltgbert.LtgBertForMaskedLM",
+ "AutoConfig": "configuration_ltgbert.LtgBertConfig",
+ "AutoModelForMaskedLM": "modeling_ltgbert.LtgBertForMaskedLM",
  "AutoModelForSequenceClassification": "modeling_ltgbert.LtgBertForSequenceClassification"
  },
  "classifier_dropout": 0.2,
@@ -22,6 +22,6 @@
  "pad_token_id": 4,
  "position_bucket_size": 32,
  "torch_dtype": "float32",
- "transformers_version": "4.43.3",
+ "transformers_version": "4.40.2",
  "vocab_size": 16384
 }

configuration_ltgbert.py

@@ -19,6 +19,30 @@
 from transformers.configuration_utils import PretrainedConfig
 
 
+LTG_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+ "bnc-bert-span": "https://huggingface.co/ltg/bnc-bert-span",
+ "bnc-bert-span-2x": "https://huggingface.co/ltg/bnc-bert-span-2x",
+ "bnc-bert-span-0.5x": "https://huggingface.co/ltg/bnc-bert-span-0.5x",
+ "bnc-bert-span-0.25x": "https://huggingface.co/ltg/bnc-bert-span-0.25x",
+ "bnc-bert-span-order": "https://huggingface.co/ltg/bnc-bert-span-order",
+ "bnc-bert-span-document": "https://huggingface.co/ltg/bnc-bert-span-document",
+ "bnc-bert-span-word": "https://huggingface.co/ltg/bnc-bert-span-word",
+ "bnc-bert-span-subword": "https://huggingface.co/ltg/bnc-bert-span-subword",
+
+ "norbert3-xs": "https://huggingface.co/ltg/norbert3-xs/config.json",
+ "norbert3-small": "https://huggingface.co/ltg/norbert3-small/config.json",
+ "norbert3-base": "https://huggingface.co/ltg/norbert3-base/config.json",
+ "norbert3-large": "https://huggingface.co/ltg/norbert3-large/config.json",
+
+ "norbert3-oversampled-base": "https://huggingface.co/ltg/norbert3-oversampled-base/config.json",
+ "norbert3-ncc-base": "https://huggingface.co/ltg/norbert3-ncc-base/config.json",
+ "norbert3-nak-base": "https://huggingface.co/ltg/norbert3-nak-base/config.json",
+ "norbert3-nb-base": "https://huggingface.co/ltg/norbert3-nb-base/config.json",
+ "norbert3-wiki-base": "https://huggingface.co/ltg/norbert3-wiki-base/config.json",
+ "norbert3-c4-base": "https://huggingface.co/ltg/norbert3-c4-base/config.json"
+}
+
+
 class LtgBertConfig(PretrainedConfig):
  r"""
  This is the configuration class to store the configuration of a [`LtgBertModel`]. It is used to
@@ -49,7 +73,7 @@ class LtgBertConfig(PretrainedConfig):
  classifier_dropout (`float`, *optional*):
  The dropout ratio for the classification head.
  """
- model_type = "bert"
+ model_type = "ltgbert"
  def __init__(
  self,
  vocab_size=16384,
@@ -80,4 +104,4 @@ class LtgBertConfig(PretrainedConfig):
  self.output_all_encoded_layers = output_all_encoded_layers
  self.position_bucket_size = position_bucket_size
  self.layer_norm_eps = layer_norm_eps
- self.classifier_dropout = classifier_dropout
+ self.classifier_dropout = classifier_dropout

modeling_ltgbert.py

@@ -39,10 +39,34 @@ from transformers.pytorch_utils import softmax_backward_data
 from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward
 
 
-_CHECKPOINT_FOR_DOC = "ltg/ltg-bert-bnc"
+_CHECKPOINT_FOR_DOC = "ltg/bnc-bert-span"
 _CONFIG_FOR_DOC = "LtgBertConfig"
 
 
+LTG_BERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
+ "bnc-bert-span",
+ "bnc-bert-span-2x",
+ "bnc-bert-span-0.5x",
+ "bnc-bert-span-0.25x",
+ "bnc-bert-span-order",
+ "bnc-bert-span-document",
+ "bnc-bert-span-word",
+ "bnc-bert-span-subword",
+
+ "norbert3-xs",
+ "norbert3-small",
+ "norbert3-base",
+ "norbert3-large",
+
+ "norbert3-oversampled-base",
+ "norbert3-ncc-base",
+ "norbert3-nak-base",
+ "norbert3-nb-base",
+ "norbert3-wiki-base",
+ "norbert3-c4-base"
+]
+
+
 class Encoder(nn.Module):
  def __init__(self, config, activation_checkpointing=False):
  super().__init__()
@@ -224,8 +248,10 @@ class Attention(nn.Module):
 
  attention_scores = torch.bmm(query, key.transpose(1, 2) * self.scale)
 
- pos = self.in_proj_qk(self.dropout(relative_embedding)) # shape: [2T-1, 2D]
- query_pos, key_pos = pos.view(-1, self.num_heads, 2*self.head_size).chunk(2, dim=2)
+ query_pos, key_pos = self.in_proj_qk(self.dropout(relative_embedding)).chunk(2, dim=-1) # shape: [2T-1, D]
+ query_pos = query_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
+ key_pos = key_pos.view(-1, self.num_heads, self.head_size) # shape: [2T-1, H, D]
+
  query = query.view(batch_size, self.num_heads, query_len, self.head_size)
  key = key.view(batch_size, self.num_heads, query_len, self.head_size)
 
@@ -367,8 +393,6 @@ class LtgBertModel(LtgBertPreTrainedModel):
  ) -> List[torch.Tensor]:
  if input_ids is not None:
  input_shape = input_ids.size()
- # elif inputs_embeds is not None:
- # input_shape = inputs_embeds.size()[:-1]
  else:
  raise ValueError("You have to specify input_ids")
 
@@ -380,9 +404,7 @@ class LtgBertModel(LtgBertPreTrainedModel):
  else:
  attention_mask = ~attention_mask.bool()
  attention_mask = attention_mask.unsqueeze(1).unsqueeze(2)
-
- # if inputs_embeds is None:
- # static_embeddings, relative_embedding = self.embedding(input_ids.t())
+ 
  static_embeddings, relative_embedding = self.embedding(input_ids.t())
  contextualized_embeddings, attention_probs = self.transformer(static_embeddings, attention_mask, relative_embedding)
  contextualized_embeddings = [e.transpose(0, 1) for e in contextualized_embeddings]
@@ -409,7 +431,8 @@ class LtgBertModel(LtgBertPreTrainedModel):
  )
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+ attention_mask=attention_mask)
 
  if not return_dict:
  return (
@@ -456,7 +479,8 @@ class LtgBertForMaskedLM(LtgBertModel):
  """
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+ attention_mask=attention_mask)
  subword_prediction = self.classifier(sequence_output)
 
  masked_lm_loss = None
@@ -554,8 +578,9 @@ class LtgBertForSequenceClassification(LtgBertModel):
  """
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, inputs_embeds,
- ~attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+ inputs_embeds=inputs_embeds,
+ attention_mask=attention_mask)
  logits = self.head(sequence_output[:, 0, :])
 
  loss = None
@@ -628,7 +653,8 @@ class LtgBertForTokenClassification(LtgBertModel):
  ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+ attention_mask=attention_mask)
  logits = self.head(sequence_output)
 
  loss = None
@@ -684,7 +710,8 @@ class LtgBertForQuestionAnswering(LtgBertModel):
  ) -> Union[Tuple[torch.Tensor], QuestionAnsweringModelOutput]:
  return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids, attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=input_ids,
+ attention_mask=attention_mask)
  logits = self.head(sequence_output)
 
  start_logits, end_logits = logits.split(1, dim=-1)
@@ -762,7 +789,8 @@ class LtgBertForMultipleChoice(LtgBertModel):
  flat_input_ids = input_ids.view(-1, input_ids.size(-1))
  flat_attention_mask = attention_mask.view(-1, attention_mask.size(-1)) if attention_mask is not None else None
 
- sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(flat_input_ids, flat_attention_mask)
+ sequence_output, contextualized_embeddings, attention_probs = self.get_contextualized_embeddings(input_ids=flat_input_ids,
+ attention_mask=flat_attention_mask)
  logits = self.head(sequence_output)
  reshaped_logits = logits.view(-1, num_choices)
 
@@ -785,3 +813,4 @@ class LtgBertForMultipleChoice(LtgBertModel):
  hidden_states=contextualized_embeddings if output_hidden_states else None,
  attentions=attention_probs if output_attentions else None
  )
+