add model

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "/content/drive/MyDrive/ColabModels/XROBERTA_USE_QA/pytorch_model/",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoModel": "modeling_roberta.XLMRobertaModel"
+  },
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "roberta": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.18.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

modeling_roberta.py

@@ -0,0 +1,24 @@
+import torch
+
+from transformers import XLMRobertaModel as XLMRobertaModelBase
+
+
+class XLMRobertaModel(XLMRobertaModelBase):
+    def __init__(self, config):
+        super().__init__(config)
+        self.question_projection = torch.nn.Linear(768, 512)
+        self.answer_projection = torch.nn.Linear(768, 512)
+
+    def _embed(self, input_ids, attention_mask, projection):
+        outputs = super().__call__(input_ids, attention_mask=attention_mask)
+        sequence_output = outputs[0]
+
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(sequence_output.size()).float()
+        embeddings = torch.sum(sequence_output * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+        return torch.tanh(projection(embeddings))
+
+    def question(self, input_ids, attention_mask):
+        return self._embed(input_ids, attention_mask, self.question_projection)
+
+    def answer(self, input_ids, attention_mask):
+        return self._embed(input_ids, attention_mask, self.answer_projection)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e3ded1f396d3f20cdb3249faefa44c373f3f59d5adaa19542eb4d20ffc3b908
+size 1115392297