code: 
  
  
  """simple docstring"""
  import math
  from typing import Optional
  
  import torch
  import torch.utils.checkpoint as checkpoint
  from torch import nn
  
  from ....modeling_utils import PreTrainedModel
  from ....utils import add_start_docstrings, logging
  from ...bert.modeling_bert import BertModel
  from .configuration_retribert import RetriBertConfig
  
  
  UpperCAmelCase   =	logging.get_logger(__name__)
  
  UpperCAmelCase   =	[
      """yjernite/retribert-base-uncased""",
      # See all RetriBert models at https://huggingface.co/models?filter=retribert
  ]
  
  
  
  
  class   lowercase__							(				A_							):
     __UpperCAmelCase       					=				RetriBertConfig
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				'''retribert'''
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Union[str, Any]:
        if isinstance(SCREAMING_SNAKE_CASE			,      nn.Linear):
           module.weight.data.normal_(mean=0.0			,      std=self.config.initializer_range)
           if module.bias is not None:
              module.bias.data.zero_()
        elif isinstance(SCREAMING_SNAKE_CASE			,      nn.Embedding):
           module.weight.data.normal_(mean=0.0			,      std=self.config.initializer_range)
           if module.padding_idx is not None:
              module.weight.data[module.padding_idx].zero_()
        elif isinstance(SCREAMING_SNAKE_CASE			,      nn.LayerNorm):
           module.bias.data.zero_()
           module.weight.data.fill_(1.0)
  
  
  
  
  
  
  
  UpperCAmelCase   =	r"""
  
      This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
      library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
      etc.)
  
      This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
      Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
      and behavior.
  
      Parameters:
          config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.
              Initializing with a config file does not load the weights associated with the model, only the
              configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
  """
  
  
  
  
  @add_start_docstrings(
      '''Bert Based model to embed queries or document for document retrieval.'''   ,A_   ,)
  class   lowercase__							(				A_							):
  
  
  
  
  
     def __init__(       self			,      SCREAMING_SNAKE_CASE)     ->     None:
        super().__init__(SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				List[str]      =		config.projection_dim
  
        _lowerCamelCase  :				Union[str, Any]      =		BertModel(SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				int      =		None if config.share_encoders else BertModel(SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				str      =		nn.Dropout(config.hidden_dropout_prob)
        _lowerCamelCase  :				List[Any]      =		nn.Linear(config.hidden_size			,      config.projection_dim			,      bias=SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Optional[int]      =		nn.Linear(config.hidden_size			,      config.projection_dim			,      bias=SCREAMING_SNAKE_CASE)
  
        _lowerCamelCase  :				Optional[Any]      =		nn.CrossEntropyLoss(reduction="""mean""")
  
        # Initialize weights and apply final processing
        self.post_init()
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE=-1			,      )     ->     Dict:
        # reproduces BERT forward pass with checkpointing
        if checkpoint_batch_size < 0 or input_ids.shape[0] < checkpoint_batch_size:
           return sent_encoder(SCREAMING_SNAKE_CASE			,      attention_mask=SCREAMING_SNAKE_CASE)[1]
        else:
           # prepare implicit variables
           _lowerCamelCase  :				str      =		input_ids.device
           _lowerCamelCase  :				Optional[int]      =		input_ids.size()
           _lowerCamelCase  :				Union[str, Any]      =		torch.zeros(SCREAMING_SNAKE_CASE			,      dtype=torch.long			,      device=SCREAMING_SNAKE_CASE)
           _lowerCamelCase  :				Optional[int]      =		[None] * sent_encoder.config.num_hidden_layers
           _lowerCamelCase  :				torch.Tensor      =		sent_encoder.get_extended_attention_mask(
               SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
  
           # define function for checkpointing
           def partial_encode(*SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				List[str]      =		sent_encoder.encoder(
                  inputs[0]			,      attention_mask=inputs[1]			,      head_mask=SCREAMING_SNAKE_CASE			,      )
              _lowerCamelCase  :				List[str]      =		encoder_outputs[0]
              _lowerCamelCase  :				List[Any]      =		sent_encoder.pooler(SCREAMING_SNAKE_CASE)
              return pooled_output
  
           # run embedding layer on everything at once
           _lowerCamelCase  :				Tuple      =		sent_encoder.embeddings(
               input_ids=SCREAMING_SNAKE_CASE			,      position_ids=SCREAMING_SNAKE_CASE			,      token_type_ids=SCREAMING_SNAKE_CASE			,      inputs_embeds=SCREAMING_SNAKE_CASE)
           # run encoding and pooling on one mini-batch at a time
           _lowerCamelCase  :				Tuple      =		[]
           for b in range(math.ceil(input_ids.shape[0] / checkpoint_batch_size)):
              _lowerCamelCase  :				str      =		embedding_output[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              _lowerCamelCase  :				Any      =		extended_attention_mask[b * checkpoint_batch_size : (b + 1) * checkpoint_batch_size]
              _lowerCamelCase  :				int      =		checkpoint.checkpoint(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
              pooled_output_list.append(SCREAMING_SNAKE_CASE)
           return torch.cat(SCREAMING_SNAKE_CASE			,      dim=0)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=-1			,      )     ->     Dict:
        _lowerCamelCase  :				int      =		self.embed_sentences_checkpointed(
            SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      self.bert_query			,      SCREAMING_SNAKE_CASE			,      )
        return self.project_query(SCREAMING_SNAKE_CASE)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=-1			,      )     ->     Union[str, Any]:
        _lowerCamelCase  :				Tuple      =		self.embed_sentences_checkpointed(
            SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      self.bert_query if self.bert_doc is None else self.bert_doc			,      SCREAMING_SNAKE_CASE			,      )
        return self.project_doc(SCREAMING_SNAKE_CASE)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = -1			,      )     ->     torch.FloatTensor:
        _lowerCamelCase  :				List[str]      =		input_ids_query.device
        _lowerCamelCase  :				Optional[int]      =		self.embed_questions(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Optional[Any]      =		self.embed_answers(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Any      =		torch.mm(SCREAMING_SNAKE_CASE			,      a_reps.t())
        _lowerCamelCase  :				Tuple      =		self.ce_loss(SCREAMING_SNAKE_CASE			,      torch.arange(compare_scores.shape[1]).to(SCREAMING_SNAKE_CASE))
        _lowerCamelCase  :				Any      =		self.ce_loss(compare_scores.t()			,      torch.arange(compare_scores.shape[0]).to(SCREAMING_SNAKE_CASE))
        _lowerCamelCase  :				Optional[int]      =		(loss_qa + loss_aq) / 2
        return loss
  
  
style_context: 
  
  
  """simple docstring"""
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  UpperCAmelCase   =	datasets.utils.logging.get_logger(__name__)
  
  
  
  
  @dataclass
  class   lowercase__							(				datasets.BuilderConfig							):
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				"utf-8"
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				10 << 20  # 10MB
     __UpperCAmelCase       					=				False
     __UpperCAmelCase       					=				"line"
  
  
  
  
  
  
  
  class   lowercase__							(				datasets.ArrowBasedBuilder							):
     __UpperCAmelCase       					=				TextConfig
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     int:
        return datasets.DatasetInfo(features=self.config.features)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Optional[int]:
        if not self.config.data_files:
           raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
        _lowerCamelCase  :				Dict      =		dl_manager.download_and_extract(self.config.data_files)
        if isinstance(SCREAMING_SNAKE_CASE			,      (str, list, tuple)):
           _lowerCamelCase  :				Optional[Any]      =		data_files
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				List[Any]      =		[files]
           _lowerCamelCase  :				List[str]      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,      gen_kwargs={"""files""": files})]
        _lowerCamelCase  :				int      =		[]
        for split_name, files in data_files.items():
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				Any      =		[files]
           _lowerCamelCase  :				Any      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE			,      gen_kwargs={"""files""": files}))
        return splits
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     pa.Table:
        if self.config.features is not None:
           _lowerCamelCase  :				Union[str, Any]      =		self.config.features.arrow_schema
           if all(not require_storage_cast(SCREAMING_SNAKE_CASE) for feature in self.config.features.values()):
              # cheaper cast
              _lowerCamelCase  :				Any      =		pa_table.cast(SCREAMING_SNAKE_CASE)
           else:
              # more expensive cast; allows str <-> int/float or str to Audio for example
              _lowerCamelCase  :				Any      =		table_cast(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
           return pa_table
        else:
           return pa_table.cast(pa.schema({"""text""": pa.string()}))
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Any:
        _lowerCamelCase  :				Optional[Any]      =		list(self.config.features) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE)):
           # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
           with open(SCREAMING_SNAKE_CASE			,      encoding=self.config.encoding			,      errors=self.config.errors) as f:
              if self.config.sample_by == "line":
                 _lowerCamelCase  :				Union[str, Any]      =		0
                 while True:
                    _lowerCamelCase  :				int      =		f.read(self.config.chunksize)
                    if not batch:
                       break
                    batch += f.readline()  # finish current line
                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                    _lowerCamelCase  :				List[str]      =		StringIO(SCREAMING_SNAKE_CASE).readlines()
                    if not self.config.keep_linebreaks:
                       _lowerCamelCase  :				Optional[Any]      =		[line.rstrip("""\n""") for line in batch]
                    _lowerCamelCase  :				Any      =		pa.Table.from_arrays([pa.array(SCREAMING_SNAKE_CASE)]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
              elif self.config.sample_by == "paragraph":
                 _lowerCamelCase  :				Optional[Any]      =		0
                 _lowerCamelCase  :				Dict      =		""""""
                 while True:
                    _lowerCamelCase  :				Union[str, Any]      =		f.read(self.config.chunksize)
                    if not new_batch:
                       break
                    batch += new_batch
                    batch += f.readline()  # finish current line
                    _lowerCamelCase  :				Any      =		batch.split("""\n\n""")
                    _lowerCamelCase  :				Dict      =		pa.Table.from_arrays(
                        [pa.array([example for example in batch[:-1] if example])]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
                    _lowerCamelCase  :				Union[str, Any]      =		batch[-1]
                 if batch:
                    _lowerCamelCase  :				Optional[int]      =		pa.Table.from_arrays([pa.array([batch])]			,      names=SCREAMING_SNAKE_CASE)
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
              elif self.config.sample_by == "document":
                 _lowerCamelCase  :				Tuple      =		f.read()
                 _lowerCamelCase  :				List[Any]      =		pa.Table.from_arrays([pa.array([text])]			,      names=SCREAMING_SNAKE_CASE)
                 yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE)
  
  