'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


__snake_case  : Dict		       =    logging.get_logger(__name__)

__snake_case  : Union[str, Any]		       =    {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__snake_case  : Optional[Any]		       =    {
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

__snake_case  : Optional[int]		       =    {
    'allenai/longformer-base-4096': 4_096,
    'allenai/longformer-large-4096': 4_096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4_096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4_096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4_096,
}




class       __UpperCAmelCase							(							_UpperCamelCase	):





       '''simple docstring'''



       __lowercase   :   List[Any]  					=							VOCAB_FILES_NAMES
       __lowercase   :   int  					=							PRETRAINED_VOCAB_FILES_MAP
       __lowercase   :   Optional[int]  					=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowercase   :   Optional[Any]  					=							['input_ids', 'attention_mask']
       __lowercase   :   Dict  					=							LongformerTokenizer

       def __init__( self       ,	_SCREAMING_SNAKE_CASE=None       ,	_SCREAMING_SNAKE_CASE=None       ,	_SCREAMING_SNAKE_CASE=None       ,	_SCREAMING_SNAKE_CASE="replace"       ,	_SCREAMING_SNAKE_CASE="<s>"       ,	_SCREAMING_SNAKE_CASE="</s>"       ,	_SCREAMING_SNAKE_CASE="</s>"       ,	_SCREAMING_SNAKE_CASE="<s>"       ,	_SCREAMING_SNAKE_CASE="<unk>"       ,	_SCREAMING_SNAKE_CASE="<pad>"       ,	_SCREAMING_SNAKE_CASE="<mask>"       ,	_SCREAMING_SNAKE_CASE=False       ,	_SCREAMING_SNAKE_CASE=True       ,	**_SCREAMING_SNAKE_CASE       ,	)						->	List[str]:
           super().__init__(
               _SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	tokenizer_file=_SCREAMING_SNAKE_CASE       ,	errors=_SCREAMING_SNAKE_CASE       ,	bos_token=_SCREAMING_SNAKE_CASE       ,	eos_token=_SCREAMING_SNAKE_CASE       ,	sep_token=_SCREAMING_SNAKE_CASE       ,	cls_token=_SCREAMING_SNAKE_CASE       ,	unk_token=_SCREAMING_SNAKE_CASE       ,	pad_token=_SCREAMING_SNAKE_CASE       ,	mask_token=_SCREAMING_SNAKE_CASE       ,	add_prefix_space=_SCREAMING_SNAKE_CASE       ,	trim_offsets=_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE       ,	)

           A_      =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
           if pre_tok_state.get('''add_prefix_space'''       ,	_SCREAMING_SNAKE_CASE					) != add_prefix_space:
               A_      =  getattr(_SCREAMING_SNAKE_CASE       ,	pre_tok_state.pop('''type'''					)					)
               A_      =  add_prefix_space
               A_      =  pre_tok_class(**_SCREAMING_SNAKE_CASE					)

           A_      =  add_prefix_space

           A_      =  '''post_processor'''
           A_      =  getattr(self.backend_tokenizer       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
           if tokenizer_component_instance:
               A_      =  json.loads(tokenizer_component_instance.__getstate__()					)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                   A_      =  tuple(state['''sep''']					)
               if "cls" in state:
                   A_      =  tuple(state['''cls''']					)

               A_      =  False

               if state.get('''add_prefix_space'''       ,	_SCREAMING_SNAKE_CASE					) != add_prefix_space:
                   A_      =  add_prefix_space
                   A_      =  True

               if state.get('''trim_offsets'''       ,	_SCREAMING_SNAKE_CASE					) != trim_offsets:
                   A_      =  trim_offsets
                   A_      =  True

               if changes_to_apply:
                   A_      =  getattr(_SCREAMING_SNAKE_CASE       ,	state.pop('''type'''					)					)
                   A_      =  component_class(**_SCREAMING_SNAKE_CASE					)
                   setattr(self.backend_tokenizer       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)

       @property
       def    __A			( self					)						->	str:
           if self._mask_token is None:
               if self.verbose:
                   logger.error('''Using mask_token, but it is not set yet.'''					)
               return None
           return str(self._mask_token					)

       @mask_token.setter
       def    __A			( self       ,	_SCREAMING_SNAKE_CASE					)						->	str:
           A_      =  AddedToken(_SCREAMING_SNAKE_CASE       ,	lstrip=_SCREAMING_SNAKE_CASE       ,	rstrip=_SCREAMING_SNAKE_CASE					) if isinstance(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					) else value
           A_      =  value

       def    __A			( self       ,	*_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)						->	BatchEncoding:
           A_      =  kwargs.get('''is_split_into_words'''       ,	_SCREAMING_SNAKE_CASE					)
           assert self.add_prefix_space or not is_split_into_words, (
               F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)

       def    __A			( self       ,	*_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)						->	BatchEncoding:
           A_      =  kwargs.get('''is_split_into_words'''       ,	_SCREAMING_SNAKE_CASE					)

           assert self.add_prefix_space or not is_split_into_words, (
               F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = None					)						->	Tuple[str]:
           A_      =  self._tokenizer.model.save(_SCREAMING_SNAKE_CASE       ,	name=_SCREAMING_SNAKE_CASE					)
           return tuple(_SCREAMING_SNAKE_CASE					)

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None					)						->	List[Any]:
           A_      =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
           if token_ids_a is None:
               return output

           return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE = None					)						->	List[int]:
           A_      =  [self.sep_token_id]
           A_      =  [self.cls_token_id]

           if token_ids_a is None:
               return len(cls + token_ids_a + sep					) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]







'''simple docstring'''




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


__snake_case  : Optional[int]		       =    logging.getLogger(__name__)




class       __UpperCAmelCase							:





       '''simple docstring'''



       def __init__( self					)						->	List[str]:
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	str:
           if not self.initialized:
               A_      =  RagRetriever(
                   _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)
               A_      =  True

       def    __A			( self					)						->	List[Any]:
           self.retriever.index.init_index()

       def    __A			( self					)						->	Dict:
           # delete the old self.retriever object before assigning the new index
           del self.retriever
           A_      =  False

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           A_  ,A_      =  self.retriever._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
           A_      =  self.retriever.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return doc_ids, retrieved_doc_embeds, doc_dicts




class       __UpperCAmelCase							(							_UpperCamelCase	):





       '''simple docstring'''



       def __init__( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None					)						->	List[Any]:
           if index is not None and index.is_initialized() and len(_SCREAMING_SNAKE_CASE					) > 0:
               raise ValueError(
                   '''When using Ray for distributed fine-tuning, '''
                   '''you\'ll need to provide the paths instead, '''
                   '''as the dataset and the index are loaded '''
                   '''separately. More info in examples/rag/use_own_knowledge_dataset.py '''					)

           super().__init__(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	init_retrieval=_SCREAMING_SNAKE_CASE       ,	)

           A_      =  retrieval_workers
           A_      =  question_encoder_tokenizer
           A_      =  generator_tokenizer
           if len(self.retrieval_workers					) > 0:
               ray.get(
                   [
                       worker.create_rag_retriever.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
                       for worker in self.retrieval_workers
                   ]					)

       def    __A			( self					)						->	List[Any]:
           logger.info('''initializing retrieval'''					)

           if len(self.retrieval_workers					) > 0:
               ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers]					)
           else:
               # Non-distributed training. Load index into this same process.
               self.index.init_index()

       def    __A			( self       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)						->	Optional[int]:
           if len(self.retrieval_workers					) > 0:
               # Select a random retrieval actor.
               A_      =  self.retrieval_workers[random.randint(0       ,	len(self.retrieval_workers					) - 1					)]
               A_  ,A_  ,A_      =  ray.get(
                   random_worker.retrieve.remote(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)					)
           else:
               A_  ,A_      =  self._main_retrieve(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE					)
               A_      =  self.index.get_doc_dicts(_SCREAMING_SNAKE_CASE					)
           return retrieved_doc_embeds, doc_ids, doc_dicts

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	List[Any]:
           return super(_SCREAMING_SNAKE_CASE       ,	cls					).get_tokenizers(_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)

       @classmethod
       def    __A			( cls       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE       ,	_SCREAMING_SNAKE_CASE=None       ,	**_SCREAMING_SNAKE_CASE					)						->	int:
           A_      =  kwargs.pop('''config'''       ,	_SCREAMING_SNAKE_CASE					) or RagConfig.from_pretrained(_SCREAMING_SNAKE_CASE       ,	**_SCREAMING_SNAKE_CASE					)
           A_      =  RagTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE       ,	config=_SCREAMING_SNAKE_CASE					)
           A_      =  rag_tokenizer.question_encoder
           A_      =  rag_tokenizer.generator

           if indexed_dataset is not None:
               A_      =  '''custom'''
               A_      =  CustomHFIndex(config.retrieval_vector_size       ,	_SCREAMING_SNAKE_CASE					)
           else:
               A_      =  cls._build_index(_SCREAMING_SNAKE_CASE					)

           return cls(
               _SCREAMING_SNAKE_CASE       ,	question_encoder_tokenizer=_SCREAMING_SNAKE_CASE       ,	generator_tokenizer=_SCREAMING_SNAKE_CASE       ,	retrieval_workers=_SCREAMING_SNAKE_CASE       ,	index=_SCREAMING_SNAKE_CASE       ,	)

       def    __A			( self					)						->	Optional[int]:
           logger.info('''re-loading the new dataset with embeddings'''					)
           # access from the training loop

           ray.get([worker.clear_object.remote() for worker in self.retrieval_workers]					)

           # build the index object again
           A_      =  self._build_index(self.config					)

           ray.get(
               [
                   worker.create_rag_retriever.remote(
                       self.config       ,	self.question_encoder_tokenizer       ,	self.generator_tokenizer       ,	_SCREAMING_SNAKE_CASE					)
                   for worker in self.retrieval_workers
               ]					)






