"""simple docstring"""




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase							 =     logging.get_logger(__name__)

__lowerCamelCase							 =     "▁"

__lowerCamelCase							 =     {"vocab_file": "sentencepiece.bpe.model"}

__lowerCamelCase							 =     {
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

__lowerCamelCase							 =     {
    "facebook/xglm-564M": 20_48,
}
class 			UpperCamelCase__(	__A       ):
						lowerCAmelCase__     :  Tuple  							=						VOCAB_FILES_NAMES
						lowerCAmelCase__     :  int  							=						PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase__     :  List[Any]  							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase__     :  str  							=						['input_ids', 'attention_mask']





						def __init__(				self						,__UpperCAmelCase						,__UpperCAmelCase="<s>"						,__UpperCAmelCase="</s>"						,__UpperCAmelCase="</s>"						,__UpperCAmelCase="<s>"						,__UpperCAmelCase="<unk>"						,__UpperCAmelCase="<pad>"						,__UpperCAmelCase = None						,**__UpperCAmelCase						,)    -> None:
													A__		   =							{} if sp_model_kwargs is None else sp_model_kwargs

													# Compatibility with the original tokenizer
													A__		   =							7
													A__		   =							[f'''<madeupword{i}>''' for i in range(self.num_madeup_words   )]

													A__		   =							kwargs.get('additional_special_tokens'						,[]   )
													kwargs["additional_special_tokens"] += [
													    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
													]

													super().__init__(
													    bos_token=__UpperCAmelCase						,eos_token=__UpperCAmelCase						,unk_token=__UpperCAmelCase						,sep_token=__UpperCAmelCase						,cls_token=__UpperCAmelCase						,pad_token=__UpperCAmelCase						,sp_model_kwargs=self.sp_model_kwargs						,**__UpperCAmelCase						,)

													A__		   =							spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.Load(str(__UpperCAmelCase   )   )
													A__		   =							vocab_file

													# Original fairseq vocab and spm vocab must be "aligned":
													# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
													# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
													# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
													# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

													# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
													A__		   =							1

													# Mimic fairseq token-to-id alignment for the first 4 token
													A__		   =							{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

													A__		   =							len(self.sp_model   )
													A__		   =							{f'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words   )}
													self.fairseq_tokens_to_ids.update(__UpperCAmelCase   )

													A__		   =							{v: k for k, v in self.fairseq_tokens_to_ids.items()}





						def __getstate__(				self   )    -> Dict:
													A__		   =							self.__dict__.copy()
													A__		   =							None
													A__		   =							self.sp_model.serialized_model_proto()
													return state





						def __setstate__(				self						,__UpperCAmelCase   )    -> Optional[Any]:
													A__		   =							d

													# for backward compatibility
													if not hasattr(self						,'sp_model_kwargs'   ):
																				A__		   =							{}

													A__		   =							spm.SentencePieceProcessor(**self.sp_model_kwargs   )
													self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase = None   )    -> List[int]:

													if token_ids_a is None:
																				return [self.sep_token_id] + token_ids_a
													A__		   =							[self.sep_token_id]
													return sep + token_ids_a + sep + sep + token_ids_a





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase = None						,__UpperCAmelCase = False   )    -> List[int]:

													if already_has_special_tokens:
																				return super().get_special_tokens_mask(
																				    token_ids_a=__UpperCAmelCase						,token_ids_a=__UpperCAmelCase						,already_has_special_tokens=__UpperCAmelCase   )

													if token_ids_a is None:
																				return [1] + ([0] * len(__UpperCAmelCase   ))
													return [1] + ([0] * len(__UpperCAmelCase   )) + [1, 1] + ([0] * len(__UpperCAmelCase   ))





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase = None   )    -> List[int]:
													A__		   =							[self.sep_token_id]

													if token_ids_a is None:
																				return len(sep + token_ids_a   ) * [0]
													return len(sep + token_ids_a + sep + sep + token_ids_a   ) * [0]





						@property
						def  snake_case__		(				self   )    -> int:
													return len(self.sp_model   ) + self.fairseq_offset + self.num_madeup_words





						def  snake_case__		(				self   )    -> Optional[Any]:
													A__		   =							{self.convert_ids_to_tokens(__UpperCAmelCase   ): i for i in range(self.vocab_size   )}
													vocab.update(self.added_tokens_encoder   )
													return vocab





						def  snake_case__		(				self						,__UpperCAmelCase   )    -> List[str]:
													return self.sp_model.encode(__UpperCAmelCase						,out_type=__UpperCAmelCase   )





						def  snake_case__		(				self						,__UpperCAmelCase   )    -> List[str]:
													if token in self.fairseq_tokens_to_ids:
																				return self.fairseq_tokens_to_ids[token]
													A__		   =							self.sp_model.PieceToId(__UpperCAmelCase   )

													# Need to return unknown token if the SP model returned 0
													return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





						def  snake_case__		(				self						,__UpperCAmelCase   )    -> int:
													if index in self.fairseq_ids_to_tokens:
																				return self.fairseq_ids_to_tokens[index]
													return self.sp_model.IdToPiece(index - self.fairseq_offset   )





						def  snake_case__		(				self						,__UpperCAmelCase   )    -> List[str]:
													A__		   =							''.join(__UpperCAmelCase   ).replace(__UpperCAmelCase						,' '   ).strip()
													return out_string





						def  snake_case__		(				self						,__UpperCAmelCase						,__UpperCAmelCase = None   )    -> Tuple[str]:
													if not os.path.isdir(__UpperCAmelCase   ):
																				logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
																				return
													A__		   =							os.path.join(
													    __UpperCAmelCase						,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

													if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCAmelCase   ) and os.path.isfile(self.vocab_file   ):
																				copyfile(self.vocab_file						,__UpperCAmelCase   )
													elif not os.path.isfile(self.vocab_file   ):
																				with open(__UpperCAmelCase						,'wb'   ) as fi:
																											A__		   =							self.sp_model.serialized_model_proto()
																											fi.write(__UpperCAmelCase   )

													return (out_vocab_file,)






"""simple docstring"""




import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
				from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
				from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
				from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


__lowerCamelCase							 =     DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

__lowerCamelCase							 =     "main"
# Default branch name
__lowerCamelCase							 =     "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
__lowerCamelCase							 =     "aaaaaaa"
# This commit does not exist, so we should 404.

__lowerCamelCase							 =     "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
__lowerCamelCase							 =     "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"






@contextlib.contextmanager
def      UpperCAmelCase    (		):






							"""simple docstring"""





							print('Welcome!' )
							yield
							print('Bye!' )






@contextlib.contextmanager
def      UpperCAmelCase    (		):






							"""simple docstring"""





							print('Bonjour!' )
							yield
							print('Au revoir!' )
class 			UpperCamelCase__(	unittest.TestCase       ):





						def  snake_case__		(				self   )    -> List[str]:
													# If the spec is missing, importlib would not be able to import the module dynamically.
													assert transformers.__spec__ is not None
													assert importlib.util.find_spec('transformers'   ) is not None






class 			UpperCamelCase__(	unittest.TestCase       ):





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> Dict:
													with ContextManagers([]   ):
																				print('Transformers are awesome!'   )
													# The print statement adds a new line at the end of the output
													self.assertEqual(mock_stdout.getvalue()						,'Transformers are awesome!\n'   )





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> List[str]:
													with ContextManagers([context_en()]   ):
																				print('Transformers are awesome!'   )
													# The output should be wrapped with an English welcome and goodbye
													self.assertEqual(mock_stdout.getvalue()						,'Welcome!\nTransformers are awesome!\nBye!\n'   )





						@unittest.mock.patch('sys.stdout'						,new_callable=io.StringIO   )
						def  snake_case__		(				self						,__UpperCAmelCase   )    -> Any:
													with ContextManagers([context_fr(), context_en()]   ):
																				print('Transformers are awesome!'   )
													# The output should be wrapped with an English and French welcome and goodbye
													self.assertEqual(mock_stdout.getvalue()						,'Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'   )





						@require_torch
						def  snake_case__		(				self   )    -> Union[str, Any]:
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels', 'next_sentence_label']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['start_positions', 'end_positions']   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )





						@require_tf
						def  snake_case__		(				self   )    -> str:
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels', 'next_sentence_label']   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,['start_positions', 'end_positions']   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,['labels']   )





						@require_flax
						def  snake_case__		(				self   )    -> List[Any]:
													# Flax models don't have labels
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )
													class 			UpperCamelCase__(	__A       ):
																			pass






													self.assertEqual(find_labels(__UpperCAmelCase   )						,[]   )





