code: 
  
  
  
  """simple docstring"""
  
  
  
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       unittest.TestCase		):
  
  
  
        def 		snake_case_      (  self  :			Tuple  ,			A  :			Union[str, Any]		):
             _UpperCAmelCase    :			List[str]              =					list(tokenizer_class.max_model_input_sizes.keys()		)
             for model_name in sa_models[:1]:
                  _UpperCAmelCase    :			Tuple              =					tokenizer_class.from_pretrained(a_		)
                  self.assertIsNotNone(a_		)
                  self.assertIsInstance(a_  ,			a_		)
                  self.assertIsInstance(a_  ,			a_		)
  
                  for special_tok in tokenizer.all_special_tokens:
                       self.assertIsInstance(a_  ,			a_		)
                       _UpperCAmelCase    :			Optional[int]              =					tokenizer.convert_tokens_to_ids(a_		)
                       self.assertIsInstance(a_  ,			a_		)
  
  
  
        def 		snake_case_      (  self  :			Union[str, Any]  ,			A  :			BatchEncoding  ,			A  :			Optional[Callable] = None		):
             _UpperCAmelCase    :			str              =					pickle.dumps(a_		)
             self.assertIsNotNone(a_		)
  
             _UpperCAmelCase    :			str              =					pickle.loads(a_		)
  
             # Ensure is_fast is correctly restored
             self.assertEqual(be_restored.is_fast  ,			be_original.is_fast		)
  
             # Ensure encodings are potentially correctly restored
             if be_original.is_fast:
                  self.assertIsNotNone(be_restored.encodings		)
             else:
                  self.assertIsNone(be_restored.encodings		)
  
             # Ensure the keys are the same
             for original_v, restored_v in zip(be_original.values()  ,			be_restored.values()		):
                  if equal_op:
                       self.assertTrue(equal_op(a_  ,			a_		)		)
                  else:
                       self.assertEqual(a_  ,			a_		)
  
  
  
        @slow
        def 		snake_case_      (  self  :			Optional[int]		):
             self.check_tokenizer_from_pretrained(a_		)
  
  
  
        def 		snake_case_      (  self  :			List[str]		):
             self.assertEqual(TensorType("tf"		)  ,			TensorType.TENSORFLOW		)
             self.assertEqual(TensorType("pt"		)  ,			TensorType.PYTORCH		)
             self.assertEqual(TensorType("np"		)  ,			TensorType.NUMPY		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			List[Any]		):
             import numpy as np
  
             _UpperCAmelCase    :			Optional[Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Optional[Any]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             # Python no tensor
             with self.subTest("BatchEncoding (Python, return_tensors=None)"		):
                  self.assert_dump_and_restore(tokenizer_p("Small example to encode"		)		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=None)"		):
                  self.assert_dump_and_restore(tokenizer_r("Small example to encode"		)		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)
  
  
  
        @require_tf
        @require_tokenizers
        def 		snake_case_      (  self  :			str		):
             import tensorflow as tf
  
             def tf_array_equals(A  :			Tuple  ,			A  :			str		):
                  return tf.reduce_all(tf.equal(a_  ,			a_		)		)
  
             _UpperCAmelCase    :			Tuple              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			int              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			a_		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			a_		)
  
  
  
        @require_torch
        @require_tokenizers
        def 		snake_case_      (  self  :			List[str]		):
             import torch
  
             _UpperCAmelCase    :			Optional[Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			List[Any]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			Optional[int]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Dict              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("Python Tokenizer"		):
                  self.assertFalse(tokenizer_p("Small example to_encode"		).is_fast		)
  
             with self.subTest("Rust Tokenizer"		):
                  self.assertTrue(tokenizer_r("Small example to_encode"		).is_fast		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			str		):
             _UpperCAmelCase    :			Optional[int]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Any              =					tokenizer_r(["Test", "\xad", "test"]  ,			is_split_into_words=a_		)
  
             self.assertEqual(encoded.word_to_tokens(0		)  ,			TokenSpan(start=1  ,			end=2		)		)
             self.assertEqual(encoded.word_to_tokens(1		)  ,			a_		)
             self.assertEqual(encoded.word_to_tokens(2		)  ,			TokenSpan(start=2  ,			end=3		)		)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"  ,			prepend_batch_axis=a_		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_torch
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			int              =					batch.convert_to_tensors(tensor_type="pt"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			Dict              =					batch.convert_to_tensors(tensor_type="pt"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="pt"  ,			prepend_batch_axis=a_		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_tf
        def 		snake_case_      (  self  :			Optional[Any]		):
             _UpperCAmelCase    :			int              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="tf"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="tf"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			int              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			Dict              =					batch.convert_to_tensors(tensor_type="tf"  ,			prepend_batch_axis=a_		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_flax
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			List[Any]              =					batch.convert_to_tensors(tensor_type="jax"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			Any              =					batch.convert_to_tensors(tensor_type="jax"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			List[str]              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			Optional[int]              =					batch.convert_to_tensors(tensor_type="jax"  ,			prepend_batch_axis=a_		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			Union[str, Any]              =					[{"input_ids": np.array([0, 1, 2]		)}, {"input_ids": np.array([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			List[str]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			int              =					tokenizer.pad(a_  ,			padding=a_		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			Optional[int]              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="np"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_torch
        def 		snake_case_      (  self  :			List[Any]		):
             import torch
  
             _UpperCAmelCase    :			Optional[int]              =					[{"input_ids": torch.tensor([0, 1, 2]		)}, {"input_ids": torch.tensor([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			int              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			Any              =					tokenizer.pad(a_  ,			padding=a_		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			Dict              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="pt"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_tf
        def 		snake_case_      (  self  :			int		):
             import tensorflow as tf
  
             _UpperCAmelCase    :			List[str]              =					[{"input_ids": tf.constant([0, 1, 2]		)}, {"input_ids": tf.constant([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			str              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			Optional[Any]              =					tokenizer.pad(a_  ,			padding=a_		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
             self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			int              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="tf"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
             self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			int              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
             PreTrainedTokenizerFast(tokenizer_object=a_		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			Dict              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
             with tempfile.TemporaryDirectory() as tmpdirname:
                  bert_tokenizer.save(os.path.join(a_  ,			"tokenizer.json"		)		)
                  PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_  ,			"tokenizer.json"		)		)
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
   from .tokenization_xglm import XGLMTokenizer
  else:
   _lowerCAmelCase       :      Tuple							=							None
  
  
  _lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)
  
  _lowerCAmelCase       :      str							=							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}
  
  _lowerCAmelCase       :      Optional[Any]							=							{
      "vocab_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
      },
      "tokenizer_file": {
          "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
      },
  }
  
  _lowerCAmelCase       :      Union[str, Any]							=							{
      "facebook/xglm-564M": 20_48,
  }
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       _UpperCamelCase		):
        __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
        __SCREAMING_SNAKE_CASE		:     List[str]				  =				PRETRAINED_VOCAB_FILES_MAP
        __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
        __SCREAMING_SNAKE_CASE		:     int				  =				['input_ids', 'attention_mask']
        __SCREAMING_SNAKE_CASE		:     str				  =				XGLMTokenizer
  
  
  
        def __init__(  self  :			List[Any]  ,			A  :			Union[str, Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Optional[Any]="<s>"  ,			A  :			Optional[int]="</s>"  ,			A  :			str="</s>"  ,			A  :			int="<s>"  ,			A  :			int="<unk>"  ,			A  :			List[str]="<pad>"  ,			**A  :			Tuple  ,			):
             # Compatibility with the original tokenizer
             _UpperCAmelCase    :			Union[str, Any]              =					7
             _UpperCAmelCase    :			Optional[int]              =					[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]
  
             _UpperCAmelCase    :			Any              =					kwargs.get("additional_special_tokens"  ,			[]		)
             kwargs["additional_special_tokens"] += [
                 word for word in madeup_words if word not in kwargs["additional_special_tokens"]
             ]
  
             super().__init__(
                 A  ,			tokenizer_file=A  ,			bos_token=A  ,			eos_token=A  ,			sep_token=A  ,			cls_token=A  ,			unk_token=A  ,			pad_token=A  ,			**A  ,			)
  
             _UpperCAmelCase    :			Any              =					vocab_file
             _UpperCAmelCase    :			Union[str, Any]              =					False if not self.vocab_file else True
  
  
  
        def 		snake_case_      (  self  :			List[str]  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
  
             if token_ids_a is None:
                  return [self.sep_token_id] + token_ids_a
             _UpperCAmelCase    :			Tuple              =					[self.sep_token_id]
             return sep + token_ids_a + sep + sep + token_ids_a
  
  
  
        def 		snake_case_      (  self  :			str  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
             _UpperCAmelCase    :			Optional[int]              =					[self.sep_token_id]
  
             if token_ids_a is None:
                  return len(sep + token_ids_a		) * [0]
             return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]
  
  
  
        def 		snake_case_      (  self  :			Dict  ,			A  :			str  ,			A  :			Optional[str] = None		):
             if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                      "tokenizer."		)
  
             if not os.path.isdir(A		):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory.'		)
                  return
             _UpperCAmelCase    :			str              =					os.path.join(
                 A  ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
  
             if os.path.abspath(self.vocab_file		) != os.path.abspath(A		):
                  copyfile(self.vocab_file  ,			A		)
  
             return (out_vocab_file,)
  