



"""simple docstring"""




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
 from tokenizers import Tokenizer
 from tokenizers.models import WordPiece






class 			UpperCAmelCase_						(       unittest.TestCase		):



      def 		snake_case_      (  self  :			Tuple  ,			A  :			Union[str, Any]		):
           _UpperCAmelCase    :			List[str]              =					list(tokenizer_class.max_model_input_sizes.keys()		)
           for model_name in sa_models[:1]:
                _UpperCAmelCase    :			Tuple              =					tokenizer_class.from_pretrained(a_		)
                self.assertIsNotNone(a_		)
                self.assertIsInstance(a_  ,			a_		)
                self.assertIsInstance(a_  ,			a_		)

                for special_tok in tokenizer.all_special_tokens:
                     self.assertIsInstance(a_  ,			a_		)
                     _UpperCAmelCase    :			Optional[int]              =					tokenizer.convert_tokens_to_ids(a_		)
                     self.assertIsInstance(a_  ,			a_		)



      def 		snake_case_      (  self  :			Union[str, Any]  ,			A  :			BatchEncoding  ,			A  :			Optional[Callable] = None		):
           _UpperCAmelCase    :			str              =					pickle.dumps(a_		)
           self.assertIsNotNone(a_		)

           _UpperCAmelCase    :			str              =					pickle.loads(a_		)

           # Ensure is_fast is correctly restored
           self.assertEqual(be_restored.is_fast  ,			be_original.is_fast		)

           # Ensure encodings are potentially correctly restored
           if be_original.is_fast:
                self.assertIsNotNone(be_restored.encodings		)
           else:
                self.assertIsNone(be_restored.encodings		)

           # Ensure the keys are the same
           for original_v, restored_v in zip(be_original.values()  ,			be_restored.values()		):
                if equal_op:
                     self.assertTrue(equal_op(a_  ,			a_		)		)
                else:
                     self.assertEqual(a_  ,			a_		)



      @slow
      def 		snake_case_      (  self  :			Optional[int]		):
           self.check_tokenizer_from_pretrained(a_		)



      def 		snake_case_      (  self  :			List[str]		):
           self.assertEqual(TensorType("tf"		)  ,			TensorType.TENSORFLOW		)
           self.assertEqual(TensorType("pt"		)  ,			TensorType.PYTORCH		)
           self.assertEqual(TensorType("np"		)  ,			TensorType.NUMPY		)



      @require_tokenizers
      def 		snake_case_      (  self  :			List[Any]		):
           import numpy as np

           _UpperCAmelCase    :			Optional[Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
           _UpperCAmelCase    :			Optional[Any]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)

           # Python no tensor
           with self.subTest("BatchEncoding (Python, return_tensors=None)"		):
                self.assert_dump_and_restore(tokenizer_p("Small example to encode"		)		)

           with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"		):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)

           with self.subTest("BatchEncoding (Rust, return_tensors=None)"		):
                self.assert_dump_and_restore(tokenizer_r("Small example to encode"		)		)

           with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"		):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)



      @require_tf
      @require_tokenizers
      def 		snake_case_      (  self  :			str		):
           import tensorflow as tf

           def tf_array_equals(A  :			Tuple  ,			A  :			str		):
                return tf.reduce_all(tf.equal(a_  ,			a_		)		)

           _UpperCAmelCase    :			Tuple              =					BertTokenizer.from_pretrained("bert-base-cased"		)
           _UpperCAmelCase    :			int              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)

           with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"		):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			a_		)

           with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"		):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			a_		)



      @require_torch
      @require_tokenizers
      def 		snake_case_      (  self  :			List[str]		):
           import torch

           _UpperCAmelCase    :			Optional[Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
           _UpperCAmelCase    :			List[Any]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)

           with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"		):
                self.assert_dump_and_restore(
                    tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)

           with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"		):
                self.assert_dump_and_restore(
                    tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)



      @require_tokenizers
      def 		snake_case_      (  self  :			Optional[int]		):
           _UpperCAmelCase    :			Optional[int]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
           _UpperCAmelCase    :			Dict              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)

           with self.subTest("Python Tokenizer"		):
                self.assertFalse(tokenizer_p("Small example to_encode"		).is_fast		)

           with self.subTest("Rust Tokenizer"		):
                self.assertTrue(tokenizer_r("Small example to_encode"		).is_fast		)



      @require_tokenizers
      def 		snake_case_      (  self  :			str		):
           _UpperCAmelCase    :			Optional[int]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
           _UpperCAmelCase    :			Any              =					tokenizer_r(["Test", "\xad", "test"]  ,			is_split_into_words=a_		)

           self.assertEqual(encoded.word_to_tokens(0		)  ,			TokenSpan(start=1  ,			end=2		)		)
           self.assertEqual(encoded.word_to_tokens(1		)  ,			a_		)
           self.assertEqual(encoded.word_to_tokens(2		)  ,			TokenSpan(start=2  ,			end=3		)		)



      def 		snake_case_      (  self  :			Optional[int]		):
           _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
           _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
           # test converting the converted
           with CaptureStderr() as cs:
                _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"		)
           self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)

           _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
           _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="np"  ,			prepend_batch_axis=a_		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)



      @require_torch
      def 		snake_case_      (  self  :			List[str]		):
           _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
           _UpperCAmelCase    :			int              =					batch.convert_to_tensors(tensor_type="pt"		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
           # test converting the converted
           with CaptureStderr() as cs:
                _UpperCAmelCase    :			Dict              =					batch.convert_to_tensors(tensor_type="pt"		)
           self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)

           _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
           _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="pt"  ,			prepend_batch_axis=a_		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)



      @require_tf
      def 		snake_case_      (  self  :			Optional[Any]		):
           _UpperCAmelCase    :			int              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
           _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="tf"		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
           # test converting the converted
           with CaptureStderr() as cs:
                _UpperCAmelCase    :			List[str]              =					batch.convert_to_tensors(tensor_type="tf"		)
           self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)

           _UpperCAmelCase    :			int              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
           _UpperCAmelCase    :			Dict              =					batch.convert_to_tensors(tensor_type="tf"  ,			prepend_batch_axis=a_		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)



      @require_flax
      def 		snake_case_      (  self  :			List[str]		):
           _UpperCAmelCase    :			List[Any]              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
           _UpperCAmelCase    :			List[Any]              =					batch.convert_to_tensors(tensor_type="jax"		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
           # test converting the converted
           with CaptureStderr() as cs:
                _UpperCAmelCase    :			Any              =					batch.convert_to_tensors(tensor_type="jax"		)
           self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)

           _UpperCAmelCase    :			List[str]              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
           _UpperCAmelCase    :			Optional[int]              =					batch.convert_to_tensors(tensor_type="jax"  ,			prepend_batch_axis=a_		)
           self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
           self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)



      def 		snake_case_      (  self  :			List[Any]		):
           _UpperCAmelCase    :			Union[str, Any]              =					[{"input_ids": np.array([0, 1, 2]		)}, {"input_ids": np.array([0, 1, 2, 3]		)}]
           _UpperCAmelCase    :			List[str]              =					BertTokenizer.from_pretrained("bert-base-cased"		)

           _UpperCAmelCase    :			int              =					tokenizer.pad(a_  ,			padding=a_		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
           self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
           _UpperCAmelCase    :			Optional[int]              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="np"		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
           self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)



      @require_torch
      def 		snake_case_      (  self  :			List[Any]		):
           import torch

           _UpperCAmelCase    :			Optional[int]              =					[{"input_ids": torch.tensor([0, 1, 2]		)}, {"input_ids": torch.tensor([0, 1, 2, 3]		)}]
           _UpperCAmelCase    :			int              =					BertTokenizer.from_pretrained("bert-base-cased"		)

           _UpperCAmelCase    :			Any              =					tokenizer.pad(a_  ,			padding=a_		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
           self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
           _UpperCAmelCase    :			Dict              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="pt"		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
           self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)



      @require_tf
      def 		snake_case_      (  self  :			int		):
           import tensorflow as tf

           _UpperCAmelCase    :			List[str]              =					[{"input_ids": tf.constant([0, 1, 2]		)}, {"input_ids": tf.constant([0, 1, 2, 3]		)}]
           _UpperCAmelCase    :			str              =					BertTokenizer.from_pretrained("bert-base-cased"		)

           _UpperCAmelCase    :			Optional[Any]              =					tokenizer.pad(a_  ,			padding=a_		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
           self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
           _UpperCAmelCase    :			int              =					tokenizer.pad(a_  ,			padding=a_  ,			return_tensors="tf"		)
           self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
           self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)



      @require_tokenizers
      def 		snake_case_      (  self  :			List[str]		):
           _UpperCAmelCase    :			int              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
           PreTrainedTokenizerFast(tokenizer_object=a_		)



      @require_tokenizers
      def 		snake_case_      (  self  :			Optional[int]		):
           _UpperCAmelCase    :			Dict              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
           with tempfile.TemporaryDirectory() as tmpdirname:
                bert_tokenizer.save(os.path.join(a_  ,			"tokenizer.json"		)		)
                PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_  ,			"tokenizer.json"		)		)





"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_xglm import XGLMTokenizer
else:
 _lowerCAmelCase       :      Tuple							=							None


_lowerCAmelCase       :      List[str]							=							logging.get_logger(__name__)

_lowerCAmelCase       :      str							=							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase       :      Optional[Any]							=							{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase       :      Union[str, Any]							=							{
    "facebook/xglm-564M": 20_48,
}






class 			UpperCAmelCase_						(       _UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:     str				  =				VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:     List[str]				  =				PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:     str				  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:     int				  =				['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:     str				  =				XGLMTokenizer



      def __init__(  self  :			List[Any]  ,			A  :			Union[str, Any]=None  ,			A  :			Optional[Any]=None  ,			A  :			Optional[Any]="<s>"  ,			A  :			Optional[int]="</s>"  ,			A  :			str="</s>"  ,			A  :			int="<s>"  ,			A  :			int="<unk>"  ,			A  :			List[str]="<pad>"  ,			**A  :			Tuple  ,			):
           # Compatibility with the original tokenizer
           _UpperCAmelCase    :			Union[str, Any]              =					7
           _UpperCAmelCase    :			Optional[int]              =					[f'<madeupword{i}>' for i in range(self.num_madeup_words		)]

           _UpperCAmelCase    :			Any              =					kwargs.get("additional_special_tokens"  ,			[]		)
           kwargs["additional_special_tokens"] += [
               word for word in madeup_words if word not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               A  ,			tokenizer_file=A  ,			bos_token=A  ,			eos_token=A  ,			sep_token=A  ,			cls_token=A  ,			unk_token=A  ,			pad_token=A  ,			**A  ,			)

           _UpperCAmelCase    :			Any              =					vocab_file
           _UpperCAmelCase    :			Union[str, Any]              =					False if not self.vocab_file else True



      def 		snake_case_      (  self  :			List[str]  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):

           if token_ids_a is None:
                return [self.sep_token_id] + token_ids_a
           _UpperCAmelCase    :			Tuple              =					[self.sep_token_id]
           return sep + token_ids_a + sep + sep + token_ids_a



      def 		snake_case_      (  self  :			str  ,			A  :			List[int]  ,			A  :			Optional[List[int]] = None		):
           _UpperCAmelCase    :			Optional[int]              =					[self.sep_token_id]

           if token_ids_a is None:
                return len(sep + token_ids_a		) * [0]
           return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]



      def 		snake_case_      (  self  :			Dict  ,			A  :			str  ,			A  :			Optional[str] = None		):
           if not self.can_save_slow_tokenizer:
                raise ValueError(
                    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                    "tokenizer."		)

           if not os.path.isdir(A		):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory.'		)
                return
           _UpperCAmelCase    :			str              =					os.path.join(
               A  ,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

           if os.path.abspath(self.vocab_file		) != os.path.abspath(A		):
                copyfile(self.vocab_file  ,			A		)

           return (out_vocab_file,)
