



'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_		       =  get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")






@require_sentencepiece
@require_tokenizers
class __A			(  lowercase__   ,   unittest.TestCase						):



 '''simple docstring'''







 __lowerCamelCase    :       Any          =       SpeechTaTokenizer
 __lowerCamelCase    :       Optional[int]          =       False
 __lowerCamelCase    :       Optional[Any]          =       True



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      super().setUp()

      # We have a SentencePiece fixture for testing
      _a      =  SpeechTaTokenizer(__lowerCamelCase       )

      _a      =  AddedToken('''<mask>''' ,   lstrip=__lowerCamelCase ,   rstrip=__lowerCamelCase       )
      _a      =  mask_token
      tokenizer.add_special_tokens({'''mask_token''': mask_token}       )
      tokenizer.add_tokens(['''<ctc_blank>''']       )

      tokenizer.save_pretrained(self.tmpdirname       )



 def 					a__  (self ,   A       )    ->     Optional[int]:


      """simple docstring"""



      _a      =  "this is a test"
      _a      =  "this is a test"
      return input_text, output_text



 def 					a__  (self ,   A ,   A=False ,   A=20 ,   A=5       )    ->     int:


      """simple docstring"""



      _a      =  self.get_input_output_texts(__lowerCamelCase       )
      _a      =  tokenizer.encode(__lowerCamelCase ,   add_special_tokens=__lowerCamelCase       )
      _a      =  tokenizer.decode(__lowerCamelCase ,   clean_up_tokenization_spaces=__lowerCamelCase       )
      return text, ids



 def 					a__  (self       )    ->     str:


      """simple docstring"""



      _a      =  "<pad>"
      _a      =  1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase       ) ,   __lowerCamelCase       )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase       ) ,   __lowerCamelCase       )



 def 					a__  (self       )    ->     Optional[int]:


      """simple docstring"""



      _a      =  list(self.get_tokenizer().get_vocab().keys()       )

      self.assertEqual(vocab_keys[0] ,   '''<s>'''       )
      self.assertEqual(vocab_keys[1] ,   '''<pad>'''       )
      self.assertEqual(vocab_keys[-4] ,   '''œ'''       )
      self.assertEqual(vocab_keys[-2] ,   '''<mask>'''       )
      self.assertEqual(vocab_keys[-1] ,   '''<ctc_blank>'''       )
      self.assertEqual(len(__lowerCamelCase       ) ,   81       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      self.assertEqual(self.get_tokenizer().vocab_size ,   79       )



 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      _a      =  self.get_tokenizers(do_lower_case=__lowerCamelCase       )
      for tokenizer in tokenizers:
           with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
                _a      =  tokenizer.vocab_size
                _a      =  len(__lowerCamelCase       )

                self.assertNotEqual(__lowerCamelCase ,   0       )

                # We usually have added tokens from the start in tests because our vocab fixtures are
                # smaller than the original vocabs - let's not assert this
                # self.assertEqual(vocab_size, all_size)

                _a      =  ["aaaaa bbbbbb", "cccccccccdddddddd"]
                _a      =  tokenizer.add_tokens(__lowerCamelCase       )
                _a      =  tokenizer.vocab_size
                _a      =  len(__lowerCamelCase       )

                self.assertNotEqual(__lowerCamelCase ,   0       )
                self.assertEqual(__lowerCamelCase ,   __lowerCamelCase       )
                self.assertEqual(__lowerCamelCase ,   len(__lowerCamelCase       )       )
                self.assertEqual(__lowerCamelCase ,   all_size + len(__lowerCamelCase       )       )

                _a      =  tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l''' ,   add_special_tokens=__lowerCamelCase       )

                self.assertGreaterEqual(len(__lowerCamelCase       ) ,   4       )
                self.assertGreater(tokens[0] ,   tokenizer.vocab_size - 1       )
                self.assertGreater(tokens[-3] ,   tokenizer.vocab_size - 1       )

                _a      =  {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
                _a      =  tokenizer.add_special_tokens(__lowerCamelCase       )
                _a      =  tokenizer.vocab_size
                _a      =  len(__lowerCamelCase       )

                self.assertNotEqual(__lowerCamelCase ,   0       )
                self.assertEqual(__lowerCamelCase ,   __lowerCamelCase       )
                self.assertEqual(__lowerCamelCase ,   len(__lowerCamelCase       )       )
                self.assertEqual(__lowerCamelCase ,   all_size_a + len(__lowerCamelCase       )       )

                _a      =  tokenizer.encode(
                    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l''' ,   add_special_tokens=__lowerCamelCase       )

                self.assertGreaterEqual(len(__lowerCamelCase       ) ,   6       )
                self.assertGreater(tokens[0] ,   tokenizer.vocab_size - 1       )
                self.assertGreater(tokens[0] ,   tokens[1]       )
                self.assertGreater(tokens[-3] ,   tokenizer.vocab_size - 1       )
                self.assertGreater(tokens[-3] ,   tokens[-4]       )
                self.assertEqual(tokens[0] ,   tokenizer.eos_token_id       )
                self.assertEqual(tokens[-3] ,   tokenizer.pad_token_id       )



 def 					a__  (self       )    ->     Optional[int]:


      """simple docstring"""



      pass



 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      pass



 def 					a__  (self       )    ->     Any:


      """simple docstring"""



      _a      =  self.get_tokenizer()

      _a      =  tokenizer.tokenize('''This is a test'''       )
      # fmt: off
      self.assertListEqual(__lowerCamelCase ,   [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']       )
      # fmt: on

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__lowerCamelCase       ) ,   [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6] ,   )

      _a      =  tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
      self.assertListEqual(
          __lowerCamelCase ,   [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']       )

      _a      =  tokenizer.convert_tokens_to_ids(__lowerCamelCase       )
      # fmt: off
      self.assertListEqual(__lowerCamelCase ,   [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]       )
      # fmt: on

      _a      =  tokenizer.convert_ids_to_tokens(__lowerCamelCase       )
      self.assertListEqual(
          __lowerCamelCase ,   [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']       )



 @slow
 def 					a__  (self       )    ->     Any:


      """simple docstring"""



      _a      =  [
          "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
          "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
          "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
          "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
          "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
          "conditioning on both left and right context in all layers.",
          "The quick brown fox jumps over the lazy dog.",
      ]

      # fmt: off
      _a      =  {
          "input_ids": [
              [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
              [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
          ],
          "attention_mask": [
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
              [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
          ]
      }
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=__lowerCamelCase ,   model_name='''microsoft/speecht5_asr''' ,   revision='''c5ef64c71905caeccde0e4462ef3f9077224c524''' ,   sequences=__lowerCamelCase ,   )







import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
    import PIL.Image

    from .features import FeatureType


A      :    Optional[List[str]]		=					None
A      :    str		=					"<" if sys.byteorder == "little" else ">"
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
A      :    str		=					[
    np.dtype("|b1"),
    np.dtype("|u1"),
    np.dtype("<u2"),
    np.dtype(">u2"),
    np.dtype("<i2"),
    np.dtype(">i2"),
    np.dtype("<u4"),
    np.dtype(">u4"),
    np.dtype("<i4"),
    np.dtype(">i4"),
    np.dtype("<f4"),
    np.dtype(">f4"),
    np.dtype("<f8"),
    np.dtype(">f8"),
]







@dataclass
class 			_lowercase :


       """simple docstring"""




       A__											=			True
       A__											=			None
       # Automatically constructed
       A__											=			"PIL.Image.Image"
       A__											=			pa.struct({"bytes": pa.binary(), "path": pa.string()})
       A__											=			field(default="Image"							,  init=lowercase__							,  repr=lowercase__)





       def __call__(   self							:			Any       ):


             '''simple docstring'''



             return self.pa_type





       def 						lowerCAmelCase     (   self							:			Optional[Any]					,      __lowerCamelCase							:			Union[str, bytes, dict, np.ndarray, "PIL.Image.Image"]       ):


             '''simple docstring'''



             if config.PIL_AVAILABLE:
                   import PIL.Image
             else:
                   raise ImportError("To support encoding images, please install 'Pillow'."       )

             if isinstance(__lowerCamelCase					,      __lowerCamelCase       ):
                   lowerCamelCase__		:				str    =  np.array(__lowerCamelCase       )

             if isinstance(__lowerCamelCase					,      __lowerCamelCase       ):
                   return {"path": value, "bytes": None}
             elif isinstance(__lowerCamelCase					,      __lowerCamelCase       ):
                   return {"path": None, "bytes": value}
             elif isinstance(__lowerCamelCase					,      np.ndarray       ):
                   # convert the image array to PNG/TIFF bytes
                   return encode_np_array(__lowerCamelCase       )
             elif isinstance(__lowerCamelCase					,      PIL.Image.Image       ):
                   # convert the PIL image to bytes (default format is PNG/TIFF)
                   return encode_pil_image(__lowerCamelCase       )
             elif value.get("path"       ) is not None and os.path.isfile(value["path"]       ):
                   # we set "bytes": None to not duplicate the data if they're already available locally
                   return {"bytes": None, "path": value.get("path"       )}
             elif value.get("bytes"       ) is not None or value.get("path"       ) is not None:
                   # store the image bytes, and path is used to infer the image format using the file extension
                   return {"bytes": value.get("bytes"       ), "path": value.get("path"       )}
             else:
                   raise ValueError(
                       f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."       )





       def 						lowerCAmelCase     (   self							:			Any					,      __lowerCamelCase							:			dict					,      __lowerCamelCase							:			List[Any]=None       ):


             '''simple docstring'''



             if not self.decode:
                   raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead."       )

             if config.PIL_AVAILABLE:
                   import PIL.Image
             else:
                   raise ImportError("To support decoding images, please install 'Pillow'."       )

             if token_per_repo_id is None:
                   lowerCamelCase__		:				Union[str, Any]    =  {}

             lowerCamelCase__    ,   lowerCamelCase__		:				Optional[int]    =  value["path"], value["bytes"]
             if bytes_ is None:
                   if path is None:
                         raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}."       )
                   else:
                         if is_local_path(__lowerCamelCase       ):
                               lowerCamelCase__		:				Union[str, Any]    =  PIL.Image.open(__lowerCamelCase       )
                         else:
                               lowerCamelCase__		:				Tuple    =  path.split("::"       )[-1]
                               try:
                                     lowerCamelCase__		:				str    =  string_to_dict(__lowerCamelCase					,      config.HUB_DATASETS_URL       )["repo_id"]
                                     lowerCamelCase__		:				Any    =  token_per_repo_id.get(__lowerCamelCase       )
                               except ValueError:
                                     lowerCamelCase__		:				int    =  None
                               with xopen(__lowerCamelCase					,      "rb"					,      use_auth_token=__lowerCamelCase       ) as f:
                                     lowerCamelCase__		:				List[str]    =  BytesIO(f.read()       )
                               lowerCamelCase__		:				Optional[int]    =  PIL.Image.open(bytes_       )
             else:
                   lowerCamelCase__		:				Dict    =  PIL.Image.open(BytesIO(bytes_       )       )
             image.load()  # to avoid "Too many open files" errors
             return image





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             from .features import Value

             return (
                 self
                 if self.decode
                 else {
                     "bytes": Value("binary"       ),
                     "path": Value("string"       ),
                 }
             )





       def 						lowerCAmelCase     (   self							:			Optional[Any]					,      __lowerCamelCase							:			Union[pa.StringArray, pa.StructArray, pa.ListArray]       ):


             '''simple docstring'''



             if pa.types.is_string(storage.type       ):
                   lowerCamelCase__		:				Dict    =  pa.array([None] * len(__lowerCamelCase       )					,      type=pa.binary()       )
                   lowerCamelCase__		:				List[str]    =  pa.StructArray.from_arrays([bytes_array, storage]					,      ["bytes", "path"]					,      mask=storage.is_null()       )
             elif pa.types.is_binary(storage.type       ):
                   lowerCamelCase__		:				List[Any]    =  pa.array([None] * len(__lowerCamelCase       )					,      type=pa.string()       )
                   lowerCamelCase__		:				Any    =  pa.StructArray.from_arrays([storage, path_array]					,      ["bytes", "path"]					,      mask=storage.is_null()       )
             elif pa.types.is_struct(storage.type       ):
                   if storage.type.get_field_index("bytes"       ) >= 0:
                         lowerCamelCase__		:				Dict    =  storage.field("bytes"       )
                   else:
                         lowerCamelCase__		:				Optional[int]    =  pa.array([None] * len(__lowerCamelCase       )					,      type=pa.binary()       )
                   if storage.type.get_field_index("path"       ) >= 0:
                         lowerCamelCase__		:				Dict    =  storage.field("path"       )
                   else:
                         lowerCamelCase__		:				Dict    =  pa.array([None] * len(__lowerCamelCase       )					,      type=pa.string()       )
                   lowerCamelCase__		:				int    =  pa.StructArray.from_arrays([bytes_array, path_array]					,      ["bytes", "path"]					,      mask=storage.is_null()       )
             elif pa.types.is_list(storage.type       ):
                   lowerCamelCase__		:				Union[str, Any]    =  pa.array(
                       [encode_np_array(np.array(__lowerCamelCase       )       )["bytes"] if arr is not None else None for arr in storage.to_pylist()]					,      type=pa.binary()					,      )
                   lowerCamelCase__		:				Dict    =  pa.array([None] * len(__lowerCamelCase       )					,      type=pa.string()       )
                   lowerCamelCase__		:				Dict    =  pa.StructArray.from_arrays(
                       [bytes_array, path_array]					,      ["bytes", "path"]					,      mask=bytes_array.is_null()       )
             return array_cast(__lowerCamelCase					,      self.pa_type       )





       def 						lowerCAmelCase     (   self							:			int					,      __lowerCamelCase							:			pa.StructArray       ):


             '''simple docstring'''



             @no_op_if_value_is_null
             def path_to_bytes(__lowerCamelCase							:			Union[str, Any]       ):
                   with xopen(__lowerCamelCase					,      "rb"       ) as f:
                         lowerCamelCase__		:				str    =  f.read()
                   return bytes_

             lowerCamelCase__		:				List[Any]    =  pa.array(
                 [
                     (path_to_bytes(x["path"]       ) if x["bytes"] is None else x["bytes"]) if x is not None else None
                     for x in storage.to_pylist()
                 ]					,      type=pa.binary()					,      )
             lowerCamelCase__		:				Optional[int]    =  pa.array(
                 [os.path.basename(__lowerCamelCase       ) if path is not None else None for path in storage.field("path"       ).to_pylist()]					,      type=pa.string()					,      )
             lowerCamelCase__		:				Tuple    =  pa.StructArray.from_arrays([bytes_array, path_array]					,      ["bytes", "path"]					,      mask=bytes_array.is_null()       )
             return array_cast(__lowerCamelCase					,      self.pa_type       )



def       lowercase_		(		):

      """simple docstring"""




      if config.PIL_AVAILABLE:
            import PIL.Image
      else:
            raise ImportError("To support encoding images, please install 'Pillow'."						)

      global _IMAGE_COMPRESSION_FORMATS
      if _IMAGE_COMPRESSION_FORMATS is None:
            PIL.Image.init()
            lowerCamelCase__		:				List[str]    =  list(set(PIL.Image.OPEN.keys()						) & set(PIL.Image.SAVE.keys()						)						)
      return _IMAGE_COMPRESSION_FORMATS

def       lowercase_		(		_A			:    "PIL.Image.Image"						):

      """simple docstring"""




      lowerCamelCase__		:				Optional[Any]    =  BytesIO()
      if image.format in list_image_compression_formats():
            lowerCamelCase__		:				int    =  image.format
      else:
            lowerCamelCase__		:				int    =  "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
      image.save(_A    ,					format=_A						)
      return buffer.getvalue()

def       lowercase_		(		_A			:    "PIL.Image.Image"						):

      """simple docstring"""




      if hasattr(_A    ,					"filename"						) and image.filename != "":
            return {"path": image.filename, "bytes": None}
      else:
            return {"path": None, "bytes": image_to_bytes(_A						)}

def       lowercase_		(		_A			:    np.ndarray						):

      """simple docstring"""




      if config.PIL_AVAILABLE:
            import PIL.Image
      else:
            raise ImportError("To support encoding images, please install 'Pillow'."						)

      lowerCamelCase__		:				int    =  array.dtype
      lowerCamelCase__		:				List[str]    =  dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
      lowerCamelCase__		:				List[str]    =  dtype.kind
      lowerCamelCase__		:				Optional[Any]    =  dtype.itemsize

      lowerCamelCase__		:				Dict    =  None

      # Multi-channel array case (only np.dtype("|u1") is allowed)
      if array.shape[2:]:
            lowerCamelCase__		:				List[Any]    =  np.dtype("|u1"						)
            if dtype_kind not in ["u", "i"]:
                  raise TypeError(
                      F"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."						)
            if dtype is not dest_dtype:
                  warnings.warn(F"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"						)
    # Exact match
      elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
            lowerCamelCase__		:				Any    =  dtype
      else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
            while dtype_itemsize >= 1:
                  lowerCamelCase__		:				Optional[Any]    =  dtype_byteorder + dtype_kind + str(_A						)
                  lowerCamelCase__		:				int    =  np.dtype(_A						)
                  if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
                        warnings.warn(F"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"						)
                        break
                  else:
                        dtype_itemsize //= 2

      if dest_dtype is None:
            raise TypeError(
                F"Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"						)
      lowerCamelCase__		:				List[Any]    =  PIL.Image.fromarray(array.astype(_A						)						)
      return {"path": None, "bytes": image_to_bytes(_A						)}


def       lowercase_		(		_A			:    Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]]						):

      """simple docstring"""




      if config.PIL_AVAILABLE:
            import PIL.Image
      else:
            raise ImportError("To support encoding images, please install 'Pillow'."						)

      if objs:
            lowerCamelCase__    ,   lowerCamelCase__		:				int    =  first_non_null_value(_A						)
            if isinstance(_A    ,					_A						):
                  return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
            if isinstance(_A    ,					np.ndarray						):
                  lowerCamelCase__		:				Optional[Any]    =  no_op_if_value_is_null(_A						)
                  return [obj_to_image_dict_func(_A						) for obj in objs]
            elif isinstance(_A    ,					PIL.Image.Image						):
                  lowerCamelCase__		:				int    =  no_op_if_value_is_null(_A						)
                  return [obj_to_image_dict_func(_A						) for obj in objs]
            else:
                  return objs
      else:
            return objs


