


'''simple docstring'''



import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class    UpperCAmelCase_  (_UpperCAmelCase					,		unittest.TestCase  ):





    """simple docstring"""






    lowerCamelCase  :					List[str]							     =			GPTaTokenizer
    lowerCamelCase  :					Dict							     =			GPTaTokenizerFast
    lowerCamelCase  :					str							     =			True
    lowerCamelCase  :					Dict							     =			{'add_prefix_space': True}
    lowerCamelCase  :					Dict							     =			False







    def       lowercase_				(							self	)				->     Optional[int]:
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         __lowerCamelCase   :	Tuple								=		[
             'l',
             'o',
             'w',
             'e',
             'r',
             's',
             't',
             'i',
             'd',
             'n',
             '\u0120',
             '\u0120l',
             '\u0120n',
             '\u0120lo',
             '\u0120low',
             'er',
             '\u0120lowest',
             '\u0120newer',
             '\u0120wider',
             '<unk>',
             '<|endoftext|>',
         ]
         __lowerCamelCase   :	Dict								=		dict(zip(SCREAMING_SNAKE_CASE_			,     range(len(SCREAMING_SNAKE_CASE_	)	)	)	)
         __lowerCamelCase   :	Union[str, Any]								=		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
         __lowerCamelCase   :	Any								=		{'unk_token': '<unk>'}

         __lowerCamelCase   :	Any								=		os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['vocab_file']	)
         __lowerCamelCase   :	Any								=		os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES['merges_file']	)
         with open(self.vocab_file			,     'w'			,     encoding='utf-8'	) as fp:
              fp.write(json.dumps(SCREAMING_SNAKE_CASE_	) + '\n'	)
         with open(self.merges_file			,     'w'			,     encoding='utf-8'	) as fp:
              fp.write('\n'.join(SCREAMING_SNAKE_CASE_	)	)







    def       lowercase_				(							self			,     **SCREAMING_SNAKE_CASE_	)				->     int:
         kwargs.update(self.special_tokens_map	)
         return GPTaTokenizer.from_pretrained(self.tmpdirname			,     **SCREAMING_SNAKE_CASE_	)







    def       lowercase_				(							self			,     **SCREAMING_SNAKE_CASE_	)				->     Union[str, Any]:
         kwargs.update(self.special_tokens_map	)
         return GPTaTokenizerFast.from_pretrained(self.tmpdirname			,     **SCREAMING_SNAKE_CASE_	)







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_	)				->     Union[str, Any]:
         __lowerCamelCase   :	str								=		'lower newer'
         __lowerCamelCase   :	Optional[int]								=		'lower newer'
         return input_text, output_text







    def       lowercase_				(							self	)				->     Tuple:
         __lowerCamelCase   :	int								=		GPTaTokenizer(self.vocab_file			,     self.merges_file			,     **self.special_tokens_map	)
         __lowerCamelCase   :	Optional[Any]								=		'lower newer'
         __lowerCamelCase   :	List[str]								=		['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
         __lowerCamelCase   :	Optional[Any]								=		tokenizer.tokenize(SCREAMING_SNAKE_CASE_			,     add_prefix_space=SCREAMING_SNAKE_CASE_	)
         self.assertListEqual(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)

         __lowerCamelCase   :	Union[str, Any]								=		tokens + [tokenizer.unk_token]
         __lowerCamelCase   :	Optional[int]								=		[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)			,     SCREAMING_SNAKE_CASE_	)







    def       lowercase_				(							self	)				->     Tuple:
         if not self.test_rust_tokenizer:
              return

         __lowerCamelCase   :	str								=		self.get_tokenizer()
         __lowerCamelCase   :	Tuple								=		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE_	)

         __lowerCamelCase   :	Tuple								=		'lower newer'

         # Testing tokenization
         __lowerCamelCase   :	Any								=		tokenizer.tokenize(SCREAMING_SNAKE_CASE_			,     add_prefix_space=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	str								=		rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_	)
         self.assertListEqual(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)

         # Testing conversion to ids without special tokens
         __lowerCamelCase   :	int								=		tokenizer.encode(SCREAMING_SNAKE_CASE_			,     add_special_tokens=SCREAMING_SNAKE_CASE_			,     add_prefix_space=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	Optional[int]								=		rust_tokenizer.encode(SCREAMING_SNAKE_CASE_			,     add_special_tokens=SCREAMING_SNAKE_CASE_	)
         self.assertListEqual(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)

         # Testing conversion to ids with special tokens
         __lowerCamelCase   :	Optional[int]								=		self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	Union[str, Any]								=		tokenizer.encode(SCREAMING_SNAKE_CASE_			,     add_prefix_space=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	int								=		rust_tokenizer.encode(SCREAMING_SNAKE_CASE_	)
         self.assertListEqual(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)

         # Testing the unknown token
         __lowerCamelCase   :	Any								=		tokens + [rust_tokenizer.unk_token]
         __lowerCamelCase   :	Optional[int]								=		[14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)			,     SCREAMING_SNAKE_CASE_	)







    def       lowercase_				(							self			,     *SCREAMING_SNAKE_CASE_			,     **SCREAMING_SNAKE_CASE_	)				->     Dict:
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass







    def       lowercase_				(							self			,     SCREAMING_SNAKE_CASE_=15	)				->     Any:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'	):
                   __lowerCamelCase   :	Any								=		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_			,     **SCREAMING_SNAKE_CASE_	)

                   # Simple input
                   __lowerCamelCase   :	str								=		'This is a simple input'
                   __lowerCamelCase   :	Union[str, Any]								=		['This is a simple input 1', 'This is a simple input 2']
                   __lowerCamelCase   :	Optional[Any]								=		('This is a simple input', 'This is a pair')
                   __lowerCamelCase   :	Optional[Any]								=		[
                       ('This is a simple input 1', 'This is a simple input 2'),
                       ('This is a simple pair 1', 'This is a simple pair 2'),
                   ]

                   # Simple input tests
                   self.assertRaises(SCREAMING_SNAKE_CASE_			,     tokenizer_r.encode			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'	)

                   # Simple input
                   self.assertRaises(SCREAMING_SNAKE_CASE_			,     tokenizer_r.encode_plus			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'	)

                   # Simple input
                   self.assertRaises(
                       SCREAMING_SNAKE_CASE_			,     tokenizer_r.batch_encode_plus			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'			,     )

                   # Pair input
                   self.assertRaises(SCREAMING_SNAKE_CASE_			,     tokenizer_r.encode			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'	)

                   # Pair input
                   self.assertRaises(SCREAMING_SNAKE_CASE_			,     tokenizer_r.encode_plus			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'	)

                   # Pair input
                   self.assertRaises(
                       SCREAMING_SNAKE_CASE_			,     tokenizer_r.batch_encode_plus			,     SCREAMING_SNAKE_CASE_			,     max_length=SCREAMING_SNAKE_CASE_			,     padding='max_length'			,     )







    def       lowercase_				(							self	)				->     Any:
         __lowerCamelCase   :	List[str]								=		GPTaTokenizer.from_pretrained(self.tmpdirname			,     pad_token='<pad>'	)

         # Simple input
         __lowerCamelCase   :	Optional[Any]								=		'This is a simple input'
         __lowerCamelCase   :	Any								=		['This is a simple input looooooooong', 'This is a simple input']
         __lowerCamelCase   :	Optional[Any]								=		('This is a simple input', 'This is a pair')
         __lowerCamelCase   :	Tuple								=		[
             ('This is a simple input loooooong', 'This is a simple input'),
             ('This is a simple pair loooooong', 'This is a simple pair'),
         ]

         __lowerCamelCase   :	List[str]								=		tokenizer.pad_token_id

         __lowerCamelCase   :	List[Any]								=		tokenizer(SCREAMING_SNAKE_CASE_			,     padding='max_length'			,     max_length=30			,     return_tensors='np'	)
         __lowerCamelCase   :	Any								=		tokenizer(SCREAMING_SNAKE_CASE_			,     padding=SCREAMING_SNAKE_CASE_			,     truncate=SCREAMING_SNAKE_CASE_			,     return_tensors='np'	)
         __lowerCamelCase   :	str								=		tokenizer(*SCREAMING_SNAKE_CASE_			,     padding='max_length'			,     max_length=60			,     return_tensors='np'	)
         __lowerCamelCase   :	List[str]								=		tokenizer(SCREAMING_SNAKE_CASE_			,     padding=SCREAMING_SNAKE_CASE_			,     truncate=SCREAMING_SNAKE_CASE_			,     return_tensors='np'	)

         # s
         # test single string max_length padding
         self.assertEqual(out_s['input_ids'].shape[-1]			,     30	)
         self.assertTrue(pad_token_id in out_s['input_ids']	)
         self.assertTrue(0 in out_s['attention_mask']	)

         # s2
         # test automatic padding
         self.assertEqual(out_sa['input_ids'].shape[-1]			,     33	)
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['input_ids'][0]	)
         self.assertFalse(0 in out_sa['attention_mask'][0]	)
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['input_ids'][1]	)
         self.assertTrue(0 in out_sa['attention_mask'][1]	)

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['input_ids'].shape[-1]			,     60	)
         self.assertTrue(pad_token_id in out_p['input_ids']	)
         self.assertTrue(0 in out_p['attention_mask']	)

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['input_ids'].shape[-1]			,     52	)
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['input_ids'][0]	)
         self.assertFalse(0 in out_pa['attention_mask'][0]	)
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['input_ids'][1]	)
         self.assertTrue(0 in out_pa['attention_mask'][1]	)







    def       lowercase_				(							self	)				->     int:
         __lowerCamelCase   :	List[str]								=		'$$$'
         __lowerCamelCase   :	Optional[int]								=		GPTaTokenizer.from_pretrained(self.tmpdirname			,     bos_token=SCREAMING_SNAKE_CASE_			,     add_bos_token=SCREAMING_SNAKE_CASE_	)

         __lowerCamelCase   :	List[Any]								=		'This is a simple input'
         __lowerCamelCase   :	int								=		['This is a simple input 1', 'This is a simple input 2']

         __lowerCamelCase   :	Optional[int]								=		tokenizer.bos_token_id

         __lowerCamelCase   :	Optional[Any]								=		tokenizer(SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	Tuple								=		tokenizer(SCREAMING_SNAKE_CASE_	)

         self.assertEqual(out_s.input_ids[0]			,     SCREAMING_SNAKE_CASE_	)
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids	)	)

         __lowerCamelCase   :	Tuple								=		tokenizer.decode(out_s.input_ids	)
         __lowerCamelCase   :	int								=		tokenizer.batch_decode(out_sa.input_ids	)

         self.assertEqual(decode_s.split()[0]			,     SCREAMING_SNAKE_CASE_	)
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa	)	)







    def       lowercase_				(							self	)				->     List[str]:
         pass







    def       lowercase_				(							self	)				->     Any:
         # TODO: change to self.get_tokenizers() when the fast version is implemented
         __lowerCamelCase   :	Tuple								=		[self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE_			,     add_bos_token=SCREAMING_SNAKE_CASE_	)]
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'	):
                   __lowerCamelCase   :	Dict								=		'Encode this.'
                   __lowerCamelCase   :	Tuple								=		'This one too please.'
                   __lowerCamelCase   :	List[Any]								=		tokenizer.encode(SCREAMING_SNAKE_CASE_			,     add_special_tokens=SCREAMING_SNAKE_CASE_	)
                   encoded_sequence += tokenizer.encode(SCREAMING_SNAKE_CASE_			,     add_special_tokens=SCREAMING_SNAKE_CASE_	)
                   __lowerCamelCase   :	Tuple								=		tokenizer.encode_plus(
                       SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_			,     add_special_tokens=SCREAMING_SNAKE_CASE_			,     return_special_tokens_mask=SCREAMING_SNAKE_CASE_			,     )
                   __lowerCamelCase   :	Optional[Any]								=		encoded_sequence_dict['input_ids']
                   __lowerCamelCase   :	Optional[Any]								=		encoded_sequence_dict['special_tokens_mask']
                   self.assertEqual(len(SCREAMING_SNAKE_CASE_	)			,     len(SCREAMING_SNAKE_CASE_	)	)

                   __lowerCamelCase   :	int								=		[
                       (x if not special_tokens_mask[i] else None) for i, x in enumerate(SCREAMING_SNAKE_CASE_	)
                   ]
                   __lowerCamelCase   :	int								=		[x for x in filtered_sequence if x is not None]
                   self.assertEqual(SCREAMING_SNAKE_CASE_			,     SCREAMING_SNAKE_CASE_	)







@require_tokenizers
class    UpperCAmelCase_  (unittest.TestCase  ):





    """simple docstring"""






    def       lowercase_				(							self	)				->     Union[str, Any]:
         # More context:
         # https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
         # https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
         # https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

         __lowerCamelCase   :	Optional[int]								=		AutoTokenizer.from_pretrained('facebook/opt-350m'			,     from_slow=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	List[Any]								=		'A photo of a cat'

         __lowerCamelCase   :	Any								=		tokenizer.encode(
             SCREAMING_SNAKE_CASE_			,     )
         self.assertEqual(SCREAMING_SNAKE_CASE_			,     [2, 2_50, 13_45, 9, 10, 47_58]	)
         tokenizer.save_pretrained('test_opt'	)

         __lowerCamelCase   :	Union[str, Any]								=		AutoTokenizer.from_pretrained('./test_opt'	)
         __lowerCamelCase   :	Optional[Any]								=		tokenizer.encode(
             SCREAMING_SNAKE_CASE_			,     )
         self.assertEqual(SCREAMING_SNAKE_CASE_			,     [2, 2_50, 13_45, 9, 10, 47_58]	)







    def       lowercase_				(							self	)				->     Optional[int]:
         __lowerCamelCase   :	Optional[Any]								=		AutoTokenizer.from_pretrained('facebook/opt-350m'			,     use_slow=SCREAMING_SNAKE_CASE_	)
         __lowerCamelCase   :	List[str]								=		'A photo of a cat'

         __lowerCamelCase   :	Union[str, Any]								=		tokenizer.encode(
             SCREAMING_SNAKE_CASE_			,     )
         # Same as above
         self.assertEqual(SCREAMING_SNAKE_CASE_			,     [2, 2_50, 13_45, 9, 10, 47_58]	)







    @unittest.skip('This test is failing because of a bug in the fast tokenizer'	)
    def       lowercase_				(							self	)				->     int:
         __lowerCamelCase   :	Union[str, Any]								=		AutoTokenizer.from_pretrained('facebook/opt-350m'			,     from_slow=SCREAMING_SNAKE_CASE_	)

         __lowerCamelCase   :	Optional[Any]								=		'bos'
         __lowerCamelCase   :	int								=		tokenizer.get_vocab()['bos']

         __lowerCamelCase   :	Union[str, Any]								=		'A photo of a cat'
         __lowerCamelCase   :	Union[str, Any]								=		tokenizer.encode(
             SCREAMING_SNAKE_CASE_			,     )
         # We changed the bos token
         self.assertEqual(SCREAMING_SNAKE_CASE_			,     [3_19_57, 2_50, 13_45, 9, 10, 47_58]	)
         tokenizer.save_pretrained('./tok'	)
         __lowerCamelCase   :	Optional[int]								=		AutoTokenizer.from_pretrained('./tok'	)
         self.assertTrue(tokenizer.is_fast	)
         __lowerCamelCase   :	Union[str, Any]								=		tokenizer.encode(
             SCREAMING_SNAKE_CASE_			,     )
         self.assertEqual(SCREAMING_SNAKE_CASE_			,     [3_19_57, 2_50, 13_45, 9, 10, 47_58]	)










'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


A__		:					List[Any]												=  {
    """configuration_megatron_bert""": ["""MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MegatronBertConfig"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__		:					int												=  [
          """MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """MegatronBertForCausalLM""",
          """MegatronBertForMaskedLM""",
          """MegatronBertForMultipleChoice""",
          """MegatronBertForNextSentencePrediction""",
          """MegatronBertForPreTraining""",
          """MegatronBertForQuestionAnswering""",
          """MegatronBertForSequenceClassification""",
          """MegatronBertForTokenClassification""",
          """MegatronBertModel""",
          """MegatronBertPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_megatron_bert import (
                MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                MegatronBertForCausalLM,
                MegatronBertForMaskedLM,
                MegatronBertForMultipleChoice,
                MegatronBertForNextSentencePrediction,
                MegatronBertForPreTraining,
                MegatronBertForQuestionAnswering,
                MegatronBertForSequenceClassification,
                MegatronBertForTokenClassification,
                MegatronBertModel,
                MegatronBertPreTrainedModel,
            )

else:
      import sys

      A__		:					List[Any]												=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






