



'''simple docstring'''





import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
  from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
  lowercase_		       =  get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase_		       =  128_022
lowercase_		       =  128_028






@require_sentencepiece
class __A			(  A   ,   unittest.TestCase						):



 '''simple docstring'''







 __lowerCamelCase    :       int          =       MaMaaaTokenizer
 __lowerCamelCase    :       Tuple          =       False
 __lowerCamelCase    :       List[str]          =       False
 __lowerCamelCase    :       Optional[int]          =       True



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      super().setUp()

      _a      =  ['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
      _a      =  dict(zip(A ,   range(len(A       )       )       )       )
      _a      =  Path(self.tmpdirname       )
      save_json(A ,   save_dir / VOCAB_FILES_NAMES['''vocab_file''']       )
      if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
           copyfile(A ,   save_dir / VOCAB_FILES_NAMES['''spm_file''']       )

      _a      =  MaMaaaTokenizer.from_pretrained(self.tmpdirname       )
      tokenizer.save_pretrained(self.tmpdirname       )



 def 					a__  (self ,   **A       )    ->     List[Any]:


      """simple docstring"""



      return MaMaaaTokenizer.from_pretrained(self.tmpdirname ,   **A       )



 def 					a__  (self ,   A       )    ->     Dict:


      """simple docstring"""



      return (
          "This is a test",
          "This is a test",
      )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  '''</s>'''
      _a      =  0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(A       ) ,   A       )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(A       ) ,   A       )



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      _a      =  self.get_tokenizer()
      _a      =  list(tokenizer.get_vocab().keys()       )

      self.assertEqual(vocab_keys[0] ,   '''</s>'''       )
      self.assertEqual(vocab_keys[1] ,   '''<unk>'''       )
      self.assertEqual(vocab_keys[-1] ,   '''<s>'''       )
      self.assertEqual(len(A       ) ,   tokenizer.vocab_size + len(tokenizer.get_added_vocab()       )       )



 @unittest.skip('''Skip this test while all models are still to be uploaded.'''       )
 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      pass



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      _a      =  self.get_tokenizer()

      _a      =  tokenizer.tokenize('''This is a test'''       )
      self.assertListEqual(A ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(A       ) ,   [2, 3, 4, 5, 6] ,   )

      _a      =  tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]       )
      self.assertListEqual(A ,   ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

      _a      =  tokenizer.convert_tokens_to_string(A       )
      self.assertEqual(A ,   '''This is a test'''       )



 @slow
 def 					a__  (self       )    ->     str:


      """simple docstring"""



      _a      =  {'''input_ids''': [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=A ,   model_name='''facebook/m2m100_418M''' ,   revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e''' ,   )






@require_torch
@require_sentencepiece
@require_tokenizers
class __A			(  unittest.TestCase						):



 '''simple docstring'''







 __lowerCamelCase    :       Any          =       'facebook/m2m100_418M'
 __lowerCamelCase    :       Tuple          =       [
     'In my opinion, there are two levels of response from the French government.',
     'NSA Affair Emphasizes Complete Lack of Debate on Intelligence',
 ]
 __lowerCamelCase    :       str          =       [
     'Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.',
     'L\'affaire NSA souligne l\'absence totale de débat sur le renseignement',
 ]

 # fmt: off
 __lowerCamelCase    :       Optional[int]          =       [EN_CODE, 593, 1_949, 115_781, 4, 71_586, 4_234, 60_633, 126_233, 432, 123_808, 15_592, 1_197, 117_132, 120_618, 5, 2]



 @classmethod
 def 					a__  (cls       )    ->     Dict:


      """simple docstring"""



      _a      =  MaMaaaTokenizer.from_pretrained(
          cls.checkpoint_name ,   src_lang='''en''' ,   tgt_lang='''fr'''       )
      _a      =  1
      return cls



 def 					a__  (self       )    ->     Union[str, Any]:


      """simple docstring"""



      self.assertEqual(self.tokenizer.get_lang_id('''ar'''       ) ,   128_006       )
      self.assertEqual(self.tokenizer.get_lang_id('''en'''       ) ,   128_022       )
      self.assertEqual(self.tokenizer.get_lang_id('''ro'''       ) ,   128_076       )
      self.assertEqual(self.tokenizer.get_lang_id('''mr'''       ) ,   128_063       )



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      _a      =  self.tokenizer.get_vocab()
      self.assertEqual(len(A       ) ,   self.tokenizer.vocab_size       )
      self.assertEqual(vocab['''<unk>'''] ,   3       )
      self.assertIn(self.tokenizer.get_lang_token('''en'''       ) ,   A       )



 def 					a__  (self       )    ->     int:


      """simple docstring"""



      _a      =  '''en'''
      _a      =  self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
      self.assertListEqual(self.expected_src_tokens ,   A       )



 def 					a__  (self       )    ->     List[str]:


      """simple docstring"""



      self.assertIn(A ,   self.tokenizer.all_special_ids       )
      # fmt: off
      _a      =  [FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
      # fmt: on
      _a      =  self.tokenizer.decode(A ,   skip_special_tokens=A       )
      _a      =  self.tokenizer.decode(generated_ids[1:] ,   skip_special_tokens=A       )
      self.assertEqual(A ,   A       )
      self.assertNotIn(self.tokenizer.eos_token ,   A       )



 def 					a__  (self       )    ->     int:


      """simple docstring"""



      _a      =  tempfile.mkdtemp()
      _a      =  self.tokenizer.lang_token_to_id
      self.tokenizer.save_pretrained(A       )
      _a      =  MaMaaaTokenizer.from_pretrained(A       )
      self.assertDictEqual(new_tok.lang_token_to_id ,   A       )



 @require_torch
 def 					a__  (self       )    ->     List[Any]:


      """simple docstring"""



      _a      =  '''en'''
      _a      =  '''fr'''

      _a      =  self.tokenizer(self.src_text ,   text_target=self.tgt_text ,   padding=A ,   return_tensors='''pt'''       )

      _a      =  shift_tokens_right(
          batch['''labels'''] ,   self.tokenizer.pad_token_id ,   self.tokenizer.eos_token_id       )

      for k in batch:
           _a      =  batch[k].tolist()
      # batch = {k: v.tolist() for k,v in batch.items()}
      # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
      # batch.decoder_inputs_ids[0][0] ==
      assert batch.input_ids[1][0] == EN_CODE
      assert batch.input_ids[1][-1] == 2
      assert batch.labels[1][0] == FR_CODE
      assert batch.labels[1][-1] == 2
      assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]



 @require_torch
 def 					a__  (self       )    ->     List[Any]:


      """simple docstring"""



      _a      =  '''mr'''
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id('''mr'''       )]       )
      self.assertListEqual(self.tokenizer.suffix_tokens ,   [self.tokenizer.eos_token_id]       )

      _a      =  '''zh'''
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id('''zh'''       )]       )
      self.assertListEqual(self.tokenizer.suffix_tokens ,   [self.tokenizer.eos_token_id]       )



 @require_torch
 def 					a__  (self       )    ->     List[Any]:


      """simple docstring"""



      _a      =  '''mr'''
      self.tokenizer._switch_to_target_mode()
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id('''mr'''       )]       )
      self.assertListEqual(self.tokenizer.suffix_tokens ,   [self.tokenizer.eos_token_id]       )
      self.tokenizer._switch_to_input_mode()
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id(self.tokenizer.src_lang       )]       )

      _a      =  '''zh'''
      self.tokenizer._switch_to_target_mode()
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id('''zh'''       )]       )
      self.assertListEqual(self.tokenizer.suffix_tokens ,   [self.tokenizer.eos_token_id]       )
      self.tokenizer._switch_to_input_mode()
      self.assertListEqual(self.tokenizer.prefix_tokens ,   [self.tokenizer.get_lang_id(self.tokenizer.src_lang       )]       )



 @require_torch
 def 					a__  (self       )    ->     int:


      """simple docstring"""



      _a      =  self.tokenizer._build_translation_inputs('''A test''' ,   return_tensors='''pt''' ,   src_lang='''en''' ,   tgt_lang='''ar'''       )

      self.assertEqual(
          nested_simplify(A       ) ,   {
              # en_XX, A, test, EOS
              '''input_ids''': [[128_022, 58, 4_183, 2]],
              '''attention_mask''': [[1, 1, 1, 1]],
              # ar_AR
              '''forced_bos_token_id''': 128_006,
          } ,   )








'''simple docstring'''





import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase




def lowerCAmelCase  (__A  ,	__A  ,	__A  ,	__A):



     """simple docstring"""



     if isinstance(__A  ,	__A):
          _a      =  np.full((len(__A), sequence_length, 2)  ,	__A)
     else:
          _a      =  np.full((len(__A), sequence_length)  ,	__A)

     for i, tensor in enumerate(__A):
          if padding_side == "right":
               if isinstance(__A  ,	__A):
                    _a      =  tensor[:sequence_length]
               else:
                    _a      =  tensor[:sequence_length]
          else:
               if isinstance(__A  ,	__A):
                    _a      =  tensor[:sequence_length]
               else:
                    _a      =  tensor[:sequence_length]

     return out_tensor.tolist()




def lowerCAmelCase  (__A):



     """simple docstring"""



     _a      =  ord(__A)
     if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
          return True
     _a      =  unicodedata.category(__A)
     if cat.startswith('''P'''):
          return True
     return False






@dataclass
class __A			(  A						):



 '''simple docstring'''







 __lowerCamelCase    :       PreTrainedTokenizerBase
 __lowerCamelCase    :       Union[bool, str, PaddingStrategy]          =       True
 __lowerCamelCase    :       Optional[int]          =       None
 __lowerCamelCase    :       Optional[int]          =       None
 __lowerCamelCase    :       int          =       -100
 __lowerCamelCase    :       str          =       "pt"



 def 					a__  (self ,   A       )    ->     List[str]:


      """simple docstring"""



      import torch

      _a      =  '''label''' if '''label''' in features[0].keys() else '''labels'''
      _a      =  [feature[label_name] for feature in features] if label_name in features[0].keys() else None
      _a      =  self.tokenizer.pad(
          A ,   padding=self.padding ,   max_length=self.max_length ,   pad_to_multiple_of=self.pad_to_multiple_of ,   return_tensors='''pt''' if labels is None else None ,   )

      if labels is None:
           return batch

      _a      =  torch.tensor(batch['''entity_ids''']       ).shape[1]
      _a      =  self.tokenizer.padding_side
      if padding_side == "right":
           _a      =  [
               list(A       ) + [self.label_pad_token_id] * (sequence_length - len(A       )) for label in labels
           ]
      else:
           _a      =  [
               [self.label_pad_token_id] * (sequence_length - len(A       )) + list(A       ) for label in labels
           ]

      _a      =  [feature['''ner_tags'''] for feature in features]
      _a      =  padding_tensor(A ,   -1 ,   A ,   A       )
      _a      =  [feature['''original_entity_spans'''] for feature in features]
      _a      =  padding_tensor(A ,   (-1, -1) ,   A ,   A       )
      _a      =  {k: torch.tensor(A ,   dtype=torch.intaa       ) for k, v in batch.items()}

      return batch



