


'''simple docstring'''
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
  from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    					=      get_tests_dir('''fixtures/test_sentencepiece.model''')

UpperCamelCase__    					=      {'''target_lang''': '''fi''', '''source_lang''': '''en'''}
UpperCamelCase__    					=      '''>>zh<<'''
UpperCamelCase__    					=      '''Helsinki-NLP/'''

if is_torch_available():
  UpperCamelCase__    					=      '''pt'''
elif is_tf_available():
  UpperCamelCase__    					=      '''tf'''
else:
  UpperCamelCase__    					=      '''jax'''




@require_sentencepiece
class 			lowerCamelCase_   (	__a     ,       unittest.TestCase      ):
      lowerCAmelCase__		     =     MarianTokenizer
      lowerCAmelCase__		     =     False
      lowerCAmelCase__		     =     True






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          super().setUp()
          UpperCAmelCase__			:		Optional[Any]													=				['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
          UpperCAmelCase__			:		List[str]													=				dict(zip(_A						,						range(len(_A ) ) ) )
          UpperCAmelCase__			:		str													=				Path(self.tmpdirname )
          save_json(_A						,						save_dir / VOCAB_FILES_NAMES['''vocab'''] )
          save_json(_A						,						save_dir / VOCAB_FILES_NAMES['''tokenizer_config_file'''] )
          if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
              copyfile(_A						,						save_dir / VOCAB_FILES_NAMES['''source_spm'''] )
              copyfile(_A						,						save_dir / VOCAB_FILES_NAMES['''target_spm'''] )

          UpperCAmelCase__			:		int													=				MarianTokenizer.from_pretrained(self.tmpdirname )
          tokenizer.save_pretrained(self.tmpdirname )






      def lowercase_							(					self				:    Dict						,						**_A				:    List[Any] ):


          '''simple docstring'''


          return MarianTokenizer.from_pretrained(self.tmpdirname						,						**_A )






      def lowercase_							(					self				:    Optional[Any]						,						_A				:    Union[str, Any] ):


          '''simple docstring'''


          return (
              "This is a test",
              "This is a test",
          )






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				'''</s>'''
          UpperCAmelCase__			:		Union[str, Any]													=				0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A )						,						_A )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A )						,						_A )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				list(self.get_tokenizer().get_vocab().keys() )

          self.assertEqual(vocab_keys[0]						,						'''</s>''' )
          self.assertEqual(vocab_keys[1]						,						'''<unk>''' )
          self.assertEqual(vocab_keys[-1]						,						'''<pad>''' )
          self.assertEqual(len(_A )						,						9 )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          self.assertEqual(self.get_tokenizer().vocab_size						,						9 )






      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		int													=				MarianTokenizer.from_pretrained(f"""{ORG_NAME}opus-mt-en-de""" )
          UpperCAmelCase__			:		str													=				en_de_tokenizer(['''I am a small frog''']						,						return_tensors=_A )
          self.assertIsInstance(_A						,						_A )
          UpperCAmelCase__			:		str													=				[38, 121, 14, 697, 38_848, 0]
          self.assertListEqual(_A						,						batch.input_ids[0] )

          UpperCAmelCase__			:		List[Any]													=				tempfile.mkdtemp()
          en_de_tokenizer.save_pretrained(_A )
          UpperCAmelCase__			:		Dict													=				[x.name for x in Path(_A ).glob('''*''' )]
          self.assertIn('''source.spm'''						,						_A )
          MarianTokenizer.from_pretrained(_A )






      def lowercase_							(					self				:    Tuple ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				self.get_tokenizer()

          UpperCAmelCase__			:		Tuple													=				tok(
              ['''I am a small frog''' * 1_000, '''I am a small frog''']						,						padding=_A						,						truncation=_A						,						return_tensors=_A )
          self.assertIsInstance(_A						,						_A )
          self.assertEqual(batch.input_ids.shape						,						(2, 512) )






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				self.get_tokenizer()
          UpperCAmelCase__			:		Union[str, Any]													=				tok(['''I am a tiny frog''', '''I am a small frog''']						,						padding=_A						,						return_tensors=_A )
          self.assertIsInstance(_A						,						_A )
          self.assertEqual(batch_smaller.input_ids.shape						,						(2, 10) )






      @slow
      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Any													=				{'''input_ids''': [[43_495, 462, 20, 42_164, 1_369, 52, 464, 132, 1_703, 492, 13, 7_491, 38_999, 6, 8, 464, 132, 1_703, 492, 13, 4_669, 37_867, 13, 7_525, 27, 1_593, 988, 13, 33_972, 7_029, 6, 20, 8_251, 383, 2, 270, 5_866, 3_788, 2, 2_353, 8_251, 12_338, 2, 13_958, 387, 2, 3_629, 6_953, 188, 2_900, 2, 13_958, 8_011, 11_501, 23, 8_460, 4_073, 34_009, 20, 435, 11_439, 27, 8, 8_460, 4_073, 6_004, 20, 9_988, 375, 27, 33, 266, 1_945, 1_076, 1_350, 37_867, 3_288, 5, 577, 1_076, 4_374, 8, 5_082, 5, 26_453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10_767, 6, 316, 304, 4_239, 3, 0], [148, 15_722, 19, 1_839, 12, 1_350, 13, 22_327, 5_082, 5_418, 47_567, 35_938, 59, 318, 19_552, 108, 2_183, 54, 14_976, 4_835, 32, 547, 1_114, 8, 315, 2_417, 5, 92, 19_088, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100], [36, 6_395, 12_570, 39_147, 11_597, 6, 266, 4, 45_405, 7_296, 3, 0, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100, 58_100]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_A						,						model_name='''Helsinki-NLP/opus-mt-en-de'''						,						revision='''1a8c2263da11e68e50938f97e10cd57820bd504c'''						,						decode_kwargs={'''use_source_tokenizer''': True}						,						)






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				MarianTokenizer.from_pretrained('''hf-internal-testing/test-marian-two-vocabs''' )

          UpperCAmelCase__			:		Tuple													=				'''Tämä on testi'''
          UpperCAmelCase__			:		int													=				'''This is a test'''

          UpperCAmelCase__			:		str													=				[76, 7, 2_047, 2]
          UpperCAmelCase__			:		Optional[Any]													=				[69, 12, 11, 940, 2]

          UpperCAmelCase__			:		int													=				tokenizer(_A ).input_ids
          self.assertListEqual(_A						,						_A )

          UpperCAmelCase__			:		List[str]													=				tokenizer(text_target=_A ).input_ids
          self.assertListEqual(_A						,						_A )

          UpperCAmelCase__			:		List[str]													=				tokenizer.decode(_A						,						skip_special_tokens=_A )
          self.assertEqual(_A						,						_A )




'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__    					=      logging.get_logger(__name__)

UpperCamelCase__    					=      '''▁'''

UpperCamelCase__    					=      {'''vocab_file''': '''sentencepiece.bpe.model'''}

UpperCamelCase__    					=      {
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

UpperCamelCase__    					=      {
    '''facebook/xglm-564M''': 2_0_4_8,
}




class 			lowerCamelCase_   (	__a      ):
      lowerCAmelCase__		     =     VOCAB_FILES_NAMES
      lowerCAmelCase__		     =     PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__		     =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__		     =     ['input_ids', 'attention_mask']






      def __init__(					self				:    Optional[int]						,						_A				:    Optional[Any]						,						_A				:    Optional[Any]="<s>"						,						_A				:    List[str]="</s>"						,						_A				:    Optional[Any]="</s>"						,						_A				:    List[str]="<s>"						,						_A				:    Tuple="<unk>"						,						_A				:    List[str]="<pad>"						,						_A				:    Optional[Dict[str, Any]] = None						,						**_A				:    Union[str, Any]						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				{} if sp_model_kwargs is None else sp_model_kwargs

          # Compatibility with the original tokenizer
          UpperCAmelCase__			:		Optional[int]													=				7
          UpperCAmelCase__			:		Optional[int]													=				[f"""<madeupword{i}>""" for i in range(self.num_madeup_words )]

          UpperCAmelCase__			:		Tuple													=				kwargs.get('''additional_special_tokens'''						,						[] )
          kwargs["additional_special_tokens"] += [
              word for word in madeup_words if word not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              bos_token=_A						,						eos_token=_A						,						unk_token=_A						,						sep_token=_A						,						cls_token=_A						,						pad_token=_A						,						sp_model_kwargs=self.sp_model_kwargs						,						**_A						,						)

          UpperCAmelCase__			:		Any													=				spm.SentencePieceProcessor(**self.sp_model_kwargs )
          self.sp_model.Load(str(_A ) )
          UpperCAmelCase__			:		List[str]													=				vocab_file

          # Original fairseq vocab and spm vocab must be "aligned":
          # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
          # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
          # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
          # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

          # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
          UpperCAmelCase__			:		Any													=				1

          # Mimic fairseq token-to-id alignment for the first 4 token
          UpperCAmelCase__			:		Any													=				{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

          UpperCAmelCase__			:		int													=				len(self.sp_model )
          UpperCAmelCase__			:		Optional[int]													=				{f"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words )}
          self.fairseq_tokens_to_ids.update(_A )

          UpperCAmelCase__			:		Tuple													=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}






      def __getstate__(					self				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				self.__dict__.copy()
          UpperCAmelCase__			:		Optional[int]													=				None
          UpperCAmelCase__			:		Optional[Any]													=				self.sp_model.serialized_model_proto()
          return state






      def __setstate__(					self				:    Union[str, Any]						,						_A				:    int ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				d

          # for backward compatibility
          if not hasattr(self						,						'''sp_model_kwargs''' ):
              UpperCAmelCase__			:		Dict													=				{}

          UpperCAmelCase__			:		str													=				spm.SentencePieceProcessor(**self.sp_model_kwargs )
          self.sp_model.LoadFromSerializedProto(self.sp_model_proto )






      def lowercase_							(					self				:    Any						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          if token_ids_a is None:
              return [self.sep_token_id] + token_ids_a
          UpperCAmelCase__			:		Dict													=				[self.sep_token_id]
          return sep + token_ids_a + sep + sep + token_ids_a






      def lowercase_							(					self				:    str						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None						,						_A				:    bool = False ):


          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_A						,						token_ids_a=_A						,						already_has_special_tokens=_A )

          if token_ids_a is None:
              return [1] + ([0] * len(_A ))
          return [1] + ([0] * len(_A )) + [1, 1] + ([0] * len(_A ))






      def lowercase_							(					self				:    Any						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				[self.sep_token_id]

          if token_ids_a is None:
              return len(sep + token_ids_a ) * [0]
          return len(sep + token_ids_a + sep + sep + token_ids_a ) * [0]






      @property
      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          return len(self.sp_model ) + self.fairseq_offset + self.num_madeup_words






      def lowercase_							(					self				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				{self.convert_ids_to_tokens(_A ): i for i in range(self.vocab_size )}
          vocab.update(self.added_tokens_encoder )
          return vocab






      def lowercase_							(					self				:    Optional[Any]						,						_A				:    str ):


          '''simple docstring'''


          return self.sp_model.encode(_A						,						out_type=_A )






      def lowercase_							(					self				:    List[str]						,						_A				:    List[Any] ):


          '''simple docstring'''


          if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
          UpperCAmelCase__			:		Union[str, Any]													=				self.sp_model.PieceToId(_A )

          # Need to return unknown token if the SP model returned 0
          return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






      def lowercase_							(					self				:    List[Any]						,						_A				:    str ):


          '''simple docstring'''


          if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
          return self.sp_model.IdToPiece(index - self.fairseq_offset )






      def lowercase_							(					self				:    int						,						_A				:    Optional[int] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				''''''.join(_A ).replace(_A						,						''' ''' ).strip()
          return out_string






      def lowercase_							(					self				:    Any						,						_A				:    str						,						_A				:    Optional[str] = None ):


          '''simple docstring'''


          if not os.path.isdir(_A ):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
              return
          UpperCAmelCase__			:		List[str]													=				os.path.join(
              _A						,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

          if os.path.abspath(self.vocab_file ) != os.path.abspath(_A ) and os.path.isfile(self.vocab_file ):
              copyfile(self.vocab_file						,						_A )
          elif not os.path.isfile(self.vocab_file ):
              with open(_A						,						'''wb''' ) as fi:
                  UpperCAmelCase__			:		List[Any]													=				self.sp_model.serialized_model_proto()
                  fi.write(_A )

          return (out_vocab_file,)
