



'''simple docstring'''





import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
   from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase											=						get_tests_dir('fixtures/test_sentencepiece.model')

_UpperCamelCase											=						{'target_lang': 'fi', 'source_lang': 'en'}
_UpperCamelCase											=						'>>zh<<'
_UpperCamelCase											=						'Helsinki-NLP/'

if is_torch_available():
   _UpperCamelCase											=						'pt'
elif is_tf_available():
   _UpperCamelCase											=						'tf'
else:
   _UpperCamelCase											=						'jax'






@require_sentencepiece
class 					lowerCamelCase_    (				SCREAMING_SNAKE_CASE_   ,   unittest.TestCase     ):
  """simple docstring"""




  a_												=MarianTokenizer
  a_												=False
  a_												=True




  def 						_lowercase     ( self      :	int   )					->					List[Any]:
    super().setUp()
    __lowerCamelCase       :       Any           = ['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
    __lowerCamelCase       :       Optional[int]           = dict(zip(_a			,	range(len(_a   )   )   )   )
    __lowerCamelCase       :       str           = Path(self.tmpdirname   )
    save_json(_a			,	save_dir / VOCAB_FILES_NAMES['vocab']   )
    save_json(_a			,	save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']   )
    if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
      copyfile(_a			,	save_dir / VOCAB_FILES_NAMES['source_spm']   )
      copyfile(_a			,	save_dir / VOCAB_FILES_NAMES['target_spm']   )

    __lowerCamelCase       :       Union[str, Any]           = MarianTokenizer.from_pretrained(self.tmpdirname   )
    tokenizer.save_pretrained(self.tmpdirname   )




  def 						_lowercase     ( self      :	int			,	**_a      :	Optional[Any]   )					->					MarianTokenizer:
    return MarianTokenizer.from_pretrained(self.tmpdirname			,	**_a   )




  def 						_lowercase     ( self      :	List[Any]			,	_a      :	Optional[Any]   )					->					int:
    return (
        "This is a test",
        "This is a test",
    )




  def 						_lowercase     ( self      :	Dict   )					->					Tuple:
    __lowerCamelCase       :       Union[str, Any]           = '</s>'
    __lowerCamelCase       :       Any           = 0

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a   )			,	_a   )
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a   )			,	_a   )




  def 						_lowercase     ( self      :	Union[str, Any]   )					->					str:
    __lowerCamelCase       :       Dict           = list(self.get_tokenizer().get_vocab().keys()   )

    self.assertEqual(vocab_keys[0]			,	'</s>'   )
    self.assertEqual(vocab_keys[1]			,	'<unk>'   )
    self.assertEqual(vocab_keys[-1]			,	'<pad>'   )
    self.assertEqual(len(_a   )			,	9   )




  def 						_lowercase     ( self      :	Tuple   )					->					str:
    self.assertEqual(self.get_tokenizer().vocab_size			,	9   )




  def 						_lowercase     ( self      :	List[str]   )					->					str:
    __lowerCamelCase       :       List[str]           = MarianTokenizer.from_pretrained(f'{ORG_NAME}opus-mt-en-de'   )
    __lowerCamelCase       :       Any           = en_de_tokenizer(['I am a small frog']			,	return_tensors=_a   )
    self.assertIsInstance(_a			,	_a   )
    __lowerCamelCase       :       List[Any]           = [38, 121, 14, 697, 3_8848, 0]
    self.assertListEqual(_a			,	batch.input_ids[0]   )

    __lowerCamelCase       :       str           = tempfile.mkdtemp()
    en_de_tokenizer.save_pretrained(_a   )
    __lowerCamelCase       :       int           = [x.name for x in Path(_a   ).glob('*'   )]
    self.assertIn('source.spm'			,	_a   )
    MarianTokenizer.from_pretrained(_a   )




  def 						_lowercase     ( self      :	Union[str, Any]   )					->					int:
    __lowerCamelCase       :       Dict           = self.get_tokenizer()

    __lowerCamelCase       :       Any           = tok(
        ['I am a small frog' * 1000, 'I am a small frog']			,	padding=_a			,	truncation=_a			,	return_tensors=_a   )
    self.assertIsInstance(_a			,	_a   )
    self.assertEqual(batch.input_ids.shape			,	(2, 512)   )




  def 						_lowercase     ( self      :	Any   )					->					Dict:
    __lowerCamelCase       :       Optional[int]           = self.get_tokenizer()
    __lowerCamelCase       :       str           = tok(['I am a tiny frog', 'I am a small frog']			,	padding=_a			,	return_tensors=_a   )
    self.assertIsInstance(_a			,	_a   )
    self.assertEqual(batch_smaller.input_ids.shape			,	(2, 10)   )




  @slow
  def 						_lowercase     ( self      :	List[Any]   )					->					Dict:
    # fmt: off
    __lowerCamelCase       :       Optional[Any]           = {'input_ids': [[4_3495, 462, 20, 4_2164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 3_8999, 6, 8, 464, 132, 1703, 492, 13, 4669, 3_7867, 13, 7525, 27, 1593, 988, 13, 3_3972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 1_2338, 2, 1_3958, 387, 2, 3629, 6953, 188, 2900, 2, 1_3958, 8011, 1_1501, 23, 8460, 4073, 3_4009, 20, 435, 1_1439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 3_7867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 2_6453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 1_0767, 6, 316, 304, 4239, 3, 0], [148, 1_5722, 19, 1839, 12, 1350, 13, 2_2327, 5082, 5418, 4_7567, 3_5938, 59, 318, 1_9552, 108, 2183, 54, 1_4976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 1_9088, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100], [36, 6395, 1_2570, 3_9147, 1_1597, 6, 266, 4, 4_5405, 7296, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=_a			,	model_name='Helsinki-NLP/opus-mt-en-de'			,	revision='1a8c2263da11e68e50938f97e10cd57820bd504c'			,	decode_kwargs={'use_source_tokenizer': True}			,	)




  def 						_lowercase     ( self      :	List[str]   )					->					Optional[Any]:
    __lowerCamelCase       :       List[str]           = MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'   )

    __lowerCamelCase       :       Dict           = 'Tämä on testi'
    __lowerCamelCase       :       Union[str, Any]           = 'This is a test'

    __lowerCamelCase       :       Dict           = [76, 7, 2047, 2]
    __lowerCamelCase       :       str           = [69, 12, 11, 940, 2]

    __lowerCamelCase       :       int           = tokenizer(_a   ).input_ids
    self.assertListEqual(_a			,	_a   )

    __lowerCamelCase       :       Tuple           = tokenizer(text_target=_a   ).input_ids
    self.assertListEqual(_a			,	_a   )

    __lowerCamelCase       :       Dict           = tokenizer.decode(_a			,	skip_special_tokens=_a   )
    self.assertEqual(_a			,	_a   )











'''simple docstring'''





def        a_     (							_lowerCAmelCase       )      ->      str:
  if not all(char in '01' for char in bin_string       ):
    raise ValueError('Non-binary value was passed to the function'       )
  if not bin_string:
    raise ValueError('Empty string was passed to the function'       )
  __lowerCamelCase       :       int           = ''
  while len(_lowerCAmelCase       ) % 3 != 0:
    __lowerCamelCase       :       str           = '0' + bin_string
  __lowerCamelCase       :       Union[str, Any]           = [
      bin_string[index : index + 3]
      for index in range(len(_lowerCAmelCase       )       )
      if index % 3 == 0
  ]
  for bin_group in bin_string_in_3_list:
    __lowerCamelCase       :       Tuple           = 0
    for index, val in enumerate(_lowerCAmelCase       ):
      oct_val += int(2 ** (2 - index) * int(_lowerCAmelCase       )       )
    oct_string += str(_lowerCAmelCase       )
  return oct_string


if __name__ == "__main__":
   from doctest import testmod

   testmod()






