code: 
  
  import json
  import os
  import unittest
  
  from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES, FSMTTokenizer
  from transformers.testing_utils import slow
  from transformers.utils import cached_property
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  # using a different tiny model than the one used for default params defined in init to ensure proper testing
  A_		:			Dict										=   'stas/tiny-wmt19-en-ru'
  class 					A_   (      _a ,     unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       FSMTTokenizer
      a__     							=       False
  
  
      def      lowerCAmelCase_						(self  )     ->      List[Any]:
          super().setUp()
  
          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          __UpperCAmelCase									=			[
              '''l''',
              '''o''',
              '''w''',
              '''e''',
              '''r''',
              '''s''',
              '''t''',
              '''i''',
              '''d''',
              '''n''',
              '''w</w>''',
              '''r</w>''',
              '''t</w>''',
              '''lo''',
              '''low''',
              '''er</w>''',
              '''low</w>''',
              '''lowest</w>''',
              '''newer</w>''',
              '''wider</w>''',
              '''<unk>''',
          ]
          __UpperCAmelCase									=			dict(zip(lowercase__     ,      range(len(lowercase__  )  )  )  )
          __UpperCAmelCase									=			['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']
  
          __UpperCAmelCase									=			['''en''', '''ru''']
          __UpperCAmelCase									=			{
              '''langs''': self.langs,
              '''src_vocab_size''': 10,
              '''tgt_vocab_size''': 20,
          }
  
          __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES['''src_vocab_file''']  )
          __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES['''tgt_vocab_file''']  )
          __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      '''tokenizer_config.json'''  )
          __UpperCAmelCase									=			os.path.join(self.tmpdirname     ,      VOCAB_FILES_NAMES['''merges_file''']  )
          with open(self.src_vocab_file     ,      '''w'''  ) as fp:
              fp.write(json.dumps(lowercase__  )  )
          with open(self.tgt_vocab_file     ,      '''w'''  ) as fp:
              fp.write(json.dumps(lowercase__  )  )
          with open(self.merges_file     ,      '''w'''  ) as fp:
              fp.write('''\n'''.join(lowercase__  )  )
          with open(lowercase__     ,      '''w'''  ) as fp:
              fp.write(json.dumps(lowercase__  )  )
  
  
      @cached_property
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          return FSMTTokenizer.from_pretrained('''facebook/wmt19-ru-en'''  )
  
  
      @cached_property
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          return FSMTTokenizer.from_pretrained('''facebook/wmt19-en-ru'''  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          __UpperCAmelCase									=			FSMTTokenizer.from_pretrained(lowercase__  )
          self.assertListEqual([tokenizer.src_lang, tokenizer.tgt_lang]     ,      ['''en''', '''ru''']  )
          self.assertEqual(tokenizer.src_vocab_size     ,      21  )
          self.assertEqual(tokenizer.tgt_vocab_size     ,      21  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          __UpperCAmelCase									=			FSMTTokenizer(self.langs     ,      self.src_vocab_file     ,      self.tgt_vocab_file     ,      self.merges_file  )
  
          __UpperCAmelCase									=			'''lower'''
          __UpperCAmelCase									=			['''low''', '''er</w>''']
          __UpperCAmelCase									=			tokenizer.tokenize(lowercase__  )
          self.assertListEqual(lowercase__     ,      lowercase__  )
  
          __UpperCAmelCase									=			tokens + ['''<unk>''']
          __UpperCAmelCase									=			[14, 15, 20]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__  )     ,      lowercase__  )
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      Any:
          __UpperCAmelCase									=			self.tokenizer_ru_en
  
          __UpperCAmelCase									=			tokenizer.encode('''sequence builders'''     ,      add_special_tokens=lowercase__  )
          __UpperCAmelCase									=			tokenizer.encode('''multi-sequence build'''     ,      add_special_tokens=lowercase__  )
  
          __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(lowercase__  )
          __UpperCAmelCase									=			tokenizer.build_inputs_with_special_tokens(lowercase__     ,      lowercase__  )
  
          assert encoded_sentence == text + [2]
          assert encoded_pair == text + [2] + text_a + [2]
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          __UpperCAmelCase									=			self.tokenizer_en_ru
          __UpperCAmelCase									=			self.tokenizer_ru_en
  
          __UpperCAmelCase									=			[
              [
                  '''Here\'s a little song I wrote. Don\'t worry, be happy.''',
                  [2_470, 39, 11, 2_349, 7_222, 70, 5_979, 7, 8_450, 1_050, 13_160, 5, 26, 6_445, 7, 2],
              ],
              ['''This is it. No more. I\'m done!''', [132, 21, 37, 7, 1_434, 86, 7, 70, 6_476, 1_305, 427, 2]],
          ]
  
          # if data needs to be recreated or added, run:
          # import torch
          # model = torch.hub.load("pytorch/fairseq", "transformer.wmt19.en-ru", checkpoint_file="model4.pt", tokenizer="moses", bpe="fastbpe")
          # for src_text, _ in targets: print(f"""[\n"{src_text}",\n {model.encode(src_text).tolist()}\n],""")
  
          for src_text, tgt_input_ids in targets:
              __UpperCAmelCase									=			tokenizer_enc.encode(lowercase__     ,      return_tensors=lowercase__  )
              self.assertListEqual(lowercase__     ,      lowercase__  )
  
              # and decode backward, using the reversed languages model
              __UpperCAmelCase									=			tokenizer_dec.decode(lowercase__     ,      skip_special_tokens=lowercase__  )
              self.assertEqual(lowercase__     ,      lowercase__  )
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          __UpperCAmelCase									=			FSMTTokenizer.from_pretrained('''facebook/wmt19-ru-en'''     ,      do_lower_case=lowercase__  )
          __UpperCAmelCase									=			tokenizer.tokenize('''USA is United States of America'''  )
          __UpperCAmelCase									=			['''us''', '''a</w>''', '''is</w>''', '''un''', '''i''', '''ted</w>''', '''st''', '''ates</w>''', '''of</w>''', '''am''', '''er''', '''ica</w>''']
          self.assertListEqual(lowercase__     ,      lowercase__  )
  
  
      @unittest.skip('''FSMTConfig.__init__  requires non-optional args'''  )
      def      lowerCAmelCase_						(self  )     ->      Any:
          pass
  
  
  
  
  
  
      @unittest.skip('''FSMTConfig.__init__  requires non-optional args'''  )
      def      lowerCAmelCase_						(self  )     ->      int:
          pass
  
  
  
  
  
style_context: 
  
  import unittest
  
  import numpy as np
  
  from transformers.testing_utils import require_torch, require_vision
  from transformers.utils import is_torch_available, is_vision_available
  
  from ...test_image_processing_common import ImageProcessingSavingTestMixin
  
  
  if is_torch_available():
         import torch
  
  if is_vision_available():
         from PIL import Image
  
         from transformers import SwinaSRImageProcessor
         from transformers.image_transforms import get_image_size
  class 					A_   (      unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      def __init__(self     ,      lowercase__     ,      lowercase__=7     ,      lowercase__=3     ,      lowercase__=18     ,      lowercase__=30     ,      lowercase__=400     ,      lowercase__=True     ,      lowercase__=1 / 255     ,      lowercase__=True     ,      lowercase__=8     ,      )     ->      List[str]:
          __UpperCAmelCase									=			parent
          __UpperCAmelCase									=			batch_size
          __UpperCAmelCase									=			num_channels
          __UpperCAmelCase									=			image_size
          __UpperCAmelCase									=			min_resolution
          __UpperCAmelCase									=			max_resolution
          __UpperCAmelCase									=			do_rescale
          __UpperCAmelCase									=			rescale_factor
          __UpperCAmelCase									=			do_pad
          __UpperCAmelCase									=			pad_size
  
  
      def      lowerCAmelCase_						(self  )     ->      int:
          return {
              "do_rescale": self.do_rescale,
              "rescale_factor": self.rescale_factor,
              "do_pad": self.do_pad,
              "pad_size": self.pad_size,
          }
  
  
  
  
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__=False     ,      lowercase__=False     ,      lowercase__=False  )     ->      Tuple:
          assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"
  
          if equal_resolution:
              __UpperCAmelCase									=			[]
              for i in range(self.batch_size  ):
                  image_inputs.append(
                      np.random.randint(
                          255     ,      size=(self.num_channels, self.max_resolution, self.max_resolution)     ,      dtype=np.uinta  )  )
          else:
              __UpperCAmelCase									=			[]
              for i in range(self.batch_size  ):
                  __UpperCAmelCase					,     __UpperCAmelCase									=			np.random.choice(np.arange(self.min_resolution     ,      self.max_resolution  )     ,      2  )
                  image_inputs.append(np.random.randint(255     ,      size=(self.num_channels, width, height)     ,      dtype=np.uinta  )  )
  
          if not numpify and not torchify:
              # PIL expects the channel dimension as last dimension
              __UpperCAmelCase									=			[Image.fromarray(np.moveaxis(lowercase__     ,      0     ,      -1  )  ) for x in image_inputs]
  
          if torchify:
              __UpperCAmelCase									=			[torch.from_numpy(lowercase__  ) for x in image_inputs]
  
          return image_inputs
  
  
  
  
  
  
  @require_torch
  @require_vision
  class 					A_   (      _a ,     unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       SwinaSRImageProcessor if is_vision_available() else None
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			SwinaSRImageProcessingTester(self  )
  
  
      @property
      def      lowerCAmelCase_						(self  )     ->      str:
          return self.image_processor_tester.prepare_image_processor_dict()
  
  
      def      lowerCAmelCase_						(self  )     ->      List[Any]:
          __UpperCAmelCase									=			self.image_processing_class(**self.image_processor_dict  )
          self.assertTrue(hasattr(lowercase__     ,      '''do_rescale'''  )  )
          self.assertTrue(hasattr(lowercase__     ,      '''rescale_factor'''  )  )
          self.assertTrue(hasattr(lowercase__     ,      '''do_pad'''  )  )
          self.assertTrue(hasattr(lowercase__     ,      '''pad_size'''  )  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          pass
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      int:
          __UpperCAmelCase					,     __UpperCAmelCase									=			get_image_size(lowercase__  )
          __UpperCAmelCase									=			self.image_processor_tester.pad_size
  
          __UpperCAmelCase									=			(old_height // size + 1) * size - old_height
          __UpperCAmelCase									=			(old_width // size + 1) * size - old_width
          return old_height + pad_height, old_width + pad_width
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          # Initialize image_processor
          __UpperCAmelCase									=			self.image_processing_class(**self.image_processor_dict  )
          # create random PIL images
          __UpperCAmelCase									=			self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__  )
          for image in image_inputs:
              self.assertIsInstance(lowercase__     ,      Image.Image  )
  
          # Test not batched input
          __UpperCAmelCase									=			image_processor(image_inputs[0]     ,      return_tensors='''pt'''  ).pixel_values
          __UpperCAmelCase					,     __UpperCAmelCase									=			self.calculate_expected_size(np.array(image_inputs[0]  )  )
          self.assertEqual(
              encoded_images.shape     ,      (
                  1,
                  self.image_processor_tester.num_channels,
                  expected_height,
                  expected_width,
              )     ,      )
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          # Initialize image_processor
          __UpperCAmelCase									=			self.image_processing_class(**self.image_processor_dict  )
          # create random numpy tensors
          __UpperCAmelCase									=			self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__     ,      numpify=lowercase__  )
          for image in image_inputs:
              self.assertIsInstance(lowercase__     ,      np.ndarray  )
  
          # Test not batched input
          __UpperCAmelCase									=			image_processor(image_inputs[0]     ,      return_tensors='''pt'''  ).pixel_values
          __UpperCAmelCase					,     __UpperCAmelCase									=			self.calculate_expected_size(image_inputs[0]  )
          self.assertEqual(
              encoded_images.shape     ,      (
                  1,
                  self.image_processor_tester.num_channels,
                  expected_height,
                  expected_width,
              )     ,      )
  
  
  
  
  
  
      def      lowerCAmelCase_						(self  )     ->      int:
          # Initialize image_processor
          __UpperCAmelCase									=			self.image_processing_class(**self.image_processor_dict  )
          # create random PyTorch tensors
          __UpperCAmelCase									=			self.image_processor_tester.prepare_inputs(equal_resolution=lowercase__     ,      torchify=lowercase__  )
          for image in image_inputs:
              self.assertIsInstance(lowercase__     ,      torch.Tensor  )
  
          # Test not batched input
          __UpperCAmelCase									=			image_processor(image_inputs[0]     ,      return_tensors='''pt'''  ).pixel_values
          __UpperCAmelCase					,     __UpperCAmelCase									=			self.calculate_expected_size(image_inputs[0]  )
          self.assertEqual(
              encoded_images.shape     ,      (
                  1,
                  self.image_processor_tester.num_channels,
                  expected_height,
                  expected_width,
              )     ,      )
  
  
  
  
  