





'''simple docstring'''


import sys
from collections.abc import Mapping

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


class 		UpperCamelCase__     (       TensorFormatter[Mapping, np.ndarray, Mapping]							):
      '''simple docstring'''






      def __init__(     self     , SCREAMING_SNAKE_CASE=None     , **SCREAMING_SNAKE_CASE )				->  Union[str, Any]:
        super().__init__(features=snake_case__ )
        __lowerCAmelCase		:    Optional[Any]			     =  np_array_kwargs




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Dict:


        if isinstance(snake_case__     , snake_case__ ):
          if column and all(
              isinstance(snake_case__     , np.ndarray ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column ):
            return np.stack(snake_case__ )
          else:
            # don't use np.array(column, dtype=object)
            # since it fails in certain cases
            # see https://stackoverflow.com/q/51005699
            __lowerCAmelCase		:    int			     =  np.empty(len(snake_case__ )     , dtype=snake_case__ )
            __lowerCAmelCase		:    Dict			     =  column
            return out
        return column




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  List[Any]:


        if isinstance(snake_case__     , (str, bytes, type(snake_case__ )) ):
          return value
        elif isinstance(snake_case__     , (np.character, np.ndarray) ) and np.issubdtype(value.dtype     , np.character ):
          return value
        elif isinstance(snake_case__     , np.number ):
          return value

        __lowerCAmelCase		:    int			     =  {}

        if isinstance(snake_case__     , np.ndarray ) and np.issubdtype(value.dtype     , np.integer ):
          __lowerCAmelCase		:    List[Any]			     =  {'dtype': np.intaa}
        elif isinstance(snake_case__     , np.ndarray ) and np.issubdtype(value.dtype     , np.floating ):
          __lowerCAmelCase		:    Dict			     =  {'dtype': np.floataa}
        elif config.PIL_AVAILABLE and "PIL" in sys.modules:
          import PIL.Image

          if isinstance(snake_case__     , PIL.Image.Image ):
            return np.asarray(snake_case__     , **self.np_array_kwargs )

        return np.asarray(snake_case__     , **{**default_dtype, **self.np_array_kwargs} )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  List[str]:


        if config.TORCH_AVAILABLE and "torch" in sys.modules:
          import torch

          if isinstance(snake_case__     , torch.Tensor ):
            return self._tensorize(data_struct.detach().cpu().numpy()[()] )
        if hasattr(snake_case__     , '__array__' ) and not isinstance(snake_case__     , (np.ndarray, np.character, np.number) ):
          __lowerCAmelCase		:    Any			     =  data_struct.__array__()
        # support for nested types like struct of list of struct
        if isinstance(snake_case__     , np.ndarray ):
          if data_struct.dtype == object:
            return self._consolidate([self.recursive_tensorize(snake_case__ ) for substruct in data_struct] )
        if isinstance(snake_case__     , (list, tuple) ):
          return self._consolidate([self.recursive_tensorize(snake_case__ ) for substruct in data_struct] )
        return self._tensorize(snake_case__ )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  List[str]:
        return map_nested(self._recursive_tensorize     , snake_case__     , map_list=snake_case__ )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Tuple:
        __lowerCAmelCase		:    List[str]			     =  self.numpy_arrow_extractor().extract_row(snake_case__ )
        __lowerCAmelCase		:    List[Any]			     =  self.python_features_decoder.decode_row(snake_case__ )
        return self.recursive_tensorize(snake_case__ )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Optional[int]:
        __lowerCAmelCase		:    List[Any]			     =  self.numpy_arrow_extractor().extract_column(snake_case__ )
        __lowerCAmelCase		:    str			     =  self.python_features_decoder.decode_column(snake_case__     , pa_table.column_names[0] )
        __lowerCAmelCase		:    Tuple			     =  self.recursive_tensorize(snake_case__ )
        __lowerCAmelCase		:    Dict			     =  self._consolidate(snake_case__ )
        return column







      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Tuple:
        __lowerCAmelCase		:    Dict			     =  self.numpy_arrow_extractor().extract_batch(snake_case__ )
        __lowerCAmelCase		:    Union[str, Any]			     =  self.python_features_decoder.decode_batch(snake_case__ )
        __lowerCAmelCase		:    List[Any]			     =  self.recursive_tensorize(snake_case__ )
        for column_name in batch:
          __lowerCAmelCase		:    Optional[int]			     =  self._consolidate(batch[column_name] )
        return batch










import unittest

import pytest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    MBartaaTokenizerFast,
    MBartConfig,
    MBartForConditionalGeneration,
    TranslationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch, slow

from .test_pipelines_common import ANY


@is_pipeline_test
class    _lowercase (unittest.TestCase   ):
				'''simple docstring'''







				lowercase__     =    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
				lowercase__     =    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__  ,			snake_case__     ):

									'''simple docstring'''


									if isinstance(model.config  ,			snake_case__     ):
														UpperCamelCase_	,     UpperCamelCase_			= list(tokenizer.lang_code_to_id.keys()     )[:2]
														UpperCamelCase_			= TranslationPipeline(model=snake_case__  ,			tokenizer=snake_case__  ,			src_lang=snake_case__  ,			tgt_lang=snake_case__     )
									else:
														UpperCamelCase_			= TranslationPipeline(model=snake_case__  ,			tokenizer=snake_case__     )
									return translator, ["Some string", "Some other text"]

				def    _lowerCamelCase       (			self  ,			snake_case__  ,			snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= translator("Some string"     )
									self.assertEqual(snake_case__  ,			[{"translation_text": ANY(snake_case__     )}]     )

									UpperCamelCase_			= translator(["Some string"]     )
									self.assertEqual(snake_case__  ,			[{"translation_text": ANY(snake_case__     )}]     )

									UpperCamelCase_			= translator(["Some string", "other string"]     )
									self.assertEqual(snake_case__  ,			[{"translation_text": ANY(snake_case__     )}, {"translation_text": ANY(snake_case__     )}]     )

				@require_torch
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= pipeline("translation_en_to_ro"  ,			model="patrickvonplaten/t5-tiny-random"  ,			framework="pt"     )
									UpperCamelCase_			= translator("This is a test string"  ,			max_length=20     )
									self.assertEqual(
									    snake_case__  ,			[
									        {
									            "translation_text": (
									                "Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide"
									                " Beide Beide"
									            )
									        }
									    ]  ,			)

				@require_tf
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= pipeline("translation_en_to_ro"  ,			model="patrickvonplaten/t5-tiny-random"  ,			framework="tf"     )
									UpperCamelCase_			= translator("This is a test string"  ,			max_length=20     )
									self.assertEqual(
									    snake_case__  ,			[
									        {
									            "translation_text": (
									                "Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide Beide"
									                " Beide Beide"
									            )
									        }
									    ]  ,			)

				@require_torch
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= pipeline("translation_en_to_de"  ,			model="patrickvonplaten/t5-tiny-random"  ,			framework="pt"     )
									UpperCamelCase_			= translator("This is a test string"  ,			max_length=20     )
									self.assertEqual(
									    snake_case__  ,			[
									        {
									            "translation_text": (
									                "monoton monoton monoton monoton monoton monoton monoton monoton monoton monoton urine urine"
									                " urine urine urine urine urine urine urine"
									            )
									        }
									    ]  ,			)

				@require_tf
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= pipeline("translation_en_to_de"  ,			model="patrickvonplaten/t5-tiny-random"  ,			framework="tf"     )
									UpperCamelCase_			= translator("This is a test string"  ,			max_length=20     )
									self.assertEqual(
									    snake_case__  ,			[
									        {
									            "translation_text": (
									                "monoton monoton monoton monoton monoton monoton monoton monoton monoton monoton urine urine"
									                " urine urine urine urine urine urine urine"
									            )
									        }
									    ]  ,			)




class    _lowercase (unittest.TestCase   ):
				'''simple docstring'''







				@require_torch
				@slow
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									with self.assertRaises(snake_case__     ):
														pipeline(task="translation_cn_to_ar"     )

									# but we do for this one
									UpperCamelCase_			= pipeline(task="translation_en_to_de"     )
									self.assertEqual(translator._preprocess_params["src_lang"]  ,			"en"     )
									self.assertEqual(translator._preprocess_params["tgt_lang"]  ,			"de"     )

				@require_torch
				@slow
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-50-many-to-many-mmt"     )
									UpperCamelCase_			= MBartaaTokenizerFast.from_pretrained("facebook/mbart-large-50-many-to-many-mmt"     )

									UpperCamelCase_			= pipeline(task="translation"  ,			model=snake_case__  ,			tokenizer=snake_case__     )
									# Missing src_lang, tgt_lang
									with self.assertRaises(snake_case__     ):
														translator("This is a test"     )

									UpperCamelCase_			= translator("This is a test"  ,			src_lang="en_XX"  ,			tgt_lang="ar_AR"     )
									self.assertEqual(snake_case__  ,			[{"translation_text": "هذا إختبار"}]     )

									UpperCamelCase_			= translator("This is a test"  ,			src_lang="en_XX"  ,			tgt_lang="hi_IN"     )
									self.assertEqual(snake_case__  ,			[{"translation_text": "यह एक परीक्षण है"}]     )

									# src_lang, tgt_lang can be defined at pipeline call time
									UpperCamelCase_			= pipeline(task="translation"  ,			model=snake_case__  ,			tokenizer=snake_case__  ,			src_lang="en_XX"  ,			tgt_lang="ar_AR"     )
									UpperCamelCase_			= translator("This is a test"     )
									self.assertEqual(snake_case__  ,			[{"translation_text": "هذا إختبار"}]     )

				@require_torch
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= "patrickvonplaten/t5-tiny-random"
									UpperCamelCase_			= pipeline(task="translation_cn_to_ar"  ,			model=snake_case__     )
									self.assertEqual(translator._preprocess_params["src_lang"]  ,			"cn"     )
									self.assertEqual(translator._preprocess_params["tgt_lang"]  ,			"ar"     )

				@require_torch
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= "patrickvonplaten/t5-tiny-random"
									with pytest.warns(snake_case__  ,			match=R".*translation_en_to_de.*"     ):
														UpperCamelCase_			= pipeline(task="translation"  ,			model=snake_case__     )
									self.assertEqual(translator.task  ,			"translation_en_to_de"     )
									self.assertEqual(translator._preprocess_params["src_lang"]  ,			"en"     )
									self.assertEqual(translator._preprocess_params["tgt_lang"]  ,			"de"     )

				@require_torch
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									with self.assertRaises(snake_case__     ):
														pipeline(task="translation"     )
