






"""simple docstring"""



import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer




class  lowerCAmelCase__ (						tf.keras.layers.Layer      ):
   def __init__(		self			:  List[Any]       , snake_case__			:  List       , snake_case__			:  bool       , snake_case__			:  int = None       , snake_case__			:  int = None       , snake_case__			:  int = None       , snake_case__			:  str = "longest"       , snake_case__			:  bool = True       , snake_case__			:  int = 5_1_2       , snake_case__			:  int = None       , snake_case__			:  bool = True       , snake_case__			:  bool = True       , snake_case__			:  bool = True       , **snake_case__			:  Optional[Any]       , ):


        '''simple docstring'''
        super().__init__()
        if use_fast_bert_tokenizer:
             UpperCAmelCase__	:						Optional[int]       =       FastBertTokenizer(
                 snake_case__       , token_out_type=tf.intaa       , lower_case_nfd_strip_accents=snake_case__       , **snake_case__       )
        else:
             UpperCAmelCase__	:						int       =       tf.lookup.StaticVocabularyTable(
                 tf.lookup.KeyValueTensorInitializer(
                     keys=snake_case__       , key_dtype=tf.string       , values=tf.range(tf.size(snake_case__       , out_type=tf.intaa       )       , dtype=tf.intaa       )       , value_dtype=tf.intaa       , )       , num_oov_buckets=1       , )
             UpperCAmelCase__	:						int       =       BertTokenizerLayer(
                 snake_case__       , token_out_type=tf.intaa       , lower_case=snake_case__       , **snake_case__       )

        UpperCAmelCase__	:						Tuple       =       vocab_list
        UpperCAmelCase__	:						Union[str, Any]       =       do_lower_case
        UpperCAmelCase__	:						Union[str, Any]       =       cls_token_id or vocab_list.index("[CLS]"       )
        UpperCAmelCase__	:						List[str]       =       sep_token_id or vocab_list.index("[SEP]"       )
        UpperCAmelCase__	:						List[Any]       =       pad_token_id or vocab_list.index("[PAD]"       )
        UpperCAmelCase__	:						Optional[Any]       =       ShrinkLongestTrimmer(max_length - 3       , axis=1       )  # Allow room for special tokens
        UpperCAmelCase__	:						Dict       =       max_length
        UpperCAmelCase__	:						Optional[Any]       =       padding
        UpperCAmelCase__	:						List[str]       =       truncation
        UpperCAmelCase__	:						Any       =       pad_to_multiple_of
        UpperCAmelCase__	:						Optional[int]       =       return_token_type_ids
        UpperCAmelCase__	:						Optional[Any]       =       return_attention_mask
   @classmethod
   def 			__a       (		cls			:  Any       , snake_case__			:  "PreTrainedTokenizerBase"       , **snake_case__			:  List[Any]       ):  # noqa: F821


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[Any]       =       kwargs.pop("do_lower_case"       , snake_case__       )
        UpperCAmelCase__	:						Any       =       tokenizer.do_lower_case if do_lower_case is None else do_lower_case
        UpperCAmelCase__	:						Optional[Any]       =       kwargs.pop("cls_token_id"       , snake_case__       )
        UpperCAmelCase__	:						Tuple       =       tokenizer.cls_token_id if cls_token_id is None else cls_token_id
        UpperCAmelCase__	:						Union[str, Any]       =       kwargs.pop("sep_token_id"       , snake_case__       )
        UpperCAmelCase__	:						Optional[Any]       =       tokenizer.sep_token_id if sep_token_id is None else sep_token_id
        UpperCAmelCase__	:						List[Any]       =       kwargs.pop("pad_token_id"       , snake_case__       )
        UpperCAmelCase__	:						Tuple       =       tokenizer.pad_token_id if pad_token_id is None else pad_token_id

        UpperCAmelCase__	:						List[str]       =       tokenizer.get_vocab()
        UpperCAmelCase__	:						Dict       =       sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]       , key=lambda snake_case__       : x[1]       )
        UpperCAmelCase__	:						int       =       [entry[0] for entry in vocab]
        return cls(
            vocab_list=snake_case__       , do_lower_case=snake_case__       , cls_token_id=snake_case__       , sep_token_id=snake_case__       , pad_token_id=snake_case__       , **snake_case__       , )
   @classmethod
   def 			__a       (		cls			:  Optional[Any]       , snake_case__			:  Union[str, os.PathLike]       , *snake_case__			:  List[Any]       , **snake_case__			:  str       ):


        '''simple docstring'''
        try:
             UpperCAmelCase__	:						List[Any]       =       BertTokenizer.from_pretrained(snake_case__       , *snake_case__       , **snake_case__       )
        except:  # noqa: E722
             from .tokenization_bert_fast import BertTokenizerFast

             UpperCAmelCase__	:						List[Any]       =       BertTokenizerFast.from_pretrained(snake_case__       , *snake_case__       , **snake_case__       )
        return cls.from_tokenizer(snake_case__       , **snake_case__       )
   def 			__a       (		self			:  Tuple       , snake_case__			:  int       ):


        '''simple docstring'''
        if self.do_lower_case:
             UpperCAmelCase__	:						List[str]       =       case_fold_utfa(snake_case__       )
        UpperCAmelCase__	:						Optional[int]       =       self.tf_tokenizer.tokenize(snake_case__       )
        return tokens.merge_dims(1       , -1       )
   def 			__a       (		self			:  Optional[int]       , snake_case__			:  Optional[int]       , snake_case__			:  int=None       , snake_case__			:  Tuple=None       , snake_case__			:  Union[str, Any]=None       , snake_case__			:  Any=None       , snake_case__			:  Dict=None       , snake_case__			:  int=None       , snake_case__			:  Any=None       , ):


        '''simple docstring'''
        if padding is None:
             UpperCAmelCase__	:						Optional[Any]       =       self.padding
        if padding not in ("longest", "max_length"):
             raise ValueError("Padding must be either 'longest' or 'max_length'!"       )
        if max_length is not None and text_pair is not None:
             # Because we have to instantiate a Trimmer to do it properly
             raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"       )
        if max_length is None:
             UpperCAmelCase__	:						Any       =       self.max_length
        if truncation is None:
             UpperCAmelCase__	:						int       =       self.truncation
        if pad_to_multiple_of is None:
             UpperCAmelCase__	:						List[Any]       =       self.pad_to_multiple_of
        if return_token_type_ids is None:
             UpperCAmelCase__	:						int       =       self.return_token_type_ids
        if return_attention_mask is None:
             UpperCAmelCase__	:						Union[str, Any]       =       self.return_attention_mask
        if not isinstance(snake_case__       , tf.Tensor       ):
             UpperCAmelCase__	:						Dict       =       tf.convert_to_tensor(snake_case__       )
        if text_pair is not None and not isinstance(snake_case__       , tf.Tensor       ):
             UpperCAmelCase__	:						int       =       tf.convert_to_tensor(snake_case__       )
        if text_pair is not None:
             if text.shape.rank > 1:
                  raise ValueError("text argument should not be multidimensional when a text pair is supplied!"       )
             if text_pair.shape.rank > 1:
                  raise ValueError("text_pair should not be multidimensional!"       )
        if text.shape.rank == 2:
             UpperCAmelCase__	:						Any       =       text[:, 0], text[:, 1]
        UpperCAmelCase__	:						Dict       =       self.unpaired_tokenize(snake_case__       )
        if text_pair is None:  # Unpaired text
             if truncation:
                  UpperCAmelCase__	:						str       =       text[:, : max_length - 2]  # Allow room for special tokens
             UpperCAmelCase__	:						int       =       combine_segments(
                 (text,)       , start_of_sequence_id=self.cls_token_id       , end_of_segment_id=self.sep_token_id       )
        else:  # Paired text
             UpperCAmelCase__	:						List[str]       =       self.unpaired_tokenize(snake_case__       )
             if truncation:
                  UpperCAmelCase__	:						Dict       =       self.paired_trimmer.trim([text, text_pair]       )
             UpperCAmelCase__	:						List[str]       =       combine_segments(
                 (text, text_pair)       , start_of_sequence_id=self.cls_token_id       , end_of_segment_id=self.sep_token_id       )
        if padding == "longest":
             UpperCAmelCase__	:						List[Any]       =       input_ids.bounding_shape(axis=1       )
             if pad_to_multiple_of is not None:
                  # No ceiling division in tensorflow, so we negate floordiv instead
                  UpperCAmelCase__	:						List[str]       =       pad_to_multiple_of * (-tf.math.floordiv(-pad_length       , snake_case__       ))
        else:
             UpperCAmelCase__	:						Tuple       =       max_length

        UpperCAmelCase__	:						Dict       =       pad_model_inputs(snake_case__       , max_seq_length=snake_case__       , pad_value=self.pad_token_id       )
        UpperCAmelCase__	:						Optional[int]       =       {"input_ids": input_ids}
        if return_attention_mask:
             UpperCAmelCase__	:						List[Any]       =       attention_mask
        if return_token_type_ids:
             UpperCAmelCase__	:						Tuple       =       pad_model_inputs(
                 snake_case__       , max_seq_length=snake_case__       , pad_value=self.pad_token_id       )
             UpperCAmelCase__	:						Tuple       =       token_type_ids
        return output




   def 			__a       (		self			:  str       ):


        '''simple docstring'''
        return {
            "vocab_list": self.vocab_list,
            "do_lower_case": self.do_lower_case,
            "cls_token_id": self.cls_token_id,
            "sep_token_id": self.sep_token_id,
            "pad_token_id": self.pad_token_id,
        }










"""simple docstring"""



import inspect
import unittest

import requests

from transformers import MgpstrConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import MgpstrForSceneTextRecognition


if is_vision_available():
 from PIL import Image

 from transformers import MgpstrProcessor




class  lowerCAmelCase__ :
   def __init__(		self			:  int       , snake_case__			:  Tuple       , snake_case__			:  Any=False       , snake_case__			:  Optional[Any]=1_3       , snake_case__			:  str=(3_2, 1_2_8)       , snake_case__			:  int=4       , snake_case__			:  int=3       , snake_case__			:  int=2_7       , snake_case__			:  Tuple=3_8       , snake_case__			:  Dict=9_9       , snake_case__			:  Union[str, Any]=9_9       , snake_case__			:  Union[str, Any]=3_2       , snake_case__			:  Optional[Any]=5       , snake_case__			:  Optional[Any]=4       , snake_case__			:  str=4.0       , snake_case__			:  List[Any]=2_5_7       , snake_case__			:  int=None       , ):


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[Any]       =       parent
        UpperCAmelCase__	:						Union[str, Any]       =       is_training
        UpperCAmelCase__	:						Tuple       =       batch_size
        UpperCAmelCase__	:						List[Any]       =       image_size
        UpperCAmelCase__	:						str       =       patch_size
        UpperCAmelCase__	:						List[str]       =       num_channels
        UpperCAmelCase__	:						Dict       =       max_token_length
        UpperCAmelCase__	:						List[Any]       =       num_character_labels
        UpperCAmelCase__	:						List[Any]       =       num_bpe_labels
        UpperCAmelCase__	:						List[Any]       =       num_wordpiece_labels
        UpperCAmelCase__	:						List[Any]       =       hidden_size
        UpperCAmelCase__	:						Optional[int]       =       num_hidden_layers
        UpperCAmelCase__	:						Union[str, Any]       =       num_attention_heads
        UpperCAmelCase__	:						Union[str, Any]       =       mlp_ratio
        UpperCAmelCase__	:						int       =       patch_embeds_hidden_size
        UpperCAmelCase__	:						int       =       output_hidden_states
   def 			__a       (		self			:  List[str]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Any       =       floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]       )
        UpperCAmelCase__	:						Optional[int]       =       self.get_config()
        return config, pixel_values
   def 			__a       (		self			:  List[Any]       ):


        '''simple docstring'''
        return MgpstrConfig(
            image_size=self.image_size       , patch_size=self.patch_size       , num_channels=self.num_channels       , max_token_length=self.max_token_length       , num_character_labels=self.num_character_labels       , num_bpe_labels=self.num_bpe_labels       , num_wordpiece_labels=self.num_wordpiece_labels       , hidden_size=self.hidden_size       , num_hidden_layers=self.num_hidden_layers       , num_attention_heads=self.num_attention_heads       , mlp_ratio=self.mlp_ratio       , output_hidden_states=self.output_hidden_states       , )
   def 			__a       (		self			:  Any       , snake_case__			:  int       , snake_case__			:  Dict       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Union[str, Any]       =       MgpstrForSceneTextRecognition(snake_case__       )
        model.to(snake_case__       )
        model.eval()
        with torch.no_grad():
             UpperCAmelCase__	:						Any       =       model(snake_case__       )
        self.parent.assertEqual(
            generated_ids[0][0].shape       , (self.batch_size, self.max_token_length, self.num_character_labels)       )




   def 			__a       (		self			:  List[str]       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[Any]       =       self.prepare_config_and_inputs()
        UpperCAmelCase__       ,							UpperCAmelCase__	:						int       =       config_and_inputs
        UpperCAmelCase__	:						List[str]       =       {"pixel_values": pixel_values}
        return config, inputs_dict




@require_torch
class  lowerCAmelCase__ (						__magic_name__       ,	__magic_name__       ,	unittest.TestCase      ):
   SCREAMING_SNAKE_CASE_         =(MgpstrForSceneTextRecognition,) if is_torch_available() else ()
   SCREAMING_SNAKE_CASE_         ={'''feature-extraction''': MgpstrForSceneTextRecognition} if is_torch_available() else {}
   SCREAMING_SNAKE_CASE_         =False

   SCREAMING_SNAKE_CASE_         =False
   SCREAMING_SNAKE_CASE_         =False
   SCREAMING_SNAKE_CASE_         =False
   SCREAMING_SNAKE_CASE_         =False
   def 			__a       (		self			:  str       ):


        '''simple docstring'''
        UpperCAmelCase__	:						int       =       MgpstrModelTester(self       )
        UpperCAmelCase__	:						List[str]       =       ConfigTester(self       , config_class=snake_case__       , has_text_modality=snake_case__       )
   def 			__a       (		self			:  Optional[Any]       ):


        '''simple docstring'''
        self.config_tester.run_common_tests()
   def 			__a       (		self			:  int       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[Any]       =       self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case__       )
   @unittest.skip(reason="MgpstrModel does not use inputs_embeds"       )
   def 			__a       (		self			:  str       ):


        '''simple docstring'''
        pass
   def 			__a       (		self			:  Union[str, Any]       ):


        '''simple docstring'''
        UpperCAmelCase__       ,							UpperCAmelCase__	:						Optional[int]       =       self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             UpperCAmelCase__	:						Dict       =       model_class(snake_case__       )
             self.assertIsInstance(model.get_input_embeddings()       , (nn.Module)       )
             UpperCAmelCase__	:						Optional[Any]       =       model.get_output_embeddings()
             self.assertTrue(x is None or isinstance(snake_case__       , nn.Linear       )       )
   def 			__a       (		self			:  List[str]       ):


        '''simple docstring'''
        UpperCAmelCase__       ,							UpperCAmelCase__	:						Any       =       self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             UpperCAmelCase__	:						List[str]       =       model_class(snake_case__       )
             UpperCAmelCase__	:						int       =       inspect.signature(model.forward       )
             # signature.parameters is an OrderedDict => so arg_names order is deterministic
             UpperCAmelCase__	:						Any       =       [*signature.parameters.keys()]

             UpperCAmelCase__	:						Optional[Any]       =       ["pixel_values"]
             self.assertListEqual(arg_names[:1]       , snake_case__       )
   @unittest.skip(reason="MgpstrModel does not support feedforward chunking"       )
   def 			__a       (		self			:  List[str]       ):


        '''simple docstring'''
        pass
   def 			__a       (		self			:  Dict       ):


        '''simple docstring'''
        UpperCAmelCase__       ,							UpperCAmelCase__	:						List[str]       =       self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             if not model_class.supports_gradient_checkpointing:
                  continue

             UpperCAmelCase__	:						Union[str, Any]       =       True
             UpperCAmelCase__	:						Union[str, Any]       =       model_class(snake_case__       )
             self.assertTrue(model.is_gradient_checkpointing       )
   def 			__a       (		self			:  Any       ):


        '''simple docstring'''
        def check_hidden_states_output(snake_case__			:  List[Any]       , snake_case__			:  List[Any]       , snake_case__			:  List[str]       ):
             UpperCAmelCase__	:						int       =       model_class(snake_case__       )
             model.to(snake_case__       )
             model.eval()

             with torch.no_grad():
                  UpperCAmelCase__	:						Tuple       =       model(**self._prepare_for_class(snake_case__       , snake_case__       )       )

             UpperCAmelCase__	:						List[str]       =       outputs.hidden_states

             UpperCAmelCase__	:						Tuple       =       getattr(
                 self.model_tester       , "expected_num_hidden_layers"       , self.model_tester.num_hidden_layers + 1       )
             self.assertEqual(len(snake_case__       )       , snake_case__       )

             self.assertListEqual(
                 list(hidden_states[0].shape[-2:]       )       , [self.model_tester.patch_embeds_hidden_size, self.model_tester.hidden_size]       , )

        UpperCAmelCase__       ,							UpperCAmelCase__	:						Dict       =       self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
             UpperCAmelCase__	:						Tuple       =       True
             check_hidden_states_output(snake_case__       , snake_case__       , snake_case__       )

             # check that output_hidden_states also work using config
             del inputs_dict["output_hidden_states"]
             UpperCAmelCase__	:						Dict       =       True

             check_hidden_states_output(snake_case__       , snake_case__       , snake_case__       )
   def 			__a       (		self			:  Dict       ):


        '''simple docstring'''
        UpperCAmelCase__       ,							UpperCAmelCase__	:						Tuple       =       self.model_tester.prepare_config_and_inputs_for_common()

        UpperCAmelCase__	:						Dict       =       _config_zero_init(snake_case__       )
        for model_class in self.all_model_classes:
             UpperCAmelCase__	:						int       =       model_class(config=snake_case__       )
             for name, param in model.named_parameters():
                  if isinstance(snake_case__       , (nn.Linear, nn.Convad, nn.LayerNorm)       ):
                       if param.requires_grad:
                            self.assertIn(
                                ((param.data.mean() * 1e9).round() / 1e9).item()       , [0.0, 1.0]       , msg=f'Parameter {name} of model {model_class} seems not properly initialized'       , )




   @unittest.skip(reason="Retain_grad is tested in individual model tests"       )
   def 			__a       (		self			:  Tuple       ):


        '''simple docstring'''
        pass




def   SCREAMING_SNAKE_CASE__					(     )->		List[str]:






     '''simple docstring'''



     UpperCAmelCase__	:						List[str]       =       "https://i.postimg.cc/ZKwLg2Gw/367-14.png"
     UpperCAmelCase__	:						Union[str, Any]       =       Image.open(requests.get(snake_case ,				stream=snake_case							).raw							).convert("RGB"							)
     return im




@require_vision
@require_torch
class  lowerCAmelCase__ (						unittest.TestCase      ):
   @slow
   def 			__a       (		self			:  Tuple       ):


        '''simple docstring'''
        UpperCAmelCase__	:						Optional[int]       =       "alibaba-damo/mgp-str-base"
        UpperCAmelCase__	:						Dict       =       MgpstrForSceneTextRecognition.from_pretrained(snake_case__       ).to(snake_case__       )
        UpperCAmelCase__	:						int       =       MgpstrProcessor.from_pretrained(snake_case__       )

        UpperCAmelCase__	:						Dict       =       prepare_img()
        UpperCAmelCase__	:						List[str]       =       processor(images=snake_case__       , return_tensors="pt"       ).pixel_values.to(snake_case__       )

        # forward pass
        with torch.no_grad():
             UpperCAmelCase__	:						List[str]       =       model(snake_case__       )

        # verify the logits
        self.assertEqual(outputs.logits[0].shape       , torch.Size((1, 2_7, 3_8)       )       )

        UpperCAmelCase__	:						List[str]       =       processor.batch_decode(outputs.logits       )
        UpperCAmelCase__	:						Dict       =       "ticket"

        self.assertEqual(out_strs["generated_text"][0]       , snake_case__       )

        UpperCAmelCase__	:						int       =       torch.tensor(
            [[[-39.5397, -44.4024, -36.1844], [-61.4709, -63.8639, -58.3454], [-74.0225, -68.5494, -71.2164]]]       , device=snake_case__       , )

        self.assertTrue(torch.allclose(outputs.logits[0][:, 1:4, 1:4]       , snake_case__       , atol=1e-4       )       )


