


'''simple docstring'''






import unittest
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory

from transformers import BertConfig, BertTokenizerFast, FeatureExtractionPipeline
from transformers.convert_graph_to_onnx import (
    convert,
    ensure_valid_input,
    generate_identified_filename,
    infer_shapes,
    quantize,
)
from transformers.testing_utils import require_tf, require_tokenizers, require_torch, slow




class        a   :
 def 							__UpperCAmelCase						(       self					, __magic_name__					, __magic_name__					, __magic_name__  )    ->							List[Any]:
      return None





class        a   :
 def 							__UpperCAmelCase						(       self					, __magic_name__					, __magic_name__					, __magic_name__					, __magic_name__  )    ->							Union[str, Any]:
      return None





class        a   (   unittest.TestCase				):
 _lowerCAmelCase           =		[
     # (model_name, model_kwargs)
     ("""bert-base-cased""", {}),
     ("""gpt2""", {"""use_cache""": False}),  # We don't support exporting GPT2 past keys anymore
 ]
 @require_tf
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							List[str]:
      for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           self._test_export(__SCREAMING_SNAKE_CASE					, 'tf'					, 12					, **__SCREAMING_SNAKE_CASE  )
 @require_torch
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							int:
      for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           self._test_export(__SCREAMING_SNAKE_CASE					, 'pt'					, 12					, **__SCREAMING_SNAKE_CASE  )
 @require_torch
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							List[Any]:
      from transformers import BertModel

      _a						=							['[UNK]', '[SEP]', '[CLS]', '[PAD]', '[MASK]', 'some', 'other', 'words']
      with NamedTemporaryFile(mode='w+t'  ) as vocab_file:
           vocab_file.write('\n'.join(__SCREAMING_SNAKE_CASE  )  )
           vocab_file.flush()
           _a						=							BertTokenizerFast(vocab_file.name  )

      with TemporaryDirectory() as bert_save_dir:
           _a						=							BertModel(BertConfig(vocab_size=len(__SCREAMING_SNAKE_CASE  )  )  )
           model.save_pretrained(__SCREAMING_SNAKE_CASE  )
           self._test_export(__SCREAMING_SNAKE_CASE					, 'pt'					, 12					, __SCREAMING_SNAKE_CASE  )
 @require_tf
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							List[str]:
      for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           _a						=							self._test_export(__SCREAMING_SNAKE_CASE					, 'tf'					, 12					, **__SCREAMING_SNAKE_CASE  )
           _a						=							quantize(Path(__SCREAMING_SNAKE_CASE  )  )

           # Ensure the actual quantized model is not bigger than the original one
           if quantized_path.stat().st_size >= Path(__SCREAMING_SNAKE_CASE  ).stat().st_size:
                self.fail('Quantized model is bigger than initial ONNX model'  )
 @require_torch
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							int:
      for model, model_kwargs in OnnxExportTestCase.MODEL_TO_TEST:
           _a						=							self._test_export(__SCREAMING_SNAKE_CASE					, 'pt'					, 12					, **__SCREAMING_SNAKE_CASE  )
           _a						=							quantize(__SCREAMING_SNAKE_CASE  )

           # Ensure the actual quantized model is not bigger than the original one
           if quantized_path.stat().st_size >= Path(__SCREAMING_SNAKE_CASE  ).stat().st_size:
                self.fail('Quantized model is bigger than initial ONNX model'  )
 def 							__UpperCAmelCase						(       self					, __magic_name__					, __magic_name__					, __magic_name__					, __magic_name__=None					, **__magic_name__  )    ->							Dict:
      try:
           # Compute path
           with TemporaryDirectory() as tempdir:
                _a						=							Path(__SCREAMING_SNAKE_CASE  ).joinpath('model.onnx'  )

           # Remove folder if exists
           if path.parent.exists():
                path.parent.rmdir()

           # Export
           convert(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, **__SCREAMING_SNAKE_CASE  )

           return path
      except Exception as e:
           self.fail(__SCREAMING_SNAKE_CASE  )
 @require_torch
 @require_tokenizers
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							Dict:
      from transformers import BertModel

      _a						=							BertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'  )  )
      _a						=							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'  )
      self._test_infer_dynamic_axis(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, 'pt'  )
 @require_tf
 @require_tokenizers
 @slow
 def 							__UpperCAmelCase						(       self  )    ->							Union[str, Any]:
      from transformers import TFBertModel

      _a						=							TFBertModel(BertConfig.from_pretrained('lysandre/tiny-bert-random'  )  )
      _a						=							BertTokenizerFast.from_pretrained('lysandre/tiny-bert-random'  )
      self._test_infer_dynamic_axis(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE					, 'tf'  )
 def 							__UpperCAmelCase						(       self					, __magic_name__					, __magic_name__					, __magic_name__  )    ->							Optional[int]:
      _a						=							FeatureExtractionPipeline(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )

      _a						=							['input_ids', 'token_type_ids', 'attention_mask', 'output_0', 'output_1']
      _a					,      _a					,      _a					,      _a						=							infer_shapes(__SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )

      # Assert all variables are present
      self.assertEqual(len(__SCREAMING_SNAKE_CASE  )					, len(__SCREAMING_SNAKE_CASE  )  )
      self.assertTrue(all(var_name in shapes for var_name in variable_names  )  )
      self.assertSequenceEqual(variable_names[:3]					, __SCREAMING_SNAKE_CASE  )
      self.assertSequenceEqual(variable_names[3:]					, __SCREAMING_SNAKE_CASE  )

      # Assert inputs are {0: batch, 1: sequence}
      for var_name in ["input_ids", "token_type_ids", "attention_mask"]:
           self.assertDictEqual(shapes[var_name]					, {0: 'batch', 1: 'sequence'}  )

      # Assert outputs are {0: batch, 1: sequence} and {0: batch}
      self.assertDictEqual(shapes['output_0']					, {0: 'batch', 1: 'sequence'}  )
      self.assertDictEqual(shapes['output_1']					, {0: 'batch'}  )
 def 							__UpperCAmelCase						(       self  )    ->							Optional[Any]:
      _a						=							['input_ids', 'attention_mask', 'token_type_ids']
      _a						=							{'input_ids': [1, 2, 3, 4], 'attention_mask': [0, 0, 0, 0], 'token_type_ids': [1, 1, 1, 1]}
      _a					,      _a						=							ensure_valid_input(FuncContiguousArgs()					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )

      # Should have exactly the same number of args (all are valid)
      self.assertEqual(len(__SCREAMING_SNAKE_CASE  )					, 3  )

      # Should have exactly the same input names
      self.assertEqual(set(__SCREAMING_SNAKE_CASE  )					, set(__SCREAMING_SNAKE_CASE  )  )

      # Parameter should be reordered according to their respective place in the function:
      # (input_ids, token_type_ids, attention_mask)
      self.assertEqual(__SCREAMING_SNAKE_CASE					, (tokens['input_ids'], tokens['token_type_ids'], tokens['attention_mask'])  )

      # Generated args are interleaved with another args (for instance parameter "past" in GPT2)
      _a					,      _a						=							ensure_valid_input(FuncNonContiguousArgs()					, __SCREAMING_SNAKE_CASE					, __SCREAMING_SNAKE_CASE  )

      # Should have exactly the one arg (all before the one not provided "some_other_args")
      self.assertEqual(len(__SCREAMING_SNAKE_CASE  )					, 1  )
      self.assertEqual(len(__SCREAMING_SNAKE_CASE  )					, 1  )

      # Should have only "input_ids"
      self.assertEqual(inputs_args[0]					, tokens['input_ids']  )
      self.assertEqual(ordered_input_names[0]					, 'input_ids'  )




 def 							__UpperCAmelCase						(       self  )    ->							List[Any]:
      _a						=							generate_identified_filename(Path('/home/something/my_fake_model.onnx'  )					, '-test'  )
      self.assertEqual('/home/something/my_fake_model-test.onnx'					, generated.as_posix()  )









"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


_SCREAMING_SNAKE_CASE					:				Union[str, Any]					    =						{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

_SCREAMING_SNAKE_CASE					:				Union[str, Any]					    =						{
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

_SCREAMING_SNAKE_CASE					:				Any					    =						{
    '''unc-nlp/lxmert-base-uncased''': 512,
}

_SCREAMING_SNAKE_CASE					:				int					    =						{
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}






class      a						(					__snake_case		):
    SCREAMING_SNAKE_CASE						:     int						= VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE						:     Any						= PRETRAINED_INIT_CONFIGURATION
    SCREAMING_SNAKE_CASE						:     Any						= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE						:     Optional[Any]						= LxmertTokenizer






    def __init__(       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]=None       ,						__SCREAMING_SNAKE_CASE     :							Optional[Any]=None       ,						__SCREAMING_SNAKE_CASE     :							str=True       ,						__SCREAMING_SNAKE_CASE     :							List[Any]="[UNK]"       ,						__SCREAMING_SNAKE_CASE     :							int="[SEP]"       ,						__SCREAMING_SNAKE_CASE     :							int="[PAD]"       ,						__SCREAMING_SNAKE_CASE     :							List[Any]="[CLS]"       ,						__SCREAMING_SNAKE_CASE     :							Any="[MASK]"       ,						__SCREAMING_SNAKE_CASE     :							List[str]=True       ,						__SCREAMING_SNAKE_CASE     :							List[str]=None       ,						**__SCREAMING_SNAKE_CASE     :							Any       ,						)     ->		Any:
      super().__init__(
          __SCREAMING_SNAKE_CASE       ,						tokenizer_file=__SCREAMING_SNAKE_CASE       ,						do_lower_case=__SCREAMING_SNAKE_CASE       ,						unk_token=__SCREAMING_SNAKE_CASE       ,						sep_token=__SCREAMING_SNAKE_CASE       ,						pad_token=__SCREAMING_SNAKE_CASE       ,						cls_token=__SCREAMING_SNAKE_CASE       ,						mask_token=__SCREAMING_SNAKE_CASE       ,						tokenize_chinese_chars=__SCREAMING_SNAKE_CASE       ,						strip_accents=__SCREAMING_SNAKE_CASE       ,						**__SCREAMING_SNAKE_CASE       ,						)

      lowerCamelCase_         =						json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
      if (
          normalizer_state.get('lowercase'       ,						__SCREAMING_SNAKE_CASE					) != do_lower_case
          or normalizer_state.get('strip_accents'       ,						__SCREAMING_SNAKE_CASE					) != strip_accents
          or normalizer_state.get('handle_chinese_chars'       ,						__SCREAMING_SNAKE_CASE					) != tokenize_chinese_chars
      ):
        lowerCamelCase_         =						getattr(__SCREAMING_SNAKE_CASE       ,						normalizer_state.pop('type'					)					)
        lowerCamelCase_         =						do_lower_case
        lowerCamelCase_         =						strip_accents
        lowerCamelCase_         =						tokenize_chinese_chars
        lowerCamelCase_         =						normalizer_class(**__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						do_lower_case






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Union[str, Any]       ,						__SCREAMING_SNAKE_CASE     :							Tuple=None					)     ->		Dict:
      lowerCamelCase_         =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output






    def    UpperCamelCase       (       self     :							str       ,						__SCREAMING_SNAKE_CASE     :							List[int]       ,						__SCREAMING_SNAKE_CASE     :							Optional[List[int]] = None					)     ->		List[int]:
      lowerCamelCase_         =						[self.sep_token_id]
      lowerCamelCase_         =						[self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep					) * [0]
      return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]






    def    UpperCamelCase       (       self     :							List[str]       ,						__SCREAMING_SNAKE_CASE     :							str       ,						__SCREAMING_SNAKE_CASE     :							Optional[str] = None					)     ->		Tuple[str]:
      lowerCamelCase_         =						self._tokenizer.model.save(__SCREAMING_SNAKE_CASE       ,						name=__SCREAMING_SNAKE_CASE					)
      return tuple(__SCREAMING_SNAKE_CASE					)




