code: import unittest
  
  from transformers import ErnieMConfig, is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_torch_available():
    import torch
  
    from transformers import (
        ErnieMForInformationExtraction,
        ErnieMForMultipleChoice,
        ErnieMForQuestionAnswering,
        ErnieMForSequenceClassification,
        ErnieMForTokenClassification,
        ErnieMModel,
    )
    from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  
  
  class 					UpperCamelCase	:
  
  
  
  
       def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					List[Any]      ,_lowerCAmelCase :					Dict=13      ,_lowerCAmelCase :					int=7      ,_lowerCAmelCase :					Optional[int]=True      ,_lowerCAmelCase :					str=True      ,_lowerCAmelCase :					Optional[int]=True      ,_lowerCAmelCase :					Tuple=99      ,_lowerCAmelCase :					Tuple=32      ,_lowerCAmelCase :					Any=5      ,_lowerCAmelCase :					Optional[Any]=4      ,_lowerCAmelCase :					int=37      ,_lowerCAmelCase :					List[str]="gelu"      ,_lowerCAmelCase :					List[Any]=0.1      ,_lowerCAmelCase :					Dict=0.1      ,_lowerCAmelCase :					Any=512      ,_lowerCAmelCase :					Union[str, Any]=16      ,_lowerCAmelCase :					Union[str, Any]=2      ,_lowerCAmelCase :					str=0.0_2      ,_lowerCAmelCase :					int=3      ,_lowerCAmelCase :					List[Any]=4      ,_lowerCAmelCase :					Tuple=None      ,):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     parent
         __snake_case		       =     batch_size
         __snake_case		       =     seq_length
         __snake_case		       =     is_training
         __snake_case		       =     use_input_mask
         __snake_case		       =     use_labels
         __snake_case		       =     vocab_size
         __snake_case		       =     hidden_size
         __snake_case		       =     num_hidden_layers
         __snake_case		       =     num_attention_heads
         __snake_case		       =     intermediate_size
         __snake_case		       =     hidden_act
         __snake_case		       =     hidden_dropout_prob
         __snake_case		       =     attention_probs_dropout_prob
         __snake_case		       =     max_position_embeddings
         __snake_case		       =     type_vocab_size
         __snake_case		       =     type_sequence_label_size
         __snake_case		       =     initializer_range
         __snake_case		       =     num_labels
         __snake_case		       =     num_choices
         __snake_case		       =     scope
  
  
  
  
       def 							UpperCamelCase_						( self :					str						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size						)
  
         __snake_case		       =     None
         if self.use_input_mask:
           __snake_case		       =     random_attention_mask([self.batch_size, self.seq_length]						)
  
         __snake_case		       =     None
         __snake_case		       =     None
         __snake_case		       =     None
         if self.use_labels:
           __snake_case		       =     ids_tensor([self.batch_size]      ,self.type_sequence_label_size						)
           __snake_case		       =     ids_tensor([self.batch_size, self.seq_length]      ,self.num_labels						)
           __snake_case		       =     ids_tensor([self.batch_size]      ,self.num_choices						)
  
         __snake_case		       =     self.get_config()
  
         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
  
  
  
  
       def 							UpperCamelCase_						( self :					Union[str, Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size						)
  
         __snake_case		       =     None
         if self.use_input_mask:
           __snake_case		       =     random_attention_mask([self.batch_size, self.seq_length]						)
         __snake_case		       =     self.get_config()
  
         return config, input_ids, input_mask
  
  
  
  
       def 							UpperCamelCase_						( self :					Any						):
  
  
         """simple docstring"""
  
  
  
  
         return ErnieMConfig(
             vocab_size=self.vocab_size      ,hidden_size=self.hidden_size      ,num_hidden_layers=self.num_hidden_layers      ,num_attention_heads=self.num_attention_heads      ,intermediate_size=self.intermediate_size      ,hidden_act=self.hidden_act      ,hidden_dropout_prob=self.hidden_dropout_prob      ,attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,max_position_embeddings=self.max_position_embeddings      ,type_vocab_size=self.type_vocab_size      ,initializer_range=self.initializer_range      ,)
  
  
  
  
       def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					Any      ,_lowerCAmelCase :					List[Any]      ,_lowerCAmelCase :					int      ,_lowerCAmelCase :					Any						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ErnieMModel(config=_A						)
         model.to(_A						)
         model.eval()
         __snake_case		       =     model(_A      ,return_dict=_A						)
         self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.hidden_size)						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Union[str, Any]      ,_lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					List[Any]      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					Tuple      ,_lowerCAmelCase :					Dict      ,_lowerCAmelCase :					List[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ErnieMForQuestionAnswering(config=_A						)
         model.to(_A						)
         model.eval()
         __snake_case		       =     model(
             _A      ,attention_mask=_A      ,start_positions=_A      ,end_positions=_A      ,)
         self.parent.assertEqual(result.start_logits.shape      ,(self.batch_size, self.seq_length)						)
         self.parent.assertEqual(result.end_logits.shape      ,(self.batch_size, self.seq_length)						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[Any]      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					Any      ,_lowerCAmelCase :					Any      ,_lowerCAmelCase :					str						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ErnieMForInformationExtraction(config=_A						)
         model.to(_A						)
         model.eval()
         __snake_case		       =     torch.ones_like(_A      ,dtype=torch.floataa						)
         __snake_case		       =     model(
             _A      ,attention_mask=_A      ,start_positions=_A      ,end_positions=_A      ,)
         self.parent.assertEqual(result.start_logits.shape      ,(self.batch_size, self.seq_length)						)
         self.parent.assertEqual(result.end_logits.shape      ,(self.batch_size, self.seq_length)						)
  
  
  
  
       def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					Optional[Any]      ,_lowerCAmelCase :					Optional[Any]      ,_lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					List[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.num_labels
         __snake_case		       =     ErnieMForSequenceClassification(_A						)
         model.to(_A						)
         model.eval()
         __snake_case		       =     model(_A      ,attention_mask=_A      ,labels=_A						)
         self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_labels)						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Union[str, Any]      ,_lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					Any      ,_lowerCAmelCase :					Tuple      ,_lowerCAmelCase :					Dict      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					List[str]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.num_labels
         __snake_case		       =     ErnieMForTokenClassification(config=_A						)
         model.to(_A						)
         model.eval()
         input_ids.to(_A						)
         input_mask.to(_A						)
         token_labels.to(_A						)
  
         __snake_case		       =     model(_A      ,attention_mask=_A      ,labels=_A						)
  
         self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.num_labels)						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					Optional[Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					int      ,_lowerCAmelCase :					List[str]      ,_lowerCAmelCase :					Dict						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.num_choices
         __snake_case		       =     ErnieMForMultipleChoice(config=_A						)
         model.to(_A						)
         model.eval()
         __snake_case		       =     input_ids.unsqueeze(1						).expand(-1      ,self.num_choices      ,-1						).contiguous()
         __snake_case		       =     input_mask.unsqueeze(1						).expand(-1      ,self.num_choices      ,-1						).contiguous()
         __snake_case		       =     model(
             _A      ,attention_mask=_A      ,labels=_A      ,)
         self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_choices)						)
  
  
  
  
  
  
  
       def 							UpperCamelCase_						( self :					List[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.prepare_config_and_inputs()
         (
             __snake_case       
         )		       =     config_and_inputs
         __snake_case		       =     {'input_ids': input_ids, 'attention_mask': input_mask}
         return config, inputs_dict
  
  
  
  
  @require_torch
  class 					UpperCamelCase	(     lowercase__			,      lowercase__			,      unittest.TestCase      ):
       __UpperCamelCase						=   (
           (
               ErnieMModel,
               ErnieMForMultipleChoice,
               ErnieMForQuestionAnswering,
               ErnieMForSequenceClassification,
               ErnieMForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       __UpperCamelCase						=   ()
       __UpperCamelCase						=   (
           {
               """feature-extraction""": ErnieMModel,
               """question-answering""": ErnieMForQuestionAnswering,
               """text-classification""": ErnieMForSequenceClassification,
               """token-classification""": ErnieMForTokenClassification,
               """zero-shot""": ErnieMForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       __UpperCamelCase						=   False
  
  
  
  
       def 							UpperCamelCase_						( self :					List[str]      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					Any      ,_lowerCAmelCase :					Optional[int]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					List[str]						):
  
  
         """simple docstring"""
  
  
  
  
         if pipeline_test_casse_name == "QAPipelineTests":
           return True
  
         return False
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ErnieMModelTester(self						)
         __snake_case		       =     ConfigTester(self      ,config_class=_A      ,hidden_size=37						)
  
  
  
  
       def 							UpperCamelCase_						( self :					List[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         self.config_tester.run_common_tests()
  
  
  
  
       def 							UpperCamelCase_						( self :					int						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         for type in ["absolute", "relative_key", "relative_key_query"]:
           __snake_case		       =     type
           self.model_tester.create_and_check_model(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					int						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					List[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Dict						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_information_extraction(*_A						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Dict						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*_A						)
  
  
  
  
  
  
  
       @slow
       def 							UpperCamelCase_						( self :					int						):
  
  
         """simple docstring"""
  
  
  
  
         for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           __snake_case		       =     ErnieMModel.from_pretrained(_A						)
           self.assertIsNotNone(_A						)
  
  
  
  
  @require_torch
  class 					UpperCamelCase	(     unittest.TestCase      ):
  
  
  
  
       @slow
       def 							UpperCamelCase_						( self :					int						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     ErnieMModel.from_pretrained("susnato/ernie-m-base_pytorch"						)
         model.eval()
         __snake_case		       =     torch.tensor([[0, 1, 2, 3, 4, 5]]						)
         __snake_case		       =     model(_A						)[0]
  
         # TODO Replace vocab size
         __snake_case		       =     768
  
         __snake_case		       =     torch.Size((1, 6, hidden_size)						)
         self.assertEqual(output.shape      ,_A						)
  
         __snake_case		       =     torch.tensor(
             [[[-0.0_0_1_2, 0.1_2_4_5, -0.0_2_1_4], [-0.0_7_4_2, 0.0_2_4_4, -0.0_7_7_1], [-0.0_3_3_3, 0.1_1_6_4, -0.1_5_5_4]]]						)
  
         self.assertTrue(torch.allclose(output[:, :3, :3]      ,_A      ,atol=1E-3						)						)
  
  
  
  
  
  
style_context: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  lowerCamelCase__           =				logging.get_logger(__name__)
  
  lowerCamelCase__           =				{
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  lowerCamelCase__           =				{
      'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
      'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
  }
  
  lowerCamelCase__           =				{
      'openai-gpt': 5_12,
  }
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   Any:
  		__snake_case		       =     text.strip()
  		if not text:
  				return []
  		__snake_case		       =     text.split()
  		return tokens
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  
  
  
  
  					def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					str=True      ,_lowerCAmelCase :					Any=None      ,_lowerCAmelCase :					Any=True      ,_lowerCAmelCase :					Dict=None      ,_lowerCAmelCase :					List[Any]=True      ,):
  
  
  							"""simple docstring"""
  
  
  
  
  							if never_split is None:
  									__snake_case		       =     []
  							__snake_case		       =     do_lower_case
  							__snake_case		       =     set(_lowerCAmelCase						)
  							__snake_case		       =     tokenize_chinese_chars
  							__snake_case		       =     strip_accents
  							__snake_case		       =     do_split_on_punc
  
  
  
  
  					def 							UpperCamelCase_						( self :					Tuple      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					Optional[int]=None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     self.never_split.union(set(_lowerCAmelCase						)						) if never_split else self.never_split
  							__snake_case		       =     self._clean_text(_lowerCAmelCase						)
  
  							# This was added on November 1st, 2018 for the multilingual and Chinese
  							# models. This is also applied to the English models now, but it doesn't
  							# matter since the English models were not trained on any Chinese data
  							# and generally don't have any Chinese data in them (there are Chinese
  							# characters in the vocabulary because Wikipedia does have some Chinese
  							# words in the English Wikipedia.).
  							if self.tokenize_chinese_chars:
  									__snake_case		       =     self._tokenize_chinese_chars(_lowerCAmelCase						)
  							# prevents treating the same character with different unicode codepoints as different characters
  							__snake_case		       =     unicodedata.normalize("NFC"      ,_lowerCAmelCase						)
  							__snake_case		       =     whitespace_tokenize(_lowerCAmelCase						)
  							__snake_case		       =     []
  							for token in orig_tokens:
  									if token not in never_split:
  											if self.do_lower_case:
  													__snake_case		       =     token.lower()
  													if self.strip_accents is not False:
  															__snake_case		       =     self._run_strip_accents(_lowerCAmelCase						)
  											elif self.strip_accents:
  													__snake_case		       =     self._run_strip_accents(_lowerCAmelCase						)
  									split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase      ,_lowerCAmelCase						)						)
  
  							__snake_case		       =     whitespace_tokenize(" ".join(_lowerCAmelCase						)						)
  							return output_tokens
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Dict						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     unicodedata.normalize("NFD"      ,_lowerCAmelCase						)
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     unicodedata.category(_lowerCAmelCase						)
  									if cat == "Mn":
  											continue
  									output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					List[Any]=None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if not self.do_split_on_punc or (never_split is not None and text in never_split):
  									return [text]
  							__snake_case		       =     list(_lowerCAmelCase						)
  							__snake_case		       =     0
  							__snake_case		       =     True
  							__snake_case		       =     []
  							while i < len(_lowerCAmelCase						):
  									__snake_case		       =     chars[i]
  									if _is_punctuation(_lowerCAmelCase						):
  											output.append([char]						)
  											__snake_case		       =     True
  									else:
  											if start_new_word:
  													output.append([]						)
  											__snake_case		       =     False
  											output[-1].append(_lowerCAmelCase						)
  									i += 1
  
  							return ["".join(_lowerCAmelCase						) for x in output]
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					str						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     ord(_lowerCAmelCase						)
  									if self._is_chinese_char(_lowerCAmelCase						):
  											output.append(" "						)
  											output.append(_lowerCAmelCase						)
  											output.append(" "						)
  									else:
  											output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					List[str]      ,_lowerCAmelCase :					Optional[int]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if (
  							    (cp >= 0X4e00 and cp <= 0X9fff)
  							    or (cp >= 0X3400 and cp <= 0X4dbf)  #
  							    or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
  							    or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
  							    or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
  							    or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
  							    or (cp >= 0Xf900 and cp <= 0Xfaff)
  							    or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
  							):  #
  									return True
  
  							return False
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Optional[int]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     ord(_lowerCAmelCase						)
  									if cp == 0 or cp == 0Xfffd or _is_control(_lowerCAmelCase						):
  											continue
  									if _is_whitespace(_lowerCAmelCase						):
  											output.append(" "						)
  									else:
  											output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   str:
  		__snake_case		       =     set()
  		__snake_case		       =     word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)	)
  				__snake_case		       =     char
  		return pairs
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   int:
  		__snake_case		       =     text.replace("—"						,						"-"	)
  		__snake_case		       =     text.replace("–"						,						"-"	)
  		__snake_case		       =     text.replace("―"						,						"-"	)
  		__snake_case		       =     text.replace("…"						,						"..."	)
  		__snake_case		       =     text.replace("´"						,						"'"	)
  		__snake_case		       =     re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"						,						r" \1 "						,						__snake_case	)
  		__snake_case		       =     re.sub(r"\s*\n\s*"						,						" \n "						,						__snake_case	)
  		__snake_case		       =     re.sub(r"[^\S\n]+"						,						" "						,						__snake_case	)
  		return text.strip()
  
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  					__UpperCamelCase						=   VOCAB_FILES_NAMES
  					__UpperCamelCase						=   PRETRAINED_VOCAB_FILES_MAP
  					__UpperCamelCase						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					__UpperCamelCase						=   ["""input_ids""", """attention_mask"""]
  
  
  
  
  					def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					Tuple      ,_lowerCAmelCase :					Tuple="<unk>"      ,**_lowerCAmelCase :					List[str]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							super().__init__(unk_token=_lowerCAmelCase      ,**_lowerCAmelCase						)
  
  							try:
  									import ftfy
  									from spacy.lang.en import English
  
  									__snake_case		       =     English()
  									__snake_case		       =     _nlp.tokenizer
  									__snake_case		       =     ftfy.fix_text
  							except ImportError:
  									logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
  									__snake_case		       =     BasicTokenizer(do_lower_case=_lowerCAmelCase						)
  									__snake_case		       =     None
  
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as vocab_handle:
  									__snake_case		       =     json.load(_lowerCAmelCase						)
  							__snake_case		       =     {v: k for k, v in self.encoder.items()}
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as merges_handle:
  									__snake_case		       =     merges_handle.read().split("\n"						)[1:-1]
  							__snake_case		       =     [tuple(merge.split()						) for merge in merges]
  							__snake_case		       =     dict(zip(_lowerCAmelCase      ,range(len(_lowerCAmelCase						)						)						)						)
  							__snake_case		       =     {}
  
  
  
  
  					@property
  					def 							UpperCamelCase_						( self :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return True
  
  
  
  
  					@property
  					def 							UpperCamelCase_						( self :					Optional[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return len(self.encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return dict(self.encoder      ,**self.added_tokens_encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Union[str, Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     tuple(token[:-1]						) + (token[-1] + "</w>",)
  							if token in self.cache:
  									return self.cache[token]
  							__snake_case		       =     get_pairs(_lowerCAmelCase						)
  
  							if not pairs:
  									return token + "</w>"
  
  							while True:
  									__snake_case		       =     min(_lowerCAmelCase      ,key=lambda _lowerCAmelCase						: self.bpe_ranks.get(_lowerCAmelCase      ,float("inf"						)						)						)
  									if bigram not in self.bpe_ranks:
  											break
  									__snake_case       ,       __snake_case		       =     bigram
  									__snake_case		       =     []
  									__snake_case		       =     0
  									while i < len(_lowerCAmelCase						):
  											try:
  													__snake_case		       =     word.index(_lowerCAmelCase      ,_lowerCAmelCase						)
  											except ValueError:
  													new_word.extend(word[i:]						)
  													break
  											else:
  													new_word.extend(word[i:j]						)
  													__snake_case		       =     j
  
  											if word[i] == first and i < len(_lowerCAmelCase						) - 1 and word[i + 1] == second:
  													new_word.append(first + second						)
  													i += 2
  											else:
  													new_word.append(word[i]						)
  													i += 1
  									__snake_case		       =     tuple(_lowerCAmelCase						)
  									__snake_case		       =     new_word
  									if len(_lowerCAmelCase						) == 1:
  											break
  									else:
  											__snake_case		       =     get_pairs(_lowerCAmelCase						)
  							__snake_case		       =     " ".join(_lowerCAmelCase						)
  							if word == "\n  </w>":
  									__snake_case		       =     "\n</w>"
  							__snake_case		       =     word
  							return word
  
  
  
  
  					def 							UpperCamelCase_						( self :					Union[str, Any]      ,_lowerCAmelCase :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							if self.fix_text is None:
  									# Using BERT's BasicTokenizer
  									__snake_case		       =     self.nlp.tokenize(_lowerCAmelCase						)
  									for token in text:
  											split_tokens.extend(list(self.bpe(_lowerCAmelCase						).split(" "						)						)						)
  							else:
  									# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  									__snake_case		       =     self.nlp(text_standardize(self.fix_text(_lowerCAmelCase						)						)						)
  									for token in text:
  											split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
  							return split_tokens
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.encoder.get(_lowerCAmelCase      ,self.encoder.get(self.unk_token						)						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.decoder.get(_lowerCAmelCase      ,self.unk_token						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Optional[int]      ,_lowerCAmelCase :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     "".join(_lowerCAmelCase						).replace("</w>"      ," "						).strip()
  							return out_string
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					Optional[Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					Optional[str] = None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if not os.path.isdir(_lowerCAmelCase						):
  									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
  									return
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)
  
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as f:
  									f.write(json.dumps(self.encoder      ,indent=2      ,sort_keys=_lowerCAmelCase      ,ensure_ascii=_lowerCAmelCase						) + "\n"						)
  
  							__snake_case		       =     0
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as writer:
  									writer.write("#version: 0.2\n"						)
  									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,key=lambda _lowerCAmelCase						: kv[1]						):
  											if index != token_index:
  													logger.warning(
  													    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  													    " Please check that the tokenizer is not corrupted!"						)
  													__snake_case		       =     token_index
  											writer.write(" ".join(_lowerCAmelCase						) + "\n"						)
  											index += 1
  
  							return vocab_file, merge_file
  
  
  
  
  
  