code: 
  
  
  from typing import Optional, Tuple
  
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_openai import OpenAIGPTTokenizer
  
  
  UpperCAmelCase      :						Any    =		logging.get_logger(__name__)
  
  UpperCAmelCase      :						List[str]    =		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}
  
  UpperCAmelCase      :						List[str]    =		{
      "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
      "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
      "tokenizer_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"},
  }
  
  UpperCAmelCase      :						int    =		{
      "openai-gpt": 5_12,
  }
  
  
  
  
  
  
  
  class 					__lowercase					(       snake_case_     ):
  
  
  
  		"""simple docstring"""
  
  		UpperCamelCase    :				Any 					=		VOCAB_FILES_NAMES
  		UpperCamelCase    :				int 					=		PRETRAINED_VOCAB_FILES_MAP
  		UpperCamelCase    :				Union[str, Any] 					=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  		UpperCamelCase    :				Union[str, Any] 					=		["input_ids", "attention_mask"]
  		UpperCamelCase    :				Optional[Any] 					=		OpenAIGPTTokenizer
  
  
  
  
  
  
  
  		def __init__(   self		,			A=None		,			A=None		,			A=None		,			A="<unk>"		,			**A  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					super().__init__(A		,			A		,			tokenizer_file=A		,			unk_token=A		,			**A  )
  
  
  
  
  
  
  
  		@property
  		def 				__A      (   self  )     ->   List[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return True
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A = None  )     ->   Tuple[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     self._tokenizer.model.save(A		,			name=A  )
  					return tuple(A  )
  
  
style_context: 
  
  
  import unittest
  
  from cer import CER
  
  
  UpperCAmelCase      :						Union[str, Any]    =		CER()
  
  
  
  
  
  
  
  class 					__lowercase					(       unittest.TestCase     ):
  
  
  
  		"""simple docstring"""
  
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""White House"""]
  					lowerCamelCase								=     ["""white house"""]
  					# S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.1818181818  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""were wolf"""]
  					lowerCamelCase								=     ["""werewolf"""]
  					# S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.1111111  ) < 1e-6  )
  
  					lowerCamelCase								=     ["""werewolf"""]
  					lowerCamelCase								=     ["""weae     wolf"""]
  					# S = 1, D = 1, I = 0, N = 8, CER = 0.25
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.25  ) < 1e-6  )
  
  					# consecutive whitespaces case 1
  					lowerCamelCase								=     ["""were wolf"""]
  					lowerCamelCase								=     ["""were               wolf"""]
  					# S = 0, D = 0, I = 0, N = 9, CER = 0
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.0  ) < 1e-6  )
  
  					# consecutive whitespaces case 2
  					lowerCamelCase								=     ["""were   wolf"""]
  					lowerCamelCase								=     ["""were               wolf"""]
  					# S = 0, D = 0, I = 0, N = 9, CER = 0
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.0  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   str:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""werewolf"""]
  					lowerCamelCase								=     ["""weaewolf"""]
  					# S = 1, D = 0, I = 0, N = 8, CER = 0.125
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.125  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Tuple:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""werewolf"""]
  					lowerCamelCase								=     ["""wereawolf"""]
  					# S = 0, D = 1, I = 0, N = 8, CER = 0.125
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.125  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""werewolf"""]
  					lowerCamelCase								=     ["""wereolf"""]
  					# S = 0, D = 0, I = 1, N = 8, CER = 0.125
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.125  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Tuple:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""werewolf"""]
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertEqual(A		,			0.0  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""werewolf""", """I am your father"""]
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertEqual(A		,			0.0  )
  
  					lowerCamelCase								=     ["""werewolf""", """I am your father""", """doge"""]
  					lowerCamelCase								=     ["""werxwolf""", """I       am your father""", """doge"""]
  					# S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.03571428  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""My hovercraft""", """is full of eels"""]
  					lowerCamelCase								=     ["""My hovercraft is full""", """ of eels"""]
  					# S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
  					# whitespace at the front of " of eels" will be strip during preporcessing
  					# so need to insert 2 whitespaces
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A		,			concatenate_texts=A  )
  					self.assertTrue(abs(char_error_rate - 0.071428  ) < 1e-6  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     ["""我能吞下玻璃而不伤身体"""]
  					lowerCamelCase								=     [""" 能吞虾玻璃而 不霜身体啦"""]
  					# S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.4545454545  ) < 1e-6  )
  
  					lowerCamelCase								=     ["""我能吞下玻璃""", """而不伤身体"""]
  					lowerCamelCase								=     ["""我    能 吞 下 玻 璃""", """而不伤身体"""]
  					# S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertTrue(abs(char_error_rate - 0.454545454545  ) < 1e-6  )
  
  					lowerCamelCase								=     ["""我能吞下玻璃而不伤身体"""]
  					lowerCamelCase								=     cer.compute(predictions=A		,			references=A  )
  					self.assertFalse(A		,			0.0  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     [""""""]
  					lowerCamelCase								=     ["""Hypothesis"""]
  					with self.assertRaises(A  ):
  								cer.compute(predictions=A		,			references=A  )
  
  
  if __name__ == "__main__":
  		unittest.main()
  
  