code: 
  
  
  
  
  
  '''simple docstring'''
  
  import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple, Union
  
  import numpy as np
  import regex as re
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  if TYPE_CHECKING:
        if is_torch_available():
              import torch
        if is_tf_available():
              import tensorflow as tf
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  
  
  lowercase_			       =				logging.get_logger(__name__)
  
  lowercase_			       =				{
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  lowercase_			       =				{
      """vocab_file""": {
          """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
      },
      """merges_file""": {
          """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
      },
  }
  
  lowercase_			       =				{
      """Salesforce/codegen-350M-mono""": 2_048,
  }
  
  
  
  
  
  @lru_cache()
  def lowerCamelCase		(  )		->Optional[int]:
      _SCREAMING_SNAKE_CASE             =    (
          list(range(ord("""!"""					)      , ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					)      , ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					)      , ord("""ÿ"""					) + 1					)					)
      )
      _SCREAMING_SNAKE_CASE             =    bs[:]
      _SCREAMING_SNAKE_CASE             =    0
      for b in range(2**8					):
          if b not in bs:
              bs.append(__lowerCamelCase					)
              cs.append(2**8 + n					)
              n += 1
      _SCREAMING_SNAKE_CASE             =    [chr(__lowerCamelCase					) for n in cs]
      return dict(zip(__lowerCamelCase      , __lowerCamelCase					)					)
  
  
  
  
  
  def lowerCamelCase		(  __lowerCamelCase				:					List[str]					)		->Union[str, Any]:
      _SCREAMING_SNAKE_CASE             =    set()
      _SCREAMING_SNAKE_CASE             =    word[0]
      for char in word[1:]:
          pairs.add((prev_char, char)					)
          _SCREAMING_SNAKE_CASE             =    char
      return pairs
  
  
  
  
  
  
  class       a_  (       snake_case_ ):
  
      '''simple docstring'''
      UpperCamelCase									=				VOCAB_FILES_NAMES
      UpperCamelCase									=				PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase									=				['''input_ids''', '''attention_mask''']
  
  
      def __init__(	self				,	A				,	A				,	A="replace"				,	A="<|endoftext|>"				,	A="<|endoftext|>"				,	A="<|endoftext|>"				,	A=None				,	A=False				,	A=False				,	**A				,	)	->     List[str]:
          _SCREAMING_SNAKE_CASE             =    AddedToken(A				,	lstrip=A				,	rstrip=A ) if isinstance(A				,	A ) else bos_token
          _SCREAMING_SNAKE_CASE             =    AddedToken(A				,	lstrip=A				,	rstrip=A ) if isinstance(A				,	A ) else eos_token
          _SCREAMING_SNAKE_CASE             =    AddedToken(A				,	lstrip=A				,	rstrip=A ) if isinstance(A				,	A ) else unk_token
          _SCREAMING_SNAKE_CASE             =    AddedToken(A				,	lstrip=A				,	rstrip=A ) if isinstance(A				,	A ) else pad_token
          super().__init__(
              errors=A				,	unk_token=A				,	bos_token=A				,	eos_token=A				,	pad_token=A				,	add_prefix_space=A				,	add_bos_token=A				,	**A				,	)
          _SCREAMING_SNAKE_CASE             =    add_bos_token
  
          with open(A				,	encoding="""utf-8""" ) as vocab_handle:
              _SCREAMING_SNAKE_CASE             =    json.load(A )
          _SCREAMING_SNAKE_CASE             =    {v: k for k, v in self.encoder.items()}
          _SCREAMING_SNAKE_CASE             =    errors  # how to handle errors in decoding
          _SCREAMING_SNAKE_CASE             =    bytes_to_unicode()
          _SCREAMING_SNAKE_CASE             =    {v: k for k, v in self.byte_encoder.items()}
          with open(A				,	encoding="""utf-8""" ) as merges_handle:
              _SCREAMING_SNAKE_CASE             =    merges_handle.read().split("""\n""" )[1:-1]
          _SCREAMING_SNAKE_CASE             =    [tuple(merge.split() ) for merge in bpe_merges]
          _SCREAMING_SNAKE_CASE             =    dict(zip(A				,	range(len(A ) ) ) )
          _SCREAMING_SNAKE_CASE             =    {}
          _SCREAMING_SNAKE_CASE             =    add_prefix_space
  
          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          _SCREAMING_SNAKE_CASE             =    re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )
  
  
      @property
      def     snake_case_(	self )	->     int:
          return len(self.encoder )
  
  
      def     snake_case_(	self )	->     Tuple:
          return dict(self.encoder				,	**self.added_tokens_encoder )
  
  
      def     snake_case_(	self				,	A )	->     List[Any]:
          if token in self.cache:
              return self.cache[token]
          _SCREAMING_SNAKE_CASE             =    tuple(A )
          _SCREAMING_SNAKE_CASE             =    get_pairs(A )
  
          if not pairs:
              return token
  
          while True:
              _SCREAMING_SNAKE_CASE             =    min(A				,	key=lambda A : self.bpe_ranks.get(A				,	float("""inf""" ) ) )
              if bigram not in self.bpe_ranks:
                  break
              _SCREAMING_SNAKE_CASE       , _SCREAMING_SNAKE_CASE             =    bigram
              _SCREAMING_SNAKE_CASE             =    []
              _SCREAMING_SNAKE_CASE             =    0
              while i < len(A ):
                  try:
                      _SCREAMING_SNAKE_CASE             =    word.index(A				,	A )
                  except ValueError:
                      new_word.extend(word[i:] )
                      break
                  else:
                      new_word.extend(word[i:j] )
                      _SCREAMING_SNAKE_CASE             =    j
  
                  if word[i] == first and i < len(A ) - 1 and word[i + 1] == second:
                      new_word.append(first + second )
                      i += 2
                  else:
                      new_word.append(word[i] )
                      i += 1
              _SCREAMING_SNAKE_CASE             =    tuple(A )
              _SCREAMING_SNAKE_CASE             =    new_word
              if len(A ) == 1:
                  break
              else:
                  _SCREAMING_SNAKE_CASE             =    get_pairs(A )
          _SCREAMING_SNAKE_CASE             =    """ """.join(A )
          _SCREAMING_SNAKE_CASE             =    word
          return word
  
  
      def     snake_case_(	self				,	A				,	A=None )	->     List[str]:
          if self.add_bos_token:
              _SCREAMING_SNAKE_CASE             =    [self.bos_token_id]
          else:
              _SCREAMING_SNAKE_CASE             =    []
  
          _SCREAMING_SNAKE_CASE             =    bos_token_ids + token_ids_a
  
          if token_ids_a is None:
              return output
  
          return output + bos_token_ids + token_ids_a
  
  
      def     snake_case_(	self				,	A )	->     str:
          _SCREAMING_SNAKE_CASE             =    []
          for token in re.findall(self.pat				,	A ):
              _SCREAMING_SNAKE_CASE             =    """""".join(
                  self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A ).split(""" """ ) )
          return bpe_tokens
  
  
      def     snake_case_(	self				,	A )	->     Any:
          return self.encoder.get(A				,	self.encoder.get(self.unk_token ) )
  
  
      def     snake_case_(	self				,	A )	->     Dict:
          return self.decoder.get(A )
  
  
      def     snake_case_(	self				,	A )	->     Dict:
          _SCREAMING_SNAKE_CASE             =    """""".join(A )
          _SCREAMING_SNAKE_CASE             =    bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""				,	errors=self.errors )
          return text
  
  
      def     snake_case_(	self				,	A				,	A = None )	->     Tuple[str]:
          if not os.path.isdir(A ):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
              return
          _SCREAMING_SNAKE_CASE             =    os.path.join(
              A				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
          _SCREAMING_SNAKE_CASE             =    os.path.join(
              A				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )
  
          with open(A				,	"""w"""				,	encoding="""utf-8""" ) as f:
              f.write(json.dumps(self.encoder				,	indent=2				,	sort_keys=A				,	ensure_ascii=A ) + """\n""" )
  
          _SCREAMING_SNAKE_CASE             =    0
          with open(A				,	"""w"""				,	encoding="""utf-8""" ) as writer:
              writer.write("""#version: 0.2\n""" )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,	key=lambda A : kv[1] ):
                  if index != token_index:
                      logger.warning(
                          f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                          """ Please check that the tokenizer is not corrupted!""" )
                      _SCREAMING_SNAKE_CASE             =    token_index
                  writer.write(""" """.join(A ) + """\n""" )
                  index += 1
  
          return vocab_file, merge_file
  
  
      def     snake_case_(	self				,	A				,	A=False				,	**A )	->     Tuple:
          _SCREAMING_SNAKE_CASE             =    kwargs.pop("""add_prefix_space"""				,	self.add_prefix_space )
          if is_split_into_words or add_prefix_space:
              _SCREAMING_SNAKE_CASE             =    """ """ + text
          return (text, kwargs)
  
  
      def     snake_case_(	self				,	A				,	A = False				,	A = None				,	A = None				,	**A				,	)	->     str:
          _SCREAMING_SNAKE_CASE             =    super()._decode(
              token_ids=A				,	skip_special_tokens=A				,	clean_up_tokenization_spaces=A				,	**A				,	)
  
          if truncate_before_pattern is not None and len(A ) > 0:
              _SCREAMING_SNAKE_CASE             =    self.truncate(A				,	A )
  
          return decoded_text
  
  
  
  
  
      def     snake_case_(	self				,	A				,	A )	->     Dict:
          def find_re(A				,	A				,	A ):
              _SCREAMING_SNAKE_CASE             =    pattern.search(A				,	A )
              return m.start() if m else -1
  
          _SCREAMING_SNAKE_CASE             =    [re.compile(A				,	re.MULTILINE ) for pattern in truncate_before_pattern]
  
          _SCREAMING_SNAKE_CASE             =    list(re.finditer("""^print"""				,	A				,	re.MULTILINE ) )
  
          if len(A ) > 1:
              _SCREAMING_SNAKE_CASE             =    completion[: prints[1].start()]
  
          _SCREAMING_SNAKE_CASE             =    list(re.finditer("""^def"""				,	A				,	re.MULTILINE ) )
  
          if len(A ) > 1:
              _SCREAMING_SNAKE_CASE             =    completion[: defs[1].start()]
  
          _SCREAMING_SNAKE_CASE             =    0
  
          _SCREAMING_SNAKE_CASE             =    [
              pos for pos in [find_re(A				,	A				,	A ) for terminal in terminals] if pos != -1
          ]
  
          if len(A ) > 0:
              return completion[: min(A )]
          else:
              return completion
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  import math
  from timeit import timeit
  
  
  
  
  
  def lowerCamelCase		(  __lowerCamelCase				:					int					)		->int:
      _SCREAMING_SNAKE_CASE             =    0
      _SCREAMING_SNAKE_CASE             =    abs(__lowerCamelCase					)
      while True:
          _SCREAMING_SNAKE_CASE             =    n // 10
          digits += 1
          if n == 0:
              break
      return digits
  
  
  
  
  
  def lowerCamelCase		(  __lowerCamelCase				:					int					)		->int:
      return 1 if n == 0 else math.floor(math.log(abs(__lowerCamelCase					)      , 10					) + 1					)
  
  
  
  
  
  def lowerCamelCase		(  __lowerCamelCase				:					int					)		->int:
      return len(str(abs(__lowerCamelCase					)					)					)
  
  
  
  
  
  def lowerCamelCase		(  )		->None:
      from collections.abc import Callable
  
      def benchmark_a_function(__lowerCamelCase				:					Callable      , __lowerCamelCase				:					int					) -> None:
          _SCREAMING_SNAKE_CASE             =    F'{func.__name__}({value})'
          _SCREAMING_SNAKE_CASE             =    timeit(F'__main__.{call}'      , setup="""import __main__"""					)
          print(F'{call}: {func(__lowerCamelCase					)} -- {timing} seconds'					)
  
      for value in (26_2144, 1125_8999_0684_2624, 126_7650_6002_2822_9401_4967_0320_5376):
          for func in (num_digits, num_digits_fast, num_digits_faster):
              benchmark_a_function(__lowerCamelCase      , __lowerCamelCase					)
          print()
  
  
  if __name__ == "__main__":
        import doctest
  
        doctest.testmod()
        benchmark()
  
  
  
  