code: 
  
  
  
  
  
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  
  
  
  
  class 				_lowerCamelCase (    tf.keras.layers.Layer    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase = None				,						UpperCAmelCase = None				,						UpperCAmelCase = None				,						UpperCAmelCase = "longest"				,						UpperCAmelCase = True				,						UpperCAmelCase = 512				,						UpperCAmelCase = None				,						UpperCAmelCase = True				,						UpperCAmelCase = True				,						UpperCAmelCase = True				,						**UpperCAmelCase				,						)       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          super().__init__()
          if use_fast_bert_tokenizer:
            __snake_case   :				Union[str, Any]        =       FastBertTokenizer(
                UpperCAmelCase				,						token_out_type=tf.intaa				,						lower_case_nfd_strip_accents=UpperCAmelCase				,						**UpperCAmelCase   )
          else:
            __snake_case   :				List[Any]        =       tf.lookup.StaticVocabularyTable(
                tf.lookup.KeyValueTensorInitializer(
                    keys=UpperCAmelCase				,						key_dtype=tf.string				,						values=tf.range(tf.size(UpperCAmelCase				,						out_type=tf.intaa   )				,						dtype=tf.intaa   )				,						value_dtype=tf.intaa				,						)				,						num_oov_buckets=1				,						)
            __snake_case   :				str        =       BertTokenizerLayer(
                UpperCAmelCase				,						token_out_type=tf.intaa				,						lower_case=UpperCAmelCase				,						**UpperCAmelCase   )
  
          __snake_case   :				int        =       vocab_list
          __snake_case   :				Dict        =       do_lower_case
          __snake_case   :				Optional[int]        =       cls_token_id or vocab_list.index("[CLS]"   )
          __snake_case   :				Any        =       sep_token_id or vocab_list.index("[SEP]"   )
          __snake_case   :				List[Any]        =       pad_token_id or vocab_list.index("[PAD]"   )
          __snake_case   :				List[str]        =       ShrinkLongestTrimmer(max_length - 3				,						axis=1   )  # Allow room for special tokens
          __snake_case   :				Tuple        =       max_length
          __snake_case   :				List[Any]        =       padding
          __snake_case   :				List[Any]        =       truncation
          __snake_case   :				Union[str, Any]        =       pad_to_multiple_of
          __snake_case   :				Union[str, Any]        =       return_token_type_ids
          __snake_case   :				Union[str, Any]        =       return_attention_mask
  
  
  
  
  
  
  
        @classmethod
        def 						UpperCAmelCase		( cls				,						UpperCAmelCase				,						**UpperCAmelCase   )       ->   Dict:  # noqa: F821
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       kwargs.pop("do_lower_case"				,						UpperCAmelCase   )
          __snake_case   :				int        =       tokenizer.do_lower_case if do_lower_case is None else do_lower_case
          __snake_case   :				Optional[Any]        =       kwargs.pop("cls_token_id"				,						UpperCAmelCase   )
          __snake_case   :				str        =       tokenizer.cls_token_id if cls_token_id is None else cls_token_id
          __snake_case   :				str        =       kwargs.pop("sep_token_id"				,						UpperCAmelCase   )
          __snake_case   :				int        =       tokenizer.sep_token_id if sep_token_id is None else sep_token_id
          __snake_case   :				List[Any]        =       kwargs.pop("pad_token_id"				,						UpperCAmelCase   )
          __snake_case   :				Optional[Any]        =       tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
          __snake_case   :				int        =       tokenizer.get_vocab()
          __snake_case   :				List[str]        =       sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]				,						key=lambda UpperCAmelCase   : x[1]   )
          __snake_case   :				Dict        =       [entry[0] for entry in vocab]
          return cls(
              vocab_list=UpperCAmelCase				,						do_lower_case=UpperCAmelCase				,						cls_token_id=UpperCAmelCase				,						sep_token_id=UpperCAmelCase				,						pad_token_id=UpperCAmelCase				,						**UpperCAmelCase				,						)
  
  
  
  
  
  
  
        @classmethod
        def 						UpperCAmelCase		( cls				,						UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          try:
            __snake_case   :				int        =       BertTokenizer.from_pretrained(UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   )
          except:  # noqa: E722
            from .tokenization_bert_fast import BertTokenizerFast
  
            __snake_case   :				Dict        =       BertTokenizerFast.from_pretrained(UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   )
          return cls.from_tokenizer(UpperCAmelCase				,						**UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if self.do_lower_case:
            __snake_case   :				int        =       case_fold_utfa(UpperCAmelCase   )
          __snake_case   :				str        =       self.tf_tokenizer.tokenize(UpperCAmelCase   )
          return tokens.merge_dims(1				,						-1   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						UpperCAmelCase=None				,						)       ->   Optional[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if padding is None:
            __snake_case   :				str        =       self.padding
          if padding not in ("longest", "max_length"):
            raise ValueError("Padding must be either \'longest\' or \'max_length\'!"   )
          if max_length is not None and text_pair is not None:
            # Because we have to instantiate a Trimmer to do it properly
            raise ValueError("max_length cannot be overridden at call time when truncating paired texts!"   )
          if max_length is None:
            __snake_case   :				Union[str, Any]        =       self.max_length
          if truncation is None:
            __snake_case   :				List[Any]        =       self.truncation
          if pad_to_multiple_of is None:
            __snake_case   :				List[Any]        =       self.pad_to_multiple_of
          if return_token_type_ids is None:
            __snake_case   :				List[str]        =       self.return_token_type_ids
          if return_attention_mask is None:
            __snake_case   :				Any        =       self.return_attention_mask
          if not isinstance(UpperCAmelCase				,						tf.Tensor   ):
            __snake_case   :				Dict        =       tf.convert_to_tensor(UpperCAmelCase   )
          if text_pair is not None and not isinstance(UpperCAmelCase				,						tf.Tensor   ):
            __snake_case   :				str        =       tf.convert_to_tensor(UpperCAmelCase   )
          if text_pair is not None:
            if text.shape.rank > 1:
              raise ValueError("text argument should not be multidimensional when a text pair is supplied!"   )
            if text_pair.shape.rank > 1:
              raise ValueError("text_pair should not be multidimensional!"   )
          if text.shape.rank == 2:
            __snake_case   :				str        =       text[:, 0], text[:, 1]
          __snake_case   :				Optional[Any]        =       self.unpaired_tokenize(UpperCAmelCase   )
          if text_pair is None:  # Unpaired text
            if truncation:
              __snake_case   :				Optional[int]        =       text[:, : max_length - 2]  # Allow room for special tokens
            __snake_case   :				Any        =       combine_segments(
                (text,)				,						start_of_sequence_id=self.cls_token_id				,						end_of_segment_id=self.sep_token_id   )
          else:  # Paired text
            __snake_case   :				Tuple        =       self.unpaired_tokenize(UpperCAmelCase   )
            if truncation:
              __snake_case   :				Optional[int]        =       self.paired_trimmer.trim([text, text_pair]   )
            __snake_case   :				Union[str, Any]        =       combine_segments(
                (text, text_pair)				,						start_of_sequence_id=self.cls_token_id				,						end_of_segment_id=self.sep_token_id   )
          if padding == "longest":
            __snake_case   :				List[Any]        =       input_ids.bounding_shape(axis=1   )
            if pad_to_multiple_of is not None:
              # No ceiling division in tensorflow, so we negate floordiv instead
              __snake_case   :				Tuple        =       pad_to_multiple_of * (-tf.math.floordiv(-pad_length				,						UpperCAmelCase   ))
          else:
            __snake_case   :				int        =       max_length
  
          __snake_case   :				Any        =       pad_model_inputs(UpperCAmelCase				,						max_seq_length=UpperCAmelCase				,						pad_value=self.pad_token_id   )
          __snake_case   :				Dict        =       {'''input_ids''': input_ids}
          if return_attention_mask:
            __snake_case   :				Optional[int]        =       attention_mask
          if return_token_type_ids:
            __snake_case   :				Tuple        =       pad_model_inputs(
                UpperCAmelCase				,						max_seq_length=UpperCAmelCase				,						pad_value=self.pad_token_id   )
            __snake_case   :				int        =       token_type_ids
          return output
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return {
              "vocab_list": self.vocab_list,
              "do_lower_case": self.do_lower_case,
              "cls_token_id": self.cls_token_id,
              "sep_token_id": self.sep_token_id,
              "pad_token_id": self.pad_token_id,
          }
  
  
style_context: 
  
  
  
  
  
  def       lowerCAmelCase__(	lowercase       :    str				) ->    int:
    __snake_case   :				List[str]        =       str(lowercase				).strip()
    if not oct_string:
      raise ValueError("Empty string was passed to the function"				)
    __snake_case   :				int        =       oct_string[0] == "-"
    if is_negative:
      __snake_case   :				Union[str, Any]        =       oct_string[1:]
    if not oct_string.isdigit() or not all(0 <= int(lowercase				) <= 7 for char in oct_string				):
      raise ValueError("Non-octal value was passed to the function"				)
    __snake_case   :				List[Any]        =       0
    for char in oct_string:
      __snake_case   :				Optional[Any]        =       8 * decimal_number + int(lowercase				)
    if is_negative:
      __snake_case   :				Any        =       -decimal_number
    return decimal_number
  
  
  if __name__ == "__main__":
       from doctest import testmod
  
       testmod()
  
  