code: 
  
  
  
  '''simple docstring'''
  
  import functools
  import operator
  
  from ...configuration_utils import PretrainedConfig
  from ...utils import logging
  
  
  UpperCamelCase_     			=       logging.get_logger(__name__)
  
  UpperCamelCase_     			=       {
      """facebook/hubert-base-ls960""": """https://huggingface.co/facebook/hubert-base-ls960/resolve/main/config.json""",
      # See all Hubert models at https://huggingface.co/models?filter=hubert
  }
  
  
  class   a_  (_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      """hubert"""
  
  
  
  
         def __init__(     self   ,   snake_case_=3_2   ,   snake_case_=7_6_8   ,   snake_case_=1_2   ,   snake_case_=1_2   ,   snake_case_=3_0_7_2   ,   snake_case_="gelu"   ,   snake_case_=0.1   ,   snake_case_=0.1   ,   snake_case_=0.1   ,   snake_case_=True   ,   snake_case_=0.0   ,   snake_case_=0.1   ,   snake_case_=0.1   ,   snake_case_=0.02   ,   snake_case_=1E-5   ,   snake_case_="group"   ,   snake_case_="gelu"   ,   snake_case_=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)   ,   snake_case_=(5, 2, 2, 2, 2, 2, 2)   ,   snake_case_=(1_0, 3, 3, 3, 3, 2, 2)   ,   snake_case_=False   ,   snake_case_=1_2_8   ,   snake_case_=1_6   ,   snake_case_=False   ,   snake_case_=True   ,   snake_case_=0.05   ,   snake_case_=1_0   ,   snake_case_=2   ,   snake_case_=0.0   ,   snake_case_=1_0   ,   snake_case_=0   ,   snake_case_="sum"   ,   snake_case_=False   ,   snake_case_=False   ,   snake_case_=2_5_6   ,   snake_case_=0   ,   snake_case_=1   ,   snake_case_=2   ,   **snake_case_   ,   ):
           super().__init__(**snake_case_   ,   pad_token_id=snake_case_   ,   bos_token_id=snake_case_   ,   eos_token_id=snake_case_  )
           _lowerCAmelCase       :  Optional[int]   				=					hidden_size
           _lowerCAmelCase       :  Optional[int]   				=					feat_extract_norm
           _lowerCAmelCase       :  int   				=					feat_extract_activation
           _lowerCAmelCase       :  List[Any]   				=					list(snake_case_  )
           _lowerCAmelCase       :  Optional[int]   				=					list(snake_case_  )
           _lowerCAmelCase       :  str   				=					list(snake_case_  )
           _lowerCAmelCase       :  Any   				=					conv_bias
           _lowerCAmelCase       :  Optional[int]   				=					num_conv_pos_embeddings
           _lowerCAmelCase       :  Optional[Any]   				=					num_conv_pos_embedding_groups
           _lowerCAmelCase       :  str   				=					len(self.conv_dim  )
           _lowerCAmelCase       :  Union[str, Any]   				=					num_hidden_layers
           _lowerCAmelCase       :  int   				=					intermediate_size
           _lowerCAmelCase       :  Dict   				=					hidden_act
           _lowerCAmelCase       :  Any   				=					num_attention_heads
           _lowerCAmelCase       :  List[Any]   				=					hidden_dropout
           _lowerCAmelCase       :  Dict   				=					attention_dropout
           _lowerCAmelCase       :  int   				=					activation_dropout
           _lowerCAmelCase       :  List[str]   				=					feat_proj_layer_norm
           _lowerCAmelCase       :  Dict   				=					feat_proj_dropout
           _lowerCAmelCase       :  List[str]   				=					final_dropout
           _lowerCAmelCase       :  Union[str, Any]   				=					layerdrop
           _lowerCAmelCase       :  Optional[Any]   				=					layer_norm_eps
           _lowerCAmelCase       :  Union[str, Any]   				=					initializer_range
           _lowerCAmelCase       :  Union[str, Any]   				=					vocab_size
           _lowerCAmelCase       :  Dict   				=					do_stable_layer_norm
           _lowerCAmelCase       :  Union[str, Any]   				=					use_weighted_layer_sum
           _lowerCAmelCase       :  List[Any]   				=					classifier_proj_size
  
           if (
               (len(self.conv_stride  ) != self.num_feat_extract_layers)
               or (len(self.conv_kernel  ) != self.num_feat_extract_layers)
               or (len(self.conv_dim  ) != self.num_feat_extract_layers)
           ):
             raise ValueError(
                 """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                 """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                 f' {len(self.conv_dim  )}`, `len(config.conv_stride) = {len(self.conv_stride  )}`,'
                 f' `len(config.conv_kernel) = {len(self.conv_kernel  )}`.'  )
  
           # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
           _lowerCAmelCase       :  int   				=					apply_spec_augment
           _lowerCAmelCase       :  int   				=					mask_time_prob
           _lowerCAmelCase       :  Union[str, Any]   				=					mask_time_length
           _lowerCAmelCase       :  Optional[int]   				=					mask_time_min_masks
           _lowerCAmelCase       :  Dict   				=					mask_feature_prob
           _lowerCAmelCase       :  Dict   				=					mask_feature_length
           _lowerCAmelCase       :  List[str]   				=					mask_feature_min_masks
  
           # ctc loss
           _lowerCAmelCase       :  Optional[Any]   				=					ctc_loss_reduction
           _lowerCAmelCase       :  Any   				=					ctc_zero_infinity
  
  
  
  
  
  
  
         @property
         def 							__UpperCamelCase       (     self  ):
           return functools.reduce(operator.mul   ,   self.conv_stride   ,   1  )
  
  
  
style_context: 
  '''simple docstring'''
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  @require_tokenizers
  class   a_  (_a							,			unittest.TestCase    ):
         __lowerCAmelCase    :				List[Any]    =      GPTNeoXJapaneseTokenizer
         __lowerCAmelCase    :				Any    =      False
         __lowerCAmelCase    :				List[Any]    =      {"""do_clean_text""": False, """add_prefix_space""": False}
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           super().setUp()
  
           _lowerCAmelCase       :  Any   				=					[
               """こん""",
               """こんに""",
               """にちは""",
               """ばんは""",
               """世界,㔺界""",
               """、""",
               """。""",
               """<BR>""",
               """<SP>""",
               """<TAB>""",
               """<URL>""",
               """<EMAIL>""",
               """<TEL>""",
               """<DATE>""",
               """<PRICE>""",
               """<BLOCK>""",
               """<KIGOU>""",
               """<U2000U2BFF>""",
               """<|emoji1|>""",
               """<unk>""",
               """<|startoftext|>""",
               """<|endoftext|>""",
           ]
           _lowerCAmelCase       :  Union[str, Any]   				=					{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
           _lowerCAmelCase       :  Any   				=					{"""unk_token""": """<unk>"""}
  
           _lowerCAmelCase       :  Optional[int]   				=					os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES["""vocab_file"""]  )
           _lowerCAmelCase       :  Any   				=					os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES["""emoji_file"""]  )
           with open(self.vocab_file   ,   """w"""   ,   encoding="""utf-8"""  ) as vocab_writer:
             vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )
           with open(self.emoji_file   ,   """w"""  ) as emoji_writer:
             emoji_writer.write(json.dumps(snake_case_  )  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   **snake_case_  ):
           kwargs.update(self.special_tokens_map  )
           return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname   ,   **snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase       :  Tuple   				=					"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
           _lowerCAmelCase       :  str   				=					"""こんにちは、世界。 \nこんばんは、世界。😀"""
           return input_text, output_text
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase				,					_lowerCAmelCase       :  str   				=					self.get_input_output_texts(snake_case_  )
           _lowerCAmelCase       :  Tuple   				=					tokenizer.encode(snake_case_   ,   add_special_tokens=snake_case_  )
           _lowerCAmelCase       :  Optional[Any]   				=					tokenizer.decode(snake_case_   ,   clean_up_tokenization_spaces=snake_case_  )
           return text, ids
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  List[Any]   				=					self.get_tokenizer()
  
           # Testing tokenization
           _lowerCAmelCase       :  Tuple   				=					"""こんにちは、世界。　こんばんは、㔺界。"""
           _lowerCAmelCase       :  Optional[Any]   				=					["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
           _lowerCAmelCase       :  Tuple   				=					tokenizer.tokenize(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
           # Testing conversion to ids without special tokens
           _lowerCAmelCase       :  Optional[int]   				=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
           _lowerCAmelCase       :  Optional[int]   				=					tokenizer.convert_tokens_to_ids(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
           # Testing conversion to ids with special tokens
           _lowerCAmelCase       :  Optional[Any]   				=					tokens + [tokenizer.unk_token]
           _lowerCAmelCase       :  List[str]   				=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
           _lowerCAmelCase       :  Dict   				=					tokenizer.convert_tokens_to_ids(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
  
  
  
         @slow
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  int   				=					self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""  )
  
           _lowerCAmelCase       :  List[Any]   				=					tokenizer.encode("""ありがとう。"""   ,   add_special_tokens=snake_case_  )
           _lowerCAmelCase       :  List[str]   				=					tokenizer.encode("""どういたしまして。"""   ,   add_special_tokens=snake_case_  )
  
           _lowerCAmelCase       :  int   				=					tokenizer.build_inputs_with_special_tokens(snake_case_  )
           _lowerCAmelCase       :  Any   				=					tokenizer.build_inputs_with_special_tokens(snake_case_   ,   snake_case_  )
  
           assert encoded_sentence == ids_a
           assert encoded_pair == ids_a + ids_a
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           # Intentionally convert some words to accommodate character fluctuations unique to Japanese
           pass
  
  
  
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           # tokenizer has no padding token
           pass
  
  
  