code: 
  '''simple docstring'''
  
  from arguments import TokenizerTrainingArguments
  from datasets import load_dataset
  from tqdm import tqdm
  
  from transformers import AutoTokenizer, HfArgumentParser
  from transformers.models.gpta.tokenization_gpta import bytes_to_unicode
  
  
  def 	_UpperCAmelCase				(  _lowerCamelCase						:       str=10		)    ->						List[Any]:
    for _ in tqdm(range(0  ,      args.n_examples  ,      _lowerCamelCase		)		):
      yield [next(_lowerCamelCase		)[args.text_column] for _ in range(_lowerCamelCase		)]
  
  
  # Configuration
  UpperCamelCase_     			=       HfArgumentParser(TokenizerTrainingArguments)
  UpperCamelCase_     			=       parser.parse_args()
  
  # Base tokenizer
  UpperCamelCase_     			=       AutoTokenizer.from_pretrained(args.base_tokenizer)
  UpperCamelCase_     			=       list(bytes_to_unicode().values())
  
  # Load dataset
  UpperCamelCase_     			=       load_dataset(args.dataset_name, split="""train""", streaming=True)
  UpperCamelCase_     			=       iter(dataset)
  
  
  # Training and saving
  UpperCamelCase_     			=       tokenizer.train_new_from_iterator(
      batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
  )
  
  
  
  new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)
  
  
  
style_context: 
  '''simple docstring'''
  
  from ..utils import DummyObject, requires_backends
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				int    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Any    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Tuple    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Any    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Any    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				int    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				int    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Any    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				str    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				int    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Any    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  
  class   a_  (metaclass=_a    ):
         __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]
  
  
  
  
         def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
           requires_backends(self   ,   ["""tokenizers"""]  )
  
  
  