
'''simple docstring'''

from arguments import TokenizerTrainingArguments
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, HfArgumentParser
from transformers.models.gpta.tokenization_gpta import bytes_to_unicode


def 	_UpperCAmelCase				(  _lowerCamelCase						:       str=10		)    ->						List[Any]:
  for _ in tqdm(range(0  ,      args.n_examples  ,      _lowerCamelCase		)		):
    yield [next(_lowerCamelCase		)[args.text_column] for _ in range(_lowerCamelCase		)]


# Configuration
UpperCamelCase_     			=       HfArgumentParser(TokenizerTrainingArguments)
UpperCamelCase_     			=       parser.parse_args()

# Base tokenizer
UpperCamelCase_     			=       AutoTokenizer.from_pretrained(args.base_tokenizer)
UpperCamelCase_     			=       list(bytes_to_unicode().values())

# Load dataset
UpperCamelCase_     			=       load_dataset(args.dataset_name, split="""train""", streaming=True)
UpperCamelCase_     			=       iter(dataset)


# Training and saving
UpperCamelCase_     			=       tokenizer.train_new_from_iterator(
    batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
)



new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)




'''simple docstring'''

from ..utils import DummyObject, requires_backends


class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				int    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Any    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Tuple    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Any    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Any    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Dict    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				int    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				int    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Any    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				str    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				int    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[int]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				List[str]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Union[str, Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Any    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )



class   a_  (metaclass=_a    ):
       __lowerCAmelCase    :				Optional[Any]    =      ["""tokenizers"""]




       def __init__(     self   ,   *snake_case_   ,   **snake_case_  ):
         requires_backends(self   ,   ["""tokenizers"""]  )


