code: 
  
  
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
   from tokenizers import Tokenizer
   from tokenizers.models import WordPiece
  
  
  
  
  
  
  class 			UpperCAmelCase_						(       unittest.TestCase		):
  
  
  
        def 		snake_case_      (  self  :			Optional[Any]  ,			A  :			int		):
             _UpperCAmelCase    :			Optional[int]              =					list(tokenizer_class.max_model_input_sizes.keys()		)
             for model_name in sa_models[:1]:
                  _UpperCAmelCase    :			Optional[Any]              =					tokenizer_class.from_pretrained(A		)
                  self.assertIsNotNone(A		)
                  self.assertIsInstance(A  ,			A		)
                  self.assertIsInstance(A  ,			A		)
  
                  for special_tok in tokenizer.all_special_tokens:
                       self.assertIsInstance(A  ,			A		)
                       _UpperCAmelCase    :			int              =					tokenizer.convert_tokens_to_ids(A		)
                       self.assertIsInstance(A  ,			A		)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]  ,			A  :			BatchEncoding  ,			A  :			Optional[Callable] = None		):
             _UpperCAmelCase    :			Tuple              =					pickle.dumps(A		)
             self.assertIsNotNone(A		)
  
             _UpperCAmelCase    :			int              =					pickle.loads(A		)
  
             # Ensure is_fast is correctly restored
             self.assertEqual(be_restored.is_fast  ,			be_original.is_fast		)
  
             # Ensure encodings are potentially correctly restored
             if be_original.is_fast:
                  self.assertIsNotNone(be_restored.encodings		)
             else:
                  self.assertIsNone(be_restored.encodings		)
  
             # Ensure the keys are the same
             for original_v, restored_v in zip(be_original.values()  ,			be_restored.values()		):
                  if equal_op:
                       self.assertTrue(equal_op(A  ,			A		)		)
                  else:
                       self.assertEqual(A  ,			A		)
  
  
  
        @slow
        def 		snake_case_      (  self  :			int		):
             self.check_tokenizer_from_pretrained(A		)
  
  
  
        def 		snake_case_      (  self  :			List[str]		):
             self.assertEqual(TensorType("tf"		)  ,			TensorType.TENSORFLOW		)
             self.assertEqual(TensorType("pt"		)  ,			TensorType.PYTORCH		)
             self.assertEqual(TensorType("np"		)  ,			TensorType.NUMPY		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Any		):
             import numpy as np
  
             _UpperCAmelCase    :			Dict              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			List[str]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             # Python no tensor
             with self.subTest("BatchEncoding (Python, return_tensors=None)"		):
                  self.assert_dump_and_restore(tokenizer_p("Small example to encode"		)		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=None)"		):
                  self.assert_dump_and_restore(tokenizer_r("Small example to encode"		)		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.NUMPY		)  ,			np.array_equal		)
  
  
  
        @require_tf
        @require_tokenizers
        def 		snake_case_      (  self  :			Dict		):
             import tensorflow as tf
  
             def tf_array_equals(A  :			Optional[Any]  ,			A  :			List[Any]		):
                  return tf.reduce_all(tf.equal(A  ,			A		)		)
  
             _UpperCAmelCase    :			Union[str, Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Dict              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			A		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.TENSORFLOW		)  ,			A		)
  
  
  
        @require_torch
        @require_tokenizers
        def 		snake_case_      (  self  :			Optional[Any]		):
             import torch
  
             _UpperCAmelCase    :			Union[str, Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			int              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"		):
                  self.assert_dump_and_restore(
                      tokenizer_p("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)
  
             with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"		):
                  self.assert_dump_and_restore(
                      tokenizer_r("Small example to encode"  ,			return_tensors=TensorType.PYTORCH		)  ,			torch.equal		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Dict		):
             _UpperCAmelCase    :			Optional[int]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Tuple              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
  
             with self.subTest("Python Tokenizer"		):
                  self.assertFalse(tokenizer_p("Small example to_encode"		).is_fast		)
  
             with self.subTest("Rust Tokenizer"		):
                  self.assertTrue(tokenizer_r("Small example to_encode"		).is_fast		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Union[str, Any]		):
             _UpperCAmelCase    :			List[str]              =					BertTokenizerFast.from_pretrained("bert-base-cased"		)
             _UpperCAmelCase    :			Union[str, Any]              =					tokenizer_r(["Test", "\xad", "test"]  ,			is_split_into_words=A		)
  
             self.assertEqual(encoded.word_to_tokens(0		)  ,			TokenSpan(start=1  ,			end=2		)		)
             self.assertEqual(encoded.word_to_tokens(1		)  ,			A		)
             self.assertEqual(encoded.word_to_tokens(2		)  ,			TokenSpan(start=2  ,			end=3		)		)
  
  
  
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			Any              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			Dict              =					batch.convert_to_tensors(tensor_type="np"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			Union[str, Any]              =					batch.convert_to_tensors(tensor_type="np"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			Tuple              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			str              =					batch.convert_to_tensors(tensor_type="np"  ,			prepend_batch_axis=A		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_torch
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			Optional[Any]              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			Union[str, Any]              =					batch.convert_to_tensors(tensor_type="pt"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			str              =					batch.convert_to_tensors(tensor_type="pt"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			Tuple              =					batch.convert_to_tensors(tensor_type="pt"  ,			prepend_batch_axis=A		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_tf
        def 		snake_case_      (  self  :			Optional[Any]		):
             _UpperCAmelCase    :			Tuple              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			Any              =					batch.convert_to_tensors(tensor_type="tf"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			Optional[int]              =					batch.convert_to_tensors(tensor_type="tf"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			str              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			Optional[Any]              =					batch.convert_to_tensors(tensor_type="tf"  ,			prepend_batch_axis=A		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        @require_flax
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			Dict              =					BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}		)
             _UpperCAmelCase    :			Union[str, Any]              =					batch.convert_to_tensors(tensor_type="jax"		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(2, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(2,)		)
             # test converting the converted
             with CaptureStderr() as cs:
                  _UpperCAmelCase    :			List[Any]              =					batch.convert_to_tensors(tensor_type="jax"		)
             self.assertFalse(len(cs.err		)  ,			msg=f'should have no warning, but got {cs.err}'		)
  
             _UpperCAmelCase    :			int              =					BatchEncoding({"inputs": [1, 2, 3], "labels": 0}		)
             _UpperCAmelCase    :			str              =					batch.convert_to_tensors(tensor_type="jax"  ,			prepend_batch_axis=A		)
             self.assertEqual(tensor_batch["inputs"].shape  ,			(1, 3)		)
             self.assertEqual(tensor_batch["labels"].shape  ,			(1,)		)
  
  
  
        def 		snake_case_      (  self  :			Tuple		):
             _UpperCAmelCase    :			Any              =					[{'''input_ids''': np.array([0, 1, 2]		)}, {'''input_ids''': np.array([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			Optional[Any]              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			List[Any]              =					tokenizer.pad(A  ,			padding=A		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			Any              =					tokenizer.pad(A  ,			padding=A  ,			return_tensors="np"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			np.ndarray		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_torch
        def 		snake_case_      (  self  :			Optional[int]		):
             import torch
  
             _UpperCAmelCase    :			Union[str, Any]              =					[{'''input_ids''': torch.tensor([0, 1, 2]		)}, {'''input_ids''': torch.tensor([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			Dict              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			Tuple              =					tokenizer.pad(A  ,			padding=A		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			Optional[int]              =					tokenizer.pad(A  ,			padding=A  ,			return_tensors="pt"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			torch.Tensor		)		)
             self.assertEqual(batch["input_ids"].tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_tf
        def 		snake_case_      (  self  :			Tuple		):
             import tensorflow as tf
  
             _UpperCAmelCase    :			List[Any]              =					[{'''input_ids''': tf.constant([0, 1, 2]		)}, {'''input_ids''': tf.constant([0, 1, 2, 3]		)}]
             _UpperCAmelCase    :			Tuple              =					BertTokenizer.from_pretrained("bert-base-cased"		)
  
             _UpperCAmelCase    :			Optional[Any]              =					tokenizer.pad(A  ,			padding=A		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
             self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
             _UpperCAmelCase    :			int              =					tokenizer.pad(A  ,			padding=A  ,			return_tensors="tf"		)
             self.assertTrue(isinstance(batch["input_ids"]  ,			tf.Tensor		)		)
             self.assertEqual(batch["input_ids"].numpy().tolist()  ,			[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			Union[str, Any]		):
             _UpperCAmelCase    :			int              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
             PreTrainedTokenizerFast(tokenizer_object=A		)
  
  
  
        @require_tokenizers
        def 		snake_case_      (  self  :			int		):
             _UpperCAmelCase    :			Any              =					Tokenizer(WordPiece(unk_token="[UNK]"		)		)
             with tempfile.TemporaryDirectory() as tmpdirname:
                  bert_tokenizer.save(os.path.join(A  ,			"tokenizer.json"		)		)
                  PreTrainedTokenizerFast(tokenizer_file=os.path.join(A  ,			"tokenizer.json"		)		)
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  from arguments import TokenizerTrainingArguments
  from datasets import load_dataset
  from tqdm import tqdm
  
  from transformers import AutoTokenizer, HfArgumentParser
  from transformers.models.gpta.tokenization_gpta import bytes_to_unicode
  
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		str=10       )						->     List[Any]:
  
  
  
  
  
       '''simple docstring'''
  
       for _ in tqdm(range(0		,						args.n_examples		,						SCREAMING_SNAKE_CASE__       )       ):
            yield [next(SCREAMING_SNAKE_CASE__       )[args.text_column] for _ in range(SCREAMING_SNAKE_CASE__       )]
  
  
  # Configuration
  _lowerCAmelCase       :      Optional[Any]							=							HfArgumentParser(TokenizerTrainingArguments)
  _lowerCAmelCase       :      Dict							=							parser.parse_args()
  
  # Base tokenizer
  _lowerCAmelCase       :      int							=							AutoTokenizer.from_pretrained(args.base_tokenizer)
  _lowerCAmelCase       :      List[str]							=							list(bytes_to_unicode().values())
  
  # Load dataset
  _lowerCAmelCase       :      Dict							=							load_dataset(args.dataset_name, split="train", streaming=True)
  _lowerCAmelCase       :      List[Any]							=							iter(dataset)
  
  
  # Training and saving
  _lowerCAmelCase       :      List[str]							=							tokenizer.train_new_from_iterator(
      batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
  )
  new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)
  