



import os
import pickle
import unittest
from collections import UserDict, namedtuple

import torch

from accelerate.test_utils.testing import require_cuda, require_torch_min_version
from accelerate.test_utils.training import RegressionModel
from accelerate.utils import (
    convert_outputs_to_fpaa,
    extract_model_from_parallel,
    find_device,
    listify,
    patch_environment,
    recursively_apply,
    send_to_device,
)


_lowerCAmelCase       :      Union[str, Any]							=							namedtuple("ExampleNamedTuple", "a b c")






class 			UpperCAmelCase_						(       unittest.TestCase		):



      def 		snake_case_      (  self  :			str		):
           _UpperCAmelCase    :			Union[str, Any]              =					torch.randn(5  ,			2		)
           _UpperCAmelCase    :			Optional[int]              =					torch.device("cuda"		) if torch.cuda.is_available() else torch.device("cpu"		)

           _UpperCAmelCase    :			Dict              =					send_to_device(A  ,			A		)
           self.assertTrue(torch.equal(resulta.cpu()  ,			A		)		)

           _UpperCAmelCase    :			List[Any]              =					send_to_device((tensor, [tensor, tensor], 1)  ,			A		)
           self.assertIsInstance(A  ,			A		)
           self.assertTrue(torch.equal(resulta[0].cpu()  ,			A		)		)
           self.assertIsInstance(resulta[1]  ,			A		)
           self.assertTrue(torch.equal(resulta[1][0].cpu()  ,			A		)		)
           self.assertTrue(torch.equal(resulta[1][1].cpu()  ,			A		)		)
           self.assertEqual(resulta[2]  ,			1		)

           _UpperCAmelCase    :			Optional[int]              =					send_to_device({"a": tensor, "b": [tensor, tensor], "c": 1}  ,			A		)
           self.assertIsInstance(A  ,			A		)
           self.assertTrue(torch.equal(resulta["a"].cpu()  ,			A		)		)
           self.assertIsInstance(resulta["b"]  ,			A		)
           self.assertTrue(torch.equal(resulta["b"][0].cpu()  ,			A		)		)
           self.assertTrue(torch.equal(resulta["b"][1].cpu()  ,			A		)		)
           self.assertEqual(resulta["c"]  ,			1		)

           _UpperCAmelCase    :			Optional[int]              =					send_to_device(ExampleNamedTuple(a=A  ,			b=[tensor, tensor]  ,			c=1		)  ,			A		)
           self.assertIsInstance(A  ,			A		)
           self.assertTrue(torch.equal(resulta.a.cpu()  ,			A		)		)
           self.assertIsInstance(resulta.b  ,			A		)
           self.assertTrue(torch.equal(resulta.b[0].cpu()  ,			A		)		)
           self.assertTrue(torch.equal(resulta.b[1].cpu()  ,			A		)		)
           self.assertEqual(resulta.c  ,			1		)

           _UpperCAmelCase    :			List[Any]              =					send_to_device(UserDict({"a": tensor, "b": [tensor, tensor], "c": 1}		)  ,			A		)
           self.assertIsInstance(A  ,			A		)
           self.assertTrue(torch.equal(resulta["a"].cpu()  ,			A		)		)
           self.assertIsInstance(resulta["b"]  ,			A		)
           self.assertTrue(torch.equal(resulta["b"][0].cpu()  ,			A		)		)
           self.assertTrue(torch.equal(resulta["b"][1].cpu()  ,			A		)		)
           self.assertEqual(resulta["c"]  ,			1		)



      def 		snake_case_      (  self  :			Union[str, Any]		):
           with self.assertRaises(A		) as cm:
                _UpperCAmelCase    :			int              =					recursively_apply(torch.tensor  ,			(torch.tensor(1		), 1)  ,			error_on_other_type=A		)
           self.assertEqual(
               str(cm.exception		)  ,			"Unsupported types (<class 'int'>) passed to `tensor`. Only nested list/tuple/dicts of objects that are valid for `is_torch_tensor` should be passed."  ,			)



      def 		snake_case_      (  self  :			List[str]		):
           _UpperCAmelCase    :			Dict              =					torch.tensor([1, 2, 3, 4, 5]		)
           self.assertEqual(listify(A		)  ,			[1, 2, 3, 4, 5]		)

           _UpperCAmelCase    :			Optional[int]              =					torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)
           self.assertEqual(listify(A		)  ,			[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]]		)

           _UpperCAmelCase    :			str              =					torch.tensor([[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)
           self.assertEqual(
               listify(A		)  ,			[[[1, 2, 3, 4, 5], [6, 7, 8, 9, 1_0]], [[1_1, 1_2, 1_3, 1_4, 1_5], [1_6, 1_7, 1_8, 1_9, 2_0]]]		)



      def 		snake_case_      (  self  :			Union[str, Any]		):
           with patch_environment(aa=1  ,			BB=2		):
                self.assertEqual(os.environ.get("AA"		)  ,			"1"		)
                self.assertEqual(os.environ.get("BB"		)  ,			"2"		)

           self.assertNotIn("AA"  ,			os.environ		)
           self.assertNotIn("BB"  ,			os.environ		)



      def 		snake_case_      (  self  :			Optional[int]		):
           _UpperCAmelCase    :			List[Any]              =					RegressionModel()
           _UpperCAmelCase    :			List[Any]              =					model.forward
           _UpperCAmelCase    :			List[Any]              =					convert_outputs_to_fpaa(model.forward		)
           _UpperCAmelCase    :			Tuple              =					extract_model_from_parallel(A  ,			keep_fpaa_wrapper=A		)
           _UpperCAmelCase    :			List[str]              =					pickle.dumps(A		)



      @require_cuda
      def 		snake_case_      (  self  :			Any		):
           _UpperCAmelCase    :			Optional[int]              =					RegressionModel()
           _UpperCAmelCase    :			Optional[int]              =					model.forward
           _UpperCAmelCase    :			Optional[Any]              =					torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
           _UpperCAmelCase    :			List[str]              =					convert_outputs_to_fpaa(model.forward		)
           _UpperCAmelCase    :			Union[str, Any]              =					extract_model_from_parallel(A  ,			keep_fpaa_wrapper=A		)
           _UpperCAmelCase    :			List[Any]              =					pickle.dumps(A		)



      @require_cuda
      @require_torch_min_version(version="2.0"		)
      def 		snake_case_      (  self  :			Optional[int]		):
           _UpperCAmelCase    :			Dict              =					RegressionModel()
           _UpperCAmelCase    :			Any              =					model.forward
           _UpperCAmelCase    :			Optional[Any]              =					torch.cuda.amp.autocast(dtype=torch.floataa		)(model.forward		)
           _UpperCAmelCase    :			Union[str, Any]              =					convert_outputs_to_fpaa(model.forward		)
           _UpperCAmelCase    :			Dict              =					torch.compile(model.forward  ,			backend="inductor"		)
           _UpperCAmelCase    :			Union[str, Any]              =					torch.randn(4  ,			1_0		).cuda()
           _UpperCAmelCase    :			Dict              =					model(A		)



      def 		snake_case_      (  self  :			str		):
           _UpperCAmelCase    :			Dict              =					RegressionModel()
           # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
           _UpperCAmelCase    :			Dict              =					torch.nn.parallel.DataParallel(A		)
           _UpperCAmelCase    :			str              =					extract_model_from_parallel(A		)

           self.assertEqual(A  ,			A		)



      @require_torch_min_version(version="2.0"		)
      def 		snake_case_      (  self  :			Tuple		):
           _UpperCAmelCase    :			str              =					RegressionModel()
           _UpperCAmelCase    :			int              =					torch.compile(A		)

           # could also do a test with DistributedDataParallel, but difficult to run on CPU or single GPU
           _UpperCAmelCase    :			Optional[Any]              =					torch.nn.parallel.DataParallel(A		)
           _UpperCAmelCase    :			Optional[Any]              =					torch.compile(A		)
           _UpperCAmelCase    :			Union[str, Any]              =					extract_model_from_parallel(A		)

           self.assertEqual(compiled_model._orig_mod  ,			compiled_model_unwrapped._orig_mod		)



      def 		snake_case_      (  self  :			Dict		):
           self.assertEqual(find_device([1, "a", torch.tensor([1, 2, 3]		)]		)  ,			torch.device("cpu"		)		)
           self.assertEqual(find_device({"a": 1, "b": torch.tensor([1, 2, 3]		)}		)  ,			torch.device("cpu"		)		)
           self.assertIsNone(find_device([1, "a"]		)		)





"""simple docstring"""




from arguments import TokenizerTrainingArguments
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, HfArgumentParser
from transformers.models.gpta.tokenization_gpta import bytes_to_unicode

def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		str=10       )						->     List[Any]:





     '''simple docstring'''

     for _ in tqdm(range(0		,						args.n_examples		,						SCREAMING_SNAKE_CASE__       )       ):
          yield [next(SCREAMING_SNAKE_CASE__       )[args.text_column] for _ in range(SCREAMING_SNAKE_CASE__       )]


# Configuration
_lowerCAmelCase       :      Optional[Any]							=							HfArgumentParser(TokenizerTrainingArguments)
_lowerCAmelCase       :      Dict							=							parser.parse_args()

# Base tokenizer
_lowerCAmelCase       :      int							=							AutoTokenizer.from_pretrained(args.base_tokenizer)
_lowerCAmelCase       :      List[str]							=							list(bytes_to_unicode().values())

# Load dataset
_lowerCAmelCase       :      Dict							=							load_dataset(args.dataset_name, split="train", streaming=True)
_lowerCAmelCase       :      List[Any]							=							iter(dataset)


# Training and saving
_lowerCAmelCase       :      List[str]							=							tokenizer.train_new_from_iterator(
    batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
)
new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)
