






"""simple docstring"""






import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
UpperCAmelCase_   : List[str]   			= logging.getLogger(__name__)



def 					_A			() ->       Optional[int]:
    """simple docstring"""






    SCREAMING_SNAKE_CASE_      :  str    						=		argparse.ArgumentParser(
        description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''							)
    parser.add_argument('''--file_path'''    ,				type=__a    ,				default='''data/dump.txt'''    ,				help='''The path to the data.'''							)
    parser.add_argument('''--tokenizer_type'''    ,				type=__a    ,				default='''bert'''    ,				choices=['''bert''', '''roberta''', '''gpt2''']							)
    parser.add_argument('''--tokenizer_name'''    ,				type=__a    ,				default='''bert-base-uncased'''    ,				help='''The tokenizer to use.'''							)
    parser.add_argument('''--dump_file'''    ,				type=__a    ,				default='''data/dump'''    ,				help='''The dump file prefix.'''							)
    SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		parser.parse_args()

    logger.info(f'Loading Tokenizer ({args.tokenizer_name})'							)
    if args.tokenizer_type == "bert":
        SCREAMING_SNAKE_CASE_      :  Dict    						=		BertTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
        SCREAMING_SNAKE_CASE_      :  str    						=		tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
    elif args.tokenizer_type == "roberta":
        SCREAMING_SNAKE_CASE_      :  List[str]    						=		RobertaTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_      :  int    						=		tokenizer.special_tokens_map['''cls_token''']  # `<s>`
        SCREAMING_SNAKE_CASE_      :  Dict    						=		tokenizer.special_tokens_map['''sep_token''']  # `</s>`
    elif args.tokenizer_type == "gpt2":
        SCREAMING_SNAKE_CASE_      :  Any    						=		GPTaTokenizer.from_pretrained(args.tokenizer_name							)
        SCREAMING_SNAKE_CASE_      :  List[str]    						=		tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
        SCREAMING_SNAKE_CASE_      :  str    						=		tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

    logger.info(f'Loading text from {args.file_path}'							)
    with open(args.file_path    ,				'''r'''    ,				encoding='''utf8'''							) as fp:
        SCREAMING_SNAKE_CASE_      :  Any    						=		fp.readlines()

    logger.info('''Start encoding'''							)
    logger.info(f'{len(__a							)} examples to process.'							)

    SCREAMING_SNAKE_CASE_      :  int    						=		[]
    SCREAMING_SNAKE_CASE_      :  int    						=		0
    SCREAMING_SNAKE_CASE_      :  Optional[int]    						=		1_00_00
    SCREAMING_SNAKE_CASE_      :  Optional[Any]    						=		time.time()
    for text in data:
        SCREAMING_SNAKE_CASE_      :  Union[str, Any]    						=		f'{bos} {text.strip()} {sep}'
        SCREAMING_SNAKE_CASE_      :  List[Any]    						=		tokenizer.encode(__a    ,				add_special_tokens=__a							)
        rslt.append(__a							)

        iter += 1
        if iter % interval == 0:
            SCREAMING_SNAKE_CASE_      :  Tuple    						=		time.time()
            logger.info(f'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'							)
            SCREAMING_SNAKE_CASE_      :  Tuple    						=		time.time()
    logger.info('''Finished binarization'''							)
    logger.info(f'{len(__a							)} examples processed.'							)

    SCREAMING_SNAKE_CASE_      :  Any    						=		f'{args.dump_file}.{args.tokenizer_name}.pickle'
    SCREAMING_SNAKE_CASE_      :  List[Any]    						=		tokenizer.vocab_size
    if vocab_size < (1 << 16):
        SCREAMING_SNAKE_CASE_      :  Any    						=		[np.uintaa(__a							) for d in rslt]
    else:
        SCREAMING_SNAKE_CASE_      :  List[str]    						=		[np.intaa(__a							) for d in rslt]
    random.shuffle(rslt_							)
    logger.info(f'Dump to {dp_file}'							)
    with open(__a    ,				'''wb'''							) as handle:
        pickle.dump(rslt_    ,				__a    ,				protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
  main()





"""simple docstring"""


import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class     snake_case	(				__snake_case,							unittest.TestCase  ):
				SCREAMING_SNAKE_CASE_   :      List[str]         =  None
				SCREAMING_SNAKE_CASE_   :      Tuple         =  BloomTokenizerFast
				SCREAMING_SNAKE_CASE_   :      str         =  BloomTokenizerFast
				SCREAMING_SNAKE_CASE_   :      int         =  True
				SCREAMING_SNAKE_CASE_   :      Tuple         =  False
				SCREAMING_SNAKE_CASE_   :      int         =  """tokenizer_file"""
				SCREAMING_SNAKE_CASE_   :      List[str]         =  {"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}

				def 							lowercase_     (	self							:				List[Any])->   Dict:



						'''simple docstring'''


						super().setUp()
						__lowerCAmelCase:       Optional[Any]          =			BloomTokenizerFast.from_pretrained("bigscience/tokenizer")
						tokenizer.save_pretrained(self.tmpdirname)

				def 							lowercase_     (	self							:				List[Any]     ,  **UpperCamelCase__							:				Union[str, Any])->   Optional[Any]:



						'''simple docstring'''


						kwargs.update(self.special_tokens_map)
						return BloomTokenizerFast.from_pretrained(self.tmpdirname     ,  **UpperCamelCase__)

				def 							lowercase_     (	self							:				Union[str, Any])->   Union[str, Any]:



						'''simple docstring'''


						__lowerCAmelCase:       str          =			self.get_rust_tokenizer()

						__lowerCAmelCase:       int          =			["The quick brown fox</s>", "jumps over the lazy dog</s>"]
						__lowerCAmelCase:       List[str]          =			[[2_1_7_5, 2_3_7_1_4, 7_3_1_7_3, 1_4_4_2_5_2, 2], [7_7, 1_3_2_6_1_9, 3_4_7_8, 3_6_8, 1_0_9_5_8_6, 3_5_4_3_3, 2]]

						__lowerCAmelCase:       List[str]          =			tokenizer.batch_encode_plus(UpperCamelCase__)["input_ids"]
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

						__lowerCAmelCase:       List[Any]          =			tokenizer.batch_decode(UpperCamelCase__)
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)

				def 							lowercase_     (	self							:				Optional[Any]     ,  UpperCamelCase__							:				Tuple=6)->   Tuple:



						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
										__lowerCAmelCase:       Optional[Any]          =			self.rust_tokenizer_class.from_pretrained(UpperCamelCase__     ,  **UpperCamelCase__)
										# tokenizer_r.pad_token = None # Hotfixing padding = None
										# Simple input
										__lowerCAmelCase:       Dict          =			"This is a simple input"
										__lowerCAmelCase:       str          =			["This is a simple input 1", "This is a simple input 2"]
										__lowerCAmelCase:       int          =			("This is a simple input", "This is a pair")
										__lowerCAmelCase:       Union[str, Any]          =			[
										    ("This is a simple input 1", "This is a simple input 2"),
										    ("This is a simple pair 1", "This is a simple pair 2"),
										]

										# Simple input tests
										try:
												tokenizer_r.encode(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)

												tokenizer_r.batch_encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.encode(UpperCamelCase__     ,  max_length=UpperCamelCase__)
												tokenizer_r.batch_encode_plus(UpperCamelCase__     ,  max_length=UpperCamelCase__)
										except ValueError:
												self.fail("Bloom Tokenizer should be able to deal with padding")

										__lowerCAmelCase:       Tuple          =			None  # Hotfixing padding = None
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Simple input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Simple input
										self.assertRaises(
										    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length"     ,  )

										# Pair input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Pair input
										self.assertRaises(UpperCamelCase__     ,  tokenizer_r.encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length")

										# Pair input
										self.assertRaises(
										    UpperCamelCase__     ,  tokenizer_r.batch_encode_plus     ,  UpperCamelCase__     ,  max_length=UpperCamelCase__     ,  padding="max_length"     ,  )

				def 							lowercase_     (	self							:				Optional[Any])->   List[str]:



						'''simple docstring'''


						__lowerCAmelCase:       Dict          =			self.get_rust_tokenizer()
						__lowerCAmelCase:       List[str]          =			load_dataset("xnli"     ,  "all_languages"     ,  split="test"     ,  streaming=UpperCamelCase__)

						__lowerCAmelCase:       Union[str, Any]          =			next(iter(UpperCamelCase__))["premise"]  # pick up one data
						__lowerCAmelCase:       Any          =			list(sample_data.values())

						__lowerCAmelCase:       int          =			list(map(tokenizer.encode     ,  UpperCamelCase__))
						__lowerCAmelCase:       str          =			[tokenizer.decode(UpperCamelCase__     ,  clean_up_tokenization_spaces=UpperCamelCase__) for x in output_tokens]
						self.assertListEqual(UpperCamelCase__     ,  UpperCamelCase__)




				def 							lowercase_     (	self							:				Optional[int])->   str:



						'''simple docstring'''


						self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)     ,  1)
						self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])     ,  1)


