





"""simple docstring"""
import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					_SCREAMING_SNAKE_CASE(   A    ,	unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_   :      Union[str, Any]							=  GPTaTokenizer
       SCREAMING_SNAKE_CASE_   :      Optional[int]							=  GPTaTokenizerFast
       SCREAMING_SNAKE_CASE_   :      Dict							=  True
       SCREAMING_SNAKE_CASE_   :      Dict							=  {'''add_prefix_space''': True}
       SCREAMING_SNAKE_CASE_   :      Union[str, Any]							=  False




       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             __SCREAMING_SNAKE_CASE   :Any						=  [
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''\u0120''',
                 '''\u0120l''',
                 '''\u0120n''',
                 '''\u0120lo''',
                 '''\u0120low''',
                 '''er''',
                 '''\u0120lowest''',
                 '''\u0120newer''',
                 '''\u0120wider''',
                 '''<unk>''',
                 '''<|endoftext|>''',
             ]
             __SCREAMING_SNAKE_CASE   :str						=  dict(zip(SCREAMING_SNAKE_CASE__		,range(len(SCREAMING_SNAKE_CASE__							)							)							)							)
             __SCREAMING_SNAKE_CASE   :Any						=  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
             __SCREAMING_SNAKE_CASE   :List[str]						=  {'''unk_token''': '''<unk>'''}

             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''vocab_file''']							)
             __SCREAMING_SNAKE_CASE   :Tuple						=  os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''merges_file''']							)
             with open(self.vocab_file		,'''w'''		,encoding='''utf-8'''							) as fp:
                   fp.write(json.dumps(SCREAMING_SNAKE_CASE__							) + '''\n'''							)
             with open(self.merges_file		,'''w'''		,encoding='''utf-8'''							) as fp:
                   fp.write('''\n'''.join(SCREAMING_SNAKE_CASE__							)							)




       def     _UpperCamelCase     (						self		,**SCREAMING_SNAKE_CASE__							)			->      str:





             """simple docstring"""






             kwargs.update(self.special_tokens_map							)
             return GPTaTokenizer.from_pretrained(self.tmpdirname		,**SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,**SCREAMING_SNAKE_CASE__							)			->      str:





             """simple docstring"""






             kwargs.update(self.special_tokens_map							)
             return GPTaTokenizerFast.from_pretrained(self.tmpdirname		,**SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[str]						=  '''lower newer'''
             __SCREAMING_SNAKE_CASE   :int						=  '''lower newer'''
             return input_text, output_text




       def     _UpperCamelCase     (						self							)			->      str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  GPTaTokenizer(self.vocab_file		,self.merges_file		,**self.special_tokens_map							)
             __SCREAMING_SNAKE_CASE   :List[str]						=  '''lower newer'''
             __SCREAMING_SNAKE_CASE   :Any						=  ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
             __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__		,add_prefix_space=SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  tokens + [tokenizer.unk_token]
             __SCREAMING_SNAKE_CASE   :List[Any]						=  [14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)		,SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self							)			->      List[str]:





             """simple docstring"""






             if not self.test_rust_tokenizer:
                   return

             __SCREAMING_SNAKE_CASE   :int						=  self.get_tokenizer()
             __SCREAMING_SNAKE_CASE   :int						=  self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Any						=  '''lower newer'''

             # Testing tokenization
             __SCREAMING_SNAKE_CASE   :int						=  tokenizer.tokenize(SCREAMING_SNAKE_CASE__		,add_prefix_space=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Dict						=  rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             # Testing conversion to ids without special tokens
             __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__		,add_prefix_space=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Any						=  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             # Testing conversion to ids with special tokens
             __SCREAMING_SNAKE_CASE   :Dict						=  self.get_rust_tokenizer(add_prefix_space=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_prefix_space=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  rust_tokenizer.encode(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             # Testing the unknown token
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokens + [rust_tokenizer.unk_token]
             __SCREAMING_SNAKE_CASE   :Dict						=  [14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)		,SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,*SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)			->      Optional[int]:





             """simple docstring"""






             pass




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__=15							)			->      Dict:





             """simple docstring"""






             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
                         __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)

                         # Simple input
                         __SCREAMING_SNAKE_CASE   :Optional[int]						=  '''This is a simple input'''
                         __SCREAMING_SNAKE_CASE   :Any						=  ['''This is a simple input 1''', '''This is a simple input 2''']
                         __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  ('''This is a simple input''', '''This is a pair''')
                         __SCREAMING_SNAKE_CASE   :List[str]						=  [
                             ('''This is a simple input 1''', '''This is a simple input 2'''),
                             ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                         ]

                         # Simple input tests
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Simple input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Simple input
                         self.assertRaises(
                             SCREAMING_SNAKE_CASE__		,tokenizer_r.batch_encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,)

                         # Pair input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Pair input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Pair input
                         self.assertRaises(
                             SCREAMING_SNAKE_CASE__		,tokenizer_r.batch_encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,)




       def     _UpperCamelCase     (						self							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Dict						=  GPTaTokenizer.from_pretrained(self.tmpdirname		,pad_token='''<pad>'''							)

             # Simple input
             __SCREAMING_SNAKE_CASE   :Dict						=  '''This is a simple input'''
             __SCREAMING_SNAKE_CASE   :Dict						=  ['''This is a simple input looooooooong''', '''This is a simple input''']
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  ('''This is a simple input''', '''This is a pair''')
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  [
                 ('''This is a simple input loooooong''', '''This is a simple input'''),
                 ('''This is a simple pair loooooong''', '''This is a simple pair'''),
             ]

             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokenizer.pad_token_id

             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokenizer(SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,max_length=30		,return_tensors='''np'''							)
             __SCREAMING_SNAKE_CASE   :str						=  tokenizer(SCREAMING_SNAKE_CASE__		,padding=SCREAMING_SNAKE_CASE__		,truncate=SCREAMING_SNAKE_CASE__		,return_tensors='''np'''							)
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokenizer(*SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,max_length=60		,return_tensors='''np'''							)
             __SCREAMING_SNAKE_CASE   :str						=  tokenizer(SCREAMING_SNAKE_CASE__		,padding=SCREAMING_SNAKE_CASE__		,truncate=SCREAMING_SNAKE_CASE__		,return_tensors='''np'''							)

             # s
             # test single string max_length padding
             self.assertEqual(out_s['''input_ids'''].shape[-1]		,30							)
             self.assertTrue(pad_token_id in out_s['''input_ids''']							)
             self.assertTrue(0 in out_s['''attention_mask''']							)

             # s2
             # test automatic padding
             self.assertEqual(out_sa['''input_ids'''].shape[-1]		,33							)
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]							)
             self.assertFalse(0 in out_sa['''attention_mask'''][0]							)
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]							)
             self.assertTrue(0 in out_sa['''attention_mask'''][1]							)

             # p
             # test single pair max_length padding
             self.assertEqual(out_p['''input_ids'''].shape[-1]		,60							)
             self.assertTrue(pad_token_id in out_p['''input_ids''']							)
             self.assertTrue(0 in out_p['''attention_mask''']							)

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa['''input_ids'''].shape[-1]		,52							)
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]							)
             self.assertFalse(0 in out_pa['''attention_mask'''][0]							)
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]							)
             self.assertTrue(0 in out_pa['''attention_mask'''][1]							)




       def     _UpperCamelCase     (						self							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[int]						=  '''$$$'''
             __SCREAMING_SNAKE_CASE   :int						=  GPTaTokenizer.from_pretrained(self.tmpdirname		,bos_token=SCREAMING_SNAKE_CASE__		,add_bos_token=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Tuple						=  '''This is a simple input'''
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  ['''This is a simple input 1''', '''This is a simple input 2''']

             __SCREAMING_SNAKE_CASE   :List[str]						=  tokenizer.bos_token_id

             __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Dict						=  tokenizer(SCREAMING_SNAKE_CASE__							)

             self.assertEqual(out_s.input_ids[0]		,SCREAMING_SNAKE_CASE__							)
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

             __SCREAMING_SNAKE_CASE   :Any						=  tokenizer.decode(out_s.input_ids							)
             __SCREAMING_SNAKE_CASE   :int						=  tokenizer.batch_decode(out_sa.input_ids							)

             self.assertEqual(decode_s.split()[0]		,SCREAMING_SNAKE_CASE__							)
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)




       def     _UpperCamelCase     (						self							)			->      List[Any]:





             """simple docstring"""






             pass




       def     _UpperCamelCase     (						self							)			->      Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  [self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,add_bos_token=SCREAMING_SNAKE_CASE__							)]
             for tokenizer in tokenizers:
                   with self.subTest(f'''{tokenizer.__class__.__name__}'''							):
                         __SCREAMING_SNAKE_CASE   :Optional[Any]						=  '''Encode this.'''
                         __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  '''This one too please.'''
                         __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
                         encoded_sequence += tokenizer.encode(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer.encode_plus(
                             SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__		,return_special_tokens_mask=SCREAMING_SNAKE_CASE__		,)
                         __SCREAMING_SNAKE_CASE   :List[Any]						=  encoded_sequence_dict['''input_ids''']
                         __SCREAMING_SNAKE_CASE   :List[str]						=  encoded_sequence_dict['''special_tokens_mask''']
                         self.assertEqual(len(SCREAMING_SNAKE_CASE__							)		,len(SCREAMING_SNAKE_CASE__							)							)

                         __SCREAMING_SNAKE_CASE   :List[Any]						=  [
                             (x if not special_tokens_mask[i] else None) for i, x in enumerate(SCREAMING_SNAKE_CASE__							)
                         ]
                         __SCREAMING_SNAKE_CASE   :Dict						=  [x for x in filtered_sequence if x is not None]
                         self.assertEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)






@require_tokenizers
class 					_SCREAMING_SNAKE_CASE(   unittest.TestCase  ):




       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[Any]						=  AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,from_slow=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Dict						=  '''A photo of a cat'''

             __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.encode(
                 SCREAMING_SNAKE_CASE__		,)
             self.assertEqual(SCREAMING_SNAKE_CASE__		,[2, 2_50, 13_45, 9, 10, 47_58]							)
             tokenizer.save_pretrained('''test_opt'''							)

             __SCREAMING_SNAKE_CASE   :int						=  AutoTokenizer.from_pretrained('''./test_opt'''							)
             __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.encode(
                 SCREAMING_SNAKE_CASE__		,)
             self.assertEqual(SCREAMING_SNAKE_CASE__		,[2, 2_50, 13_45, 9, 10, 47_58]							)




       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,use_slow=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :List[Any]						=  '''A photo of a cat'''

             __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer.encode(
                 SCREAMING_SNAKE_CASE__		,)
             # Same as above
             self.assertEqual(SCREAMING_SNAKE_CASE__		,[2, 2_50, 13_45, 9, 10, 47_58]							)




       @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''							)
       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Dict						=  AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,from_slow=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  '''bos'''
             __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.get_vocab()['''bos''']

             __SCREAMING_SNAKE_CASE   :List[str]						=  '''A photo of a cat'''
             __SCREAMING_SNAKE_CASE   :Tuple						=  tokenizer.encode(
                 SCREAMING_SNAKE_CASE__		,)
             # We changed the bos token
             self.assertEqual(SCREAMING_SNAKE_CASE__		,[3_19_57, 2_50, 13_45, 9, 10, 47_58]							)
             tokenizer.save_pretrained('''./tok'''							)
             __SCREAMING_SNAKE_CASE   :Any						=  AutoTokenizer.from_pretrained('''./tok'''							)
             self.assertTrue(tokenizer.is_fast							)
             __SCREAMING_SNAKE_CASE   :str						=  tokenizer.encode(
                 SCREAMING_SNAKE_CASE__		,)
             self.assertEqual(SCREAMING_SNAKE_CASE__		,[3_19_57, 2_50, 13_45, 9, 10, 47_58]							)






"""simple docstring"""
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
lowerCamelCase_								=		logging.getLogger(__name__)



def      __lowerCamelCase   (							)			->      int:
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  argparse.ArgumentParser(
          description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''							)
      parser.add_argument('''--file_path''' ,   type=a_ ,   default='''data/dump.txt''' ,   help='''The path to the data.'''							)
      parser.add_argument('''--tokenizer_type''' ,   type=a_ ,   default='''bert''' ,   choices=['''bert''', '''roberta''', '''gpt2''']							)
      parser.add_argument('''--tokenizer_name''' ,   type=a_ ,   default='''bert-base-uncased''' ,   help='''The tokenizer to use.'''							)
      parser.add_argument('''--dump_file''' ,   type=a_ ,   default='''data/dump''' ,   help='''The dump file prefix.'''							)
      __SCREAMING_SNAKE_CASE   :Any						=  parser.parse_args()

      logger.info(f'''Loading Tokenizer ({args.tokenizer_name})'''							)
      if args.tokenizer_type == "bert":
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  BertTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            __SCREAMING_SNAKE_CASE   :str						=  RobertaTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''cls_token''']  # `<s>`
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''sep_token''']  # `</s>`
      elif args.tokenizer_type == "gpt2":
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  GPTaTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

      logger.info(f'''Loading text from {args.file_path}'''							)
      with open(args.file_path ,   '''r''' ,   encoding='''utf8'''							) as fp:
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  fp.readlines()

      logger.info('''Start encoding'''							)
      logger.info(f'''{len(a_							)} examples to process.'''							)

      __SCREAMING_SNAKE_CASE   :Optional[int]						=  []
      __SCREAMING_SNAKE_CASE   :List[str]						=  0
      __SCREAMING_SNAKE_CASE   :Optional[Any]						=  1_00_00
      __SCREAMING_SNAKE_CASE   :List[Any]						=  time.time()
      for text in data:
            __SCREAMING_SNAKE_CASE   :Any						=  f'''{bos} {text.strip()} {sep}'''
            __SCREAMING_SNAKE_CASE   :int						=  tokenizer.encode(a_ ,   add_special_tokens=a_							)
            rslt.append(a_							)

            iter += 1
            if iter % interval == 0:
                  __SCREAMING_SNAKE_CASE   :Any						=  time.time()
                  logger.info(f'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''							)
                  __SCREAMING_SNAKE_CASE   :Any						=  time.time()
      logger.info('''Finished binarization'''							)
      logger.info(f'''{len(a_							)} examples processed.'''							)

      __SCREAMING_SNAKE_CASE   :Optional[int]						=  f'''{args.dump_file}.{args.tokenizer_name}.pickle'''
      __SCREAMING_SNAKE_CASE   :str						=  tokenizer.vocab_size
      if vocab_size < (1 << 16):
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  [np.uintaa(a_							) for d in rslt]
      else:
            __SCREAMING_SNAKE_CASE   :List[Any]						=  [np.intaa(a_							) for d in rslt]
      random.shuffle(rslt_							)
      logger.info(f'''Dump to {dp_file}'''							)
      with open(a_ ,   '''wb'''							) as handle:
            pickle.dump(rslt_ ,   a_ ,   protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
  main()