





"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


a_ =			logging.get_logger(__name__)

a_ =			{"""vocab_file""": """vocab.txt"""}

a_ =			{
    """vocab_file""": {
        """YituTech/conv-bert-base""": """https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt""",
        """YituTech/conv-bert-medium-small""": (
            """https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"""
        ),
        """YituTech/conv-bert-small""": """https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt""",
    }
}

a_ =			{
    """YituTech/conv-bert-base""": 512,
    """YituTech/conv-bert-medium-small""": 512,
    """YituTech/conv-bert-small""": 512,
}


a_ =			{
    """YituTech/conv-bert-base""": {"""do_lower_case""": True},
    """YituTech/conv-bert-medium-small""": {"""do_lower_case""": True},
    """YituTech/conv-bert-small""": {"""do_lower_case""": True},
}






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 _lowerCamelCase        =     VOCAB_FILES_NAMES
 _lowerCamelCase        =     PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase        =     PRETRAINED_INIT_CONFIGURATION
 _lowerCamelCase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase        =     ConvBertTokenizer

 def __init__(    self					,						__lowerCamelCase=None					,						__lowerCamelCase=None					,						__lowerCamelCase=True					,						__lowerCamelCase="[UNK]"					,						__lowerCamelCase="[SEP]"					,						__lowerCamelCase="[PAD]"					,						__lowerCamelCase="[CLS]"					,						__lowerCamelCase="[MASK]"					,						__lowerCamelCase=True					,						__lowerCamelCase=None					,						**__lowerCamelCase					,						):



     '''simple docstring'''




     super().__init__(
         __lowerCamelCase					,						tokenizer_file=__lowerCamelCase					,						do_lower_case=__lowerCamelCase					,						unk_token=__lowerCamelCase					,						sep_token=__lowerCamelCase					,						pad_token=__lowerCamelCase					,						cls_token=__lowerCamelCase					,						mask_token=__lowerCamelCase					,						tokenize_chinese_chars=__lowerCamelCase					,						strip_accents=__lowerCamelCase					,						**__lowerCamelCase					,						)

     __A    :						List[Any]						 =     json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
     if (
         normalizer_state.get('''lowercase'''					,						__lowerCamelCase  ) != do_lower_case
         or normalizer_state.get('''strip_accents'''					,						__lowerCamelCase  ) != strip_accents
         or normalizer_state.get('''handle_chinese_chars'''					,						__lowerCamelCase  ) != tokenize_chinese_chars
     ):
         __A    :						List[Any]						 =     getattr(__lowerCamelCase					,						normalizer_state.pop('''type'''  )  )
         __A    :						Tuple						 =     do_lower_case
         __A    :						Union[str, Any]						 =     strip_accents
         __A    :						str						 =     tokenize_chinese_chars
         __A    :						Optional[Any]						 =     normalizer_class(**__lowerCamelCase  )

     __A    :						Union[str, Any]						 =     do_lower_case

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase=None  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

     return output

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     [self.sep_token_id]
     __A    :						Union[str, Any]						 =     [self.cls_token_id]
     if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
     return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase = None  ):



     '''simple docstring'''




     __A    :						Dict						 =     self._tokenizer.model.save(__lowerCamelCase					,						name=__lowerCamelCase  )
     return tuple(__lowerCamelCase  )











"""simple docstring"""



import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


a_ =			50000
a_ =			5000

a_ ,			a_ =			os.path.split(__file__)
a_ =			os.path.join(RESULTS_BASEPATH, """results""", RESULTS_FILENAME.replace(""".py""", """.json"""))







@get_duration
def 		__lowercase     (   snake_case_  :  datasets.Dataset				,snake_case_  :  Tuple						)   ->Tuple:



    '''simple docstring'''







    for i in range(snake_case_						):
        __A    :						int						 =     dataset[i]







@get_duration
def 		__lowercase     (   snake_case_  :  datasets.Dataset				,snake_case_  :  Optional[Any]				,snake_case_  :  int						)   ->Tuple:



    '''simple docstring'''







    for i in range(0				,len(snake_case_						)				,snake_case_						):
        __A    :						List[str]						 =     dataset[i : i + batch_size]







@get_duration
def 		__lowercase     (   snake_case_  :  datasets.Dataset				,snake_case_  :  List[Any]				,snake_case_  :  Any						)   ->int:



    '''simple docstring'''







    with dataset.formatted_as(type=snake_case_						):
        for i in range(snake_case_						):
            __A    :						Union[str, Any]						 =     dataset[i]







@get_duration
def 		__lowercase     (   snake_case_  :  datasets.Dataset				,snake_case_  :  Any				,snake_case_  :  Union[str, Any]				,snake_case_  :  Optional[int]						)   ->Union[str, Any]:



    '''simple docstring'''







    with dataset.formatted_as(type=snake_case_						):
        for i in range(0				,snake_case_				,snake_case_						):
            __A    :						Dict						 =     dataset[i : i + batch_size]







def 		__lowercase     (   )   ->Optional[int]:



    '''simple docstring'''







    __A    :						int						 =     {'''num examples''': SPEED_TEST_N_EXAMPLES}
    __A    :						Optional[int]						 =     [
        (read, {'''length''': SMALL_TEST}),
        (read, {'''length''': SPEED_TEST_N_EXAMPLES}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 10}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 100}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1000}),
        (read_formatted, {'''type''': '''numpy''', '''length''': SMALL_TEST}),
        (read_formatted, {'''type''': '''pandas''', '''length''': SMALL_TEST}),
        (read_formatted, {'''type''': '''torch''', '''length''': SMALL_TEST}),
        (read_formatted, {'''type''': '''tensorflow''', '''length''': SMALL_TEST}),
        (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 10}),
        (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1000}),
    ]

    __A    :						int						 =     [
        (read, {'''length''': SMALL_TEST}),
        (read, {'''length''': SPEED_TEST_N_EXAMPLES}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 10}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 100}),
        (read_batch, {'''length''': SPEED_TEST_N_EXAMPLES, '''batch_size''': 1000}),
        (read_formatted, {'''type''': '''numpy''', '''length''': SMALL_TEST}),
        (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 10}),
        (read_formatted_batch, {'''type''': '''numpy''', '''length''': SMALL_TEST, '''batch_size''': 1000}),
    ]
    with tempfile.TemporaryDirectory() as tmp_dir:
        print('''generating dataset'''						)
        __A    :						Any						 =     datasets.Features(
            {'''list''': datasets.Sequence(datasets.Value('''float32'''						)						), '''numbers''': datasets.Value('''float32'''						)}						)
        __A    :						List[Any]						 =     generate_example_dataset(
            os.path.join(snake_case_				,'''dataset.arrow'''						)				,snake_case_				,num_examples=snake_case_				,seq_shapes={'''list''': (100,)}				,)
        print('''first set of iterations'''						)
        for func, kwargs in functions:
            print(func.__name__				,str(snake_case_						)						)
            __A    :						Dict						 =     func(snake_case_				,**snake_case_						)

        print('''shuffling dataset'''						)
        __A    :						int						 =     dataset.shuffle()
        print('''Second set of iterations (after shuffling'''						)
        for func, kwargs in functions_shuffled:
            print('''shuffled '''				,func.__name__				,str(snake_case_						)						)
            __A    :						Optional[Any]						 =     func(
                snake_case_				,**snake_case_						)

    with open(snake_case_				,'''wb'''						) as f:
        f.write(json.dumps(snake_case_						).encode('''utf-8'''						)						)


if __name__ == "__main__":  # useful to run the profiler
       benchmark_iterating()




