
'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowercase    :							List[str]      =					logging.get_logger(__name__)

lowercase    :							List[str]      =					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase    :							Any      =					{
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase    :							str      =					{
    '''distilbert-base-uncased''': 512,
    '''distilbert-base-uncased-distilled-squad''': 512,
    '''distilbert-base-cased''': 512,
    '''distilbert-base-cased-distilled-squad''': 512,
    '''distilbert-base-german-cased''': 512,
    '''distilbert-base-multilingual-cased''': 512,
}


lowercase    :							str      =					{
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}


class 						__UpperCAmelCase      (					lowerCamelCase_						):
							__lowercase      		=       VOCAB_FILES_NAMES
							__lowercase      		=       PRETRAINED_VOCAB_FILES_MAP
							__lowercase      		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__lowercase      		=       PRETRAINED_INIT_CONFIGURATION
							__lowercase      		=       ["""input_ids""", """attention_mask"""]
							__lowercase      		=       DistilBertTokenizer


							def __init__(       self    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_="[UNK]"    ,   lowerCAmelCase_="[SEP]"    ,   lowerCAmelCase_="[PAD]"    ,   lowerCAmelCase_="[CLS]"    ,   lowerCAmelCase_="[MASK]"    ,   lowerCAmelCase_=True    ,   lowerCAmelCase_=None    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								super().__init__(
								    _lowercase    ,   tokenizer_file=_lowercase    ,   do_lower_case=_lowercase    ,   unk_token=_lowercase    ,   sep_token=_lowercase    ,   pad_token=_lowercase    ,   cls_token=_lowercase    ,   mask_token=_lowercase    ,   tokenize_chinese_chars=_lowercase    ,   strip_accents=_lowercase    ,   **_lowercase    ,   )

								_snake_case   						=		json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
								if (
								    normalizer_state.get('lowercase'    ,   _lowercase  ) != do_lower_case
								    or normalizer_state.get('strip_accents'    ,   _lowercase  ) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'    ,   _lowercase  ) != tokenize_chinese_chars
								):
									_snake_case   						=		getattr(_lowercase    ,   normalizer_state.pop('type'  )  )
									_snake_case   						=		do_lower_case
									_snake_case   						=		strip_accents
									_snake_case   						=		tokenize_chinese_chars
									_snake_case   						=		normalizer_class(**_lowercase  )

								_snake_case   						=		do_lower_case


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_=None  ):





								"""simple docstring"""






								_snake_case   						=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		[self.sep_token_id]
								_snake_case   						=		[self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		self._tokenizer.model.save(_lowercase    ,   name=_lowercase  )
								return tuple(_lowercase  )


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


__lowerCamelCase			: Union[str, Any]							   =		'''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
   import torch

   print('''Torch version:''', torch.__version__)
   print('''Cuda available:''', torch.cuda.is_available())
   print('''Cuda version:''', torch.version.cuda)
   print('''CuDNN version:''', torch.backends.cudnn.version())
   print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
   print('''Torch version:''', None)

try:
   import transformers

   print('''transformers version:''', transformers.__version__)
except ImportError:
   print('''transformers version:''', None)
