


'''simple docstring'''


import os
import re
import sys
import traceback
import warnings
from pathlib import Path
from typing import Dict, Optional, Union
from uuid import uuida

from huggingface_hub import HfFolder, ModelCard, ModelCardData, hf_hub_download, whoami
from huggingface_hub.file_download import REGEX_COMMIT_HASH
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    is_jinja_available,
)
from packaging import version
from requests import HTTPError

from .. import __version__
from .constants import (
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    SAFETENSORS_WEIGHTS_NAME,
    WEIGHTS_NAME,
)
from .import_utils import (
    ENV_VARS_TRUE_VALUES,
    _flax_version,
    _jax_version,
    _onnxruntime_version,
    _torch_version,
    is_flax_available,
    is_onnx_available,
    is_torch_available,
)
from .logging import get_logger


lowerCamelCase__					=   get_logger(__name__)


lowerCamelCase__					=   Path(__file__).parent / 'model_card_template.md'
lowerCamelCase__					=   uuida().hex
lowerCamelCase__					=   os.getenv('HF_HUB_OFFLINE', '').upper() in ENV_VARS_TRUE_VALUES
lowerCamelCase__					=   os.getenv('DISABLE_TELEMETRY', '').upper() in ENV_VARS_TRUE_VALUES
lowerCamelCase__					=   HUGGINGFACE_CO_RESOLVE_ENDPOINT + '/api/telemetry/'






def        __lowerCAmelCase       (__lowerCAmelCase = None      ):
		_UpperCAmelCase						:				str	    =					F"""diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}"""
		if DISABLE_TELEMETRY or HF_HUB_OFFLINE:
				return ua + "; telemetry/off"
		if is_torch_available():
				ua += F"""; torch/{_torch_version}"""
		if is_flax_available():
				ua += F"""; jax/{_jax_version}"""
				ua += F"""; flax/{_flax_version}"""
		if is_onnx_available():
				ua += F"""; onnxruntime/{_onnxruntime_version}"""
		# CI will set this value to True
		if os.environ.get("DIFFUSERS_IS_CI"					,       ""      ).upper() in ENV_VARS_TRUE_VALUES:
				ua += "; is_ci/true"
		if isinstance(__lowerCAmelCase					,       __lowerCAmelCase      ):
				ua += "; " + "; ".join(F"""{k}/{v}""" for k, v in user_agent.items()      )
		elif isinstance(__lowerCAmelCase					,       __lowerCAmelCase      ):
				ua += "; " + user_agent
		return ua






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase = None					,       __lowerCAmelCase = None      ):
		if token is None:
				_UpperCAmelCase						:				List[str]	    =					HfFolder.get_token()
		if organization is None:
				_UpperCAmelCase						:				int	    =					whoami(__lowerCAmelCase      )["name"]
				return F"""{username}/{model_id}"""
		else:
				return F"""{organization}/{model_id}"""






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		if not is_jinja_available():
				raise ValueError(
				    "Modelcard rendering is based on Jinja templates."
				    " Please make sure to have `jinja` installed before using `create_model_card`."
				    " To install it, please run `pip install Jinja2`."      )

		if hasattr(__lowerCAmelCase					,       "local_rank"      ) and args.local_rank not in [-1, 0]:
				return

		_UpperCAmelCase						:				str	    =					args.hub_token if hasattr(__lowerCAmelCase					,       "hub_token"      ) else None
		_UpperCAmelCase						:				Tuple	    =					get_full_repo_name(__lowerCAmelCase					,       token=__lowerCAmelCase      )

		_UpperCAmelCase						:				int	    =					ModelCard.from_template(
		    card_data=ModelCardData(  # Card metadata object that will be converted to YAML block
		        language="en"					,       license="apache-2.0"					,       library_name="diffusers"					,       tags=[]					,       datasets=args.dataset_name					,       metrics=[]					,       )					,       template_path=__lowerCAmelCase					,       model_name=__lowerCAmelCase					,       repo_name=__lowerCAmelCase					,       dataset_name=args.dataset_name if hasattr(__lowerCAmelCase					,       "dataset_name"      ) else None					,       learning_rate=args.learning_rate					,       train_batch_size=args.train_batch_size					,       eval_batch_size=args.eval_batch_size					,       gradient_accumulation_steps=(
		        args.gradient_accumulation_steps if hasattr(__lowerCAmelCase					,       "gradient_accumulation_steps"      ) else None
		    )					,       adam_betaa=args.adam_betaa if hasattr(__lowerCAmelCase					,       "adam_beta1"      ) else None					,       adam_betaa=args.adam_betaa if hasattr(__lowerCAmelCase					,       "adam_beta2"      ) else None					,       adam_weight_decay=args.adam_weight_decay if hasattr(__lowerCAmelCase					,       "adam_weight_decay"      ) else None					,       adam_epsilon=args.adam_epsilon if hasattr(__lowerCAmelCase					,       "adam_epsilon"      ) else None					,       lr_scheduler=args.lr_scheduler if hasattr(__lowerCAmelCase					,       "lr_scheduler"      ) else None					,       lr_warmup_steps=args.lr_warmup_steps if hasattr(__lowerCAmelCase					,       "lr_warmup_steps"      ) else None					,       ema_inv_gamma=args.ema_inv_gamma if hasattr(__lowerCAmelCase					,       "ema_inv_gamma"      ) else None					,       ema_power=args.ema_power if hasattr(__lowerCAmelCase					,       "ema_power"      ) else None					,       ema_max_decay=args.ema_max_decay if hasattr(__lowerCAmelCase					,       "ema_max_decay"      ) else None					,       mixed_precision=args.mixed_precision					,       )

		_UpperCAmelCase						:				int	    =					os.path.join(args.output_dir					,       "README.md"      )
		model_card.save(__lowerCAmelCase      )






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase = None      ):
		if resolved_file is None or commit_hash is not None:
				return commit_hash
		_UpperCAmelCase						:				str	    =					str(Path(__lowerCAmelCase      ).as_posix()      )
		_UpperCAmelCase						:				Union[str, Any]	    =					re.search(R"snapshots/([^/]+)/"					,       __lowerCAmelCase      )
		if search is None:
				return None
		_UpperCAmelCase						:				Any	    =					search.groups()[0]
		return commit_hash if REGEX_COMMIT_HASH.match(__lowerCAmelCase      ) else None


# Old default cache path, potentially to be migrated.
# This logic was more or less taken from `transformers`, with the following differences:
# - Diffusers doesn't use custom environment variables to specify the cache path.
# - There is no need to migrate the cache format, just move the files to the new location.
lowerCamelCase__					=   os.path.expanduser(
    os.getenv('HF_HOME', os.path.join(os.getenv('XDG_CACHE_HOME', '~/.cache'), 'huggingface'))
)
lowerCamelCase__					=   os.path.join(hf_cache_home, 'diffusers')






def        __lowerCAmelCase       (__lowerCAmelCase = None					,       __lowerCAmelCase = None      ):
		if new_cache_dir is None:
				_UpperCAmelCase						:				Union[str, Any]	    =					DIFFUSERS_CACHE
		if old_cache_dir is None:
				_UpperCAmelCase						:				Any	    =					old_diffusers_cache

		_UpperCAmelCase						:				Optional[int]	    =					Path(__lowerCAmelCase      ).expanduser()
		_UpperCAmelCase						:				Union[str, Any]	    =					Path(__lowerCAmelCase      ).expanduser()
		for old_blob_path in old_cache_dir.glob("**/blobs/*"      ):
				if old_blob_path.is_file() and not old_blob_path.is_symlink():
						_UpperCAmelCase						:				Union[str, Any]	    =					new_cache_dir / old_blob_path.relative_to(__lowerCAmelCase      )
						new_blob_path.parent.mkdir(parents=__lowerCAmelCase					,       exist_ok=__lowerCAmelCase      )
						os.replace(__lowerCAmelCase					,       __lowerCAmelCase      )
						try:
								os.symlink(__lowerCAmelCase					,       __lowerCAmelCase      )
						except OSError:
								logger.warning(
								    "Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded."      )
		# At this point, old_cache_dir contains symlinks to the new cache (it can still be used).


lowerCamelCase__					=   os.path.join(DIFFUSERS_CACHE, 'version_diffusers_cache.txt')
if not os.path.isfile(cache_version_file):
						lowerCamelCase__					=   0
else:
						with open(cache_version_file) as f:
												try:
																		lowerCamelCase__					=   int(f.read())
												except ValueError:
																		lowerCamelCase__					=   0

if cache_version < 1:
						lowerCamelCase__					=   os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
						if old_cache_is_not_empty:
												logger.warning(
												    'The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your '
												    'existing cached models. This is a one-time operation, you can interrupt it or run it '
												    'later by calling `diffusers.utils.hub_utils.move_cache()`.'
												)
												try:
																		move_cache()
												except Exception as e:
																		lowerCamelCase__					=   '\n'.join(traceback.format_tb(e.__traceback__))
																		logger.error(
																		    F'''There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease '''
																		    'file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole '
																		    'message and we will do our best to help.'
																		)

if cache_version < 1:
						try:
												os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
												with open(cache_version_file, 'w') as f:
																		f.write('1')
						except Exception:
												logger.warning(
												    F'''There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure '''
												    'the directory exists and can be written to.'
												)






def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase = None      ):
		if variant is not None:
				_UpperCAmelCase						:				str	    =					weights_name.split("."      )
				_UpperCAmelCase						:				int	    =					splits[:-1] + [variant] + splits[-1:]
				_UpperCAmelCase						:				Union[str, Any]	    =					".".join(__lowerCAmelCase      )

		return weights_name






def        __lowerCAmelCase       (__lowerCAmelCase					,       *,
    __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase=None					,       ):
		_UpperCAmelCase						:				str	    =					str(__lowerCAmelCase      )
		if os.path.isfile(__lowerCAmelCase      ):
				return pretrained_model_name_or_path
		elif os.path.isdir(__lowerCAmelCase      ):
				if os.path.isfile(os.path.join(__lowerCAmelCase					,       __lowerCAmelCase      )      ):
						# Load from a PyTorch checkpoint
						_UpperCAmelCase						:				str	    =					os.path.join(__lowerCAmelCase					,       __lowerCAmelCase      )
						return model_file
				elif subfolder is not None and os.path.isfile(
				    os.path.join(__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      )      ):
						_UpperCAmelCase						:				Tuple	    =					os.path.join(__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      )
						return model_file
				else:
						raise EnvironmentError(
						    F"""Error no file named {weights_name} found in directory {pretrained_model_name_or_path}."""      )
		else:
				# 1. First check if deprecated way of loading from branches is used
				if (
				    revision in DEPRECATED_REVISION_ARGS
				    and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME)
				    and version.parse(version.parse(__lowerCAmelCase      ).base_version      ) >= version.parse("0.20.0"      )
				):
						try:
								_UpperCAmelCase						:				List[str]	    =					hf_hub_download(
								    __lowerCAmelCase					,       filename=_add_variant(__lowerCAmelCase					,       __lowerCAmelCase      )					,       cache_dir=__lowerCAmelCase					,       force_download=__lowerCAmelCase					,       proxies=__lowerCAmelCase					,       resume_download=__lowerCAmelCase					,       local_files_only=__lowerCAmelCase					,       use_auth_token=__lowerCAmelCase					,       user_agent=__lowerCAmelCase					,       subfolder=__lowerCAmelCase					,       revision=revision or commit_hash					,       )
								warnings.warn(
								    F"""Loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'` is deprecated. Loading instead from `revision='main'` with `variant={revision}`. Loading model variants via `revision='{revision}'` will be removed in diffusers v1. Please use `variant='{revision}'` instead."""					,       __lowerCAmelCase					,       )
								return model_file
						except:  # noqa: E722
								warnings.warn(
								    F"""You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant='{revision}'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(__lowerCAmelCase					,       __lowerCAmelCase      )} file in the 'main' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title '{pretrained_model_name_or_path} is missing {_add_variant(__lowerCAmelCase					,       __lowerCAmelCase      )}' so that the correct variant file can be added."""					,       __lowerCAmelCase					,       )
				try:
						# 2. Load model file as usual
						_UpperCAmelCase						:				int	    =					hf_hub_download(
						    __lowerCAmelCase					,       filename=__lowerCAmelCase					,       cache_dir=__lowerCAmelCase					,       force_download=__lowerCAmelCase					,       proxies=__lowerCAmelCase					,       resume_download=__lowerCAmelCase					,       local_files_only=__lowerCAmelCase					,       use_auth_token=__lowerCAmelCase					,       user_agent=__lowerCAmelCase					,       subfolder=__lowerCAmelCase					,       revision=revision or commit_hash					,       )
						return model_file

				except RepositoryNotFoundError:
						raise EnvironmentError(
						    F"""{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier """
						    "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a "
						    "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli "
						    "login`."      )
				except RevisionNotFoundError:
						raise EnvironmentError(
						    F"""{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for """
						    "this model name. Check the model page at "
						    F"""'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions."""      )
				except EntryNotFoundError:
						raise EnvironmentError(
						    F"""{pretrained_model_name_or_path} does not appear to have a file named {weights_name}."""      )
				except HTTPError as err:
						raise EnvironmentError(
						    F"""There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}"""      )
				except ValueError:
						raise EnvironmentError(
						    F"""We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it"""
						    F""" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a"""
						    F""" directory containing a file named {weights_name} or"""
						    " \nCheckout your internet connection or see how to run the library in"
						    " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'."      )
				except EnvironmentError:
						raise EnvironmentError(
						    F"""Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from """
						    "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
						    F"""Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory """
						    F"""containing a file named {weights_name}"""      )







'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCamelCase__					=   logging.get_logger(__name__)

lowerCamelCase__					=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__					=   {
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase__					=   {
    'bert-base-uncased': 512,
    'bert-large-uncased': 512,
    'bert-base-cased': 512,
    'bert-large-cased': 512,
    'bert-base-multilingual-uncased': 512,
    'bert-base-multilingual-cased': 512,
    'bert-base-chinese': 512,
    'bert-base-german-cased': 512,
    'bert-large-uncased-whole-word-masking': 512,
    'bert-large-cased-whole-word-masking': 512,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 512,
    'bert-large-cased-whole-word-masking-finetuned-squad': 512,
    'bert-base-cased-finetuned-mrpc': 512,
    'bert-base-german-dbmdz-cased': 512,
    'bert-base-german-dbmdz-uncased': 512,
    'TurkuNLP/bert-base-finnish-cased-v1': 512,
    'TurkuNLP/bert-base-finnish-uncased-v1': 512,
    'wietsedv/bert-base-dutch-cased': 512,
}

lowerCamelCase__					=   {
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Union[str, Any]				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Dict				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		Tuple				      =   PRETRAINED_INIT_CONFIGURATION
			lowerCAmelCase       :		Optional[int]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		Any				      =   BertTokenizer




			def __init__(						self     :   int		,     lowerCamelCase__     :   Optional[Any]=None		,     lowerCamelCase__     :   Optional[int]=None		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Tuple="[UNK]"		,     lowerCamelCase__     :   str="[SEP]"		,     lowerCamelCase__     :   Optional[Any]="[PAD]"		,     lowerCamelCase__     :   List[str]="[CLS]"		,     lowerCamelCase__     :   Union[str, Any]="[MASK]"		,     lowerCamelCase__     :   str=True		,     lowerCamelCase__     :   Dict=None		,     **lowerCamelCase__     :   Union[str, Any]		,     )      ->Tuple:



					'''simple docstring'''







					super().__init__(
					    lowerCamelCase__		,     tokenizer_file=lowerCamelCase__		,     do_lower_case=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     tokenize_chinese_chars=lowerCamelCase__		,     strip_accents=lowerCamelCase__		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Optional[Any]	    =					json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
					if (
					    normalizer_state.get("lowercase"		,     lowerCamelCase__       ) != do_lower_case
					    or normalizer_state.get("strip_accents"		,     lowerCamelCase__       ) != strip_accents
					    or normalizer_state.get("handle_chinese_chars"		,     lowerCamelCase__       ) != tokenize_chinese_chars
					):
							_UpperCAmelCase						:				str	    =					getattr(lowerCamelCase__		,     normalizer_state.pop("type"       )       )
							_UpperCAmelCase						:				Optional[Any]	    =					do_lower_case
							_UpperCAmelCase						:				Any	    =					strip_accents
							_UpperCAmelCase						:				List[Any]	    =					tokenize_chinese_chars
							_UpperCAmelCase						:				int	    =					normalizer_class(**lowerCamelCase__       )

					_UpperCAmelCase						:				Union[str, Any]	    =					do_lower_case




			def 							lowerCAmelCase__			(						self     :   List[str]		,     lowerCamelCase__     :   List[str]		,     lowerCamelCase__     :   str=None       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output




			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Tuple	    =					[self.sep_token_id]
					_UpperCAmelCase						:				int	    =					[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






			def 							lowerCAmelCase__			(						self     :   List[Any]		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					self._tokenizer.model.save(lowerCamelCase__		,     name=lowerCamelCase__       )
					return tuple(lowerCamelCase__       )



