
"""simple docstring"""
import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase		:    Union[str, Any]			=       logging.get_logger(__name__)

lowercase		:    Optional[int]			=       {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowercase		:    Any			=       {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

lowercase		:    Tuple			=       {
    'openai-gpt': 5_12,
}


def      lowerCamelCase_     (						lowerCamelCase_			)  ->      Union[str, Any]:


					'''simple docstring'''
					__a:    Tuple		     =       text.strip()
					if not text:
										return []
					__a:    Optional[Any]		     =       text.split()
					return tokens




class  _snake_case	(    _lowercase							):






							def __init__(       self    :      int    ,			snake_case__    :      Optional[Any]=True    ,			snake_case__    :      Any=None    ,			snake_case__    :      Optional[int]=True    ,			snake_case__    :      Any=None    ,			snake_case__    :      Union[str, Any]=True    ,			):






												"""simple docstring"""



												if never_split is None:
																	__a:    Any		     =       []
												__a:    Optional[Any]		     =       do_lower_case
												__a:    Any		     =       set(snake_case__     )
												__a:    Dict		     =       tokenize_chinese_chars
												__a:    List[str]		     =       strip_accents
												__a:    List[Any]		     =       do_split_on_punc






							def 				__lowercase	(       self    :      Any    ,			snake_case__    :      Dict    ,			snake_case__    :      Optional[int]=None     ):






												"""simple docstring"""



												__a:    Optional[int]		     =       self.never_split.union(set(snake_case__     )     ) if never_split else self.never_split
												__a:    Dict		     =       self._clean_text(snake_case__     )

												# This was added on November 1st, 2018 for the multilingual and Chinese
												# models. This is also applied to the English models now, but it doesn't
												# matter since the English models were not trained on any Chinese data
												# and generally don't have any Chinese data in them (there are Chinese
												# characters in the vocabulary because Wikipedia does have some Chinese
												# words in the English Wikipedia.).
												if self.tokenize_chinese_chars:
																	__a:    Tuple		     =       self._tokenize_chinese_chars(snake_case__     )
												# prevents treating the same character with different unicode codepoints as different characters
												__a:    Tuple		     =       unicodedata.normalize("""NFC"""    ,			snake_case__     )
												__a:    Dict		     =       whitespace_tokenize(snake_case__     )
												__a:    int		     =       []
												for token in orig_tokens:
																	if token not in never_split:
																						if self.do_lower_case:
																											__a:    int		     =       token.lower()
																											if self.strip_accents is not False:
																																__a:    Optional[Any]		     =       self._run_strip_accents(snake_case__     )
																						elif self.strip_accents:
																											__a:    Union[str, Any]		     =       self._run_strip_accents(snake_case__     )
																	split_tokens.extend(self._run_split_on_punc(snake_case__    ,			snake_case__     )     )

												__a:    Optional[Any]		     =       whitespace_tokenize(""" """.join(snake_case__     )     )
												return output_tokens






							def 				__lowercase	(       self    :      List[Any]    ,			snake_case__    :      Tuple     ):






												"""simple docstring"""



												__a:    Dict		     =       unicodedata.normalize("""NFD"""    ,			snake_case__     )
												__a:    List[str]		     =       []
												for char in text:
																	__a:    List[str]		     =       unicodedata.category(snake_case__     )
																	if cat == "Mn":
																						continue
																	output.append(snake_case__     )
												return "".join(snake_case__     )






							def 				__lowercase	(       self    :      int    ,			snake_case__    :      Dict    ,			snake_case__    :      str=None     ):






												"""simple docstring"""



												if not self.do_split_on_punc or (never_split is not None and text in never_split):
																	return [text]
												__a:    Union[str, Any]		     =       list(snake_case__     )
												__a:    List[str]		     =       0
												__a:    Optional[Any]		     =       True
												__a:    str		     =       []
												while i < len(snake_case__     ):
																	__a:    int		     =       chars[i]
																	if _is_punctuation(snake_case__     ):
																						output.append([char]     )
																						__a:    int		     =       True
																	else:
																						if start_new_word:
																											output.append([]     )
																						__a:    Optional[Any]		     =       False
																						output[-1].append(snake_case__     )
																	i += 1

												return ["".join(snake_case__     ) for x in output]






							def 				__lowercase	(       self    :      List[Any]    ,			snake_case__    :      Optional[int]     ):






												"""simple docstring"""



												__a:    Any		     =       []
												for char in text:
																	__a:    Optional[int]		     =       ord(snake_case__     )
																	if self._is_chinese_char(snake_case__     ):
																						output.append(""" """     )
																						output.append(snake_case__     )
																						output.append(""" """     )
																	else:
																						output.append(snake_case__     )
												return "".join(snake_case__     )






							def 				__lowercase	(       self    :      Tuple    ,			snake_case__    :      Union[str, Any]     ):






												"""simple docstring"""



												if (
												    (cp >= 0x4E_00 and cp <= 0x9F_FF)
												    or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
												    or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
												    or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
												    or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
												    or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
												    or (cp >= 0xF9_00 and cp <= 0xFA_FF)
												    or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
												):  #
																	return True

												return False






							def 				__lowercase	(       self    :      List[str]    ,			snake_case__    :      Optional[Any]     ):






												"""simple docstring"""



												__a:    str		     =       []
												for char in text:
																	__a:    int		     =       ord(snake_case__     )
																	if cp == 0 or cp == 0xFF_FD or _is_control(snake_case__     ):
																						continue
																	if _is_whitespace(snake_case__     ):
																						output.append(""" """     )
																	else:
																						output.append(snake_case__     )
												return "".join(snake_case__     )






def      lowerCamelCase_     (						lowerCamelCase_			)  ->      int:


					'''simple docstring'''
					__a:    Dict		     =       set()
					__a:    Optional[int]		     =       word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)			)
										__a:    List[Any]		     =       char
					return pairs


def      lowerCamelCase_     (						lowerCamelCase_			)  ->      Tuple:


					'''simple docstring'''
					__a:    Tuple		     =       text.replace("""—"""						,					"""-"""			)
					__a:    List[str]		     =       text.replace("""–"""						,					"""-"""			)
					__a:    Tuple		     =       text.replace("""―"""						,					"""-"""			)
					__a:    Optional[int]		     =       text.replace("""…"""						,					"""..."""			)
					__a:    Any		     =       text.replace("""´"""						,					"""'"""			)
					__a:    int		     =       re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""						,					r""" \1 """						,					lowerCamelCase_			)
					__a:    Union[str, Any]		     =       re.sub(r"""\s*\n\s*"""						,					""" \n """						,					lowerCamelCase_			)
					__a:    Optional[int]		     =       re.sub(r"""[^\S\n]+"""						,					""" """						,					lowerCamelCase_			)
					return text.strip()




class  _snake_case	(    _lowercase							):
							SCREAMING_SNAKE_CASE		  =    VOCAB_FILES_NAMES
							SCREAMING_SNAKE_CASE		  =    PRETRAINED_VOCAB_FILES_MAP
							SCREAMING_SNAKE_CASE		  =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							SCREAMING_SNAKE_CASE		  =    ['''input_ids''', '''attention_mask''']






							def __init__(       self    :      Tuple    ,			snake_case__    :      Tuple    ,			snake_case__    :      str    ,			snake_case__    :      List[str]="<unk>"    ,			**snake_case__    :      Union[str, Any]     ):






												"""simple docstring"""



												super().__init__(unk_token=snake_case__    ,			**snake_case__     )

												try:
																	import ftfy
																	from spacy.lang.en import English

																	__a:    List[Any]		     =       English()
																	__a:    str		     =       _nlp.tokenizer
																	__a:    Tuple		     =       ftfy.fix_text
												except ImportError:
																	logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""     )
																	__a:    Tuple		     =       BasicTokenizer(do_lower_case=snake_case__     )
																	__a:    Tuple		     =       None

												with open(snake_case__    ,			encoding="""utf-8"""     ) as vocab_handle:
																	__a:    Optional[Any]		     =       json.load(snake_case__     )
												__a:    List[Any]		     =       {v: k for k, v in self.encoder.items()}
												with open(snake_case__    ,			encoding="""utf-8"""     ) as merges_handle:
																	__a:    Dict		     =       merges_handle.read().split("""\n"""     )[1:-1]
												__a:    List[Any]		     =       [tuple(merge.split()     ) for merge in merges]
												__a:    List[Any]		     =       dict(zip(snake_case__    ,			range(len(snake_case__     )     )     )     )
												__a:    Tuple		     =       {}






							@property
							def 				__lowercase	(       self    :      Any     ):






												"""simple docstring"""



												return True






							@property
							def 				__lowercase	(       self    :      Tuple     ):






												"""simple docstring"""



												return len(self.encoder     )






							def 				__lowercase	(       self    :      Optional[Any]     ):






												"""simple docstring"""



												return dict(self.encoder    ,			**self.added_tokens_encoder     )






							def 				__lowercase	(       self    :      Union[str, Any]    ,			snake_case__    :      Any     ):






												"""simple docstring"""



												__a:    Union[str, Any]		     =       tuple(token[:-1]     ) + (token[-1] + """</w>""",)
												if token in self.cache:
																	return self.cache[token]
												__a:    Optional[Any]		     =       get_pairs(snake_case__     )

												if not pairs:
																	return token + "</w>"

												while True:
																	__a:    int		     =       min(snake_case__    ,			key=lambda snake_case__     : self.bpe_ranks.get(snake_case__    ,			float("""inf"""     )     )     )
																	if bigram not in self.bpe_ranks:
																						break
																	__a     ,					__a:    Tuple		     =       bigram
																	__a:    int		     =       []
																	__a:    Union[str, Any]		     =       0
																	while i < len(snake_case__     ):
																						try:
																											__a:    List[Any]		     =       word.index(snake_case__    ,			snake_case__     )
																						except ValueError:
																											new_word.extend(word[i:]     )
																											break
																						else:
																											new_word.extend(word[i:j]     )
																											__a:    List[str]		     =       j

																						if word[i] == first and i < len(snake_case__     ) - 1 and word[i + 1] == second:
																											new_word.append(first + second     )
																											i += 2
																						else:
																											new_word.append(word[i]     )
																											i += 1
																	__a:    Any		     =       tuple(snake_case__     )
																	__a:    Optional[Any]		     =       new_word
																	if len(snake_case__     ) == 1:
																						break
																	else:
																						__a:    Union[str, Any]		     =       get_pairs(snake_case__     )
												__a:    List[str]		     =       """ """.join(snake_case__     )
												if word == "\n  </w>":
																	__a:    Union[str, Any]		     =       """\n</w>"""
												__a:    Any		     =       word
												return word






							def 				__lowercase	(       self    :      Any    ,			snake_case__    :      int     ):






												"""simple docstring"""



												__a:    int		     =       []
												if self.fix_text is None:
																	# Using BERT's BasicTokenizer
																	__a:    List[Any]		     =       self.nlp.tokenize(snake_case__     )
																	for token in text:
																						split_tokens.extend(list(self.bpe(snake_case__     ).split(""" """     )     )     )
												else:
																	# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																	__a:    Optional[int]		     =       self.nlp(text_standardize(self.fix_text(snake_case__     )     )     )
																	for token in text:
																						split_tokens.extend(list(self.bpe(token.text.lower()     ).split(""" """     )     )     )
												return split_tokens






							def 				__lowercase	(       self    :      Union[str, Any]    ,			snake_case__    :      int     ):






												"""simple docstring"""



												return self.encoder.get(snake_case__    ,			self.encoder.get(self.unk_token     )     )






							def 				__lowercase	(       self    :      int    ,			snake_case__    :      Optional[int]     ):






												"""simple docstring"""



												return self.decoder.get(snake_case__    ,			self.unk_token     )






							def 				__lowercase	(       self    :      Optional[int]    ,			snake_case__    :      Tuple     ):






												"""simple docstring"""



												__a:    int		     =       """""".join(snake_case__     ).replace("""</w>"""    ,			""" """     ).strip()
												return out_string






							def 				__lowercase	(       self    :      List[Any]    ,			snake_case__    :      str    ,			snake_case__    :      Optional[str] = None     ):






												"""simple docstring"""



												if not os.path.isdir(snake_case__     ):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
																	return
												__a:    List[Any]		     =       os.path.join(
												    snake_case__    ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )
												__a:    List[Any]		     =       os.path.join(
												    snake_case__    ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]     )

												with open(snake_case__    ,			"""w"""    ,			encoding="""utf-8"""     ) as f:
																	f.write(json.dumps(self.encoder    ,			indent=2    ,			sort_keys=snake_case__    ,			ensure_ascii=snake_case__     ) + """\n"""     )

												__a:    Optional[int]		     =       0
												with open(snake_case__    ,			"""w"""    ,			encoding="""utf-8"""     ) as writer:
																	writer.write("""#version: 0.2\n"""     )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,			key=lambda snake_case__     : kv[1]     ):
																						if index != token_index:
																											logger.warning(
																											    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																											    """ Please check that the tokenizer is not corrupted!"""     )
																											__a:    Union[str, Any]		     =       token_index
																						writer.write(""" """.join(snake_case__     ) + """\n"""     )
																						index += 1

												return vocab_file, merge_file


"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


lowercase		:    Dict			=       {
    'configuration_mobilenet_v1': [
        'MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MobileNetV1Config',
        'MobileNetV1OnnxConfig',
    ],
}

try:
			if not is_vision_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase		:    Optional[int]			=       ['MobileNetV1FeatureExtractor']
			lowercase		:    Union[str, Any]			=       ['MobileNetV1ImageProcessor']

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			lowercase		:    Tuple			=       [
			    'MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'MobileNetV1ForImageClassification',
			    'MobileNetV1Model',
			    'MobileNetV1PreTrainedModel',
			    'load_tf_weights_in_mobilenet_v1',
			]


if TYPE_CHECKING:
			from .configuration_mobilenet_va import (
			    MOBILENET_V1_PRETRAINED_CONFIG_ARCHIVE_MAP,
			    MobileNetVaConfig,
			    MobileNetVaOnnxConfig,
			)

			try:
						if not is_vision_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .feature_extraction_mobilenet_va import MobileNetVaFeatureExtractor
						from .image_processing_mobilenet_va import MobileNetVaImageProcessor

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_mobilenet_va import (
						    MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST,
						    MobileNetVaForImageClassification,
						    MobileNetVaModel,
						    MobileNetVaPreTrainedModel,
						    load_tf_weights_in_mobilenet_va,
						)


else:
			import sys

			lowercase		:    Union[str, Any]			=       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
