





"""simple docstring"""



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

a_				=    logging.get_logger(__name__)

a_				=    {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a_				=    {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
}

a_				=    {
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}







@lru_cache()
def 	__lowercase  (     )     ->Tuple:



				'''simple docstring'''







				__A					:		Union[str, Any]						=  (
				    list(range(ord('''!'''		)       ,ord('''~'''		) + 1		)		) + list(range(ord('''¡'''		)       ,ord('''¬'''		) + 1		)		) + list(range(ord('''®'''		)       ,ord('''ÿ'''		) + 1		)		)
				)
				__A					:		Any						=  bs[:]
				__A					:		Dict						=  0
				for b in range(2**8		):
								if b not in bs:
												bs.append(lowercase_		)
												cs.append(2**8 + n		)
												n += 1
				__A					:		Optional[int]						=  [chr(lowercase_		) for n in cs]
				return dict(zip(lowercase_       ,lowercase_		)		)







def 	__lowercase  (     snake_case_						:						List[Any]		)     ->Dict:



				'''simple docstring'''







				__A					:		Tuple						=  set()
				__A					:		int						=  word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)		)
								__A					:		str						=  char
				return pairs






class 	__snake_case      (	__UpperCAmelCase  ):





	"""simple docstring"""


	_lowerCamelCase				 =  VOCAB_FILES_NAMES
	_lowerCamelCase				 =  PRETRAINED_VOCAB_FILES_MAP
	_lowerCamelCase				 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowerCamelCase				 =  ["""input_ids""", """attention_mask"""]

	def __init__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase      ,		__lowerCamelCase="replace"      ,		__lowerCamelCase="<|endoftext|>"      ,		__lowerCamelCase="<|endoftext|>"      ,		__lowerCamelCase="<|endoftext|>"      ,		__lowerCamelCase=None      ,		__lowerCamelCase=False      ,		__lowerCamelCase=False      ,		**__lowerCamelCase      ,		):



					'''simple docstring'''




					__A					:		Any						=  AddedToken(lowercase_      ,		lstrip=lowercase_      ,		rstrip=lowercase_						) if isinstance(lowercase_      ,		lowercase_						) else bos_token
					__A					:		int						=  AddedToken(lowercase_      ,		lstrip=lowercase_      ,		rstrip=lowercase_						) if isinstance(lowercase_      ,		lowercase_						) else eos_token
					__A					:		Optional[Any]						=  AddedToken(lowercase_      ,		lstrip=lowercase_      ,		rstrip=lowercase_						) if isinstance(lowercase_      ,		lowercase_						) else unk_token
					__A					:		str						=  AddedToken(lowercase_      ,		lstrip=lowercase_      ,		rstrip=lowercase_						) if isinstance(lowercase_      ,		lowercase_						) else pad_token
					super().__init__(
					    errors=lowercase_      ,		unk_token=lowercase_      ,		bos_token=lowercase_      ,		eos_token=lowercase_      ,		pad_token=lowercase_      ,		add_prefix_space=lowercase_      ,		add_bos_token=lowercase_      ,		**lowercase_      ,		)
					__A					:		Optional[Any]						=  add_bos_token

					with open(lowercase_      ,		encoding='''utf-8'''						) as vocab_handle:
									__A					:		str						=  json.load(lowercase_						)
					__A					:		Any						=  {v: k for k, v in self.encoder.items()}
					__A					:		List[str]						=  errors  # how to handle errors in decoding
					__A					:		Union[str, Any]						=  bytes_to_unicode()
					__A					:		Union[str, Any]						=  {v: k for k, v in self.byte_encoder.items()}
					with open(lowercase_      ,		encoding='''utf-8'''						) as merges_handle:
									__A					:		Dict						=  merges_handle.read().split('''\n'''						)[1:-1]
					__A					:		List[Any]						=  [tuple(merge.split()						) for merge in bpe_merges]
					__A					:		int						=  dict(zip(lowercase_      ,		range(len(lowercase_						)						)						)						)
					__A					:		Tuple						=  {}
					__A					:		Dict						=  add_prefix_space

					# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
					__A					:		Union[str, Any]						=  re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)

	@property
	def       UpperCamelCase__(					self						):



					'''simple docstring'''




					return len(self.encoder						)

	def       UpperCamelCase__(					self						):



					'''simple docstring'''




					return dict(self.encoder      ,		**self.added_tokens_encoder						)

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					if token in self.cache:
									return self.cache[token]
					__A					:		List[str]						=  tuple(lowercase_						)
					__A					:		Union[str, Any]						=  get_pairs(lowercase_						)

					if not pairs:
									return token

					while True:
									__A					:		List[str]						=  min(lowercase_      ,		key=lambda __lowerCamelCase						: self.bpe_ranks.get(lowercase_      ,		float('''inf'''						)						)						)
									if bigram not in self.bpe_ranks:
													break
									__A					:		Optional[int]						=  bigram
									__A					:		Any						=  []
									__A					:		str						=  0
									while i < len(lowercase_						):
													try:
																	__A					:		List[Any]						=  word.index(lowercase_      ,		lowercase_						)
													except ValueError:
																	new_word.extend(word[i:]						)
																	break
													else:
																	new_word.extend(word[i:j]						)
																	__A					:		str						=  j

													if word[i] == first and i < len(lowercase_						) - 1 and word[i + 1] == second:
																	new_word.append(first + second						)
																	i += 2
													else:
																	new_word.append(word[i]						)
																	i += 1
									__A					:		Any						=  tuple(lowercase_						)
									__A					:		Optional[int]						=  new_word
									if len(lowercase_						) == 1:
													break
									else:
													__A					:		Dict						=  get_pairs(lowercase_						)
					__A					:		int						=  ''' '''.join(lowercase_						)
					__A					:		str						=  word
					return word

	def       UpperCamelCase__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase=None						):



					'''simple docstring'''




					if self.add_bos_token:
									__A					:		Dict						=  [self.bos_token_id]
					else:
									__A					:		Optional[Any]						=  []

					__A					:		Dict						=  bos_token_ids + token_ids_a

					if token_ids_a is None:
									return output

					return output + bos_token_ids + token_ids_a

	def       UpperCamelCase__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase = None      ,		__lowerCamelCase = False						):



					'''simple docstring'''




					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowercase_      ,		token_ids_a=lowercase_      ,		already_has_special_tokens=lowercase_						)

					if not self.add_bos_token:
									return super().get_special_tokens_mask(
									    token_ids_a=lowercase_      ,		token_ids_a=lowercase_      ,		already_has_special_tokens=lowercase_						)

					if token_ids_a is None:
									return [1] + ([0] * len(lowercase_						))
					return [1] + ([0] * len(lowercase_						)) + [1] + ([0] * len(lowercase_						))

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					__A					:		List[str]						=  []
					for token in re.findall(self.pat      ,		lowercase_						):
									__A					:		List[Any]						=  ''''''.join(
									    self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_						).split(''' '''						)						)
					return bpe_tokens

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					return self.encoder.get(lowercase_      ,		self.encoder.get(self.unk_token						)						)

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					return self.decoder.get(lowercase_						)

	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					__A					:		int						=  ''''''.join(lowercase_						)
					__A					:		List[str]						=  bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8'''      ,		errors=self.errors						)
					return text

	def       UpperCamelCase__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase = None						):



					'''simple docstring'''




					if not os.path.isdir(lowercase_						):
									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
									return
					__A					:		int						=  os.path.join(
					    lowercase_      ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
					__A					:		Optional[int]						=  os.path.join(
					    lowercase_      ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

					with open(lowercase_      ,		'''w'''      ,		encoding='''utf-8'''						) as f:
									f.write(json.dumps(self.encoder      ,		indent=2      ,		sort_keys=lowercase_      ,		ensure_ascii=lowercase_						) + '''\n'''						)

					__A					:		Optional[int]						=  0
					with open(lowercase_      ,		'''w'''      ,		encoding='''utf-8'''						) as writer:
									writer.write('''#version: 0.2\n'''						)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,		key=lambda __lowerCamelCase						: kv[1]						):
													if index != token_index:
																	logger.warning(
																	    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																	    ''' Please check that the tokenizer is not corrupted!'''						)
																	__A					:		Optional[Any]						=  token_index
													writer.write(''' '''.join(lowercase_						) + '''\n'''						)
													index += 1

					return vocab_file, merge_file

	def       UpperCamelCase__(					self      ,		__lowerCamelCase      ,		__lowerCamelCase=False      ,		**__lowerCamelCase						):



					'''simple docstring'''




					__A					:		List[str]						=  kwargs.pop('''add_prefix_space'''      ,		self.add_prefix_space						)
					if is_split_into_words or add_prefix_space:
									__A					:		int						=  ''' ''' + text
					return (text, kwargs)







	def       UpperCamelCase__(					self      ,		__lowerCamelCase						):



					'''simple docstring'''




					__A					:		List[str]						=  []
					for is_user, text in conversation.iter_texts():
									input_ids.extend(self.encode(lowercase_      ,		add_special_tokens=lowercase_						) + [self.eos_token_id]						)
					if len(lowercase_						) > self.model_max_length:
									__A					:		Tuple						=  input_ids[-self.model_max_length :]
					return input_ids











"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_				=    {
    """configuration_layoutlm""": ["""LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LayoutLMConfig""", """LayoutLMOnnxConfig"""],
    """tokenization_layoutlm""": ["""LayoutLMTokenizer"""],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_				=    ["""LayoutLMTokenizerFast"""]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_				=    [
           """LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """LayoutLMForMaskedLM""",
           """LayoutLMForSequenceClassification""",
           """LayoutLMForTokenClassification""",
           """LayoutLMForQuestionAnswering""",
           """LayoutLMModel""",
           """LayoutLMPreTrainedModel""",
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a_				=    [
           """TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """TFLayoutLMForMaskedLM""",
           """TFLayoutLMForSequenceClassification""",
           """TFLayoutLMForTokenClassification""",
           """TFLayoutLMForQuestionAnswering""",
           """TFLayoutLMMainLayer""",
           """TFLayoutLMModel""",
           """TFLayoutLMPreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMOnnxConfig
       from .tokenization_layoutlm import LayoutLMTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_layoutlm_fast import LayoutLMTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_layoutlm import (
                  LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  LayoutLMForMaskedLM,
                  LayoutLMForQuestionAnswering,
                  LayoutLMForSequenceClassification,
                  LayoutLMForTokenClassification,
                  LayoutLMModel,
                  LayoutLMPreTrainedModel,
              )
       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_layoutlm import (
                  TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFLayoutLMForMaskedLM,
                  TFLayoutLMForQuestionAnswering,
                  TFLayoutLMForSequenceClassification,
                  TFLayoutLMForTokenClassification,
                  TFLayoutLMMainLayer,
                  TFLayoutLMModel,
                  TFLayoutLMPreTrainedModel,
              )

else:
       import sys

       a_				=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




