

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__				=		logging.get_logger(__name__)


a__				=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

a__				=		{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

a__				=		{
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def    __UpperCAmelCase (	)			->     Union[str, Any]:

					"""simple docstring"""

					_a	:    List[Any]         =		(
					    list(range(ord('''!'''	)		,ord('''~'''	) + 1	)	) + list(range(ord('''¡'''	)		,ord('''¬'''	) + 1	)	) + list(range(ord('''®'''	)		,ord('''ÿ'''	) + 1	)	)
					)
					_a	:    Union[str, Any]         =		bs[:]
					_a	:    int         =		0
					for b in range(2**8	):
										if b not in bs:
															bs.append(__a	)
															cs.append(2**8 + n	)
															n += 1
					_a	:    str         =		[chr(__a	) for n in cs]
					return dict(zip(__a		,__a	)	)



def    __UpperCAmelCase (	__a      :			Tuple	)			->     int:

					"""simple docstring"""

					_a	:    Any         =		set()
					_a	:    Optional[int]         =		word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)	)
										_a	:    List[str]         =		char
					return pairs







class 				UpperCAmelCase_		(			__lowercase   ):
							"""simple docstring"""







							UpperCAmelCase__       :						List[str]	    =    VOCAB_FILES_NAMES
							UpperCAmelCase__       :						str	    =    PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase__       :						Any	    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase__       :						List[Any]	    =    ["input_ids", "attention_mask"]


							def __init__(			self       ,							_a       ,							_a       ,							_a="replace"       ,							_a="<s>"       ,							_a="</s>"       ,							_a="</s>"       ,							_a="<s>"       ,							_a="<unk>"       ,							_a="<pad>"       ,							_a="<mask>"       ,							_a=False       ,							**_a       ,							)    ->    List[Any]:
												_a	:    int         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else bos_token
												_a	:    Dict         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else eos_token
												_a	:    List[str]         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else sep_token
												_a	:    Optional[int]         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else cls_token
												_a	:    Tuple         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else unk_token
												_a	:    List[str]         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else pad_token

												# Mask token behave like a normal word, i.e. include the space before it
												_a	:    Any         =		AddedToken(_a       ,							lstrip=_a       ,							rstrip=_a       ) if isinstance(_a       ,							_a       ) else mask_token

												super().__init__(
												    errors=_a       ,							bos_token=_a       ,							eos_token=_a       ,							unk_token=_a       ,							sep_token=_a       ,							cls_token=_a       ,							pad_token=_a       ,							mask_token=_a       ,							add_prefix_space=_a       ,							**_a       ,							)

												with open(_a       ,							encoding='''utf-8'''       ) as vocab_handle:
																	_a	:    Optional[Any]         =		json.load(_a       )
												_a	:    Optional[int]         =		{v: k for k, v in self.encoder.items()}
												_a	:    str         =		errors  # how to handle errors in decoding
												_a	:    Optional[Any]         =		bytes_to_unicode()
												_a	:    List[Any]         =		{v: k for k, v in self.byte_encoder.items()}
												with open(_a       ,							encoding='''utf-8'''       ) as merges_handle:
																	_a	:    Any         =		merges_handle.read().split('''\n'''       )[1:-1]
												_a	:    Union[str, Any]         =		[tuple(merge.split()       ) for merge in bpe_merges]
												_a	:    Any         =		dict(zip(_a       ,							range(len(_a       )       )       )       )
												_a	:    List[str]         =		{}
												_a	:    Optional[int]         =		add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												_a	:    str         =		re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )


							@property
							def 				__lowercase				(			self       )    ->    str:
												return len(self.encoder       )


							def 				__lowercase				(			self       )    ->    Union[str, Any]:
												return dict(self.encoder       ,							**self.added_tokens_encoder       )


							def 				__lowercase				(			self       ,							_a       )    ->    str:
												if token in self.cache:
																	return self.cache[token]
												_a	:    Dict         =		tuple(_a       )
												_a	:    List[str]         =		get_pairs(_a       )

												if not pairs:
																	return token

												while True:
																	_a	:    List[str]         =		min(_a       ,							key=lambda _a       : self.bpe_ranks.get(_a       ,							float('''inf'''       )       )       )
																	if bigram not in self.bpe_ranks:
																						break
																	_a      ,					_a	:    int         =		bigram
																	_a	:    Optional[int]         =		[]
																	_a	:    List[Any]         =		0
																	while i < len(_a       ):
																						try:
																											_a	:    List[Any]         =		word.index(_a       ,							_a       )
																						except ValueError:
																											new_word.extend(word[i:]       )
																											break
																						else:
																											new_word.extend(word[i:j]       )
																											_a	:    List[str]         =		j

																						if word[i] == first and i < len(_a       ) - 1 and word[i + 1] == second:
																											new_word.append(first + second       )
																											i += 2
																						else:
																											new_word.append(word[i]       )
																											i += 1
																	_a	:    Any         =		tuple(_a       )
																	_a	:    List[str]         =		new_word
																	if len(_a       ) == 1:
																						break
																	else:
																						_a	:    Any         =		get_pairs(_a       )
												_a	:    Dict         =		''' '''.join(_a       )
												_a	:    str         =		word
												return word


							def 				__lowercase				(			self       ,							_a       )    ->    Optional[int]:
												_a	:    Optional[Any]         =		[]
												for token in re.findall(self.pat       ,							_a       ):
																	_a	:    Union[str, Any]         =		''''''.join(
																	    self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_a       ).split(''' '''       )       )
												return bpe_tokens


							def 				__lowercase				(			self       ,							_a       )    ->    List[str]:
												return self.encoder.get(_a       ,							self.encoder.get(self.unk_token       )       )


							def 				__lowercase				(			self       ,							_a       )    ->    Any:
												return self.decoder.get(_a       )


							def 				__lowercase				(			self       ,							_a       )    ->    Optional[Any]:
												_a	:    Any         =		''''''.join(_a       )
												_a	:    List[str]         =		bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''       ,							errors=self.errors       )
												return text


							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    Tuple[str]:
												if not os.path.isdir(_a       ):
																	logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
																	return
												_a	:    Any         =		os.path.join(
												    _a       ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
												_a	:    Any         =		os.path.join(
												    _a       ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

												with open(_a       ,							'''w'''       ,							encoding='''utf-8'''       ) as f:
																	f.write(json.dumps(self.encoder       ,							indent=2       ,							sort_keys=_a       ,							ensure_ascii=_a       ) + '''\n'''       )

												_a	:    Optional[int]         =		0
												with open(_a       ,							'''w'''       ,							encoding='''utf-8'''       ) as writer:
																	writer.write('''#version: 0.2\n'''       )
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,							key=lambda _a       : kv[1]       ):
																						if index != token_index:
																											logger.warning(
																											    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																											    ''' Please check that the tokenizer is not corrupted!'''       )
																											_a	:    List[str]         =		token_index
																						writer.write(''' '''.join(_a       ) + '''\n'''       )
																						index += 1

												return vocab_file, merge_file


							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    List[int]:
												if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												_a	:    Dict         =		[self.cls_token_id]
												_a	:    Optional[Any]         =		[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep


							def 				__lowercase				(			self       ,							_a       ,							_a = None       ,							_a = False       )    ->    List[int]:
												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=_a       ,							token_ids_a=_a       ,							already_has_special_tokens=_a       )

												if token_ids_a is None:
																	return [1] + ([0] * len(_a       )) + [1]
												return [1] + ([0] * len(_a       )) + [1, 1] + ([0] * len(_a       )) + [1]


							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    List[int]:
												_a	:    List[str]         =		[self.sep_token_id]
												_a	:    Any         =		[self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep       ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]




							def 				__lowercase				(			self       ,							_a       ,							_a=False       ,							**_a       )    ->    Dict:
												_a	:    Optional[int]         =		kwargs.pop('''add_prefix_space'''       ,							self.add_prefix_space       )
												if (is_split_into_words or add_prefix_space) and (len(_a       ) > 0 and not text[0].isspace()):
																	_a	:    Tuple         =		''' ''' + text
												return (text, kwargs)






import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
			import torch

if is_vision_available():
			import PIL


a__				=		logging.get_logger(__name__)



def    __UpperCAmelCase (	__a      :			np.ndarray		,__a      :			Union[int, Iterable[int]]		,__a      :			bool		,__a      :			int	)			->     Tuple[int, int]:

					"""simple docstring"""

					def constraint_to_multiple_of(__a      :			List[str]		,__a      :			Dict		,__a      :			Any=0		,__a      :			int=None	):
										_a	:    Dict         =		round(val / multiple	) * multiple

										if max_val is not None and x > max_val:
															_a	:    Any         =		math.floor(val / multiple	) * multiple

										if x < min_val:
															_a	:    Dict         =		math.ceil(val / multiple	) * multiple

										return x

					_a	:    Union[str, Any]         =		(output_size, output_size) if isinstance(__a		,__a	) else output_size

					_a      ,					_a	:    List[Any]         =		get_image_size(__a	)
					_a      ,					_a	:    Any         =		output_size

					# determine new height and width
					_a	:    Union[str, Any]         =		output_height / input_height
					_a	:    Tuple         =		output_width / input_width

					if keep_aspect_ratio:
										# scale as little as possible
										if abs(1 - scale_width	) < abs(1 - scale_height	):
															# fit width
															_a	:    Optional[Any]         =		scale_width
										else:
															# fit height
															_a	:    Tuple         =		scale_height

					_a	:    Optional[Any]         =		constraint_to_multiple_of(scale_height * input_height		,multiple=__a	)
					_a	:    int         =		constraint_to_multiple_of(scale_width * input_width		,multiple=__a	)

					return (new_height, new_width)







class 				UpperCAmelCase_		(			__lowercase   ):
							"""simple docstring"""







							UpperCAmelCase__       :						Optional[Any]	    =    ["pixel_values"]


							def __init__(			self       ,							_a = True       ,							_a = None       ,							_a = PILImageResampling.BILINEAR       ,							_a = False       ,							_a = 1       ,							_a = True       ,							_a = 1 / 2_5_5       ,							_a = True       ,							_a = None       ,							_a = None       ,							**_a       ,							)    ->    None:
												super().__init__(**_a       )
												_a	:    Optional[int]         =		size if size is not None else {'''height''': 3_8_4, '''width''': 3_8_4}
												_a	:    Optional[Any]         =		get_size_dict(_a       )
												_a	:    Any         =		do_resize
												_a	:    Dict         =		size
												_a	:    str         =		keep_aspect_ratio
												_a	:    Any         =		ensure_multiple_of
												_a	:    Optional[Any]         =		resample
												_a	:    List[Any]         =		do_rescale
												_a	:    int         =		rescale_factor
												_a	:    Any         =		do_normalize
												_a	:    Union[str, Any]         =		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
												_a	:    List[str]         =		image_std if image_std is not None else IMAGENET_STANDARD_STD


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a = False       ,							_a = 1       ,							_a = PILImageResampling.BICUBIC       ,							_a = None       ,							**_a       ,							)    ->    np.ndarray:
												_a	:    str         =		get_size_dict(_a       )
												if "height" not in size or "width" not in size:
																	raise ValueError(F"""The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"""       )
												_a	:    Optional[Any]         =		get_resize_output_image_size(
												    _a       ,							output_size=(size['''height'''], size['''width'''])       ,							keep_aspect_ratio=_a       ,							multiple=_a       ,							)
												return resize(_a       ,							size=_a       ,							resample=_a       ,							data_format=_a       ,							**_a       )


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a = None       ,							**_a       ,							)    ->    int:
												return rescale(_a       ,							scale=_a       ,							data_format=_a       ,							**_a       )


							def 				__lowercase				(			self       ,							_a       ,							_a       ,							_a       ,							_a = None       ,							**_a       ,							)    ->    np.ndarray:
												return normalize(_a       ,							mean=_a       ,							std=_a       ,							data_format=_a       ,							**_a       )


							def 				__lowercase				(			self       ,							_a       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = None       ,							_a = ChannelDimension.FIRST       ,							**_a       ,							)    ->    PIL.Image.Image:
												_a	:    Optional[int]         =		do_resize if do_resize is not None else self.do_resize
												_a	:    Union[str, Any]         =		size if size is not None else self.size
												_a	:    str         =		get_size_dict(_a       )
												_a	:    List[Any]         =		keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
												_a	:    Optional[Any]         =		ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
												_a	:    str         =		resample if resample is not None else self.resample
												_a	:    str         =		do_rescale if do_rescale is not None else self.do_rescale
												_a	:    Union[str, Any]         =		rescale_factor if rescale_factor is not None else self.rescale_factor
												_a	:    List[Any]         =		do_normalize if do_normalize is not None else self.do_normalize
												_a	:    str         =		image_mean if image_mean is not None else self.image_mean
												_a	:    Tuple         =		image_std if image_std is not None else self.image_std

												_a	:    Dict         =		make_list_of_images(_a       )

												if not valid_images(_a       ):
																	raise ValueError(
																	    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																	    '''torch.Tensor, tf.Tensor or jax.ndarray.'''       )

												if do_resize and size is None or resample is None:
																	raise ValueError('''Size and resample must be specified if do_resize is True.'''       )

												if do_rescale and rescale_factor is None:
																	raise ValueError('''Rescale factor must be specified if do_rescale is True.'''       )

												if do_normalize and (image_mean is None or image_std is None):
																	raise ValueError('''Image mean and std must be specified if do_normalize is True.'''       )

												# All transformations expect numpy arrays.
												_a	:    Dict         =		[to_numpy_array(_a       ) for image in images]

												if do_resize:
																	_a	:    int         =		[self.resize(image=_a       ,							size=_a       ,							resample=_a       ) for image in images]

												if do_rescale:
																	_a	:    Optional[Any]         =		[self.rescale(image=_a       ,							scale=_a       ) for image in images]

												if do_normalize:
																	_a	:    Optional[int]         =		[self.normalize(image=_a       ,							mean=_a       ,							std=_a       ) for image in images]

												_a	:    int         =		[to_channel_dimension_format(_a       ,							_a       ) for image in images]

												_a	:    Tuple         =		{'''pixel_values''': images}
												return BatchFeature(data=_a       ,							tensor_type=_a       )




							def 				__lowercase				(			self       ,							_a       ,							_a = None       )    ->    Any:
												_a	:    Optional[Any]         =		outputs.logits

												# Resize logits and compute semantic segmentation maps
												if target_sizes is not None:
																	if len(_a       ) != len(_a       ):
																						raise ValueError(
																						    '''Make sure that you pass in as many target sizes as the batch dimension of the logits'''       )

																	if is_torch_tensor(_a       ):
																						_a	:    List[Any]         =		target_sizes.numpy()

																	_a	:    str         =		[]

																	for idx in range(len(_a       )       ):
																						_a	:    str         =		torch.nn.functional.interpolate(
																						    logits[idx].unsqueeze(dim=0       )       ,							size=target_sizes[idx]       ,							mode='''bilinear'''       ,							align_corners=_a       )
																						_a	:    Union[str, Any]         =		resized_logits[0].argmax(dim=0       )
																						semantic_segmentation.append(_a       )
												else:
																	_a	:    Tuple         =		logits.argmax(dim=1       )
																	_a	:    Union[str, Any]         =		[semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]       )]

												return semantic_segmentation



