

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase_       :				Optional[Any]		   =      {
    '''configuration_prophetnet''': ['''PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ProphetNetConfig'''],
    '''tokenization_prophetnet''': ['''ProphetNetTokenizer'''],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_       :				List[str]		   =      [
			    '''PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''ProphetNetDecoder''',
			    '''ProphetNetEncoder''',
			    '''ProphetNetForCausalLM''',
			    '''ProphetNetForConditionalGeneration''',
			    '''ProphetNetModel''',
			    '''ProphetNetPreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_prophetnet import PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ProphetNetConfig
			from .tokenization_prophetnet import ProphetNetTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_prophetnet import (
						    PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    ProphetNetDecoder,
						    ProphetNetEncoder,
						    ProphetNetForCausalLM,
						    ProphetNetForConditionalGeneration,
						    ProphetNetModel,
						    ProphetNetPreTrainedModel,
						)

else:
			import sys

			UpperCAmelCase_       :				Any		   =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)








import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer






class     lowerCamelCase_			(  tf.keras.layers.Layer  ):




	def __init__(    self	:    int				,			__A	:    List				,			__A	:    bool				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    str = "longest"				,			__A	:    bool = True				,			__A	:    int = 512				,			__A	:    int = None				,			__A	:    bool = True				,			__A	:    bool = True				,			__A	:    bool = True				,			**__A	:    Any				,			):
		super().__init__()
		if use_fast_bert_tokenizer:
			__A   : Optional[int]								= FastBertTokenizer(
			    __A				,			token_out_type=tf.intaa				,			lower_case_nfd_strip_accents=__A				,			**__A       )
		else:
			__A   : Optional[Any]								= tf.lookup.StaticVocabularyTable(
			    tf.lookup.KeyValueTensorInitializer(
			        keys=__A				,			key_dtype=tf.string				,			values=tf.range(tf.size(__A				,			out_type=tf.intaa       )				,			dtype=tf.intaa       )				,			value_dtype=tf.intaa				,			)				,			num_oov_buckets=1				,			)
			__A   : Any								= BertTokenizerLayer(
			    __A				,			token_out_type=tf.intaa				,			lower_case=__A				,			**__A       )

		__A   : Union[str, Any]								= vocab_list
		__A   : List[str]								= do_lower_case
		__A   : List[Any]								= cls_token_id or vocab_list.index("""[CLS]"""       )
		__A   : Tuple								= sep_token_id or vocab_list.index("""[SEP]"""       )
		__A   : str								= pad_token_id or vocab_list.index("""[PAD]"""       )
		__A   : Optional[int]								= ShrinkLongestTrimmer(max_length - 3				,			axis=1       )  # Allow room for special tokens
		__A   : Union[str, Any]								= max_length
		__A   : List[Any]								= padding
		__A   : Optional[int]								= truncation
		__A   : Tuple								= pad_to_multiple_of
		__A   : Optional[Any]								= return_token_type_ids
		__A   : int								= return_attention_mask




	@classmethod
	def 					lowerCAmelCase_					(    cls	:    List[Any]				,			__A	:    "PreTrainedTokenizerBase"				,			**__A	:    List[str]       ):  # noqa: F821
		__A   : Any								= kwargs.pop("""do_lower_case"""				,			__A       )
		__A   : Dict								= tokenizer.do_lower_case if do_lower_case is None else do_lower_case
		__A   : List[str]								= kwargs.pop("""cls_token_id"""				,			__A       )
		__A   : str								= tokenizer.cls_token_id if cls_token_id is None else cls_token_id
		__A   : str								= kwargs.pop("""sep_token_id"""				,			__A       )
		__A   : Dict								= tokenizer.sep_token_id if sep_token_id is None else sep_token_id
		__A   : Tuple								= kwargs.pop("""pad_token_id"""				,			__A       )
		__A   : List[Any]								= tokenizer.pad_token_id if pad_token_id is None else pad_token_id

		__A   : Union[str, Any]								= tokenizer.get_vocab()
		__A   : Any								= sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]				,			key=lambda __A       : x[1]       )
		__A   : int								= [entry[0] for entry in vocab]
		return cls(
		    vocab_list=__A				,			do_lower_case=__A				,			cls_token_id=__A				,			sep_token_id=__A				,			pad_token_id=__A				,			**__A				,			)




	@classmethod
	def 					lowerCAmelCase_					(    cls	:    List[str]				,			__A	:    Union[str, os.PathLike]				,			*__A	:    Tuple				,			**__A	:    Optional[int]       ):
		try:
			__A   : int								= BertTokenizer.from_pretrained(__A				,			*__A				,			**__A       )
		except:  # noqa: E722
			from .tokenization_bert_fast import BertTokenizerFast

			__A   : Optional[int]								= BertTokenizerFast.from_pretrained(__A				,			*__A				,			**__A       )
		return cls.from_tokenizer(__A				,			**__A       )




	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    Optional[Any]       ):
		if self.do_lower_case:
			__A   : Tuple								= case_fold_utfa(__A       )
		__A   : Optional[int]								= self.tf_tokenizer.tokenize(__A       )
		return tokens.merge_dims(1				,			-1       )




	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Optional[Any]				,			__A	:    Dict=None				,			__A	:    List[Any]=None				,			__A	:    str=None				,			__A	:    Union[str, Any]=None				,			__A	:    Union[str, Any]=None				,			__A	:    int=None				,			__A	:    Any=None				,			):
		if padding is None:
			__A   : List[Any]								= self.padding
		if padding not in ("longest", "max_length"):
			raise ValueError("""Padding must be either 'longest' or 'max_length'!"""       )
		if max_length is not None and text_pair is not None:
			# Because we have to instantiate a Trimmer to do it properly
			raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""       )
		if max_length is None:
			__A   : Any								= self.max_length
		if truncation is None:
			__A   : int								= self.truncation
		if pad_to_multiple_of is None:
			__A   : List[str]								= self.pad_to_multiple_of
		if return_token_type_ids is None:
			__A   : int								= self.return_token_type_ids
		if return_attention_mask is None:
			__A   : Any								= self.return_attention_mask
		if not isinstance(__A				,			tf.Tensor       ):
			__A   : str								= tf.convert_to_tensor(__A       )
		if text_pair is not None and not isinstance(__A				,			tf.Tensor       ):
			__A   : Optional[Any]								= tf.convert_to_tensor(__A       )
		if text_pair is not None:
			if text.shape.rank > 1:
				raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""       )
			if text_pair.shape.rank > 1:
				raise ValueError("""text_pair should not be multidimensional!"""       )
		if text.shape.rank == 2:
			__A						,							__A   : Optional[int]								= text[:, 0], text[:, 1]
		__A   : List[Any]								= self.unpaired_tokenize(__A       )
		if text_pair is None:  # Unpaired text
			if truncation:
				__A   : Union[str, Any]								= text[:, : max_length - 2]  # Allow room for special tokens
			__A						,							__A   : List[Any]								= combine_segments(
			    (text,)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
		else:  # Paired text
			__A   : Tuple								= self.unpaired_tokenize(__A       )
			if truncation:
				__A						,							__A   : Optional[int]								= self.paired_trimmer.trim([text, text_pair]       )
			__A						,							__A   : int								= combine_segments(
			    (text, text_pair)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
		if padding == "longest":
			__A   : str								= input_ids.bounding_shape(axis=1       )
			if pad_to_multiple_of is not None:
				# No ceiling division in tensorflow, so we negate floordiv instead
				__A   : Optional[Any]								= pad_to_multiple_of * (-tf.math.floordiv(-pad_length				,			__A       ))
		else:
			__A   : Dict								= max_length

		__A						,							__A   : str								= pad_model_inputs(__A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
		__A   : List[str]								= {"""input_ids""": input_ids}
		if return_attention_mask:
			__A   : Any								= attention_mask
		if return_token_type_ids:
			__A						,							__A   : Tuple								= pad_model_inputs(
			    __A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
			__A   : List[str]								= token_type_ids
		return output




	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
		return {
		    "vocab_list": self.vocab_list,
		    "do_lower_case": self.do_lower_case,
		    "cls_token_id": self.cls_token_id,
		    "sep_token_id": self.sep_token_id,
		    "pad_token_id": self.pad_token_id,
		}





