
import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


lowercase  :					Optional[int]						    =							get_logger(__name__)


lowercase  :					Tuple						    =							R"\n    Args:\n        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):\n            Indices of input sequence tokens in the vocabulary.\n\n            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and\n            [`PreTrainedTokenizer.__call__`] for details.\n\n            [What are input IDs?](../glossary#input-ids)\n        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):\n            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam\n            search or log softmax for each vocabulary token when using beam search\n        kwargs (`Dict[str, Any]`, *optional*):\n            Additional logits processor specific kwargs.\n\n    Return:\n        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.\n\n"

class __snake_case		:
    @add_start_docstrings(snake_case      )
    def __call__(  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           raise NotImplementedError(
               f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."      )






class __snake_case		:
    @add_start_docstrings(snake_case      )
    def __call__(  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           raise NotImplementedError(
               f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."      )






class __snake_case		(      lowerCAmelCase     ):
    @add_start_docstrings(snake_case      )
    def __call__(  self		,snake_case		,snake_case		,snake_case		,**snake_case      ):



           '''simple docstring'''







           for processor in self:
                  lowercase							:     Union[str, Any]       	=      inspect.signature(processor.__call__      ).parameters
                  if len(snake_case      ) > 3:
                         if not all(arg in kwargs for arg in list(function_args.keys()      )[2:]      ):
                                raise ValueError(
                                    f"Make sure that all the required parameters: {list(function_args.keys()      )} for "
                                    f"{processor.__class__} are passed to the logits processor."      )
                         lowercase							:     List[Any]       	=      processor(snake_case		,snake_case		,snake_case		,**snake_case      )
                  else:
                         lowercase							:     Dict       	=      processor(snake_case		,snake_case		,snake_case      )
           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case      ):



           '''simple docstring'''







           if not isinstance(snake_case		,snake_case      ) or not (temperature > 0):
                  raise ValueError(f"`temperature` has to be a strictly positive float, but is {temperature}"      )

           lowercase							:     Optional[Any]       	=      temperature
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      scores / self.temperature
           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case = -float("""Inf"""      )		,snake_case = 1      ):



           '''simple docstring'''







           if not isinstance(snake_case		,snake_case      ) or (top_p < 0 or top_p > 1.0):
                  raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}"      )
           if not isinstance(snake_case		,snake_case      ) or (min_tokens_to_keep < 1):
                  raise ValueError(f"`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}"      )

           lowercase							:     Optional[Any]       	=      top_p
           lowercase							:     List[str]       	=      filter_value
           lowercase							:     Any       	=      min_tokens_to_keep
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase	,			lowercase							:     str       	=      lax.top_k(snake_case		,scores.shape[-1]      )

           lowercase							:     Optional[Any]       	=      jnp.full_like(snake_case		,self.filter_value      )
           lowercase							:     Optional[int]       	=      jax.nn.softmax(snake_case		,axis=-1      ).cumsum(axis=-1      )
           lowercase							:     Dict       	=      cumulative_probs < self.top_p

           # include the token that is higher than top_p as well
           lowercase							:     List[Any]       	=      jnp.roll(snake_case		,1      )
           score_mask |= score_mask.at[:, 0].set(snake_case      )

           # min tokens to keep
           lowercase							:     Dict       	=      score_mask.at[:, : self.min_tokens_to_keep].set(snake_case      )

           lowercase							:     Dict       	=      jnp.where(snake_case		,snake_case		,snake_case      )
           lowercase							:     Optional[Any]       	=      jax.lax.sort_key_val(snake_case		,snake_case      )[-1]

           return next_scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case = -float("""Inf"""      )		,snake_case = 1      ):



           '''simple docstring'''







           if not isinstance(snake_case		,snake_case      ) or top_k <= 0:
                  raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}"      )

           lowercase							:     Tuple       	=      max(snake_case		,snake_case      )
           lowercase							:     str       	=      filter_value
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase	,			lowercase							:     List[str]       	=      scores.shape
           lowercase							:     List[str]       	=      jnp.full(batch_size * vocab_size		,self.filter_value      )

           lowercase							:     List[str]       	=      min(self.top_k		,scores.shape[-1]      )  # Safety check
           lowercase	,			lowercase							:     Dict       	=      lax.top_k(snake_case		,snake_case      )
           lowercase							:     List[str]       	=      jnp.broadcast_to((jnp.arange(snake_case      ) * vocab_size)[:, None]		,(batch_size, topk)      ).flatten()
           lowercase							:     str       	=      topk_scores.flatten()
           lowercase							:     Any       	=      topk_indices.flatten() + shift

           lowercase							:     int       	=      next_scores_flat.at[topk_indices_flat].set(snake_case      )
           lowercase							:     Optional[int]       	=      next_scores_flat.reshape(snake_case		,snake_case      )
           return next_scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Tuple       	=      bos_token_id
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      jnp.full(scores.shape		,-float("""inf"""      )      )

           lowercase							:     Tuple       	=      1 - jnp.bool_(cur_len - 1      )

           lowercase							:     Tuple       	=      jnp.where(snake_case		,new_scores.at[:, self.bos_token_id].set(0      )		,snake_case      )

           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Optional[int]       	=      max_length
           lowercase							:     Optional[Any]       	=      eos_token_id
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      jnp.full(scores.shape		,-float("""inf"""      )      )

           lowercase							:     int       	=      1 - jnp.bool_(cur_len - self.max_length + 1      )

           lowercase							:     Any       	=      jnp.where(snake_case		,new_scores.at[:, self.eos_token_id].set(0      )		,snake_case      )

           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           if not isinstance(snake_case		,snake_case      ) or min_length < 0:
                  raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}"      )

           if not isinstance(snake_case		,snake_case      ) or eos_token_id < 0:
                  raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}"      )

           lowercase							:     Dict       	=      min_length
           lowercase							:     str       	=      eos_token_id
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     str       	=      1 - jnp.clip(cur_len - self.min_length		,0		,1      )

           lowercase							:     List[Any]       	=      jnp.where(snake_case		,scores.at[:, self.eos_token_id].set(-float("""inf"""      )      )		,snake_case      )

           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Dict       	=      list(snake_case      )
           lowercase							:     List[str]       	=      begin_index
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      1 - jnp.bool_(cur_len - self.begin_index      )

           lowercase							:     Optional[int]       	=      jnp.where(snake_case		,scores.at[:, self.begin_suppress_tokens].set(-float("""inf"""      )      )		,snake_case      )

           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     List[str]       	=      list(snake_case      )
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Union[str, Any]       	=      scores.at[..., self.suppress_tokens].set(-float("""inf"""      )      )

           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case      ):



           '''simple docstring'''







           lowercase							:     int       	=      dict(snake_case      )
           # Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
           # index of the array corresponds to the index of the token to be forced, for XLA compatibility.
           # Indexes without forced tokens will have a negative value.
           lowercase							:     Any       	=      jnp.ones((max(force_token_map.keys()      ) + 1)		,dtype=jnp.intaa      ) * -1
           for index, token in force_token_map.items():
                  if token is not None:
                         lowercase							:     List[Any]       	=      force_token_array.at[index].set(snake_case      )
           lowercase							:     Dict       	=      jnp.intaa(snake_case      )
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           def _force_token(snake_case      ):
                  lowercase							:     str       	=      scores.shape[0]
                  lowercase							:     str       	=      self.force_token_array[generation_idx]

                  lowercase							:     Optional[int]       	=      jnp.ones_like(snake_case		,dtype=scores.dtype      ) * -float("""inf"""      )
                  lowercase							:     Dict       	=      jnp.zeros((batch_size, 1)		,dtype=scores.dtype      )
                  lowercase							:     Any       	=      lax.dynamic_update_slice(snake_case		,snake_case		,(0, current_token)      )
                  return new_scores

           lowercase							:     Any       	=      lax.cond(
               cur_len >= self.force_token_array.shape[0]		,lambda: scores		,lambda: lax.cond(
                   self.force_token_array[cur_len] >= 0		,lambda: _force_token(snake_case      )		,lambda: scores		,)		,)
           return scores






class __snake_case		(      lowerCAmelCase     ):
    def __init__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Optional[Any]       	=      generate_config.eos_token_id
           lowercase							:     List[Any]       	=      generate_config.no_timestamps_token_id
           lowercase							:     int       	=      generate_config.no_timestamps_token_id + 1

           lowercase							:     Optional[int]       	=      decoder_input_length + 1

           if generate_config.is_multilingual:
                  # room for language token and task token
                  self.begin_index += 2
           if hasattr(snake_case		,"""max_initial_timestamp_index"""      ):
                  lowercase							:     List[Any]       	=      generate_config.max_initial_timestamp_index
           else:
                  lowercase							:     Optional[Any]       	=      model_config.vocab_size
           if self.max_initial_timestamp_index is None:
                  lowercase							:     Optional[int]       	=      model_config.vocab_size
    def __call__(  self		,snake_case		,snake_case		,snake_case      ):



           '''simple docstring'''







           lowercase							:     Any       	=      scores.at[:, self.no_timestamps_token_id].set(-float("""inf"""      )      )

           def handle_pairs(snake_case		,snake_case      ):
                  lowercase							:     int       	=      jnp.where((cur_len - self.begin_index) >= 1		,snake_case		,snake_case      )
                  lowercase							:     Union[str, Any]       	=      jnp.where(
                      input_ids_k[cur_len - 1] >= self.timestamp_begin		,True and last_was_timestamp		,snake_case		,)

                  lowercase							:     List[Any]       	=      jnp.where((cur_len - self.begin_index) < 2		,snake_case		,snake_case      )
                  lowercase							:     Tuple       	=      jnp.where(
                      input_ids_k[cur_len - 2] >= self.timestamp_begin		,snake_case		,snake_case		,)

                  return jnp.where(
                      snake_case		,jnp.where(
                          penultimate_was_timestamp > 0		,scores_k.at[self.timestamp_begin :].set(-float("""inf"""      )      )		,scores_k.at[: self.eos_token_id].set(-float("""inf"""      )      )		,)		,snake_case		,)

           lowercase							:     Optional[int]       	=      jax.vmap(snake_case      )(snake_case		,snake_case      )

           lowercase							:     Optional[Any]       	=      jnp.where(cur_len == self.begin_index		,snake_case		,snake_case      )
           lowercase							:     Tuple       	=      jnp.where(
               self.max_initial_timestamp_index is not None		,True and apply_max_initial_timestamp		,snake_case		,)

           lowercase							:     int       	=      self.timestamp_begin + self.max_initial_timestamp_index

           lowercase							:     Optional[int]       	=      jnp.where(
               snake_case		,scores.at[:, last_allowed + 1 :].set(-float("""inf"""      )      )		,snake_case		,)

           # if sum of probability over timestamps is above any other token, sample timestamp
           lowercase							:     List[str]       	=      jax.nn.log_softmax(snake_case		,axis=-1      )

           def handle_cumulative_probs(snake_case		,snake_case      ):
                  lowercase							:     Optional[Any]       	=      jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]		,axis=-1      )
                  lowercase							:     List[Any]       	=      jnp.max(logprobs_k[: self.timestamp_begin]      )
                  return jnp.where(
                      timestamp_logprob > max_text_token_logprob		,scores_k.at[: self.timestamp_begin].set(-float("""inf"""      )      )		,snake_case		,)

           lowercase							:     Tuple       	=      jax.vmap(snake_case      )(snake_case		,snake_case      )

           return scores






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__lowerCAmelCase	(	__magic_name__      ,       unittest.TestCase ):
							UpperCamelCase__					     =  CLIPTokenizer
							UpperCamelCase__					     =  CLIPTokenizerFast
							UpperCamelCase__					     =  True
							UpperCamelCase__					     =  {}
							UpperCamelCase__					     =  False
							def       lowerCamelCase__							(						self						:str				):







								'''simple docstring'''







								super().setUp()

								# fmt: off
								a										=							["""l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
								# fmt: on
								a										=							dict(zip(__magic_name__		,      range(len(__magic_name__				)				)				)				)
								a										=							["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>"""]
								a										=							{"""unk_token""": """<unk>"""}

								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""vocab_file"""]				)
								a										=							os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""merges_file"""]				)
								with open(self.vocab_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write(json.dumps(__magic_name__				) + """\n"""				)
								with open(self.merges_file		,      """w"""		,      encoding="""utf-8"""				) as fp:
									fp.write("""\n""".join(__magic_name__				)				)
							def       lowerCamelCase__							(						self						:Optional[Any]		,      **__magic_name__						:Union[str, Any]				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CLIPTokenizer.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:int		,      **__magic_name__						:Optional[Any]				):







								'''simple docstring'''







								kwargs.update(self.special_tokens_map				)
								return CLIPTokenizerFast.from_pretrained(self.tmpdirname		,      **__magic_name__				)
							def       lowerCamelCase__							(						self						:int		,      __magic_name__						:List[str]				):







								'''simple docstring'''







								a										=							"""lower newer"""
								a										=							"""lower newer"""
								return input_text, output_text
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								a										=							CLIPTokenizer(self.vocab_file		,      self.merges_file		,      **self.special_tokens_map				)
								a										=							"""lower newer"""
								a										=							["""lo""", """w""", """er</w>""", """n""", """e""", """w""", """er</w>"""]
								a										=							tokenizer.tokenize(__magic_name__				)
								self.assertListEqual(__magic_name__		,      __magic_name__				)

								a										=							tokens + [tokenizer.unk_token]
								a										=							[10, 2, 16, 9, 3, 2, 16, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)		,      __magic_name__				)
							@require_ftfy
							def       lowerCamelCase__							(						self						:List[Any]				):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
										a										=							self.tokenizer_class.from_pretrained(__magic_name__		,      **__magic_name__				)
										a										=							self.rust_tokenizer_class.from_pretrained(__magic_name__		,      **__magic_name__				)

										a										=							"""A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."""
										a										=							tokenizer_s.tokenize(__magic_name__				)
										a										=							tokenizer_r.tokenize(__magic_name__				)

										self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
										# with Tilde) encoded in 2 different ways
										a										=							"""xa\u0303y""" + """ """ + """x\xe3y"""
										a										=							tokenizer_s.tokenize(__magic_name__				)
										a										=							tokenizer_r.tokenize(__magic_name__				)

										self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on unicode of space type
										a										=							[
										    """\u0009""",  # (horizontal tab, '\t')
										    """\u000B""",  # (vertical tab)
										    """\u000C""",  # (form feed)
										    """\u0020""",  # (space, ' ')
										    """\u200E""",  # (left-to-right mark):w
										    """\u200F""",  # (right-to-left mark)
										]
										for unicode_seq in spaces_unicodes:
											a										=							tokenizer_s.tokenize(__magic_name__				)
											a										=							tokenizer_r.tokenize(__magic_name__				)

											self.assertListEqual(__magic_name__		,      __magic_name__				)

										# Test that the tokenization is identical on unicode of line break type
										a										=							[
										    """\u000A""",  # (line feed, '\n')
										    """\r\n""",  # (carriage return and line feed, '\r\n')
										    """\u000D""",  # (carriage return, '\r')
										    """\r""",  # (carriage return, '\r')
										    """\u000D""",  # (carriage return, '\r')
										    """\u2028""",  # (line separator)
										    """\u2029""",  # (paragraph separator)
										    # "\u0085", # (next line)
										]

										# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
										# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
										# space (and thus into an empty list).

										for unicode_seq in line_break_unicodes:
											a										=							tokenizer_s.tokenize(__magic_name__				)
											a										=							tokenizer_r.tokenize(__magic_name__				)

											self.assertListEqual(__magic_name__		,      __magic_name__				)
							def       lowerCamelCase__							(						self						:int				):







								'''simple docstring'''







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
										a										=							"""hello"""  # `hello` is a token in the vocabulary of `pretrained_name`
										a										=							F'{text_of_1_token} {text_of_1_token}'

										a										=							self.rust_tokenizer_class.from_pretrained(
										    __magic_name__		,      use_fast=__magic_name__		,      )
										a										=							tokenizer_r(__magic_name__		,      return_offsets_mapping=__magic_name__		,      add_special_tokens=__magic_name__				)
										self.assertEqual(encoding.offset_mapping[0]		,      (0, len(__magic_name__				))				)
										self.assertEqual(
										    encoding.offset_mapping[1]		,      (len(__magic_name__				) + 1, len(__magic_name__				) + 1 + len(__magic_name__				))		,      )

										a										=							F' {text}'

										a										=							self.rust_tokenizer_class.from_pretrained(
										    __magic_name__		,      use_fast=__magic_name__		,      )
										a										=							tokenizer_r(__magic_name__		,      return_offsets_mapping=__magic_name__		,      add_special_tokens=__magic_name__				)
										self.assertEqual(encoding.offset_mapping[0]		,      (1, 1 + len(__magic_name__				))				)
										self.assertEqual(
										    encoding.offset_mapping[1]		,      (1 + len(__magic_name__				) + 1, 1 + len(__magic_name__				) + 1 + len(__magic_name__				))		,      )
							def       lowerCamelCase__							(						self						:List[str]				):







								'''simple docstring'''







								with self.assertRaises(__magic_name__				) as context:
									self.rust_tokenizer_class.from_pretrained("""robot-test/old-clip-tokenizer"""				)

								self.assertTrue(
								    context.exception.args[0].startswith(
								        """The `backend_tokenizer` provided does not match the expected format."""				)				)
							@require_ftfy
							def       lowerCamelCase__							(						self						:Tuple				):







								'''simple docstring'''







								super().test_tokenization_python_rust_equals()





							def       lowerCamelCase__							(						self						:Optional[int]				):







								'''simple docstring'''







								pass

