





'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__    :      List[Any]						=    logging.get_logger(__name__)

lowercase__    :      Optional[Any]						=    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase__    :      Tuple						=    {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase__    :      Tuple						=    {
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

lowercase__    :      List[Any]						=    {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}


class  SCREAMING_SNAKE_CASE    (a__							):
     lowerCAmelCase					=       VOCAB_FILES_NAMES
     lowerCAmelCase					=       PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase					=       PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase					=       BertTokenizer


     def __init__(						self						,	_UpperCAmelCase=None						,	_UpperCAmelCase=None						,	_UpperCAmelCase=True						,	_UpperCAmelCase="[UNK]"						,	_UpperCAmelCase="[SEP]"						,	_UpperCAmelCase="[PAD]"						,	_UpperCAmelCase="[CLS]"						,	_UpperCAmelCase="[MASK]"						,	_UpperCAmelCase=True						,	_UpperCAmelCase=None						,	**_UpperCAmelCase						,	):

          '''simple docstring'''






          super().__init__(
              _UpperCAmelCase						,	tokenizer_file=_UpperCAmelCase						,	do_lower_case=_UpperCAmelCase						,	unk_token=_UpperCAmelCase						,	sep_token=_UpperCAmelCase						,	pad_token=_UpperCAmelCase						,	cls_token=_UpperCAmelCase						,	mask_token=_UpperCAmelCase						,	tokenize_chinese_chars=_UpperCAmelCase						,	strip_accents=_UpperCAmelCase						,	**_UpperCAmelCase						,	)

          __A						: str      =       json.loads(self.backend_tokenizer.normalizer.__getstate__())
          if (
              normalizer_state.get('lowercase'						,	_UpperCAmelCase) != do_lower_case
              or normalizer_state.get('strip_accents'						,	_UpperCAmelCase) != strip_accents
              or normalizer_state.get('handle_chinese_chars'						,	_UpperCAmelCase) != tokenize_chinese_chars
          ):
               __A						: str      =       getattr(_UpperCAmelCase						,	normalizer_state.pop('type'))
               __A						: Optional[Any]      =       do_lower_case
               __A						: str      =       strip_accents
               __A						: Union[str, Any]      =       tokenize_chinese_chars
               __A						: Dict      =       normalizer_class(**_UpperCAmelCase)

          __A						: str      =       do_lower_case


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase=None):

          '''simple docstring'''






          __A						: Optional[Any]      =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase = None):

          '''simple docstring'''






          __A						: str      =       [self.sep_token_id]
          __A						: Tuple      =       [self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]
          return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase						,	_UpperCAmelCase = None):

          '''simple docstring'''






          __A						: Optional[Any]      =       self._tokenizer.model.save(_UpperCAmelCase						,	name=_UpperCAmelCase)
          return tuple(_UpperCAmelCase)






'''simple docstring'''




import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import CLIPImageProcessor, CLIPProcessor


@require_vision
class  SCREAMING_SNAKE_CASE    (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[Any]      =       tempfile.mkdtemp()

          # fmt: off
          __A						: Optional[int]      =       ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
          # fmt: on
          __A						: Tuple      =       dict(zip(_UpperCAmelCase						,	range(len(_UpperCAmelCase))))
          __A						: Any      =       ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
          __A						: List[str]      =       {'unk_token': '<unk>'}

          __A						: int      =       os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES['vocab_file'])
          __A						: Optional[int]      =       os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES['merges_file'])
          with open(self.vocab_file						,	'w'						,	encoding='utf-8') as fp:
               fp.write(json.dumps(_UpperCAmelCase) + '\n')
          with open(self.merges_file						,	'w'						,	encoding='utf-8') as fp:
               fp.write('\n'.join(_UpperCAmelCase))

          __A						: str      =       {
              'do_resize': True,
              'size': 20,
              'do_center_crop': True,
              'crop_size': 18,
              'do_normalize': True,
              'image_mean': [0.48145466, 0.4578275, 0.40821073],
              'image_std': [0.26862954, 0.26130258, 0.27577711],
          }
          __A						: Union[str, Any]      =       os.path.join(self.tmpdirname						,	_UpperCAmelCase)
          with open(self.image_processor_file						,	'w'						,	encoding='utf-8') as fp:
               json.dump(_UpperCAmelCase						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,	**_UpperCAmelCase):

          '''simple docstring'''






          return CLIPTokenizer.from_pretrained(self.tmpdirname						,	**_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,	**_UpperCAmelCase):

          '''simple docstring'''






          return CLIPTokenizerFast.from_pretrained(self.tmpdirname						,	**_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self						,	**_UpperCAmelCase):

          '''simple docstring'''






          return CLIPImageProcessor.from_pretrained(self.tmpdirname						,	**_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          shutil.rmtree(self.tmpdirname)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Tuple      =       [np.random.randint(255						,	size=(3, 30, 400)						,	dtype=np.uinta)]

          __A						: Optional[int]      =       [Image.fromarray(np.moveaxis(_UpperCAmelCase						,	0						,	-1)) for x in image_inputs]

          return image_inputs


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: str      =       self.get_tokenizer()
          __A						: int      =       self.get_rust_tokenizer()
          __A						: Any      =       self.get_image_processor()

          __A						: Union[str, Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)
          processor_slow.save_pretrained(self.tmpdirname)
          __A						: Dict      =       CLIPProcessor.from_pretrained(self.tmpdirname						,	use_fast=_UpperCAmelCase)

          __A						: Union[str, Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)
          processor_fast.save_pretrained(self.tmpdirname)
          __A						: Union[str, Any]      =       CLIPProcessor.from_pretrained(self.tmpdirname)

          self.assertEqual(processor_slow.tokenizer.get_vocab()						,	tokenizer_slow.get_vocab())
          self.assertEqual(processor_fast.tokenizer.get_vocab()						,	tokenizer_fast.get_vocab())
          self.assertEqual(tokenizer_slow.get_vocab()						,	tokenizer_fast.get_vocab())
          self.assertIsInstance(processor_slow.tokenizer						,	_UpperCAmelCase)
          self.assertIsInstance(processor_fast.tokenizer						,	_UpperCAmelCase)

          self.assertEqual(processor_slow.image_processor.to_json_string()						,	image_processor.to_json_string())
          self.assertEqual(processor_fast.image_processor.to_json_string()						,	image_processor.to_json_string())
          self.assertIsInstance(processor_slow.image_processor						,	_UpperCAmelCase)
          self.assertIsInstance(processor_fast.image_processor						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       CLIPProcessor(tokenizer=self.get_tokenizer()						,	image_processor=self.get_image_processor())
          processor.save_pretrained(self.tmpdirname)

          __A						: Dict      =       self.get_tokenizer(bos_token='(BOS)'						,	eos_token='(EOS)')
          __A						: Any      =       self.get_image_processor(do_normalize=_UpperCAmelCase						,	padding_value=1.0)

          __A						: Tuple      =       CLIPProcessor.from_pretrained(
              self.tmpdirname						,	bos_token='(BOS)'						,	eos_token='(EOS)'						,	do_normalize=_UpperCAmelCase						,	padding_value=1.0)

          self.assertEqual(processor.tokenizer.get_vocab()						,	tokenizer_add_kwargs.get_vocab())
          self.assertIsInstance(processor.tokenizer						,	_UpperCAmelCase)

          self.assertEqual(processor.image_processor.to_json_string()						,	image_processor_add_kwargs.to_json_string())
          self.assertIsInstance(processor.image_processor						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[int]      =       self.get_image_processor()
          __A						: List[Any]      =       self.get_tokenizer()

          __A						: List[Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)

          __A						: List[str]      =       self.prepare_image_inputs()

          __A						: Dict      =       image_processor(_UpperCAmelCase						,	return_tensors='np')
          __A						: str      =       processor(images=_UpperCAmelCase						,	return_tensors='np')

          for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum()						,	input_processor[key].sum()						,	delta=1e-2)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: List[Any]      =       self.get_image_processor()
          __A						: List[str]      =       self.get_tokenizer()

          __A						: Optional[Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)

          __A						: Dict      =       'lower newer'

          __A						: List[Any]      =       processor(text=_UpperCAmelCase)

          __A						: Optional[int]      =       tokenizer(_UpperCAmelCase)

          for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]						,	encoded_processor[key])


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Dict      =       self.get_image_processor()
          __A						: Optional[int]      =       self.get_tokenizer()

          __A						: Optional[Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)

          __A						: int      =       'lower newer'
          __A						: Any      =       self.prepare_image_inputs()

          __A						: Tuple      =       processor(text=_UpperCAmelCase						,	images=_UpperCAmelCase)

          self.assertListEqual(list(inputs.keys())						,	['input_ids', 'attention_mask', 'pixel_values'])

          # test if it raises when no input is passed
          with pytest.raises(_UpperCAmelCase):
               processor()


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Any      =       self.get_image_processor()
          __A						: str      =       self.get_tokenizer()

          __A						: Optional[Any]      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)

          __A						: Dict      =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          __A						: str      =       processor.batch_decode(_UpperCAmelCase)
          __A						: Optional[Any]      =       tokenizer.batch_decode(_UpperCAmelCase)

          self.assertListEqual(_UpperCAmelCase						,	_UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						: Optional[int]      =       self.get_image_processor()
          __A						: int      =       self.get_tokenizer()

          __A						: Any      =       CLIPProcessor(tokenizer=_UpperCAmelCase						,	image_processor=_UpperCAmelCase)

          __A						: Any      =       'lower newer'
          __A						: Any      =       self.prepare_image_inputs()

          __A						: Any      =       processor(text=_UpperCAmelCase						,	images=_UpperCAmelCase)

          self.assertListEqual(list(inputs.keys())						,	processor.model_input_names)