


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


a				:							str					=			logging.get_logger(__name__)

a				:							List[Any]					=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

a				:							int					=			{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

a				:							str					=			{
    'Salesforce/codegen-350M-mono': 2048,
}






@lru_cache()
def    _UpperCAmelCase	(   )		->Optional[Any]:


 '''simple docstring'''
 snake_case   : Optional[Any]	  =      (
     list(range(ord('''!'''       ) ,     ord('''~'''       ) + 1       )       ) + list(range(ord('''¡'''       ) ,     ord('''¬'''       ) + 1       )       ) + list(range(ord('''®'''       ) ,     ord('''ÿ'''       ) + 1       )       )
 )
 snake_case   : Optional[Any]	  =      bs[:]
 snake_case   : Tuple	  =      0
 for b in range(2**8       ):
  if b not in bs:
   bs.append(__lowerCamelCase       )
   cs.append(2**8 + n       )
   n += 1
 snake_case   : List[Any]	  =      [chr(__lowerCamelCase       ) for n in cs]
 return dict(zip(__lowerCamelCase ,     __lowerCamelCase       )       )






def    _UpperCAmelCase	(   __lowerCamelCase       )		->Any:


 '''simple docstring'''
 snake_case   : Tuple	  =      set()
 snake_case   : Union[str, Any]	  =      word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)       )
  snake_case   : Union[str, Any]	  =      char
 return pairs





class      lowerCamelCase	(  SCREAMING_SNAKE_CASE     ):
       _UpperCamelCase    		=      VOCAB_FILES_NAMES
       _UpperCamelCase    		=      PRETRAINED_VOCAB_FILES_MAP
       _UpperCamelCase    		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _UpperCamelCase    		=      ["""input_ids""", """attention_mask"""]

       def __init__(			self	,						a__	,						a__	,						a__="replace"	,						a__="<|endoftext|>"	,						a__="<|endoftext|>"	,						a__="<|endoftext|>"	,						a__=None	,						a__=False	,						a__=False	,						**a__	,						):






        '''simple docstring'''






        snake_case   : Dict	  =      AddedToken(a__	,						lstrip=a__	,						rstrip=a__					) if isinstance(a__	,						a__					) else bos_token
        snake_case   : Optional[Any]	  =      AddedToken(a__	,						lstrip=a__	,						rstrip=a__					) if isinstance(a__	,						a__					) else eos_token
        snake_case   : Tuple	  =      AddedToken(a__	,						lstrip=a__	,						rstrip=a__					) if isinstance(a__	,						a__					) else unk_token
        snake_case   : Dict	  =      AddedToken(a__	,						lstrip=a__	,						rstrip=a__					) if isinstance(a__	,						a__					) else pad_token
        super().__init__(
            errors=a__	,						unk_token=a__	,						bos_token=a__	,						eos_token=a__	,						pad_token=a__	,						add_prefix_space=a__	,						add_bos_token=a__	,						**a__	,						)
        snake_case   : Optional[Any]	  =      add_bos_token

        with open(a__	,						encoding='''utf-8'''					) as vocab_handle:
         snake_case   : Optional[int]	  =      json.load(a__					)
        snake_case   : List[Any]	  =      {v: k for k, v in self.encoder.items()}
        snake_case   : Optional[int]	  =      errors  # how to handle errors in decoding
        snake_case   : int	  =      bytes_to_unicode()
        snake_case   : Tuple	  =      {v: k for k, v in self.byte_encoder.items()}
        with open(a__	,						encoding='''utf-8'''					) as merges_handle:
         snake_case   : Any	  =      merges_handle.read().split('''\n'''					)[1:-1]
        snake_case   : List[Any]	  =      [tuple(merge.split()					) for merge in bpe_merges]
        snake_case   : Tuple	  =      dict(zip(a__	,						range(len(a__					)					)					)					)
        snake_case   : List[Any]	  =      {}
        snake_case   : Any	  =      add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        snake_case   : Union[str, Any]	  =      re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''					)

       @property
       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        return len(self.encoder					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        return dict(self.encoder	,						**self.added_tokens_encoder					)

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        if token in self.cache:
         return self.cache[token]
        snake_case   : Tuple	  =      tuple(a__					)
        snake_case   : Tuple	  =      get_pairs(a__					)

        if not pairs:
         return token

        while True:
         snake_case   : Tuple	  =      min(a__	,						key=lambda a__					: self.bpe_ranks.get(a__	,						float('''inf'''					)					)					)
         if bigram not in self.bpe_ranks:
          break
         snake_case  ,					snake_case   : Tuple	  =      bigram
         snake_case   : str	  =      []
         snake_case   : List[Any]	  =      0
         while i < len(a__					):
          try:
           snake_case   : Optional[int]	  =      word.index(a__	,						a__					)
          except ValueError:
           new_word.extend(word[i:]					)
           break
          else:
           new_word.extend(word[i:j]					)
           snake_case   : int	  =      j

          if word[i] == first and i < len(a__					) - 1 and word[i + 1] == second:
           new_word.append(first + second					)
           i += 2
          else:
           new_word.append(word[i]					)
           i += 1
         snake_case   : Optional[Any]	  =      tuple(a__					)
         snake_case   : Optional[Any]	  =      new_word
         if len(a__					) == 1:
          break
         else:
          snake_case   : Union[str, Any]	  =      get_pairs(a__					)
        snake_case   : Optional[int]	  =      ''' '''.join(a__					)
        snake_case   : str	  =      word
        return word

       def 				UpperCAmelCase				(			self	,						a__	,						a__=None					):






        '''simple docstring'''






        if self.add_bos_token:
         snake_case   : Any	  =      [self.bos_token_id]
        else:
         snake_case   : Tuple	  =      []

        snake_case   : List[Any]	  =      bos_token_ids + token_ids_a

        if token_ids_a is None:
         return output

        return output + bos_token_ids + token_ids_a

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        snake_case   : List[str]	  =      []
        for token in re.findall(self.pat	,						a__					):
         snake_case   : List[str]	  =      ''''''.join(
             self.byte_encoder[b] for b in token.encode('''utf-8'''					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
         bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a__					).split(''' '''					)					)
        return bpe_tokens

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        return self.encoder.get(a__	,						self.encoder.get(self.unk_token					)					)

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        return self.decoder.get(a__					)

       def 				UpperCAmelCase				(			self	,						a__					):






        '''simple docstring'''






        snake_case   : str	  =      ''''''.join(a__					)
        snake_case   : str	  =      bytearray([self.byte_decoder[c] for c in text]					).decode('''utf-8'''	,						errors=self.errors					)
        return text

       def 				UpperCAmelCase				(			self	,						a__	,						a__ = None					):






        '''simple docstring'''






        if not os.path.isdir(a__					):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
         return
        snake_case   : Any	  =      os.path.join(
            a__	,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
        snake_case   : str	  =      os.path.join(
            a__	,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']					)

        with open(a__	,						'''w'''	,						encoding='''utf-8'''					) as f:
         f.write(json.dumps(self.encoder	,						indent=2	,						sort_keys=a__	,						ensure_ascii=a__					) + '''\n'''					)

        snake_case   : Tuple	  =      0
        with open(a__	,						'''w'''	,						encoding='''utf-8'''					) as writer:
         writer.write('''#version: 0.2\n'''					)
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,						key=lambda a__					: kv[1]					):
          if index != token_index:
           logger.warning(
               f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
               ''' Please check that the tokenizer is not corrupted!'''					)
           snake_case   : Dict	  =      token_index
          writer.write(''' '''.join(a__					) + '''\n'''					)
          index += 1

        return vocab_file, merge_file

       def 				UpperCAmelCase				(			self	,						a__	,						a__=False	,						**a__					):






        '''simple docstring'''






        snake_case   : List[str]	  =      kwargs.pop('''add_prefix_space'''	,						self.add_prefix_space					)
        if is_split_into_words or add_prefix_space:
         snake_case   : Union[str, Any]	  =      ''' ''' + text
        return (text, kwargs)

       def 				UpperCAmelCase				(			self	,						a__	,						a__ = False	,						a__ = None	,						a__ = None	,						**a__	,						):






        '''simple docstring'''






        snake_case   : Tuple	  =      super()._decode(
            token_ids=a__	,						skip_special_tokens=a__	,						clean_up_tokenization_spaces=a__	,						**a__	,						)

        if truncate_before_pattern is not None and len(a__					) > 0:
         snake_case   : Any	  =      self.truncate(a__	,						a__					)

        return decoded_text



       def 				UpperCAmelCase				(			self	,						a__	,						a__					):






        '''simple docstring'''






        def find_re(a__	,						a__	,						a__					):
         snake_case   : List[str]	  =      pattern.search(a__	,						a__					)
         return m.start() if m else -1

        snake_case   : int	  =      [re.compile(a__	,						re.MULTILINE					) for pattern in truncate_before_pattern]

        snake_case   : Dict	  =      list(re.finditer('''^print'''	,						a__	,						re.MULTILINE					)					)

        if len(a__					) > 1:
         snake_case   : int	  =      completion[: prints[1].start()]

        snake_case   : Any	  =      list(re.finditer('''^def'''	,						a__	,						re.MULTILINE					)					)

        if len(a__					) > 1:
         snake_case   : Dict	  =      completion[: defs[1].start()]

        snake_case   : Optional[Any]	  =      0

        snake_case   : List[Any]	  =      [
            pos for pos in [find_re(a__	,						a__	,						a__					) for terminal in terminals] if pos != -1
        ]

        if len(a__					) > 0:
         return completion[: min(a__					)]
        else:
         return completion










import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import MobileNetVaImageProcessor

class      lowerCamelCase	(  unittest.TestCase     ):

       def __init__(			self	,						a__	,						a__=7	,						a__=3	,						a__=18	,						a__=30	,						a__=4_00	,						a__=True	,						a__=None	,						a__=True	,						a__=None	,						):






        '''simple docstring'''






        snake_case   : str	  =      size if size is not None else {'''shortest_edge''': 20}
        snake_case   : List[str]	  =      crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
        snake_case   : Optional[Any]	  =      parent
        snake_case   : Dict	  =      batch_size
        snake_case   : List[str]	  =      num_channels
        snake_case   : Union[str, Any]	  =      image_size
        snake_case   : Optional[Any]	  =      min_resolution
        snake_case   : Union[str, Any]	  =      max_resolution
        snake_case   : Optional[int]	  =      do_resize
        snake_case   : Optional[int]	  =      size
        snake_case   : Optional[Any]	  =      do_center_crop
        snake_case   : Optional[int]	  =      crop_size



       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        return {
            "do_resize": self.do_resize,
            "size": self.size,
            "do_center_crop": self.do_center_crop,
            "crop_size": self.crop_size,
        }







@require_torch
@require_vision
class      lowerCamelCase	(  SCREAMING_SNAKE_CASE,			unittest.TestCase     ):
       _UpperCamelCase    		=      MobileNetVaImageProcessor if is_vision_available() else None

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : List[str]	  =      MobileNetVaImageProcessingTester(self					)

       @property
       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        return self.image_processor_tester.prepare_image_processor_dict()

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Union[str, Any]	  =      self.image_processing_class(**self.image_processor_dict					)
        self.assertTrue(hasattr(a__	,						'''do_resize'''					)					)
        self.assertTrue(hasattr(a__	,						'''size'''					)					)
        self.assertTrue(hasattr(a__	,						'''do_center_crop'''					)					)
        self.assertTrue(hasattr(a__	,						'''center_crop'''					)					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : str	  =      self.image_processing_class.from_dict(self.image_processor_dict					)
        self.assertEqual(image_processor.size	,						{'''shortest_edge''': 20}					)
        self.assertEqual(image_processor.crop_size	,						{'''height''': 18, '''width''': 18}					)

        snake_case   : Dict	  =      self.image_processing_class.from_dict(self.image_processor_dict	,						size=42	,						crop_size=84					)
        self.assertEqual(image_processor.size	,						{'''shortest_edge''': 42}					)
        self.assertEqual(image_processor.crop_size	,						{'''height''': 84, '''width''': 84}					)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        pass

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Union[str, Any]	  =      self.image_processing_class(**self.image_processor_dict					)
        # create random PIL images
        snake_case   : Tuple	  =      prepare_image_inputs(self.image_processor_tester	,						equal_resolution=a__					)
        for image in image_inputs:
         self.assertIsInstance(a__	,						Image.Image					)

        # Test not batched input
        snake_case   : List[str]	  =      image_processing(image_inputs[0]	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)

        # Test batched
        snake_case   : List[Any]	  =      image_processing(a__	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)

       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : Optional[int]	  =      self.image_processing_class(**self.image_processor_dict					)
        # create random numpy tensors
        snake_case   : Tuple	  =      prepare_image_inputs(self.image_processor_tester	,						equal_resolution=a__	,						numpify=a__					)
        for image in image_inputs:
         self.assertIsInstance(a__	,						np.ndarray					)

        # Test not batched input
        snake_case   : Dict	  =      image_processing(image_inputs[0]	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)

        # Test batched
        snake_case   : List[Any]	  =      image_processing(a__	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)



       def 				UpperCAmelCase				(			self					):






        '''simple docstring'''






        snake_case   : List[str]	  =      self.image_processing_class(**self.image_processor_dict					)
        # create random PyTorch tensors
        snake_case   : List[Any]	  =      prepare_image_inputs(self.image_processor_tester	,						equal_resolution=a__	,						torchify=a__					)
        for image in image_inputs:
         self.assertIsInstance(a__	,						torch.Tensor					)

        # Test not batched input
        snake_case   : List[str]	  =      image_processing(image_inputs[0]	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                1,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)

        # Test batched
        snake_case   : Any	  =      image_processing(a__	,						return_tensors='''pt'''					).pixel_values
        self.assertEqual(
            encoded_images.shape	,						(
                self.image_processor_tester.batch_size,
                self.image_processor_tester.num_channels,
                self.image_processor_tester.crop_size['''height'''],
                self.image_processor_tester.crop_size['''width'''],
            )	,						)






