






from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin


@dataclass
class 		lowerCAmelCase_				(  UpperCamelCase ):
     __lowerCamelCase       :    torch.FloatTensor




class 		lowerCAmelCase_				(  UpperCamelCase				,     UpperCamelCase ):





     @register_to_config
     def __init__(	self							, _UpperCAmelCase = 16							, _UpperCAmelCase = 88							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = 1							, _UpperCAmelCase = 0.0							, _UpperCAmelCase = 32							, _UpperCAmelCase = None							, _UpperCAmelCase = False							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = "geglu"							, _UpperCAmelCase = None							, _UpperCAmelCase = False							, _UpperCAmelCase = False							, _UpperCAmelCase = False							, _UpperCAmelCase = "layer_norm"							, _UpperCAmelCase = True							, )		->				Dict:
        super().__init__()
        lowerCamelCase									=use_linear_projection
        lowerCamelCase									=num_attention_heads
        lowerCamelCase									=attention_head_dim
        lowerCamelCase									=num_attention_heads * attention_head_dim

        # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
        # Define whether input is continuous or discrete depending on configuration
        lowerCamelCase									=(in_channels is not None) and (patch_size is None)
        lowerCamelCase									=num_vector_embeds is not None
        lowerCamelCase									=in_channels is not None and patch_size is not None

        if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
           lowerCamelCase									=(
               F'''The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or'''
               """ incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config."""
               """ Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect"""
               """ results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it"""
               """ would be very nice if you could open a Pull request for the `transformer/config.json` file"""
           )
           deprecate("""norm_type!=num_embeds_ada_norm"""							, """1.0.0"""							, _UpperCAmelCase							, standard_warn=_UpperCAmelCase  )
           lowerCamelCase									="""ada_norm"""

        if self.is_input_continuous and self.is_input_vectorized:
           raise ValueError(
               F'''Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make'''
               """ sure that either `in_channels` or `num_vector_embeds` is None."""  )
        elif self.is_input_vectorized and self.is_input_patches:
           raise ValueError(
               F'''Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make'''
               """ sure that either `num_vector_embeds` or `num_patches` is None."""  )
        elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
           raise ValueError(
               F'''Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:'''
               F''' {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None.'''  )

        # 2. Define input layers
        if self.is_input_continuous:
           lowerCamelCase									=in_channels

           lowerCamelCase									=torch.nn.GroupNorm(num_groups=_UpperCAmelCase							, num_channels=_UpperCAmelCase							, eps=1E-6							, affine=_UpperCAmelCase  )
           if use_linear_projection:
              lowerCamelCase									=nn.Linear(_UpperCAmelCase							, _UpperCAmelCase  )
           else:
              lowerCamelCase									=nn.Convad(_UpperCAmelCase							, _UpperCAmelCase							, kernel_size=1							, stride=1							, padding=0  )
        elif self.is_input_vectorized:
           assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
           assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

           lowerCamelCase									=sample_size
           lowerCamelCase									=sample_size
           lowerCamelCase									=num_vector_embeds
           lowerCamelCase									=self.height * self.width

           lowerCamelCase									=ImagePositionalEmbeddings(
               num_embed=_UpperCAmelCase							, embed_dim=_UpperCAmelCase							, height=self.height							, width=self.width  )
        elif self.is_input_patches:
           assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

           lowerCamelCase									=sample_size
           lowerCamelCase									=sample_size

           lowerCamelCase									=patch_size
           lowerCamelCase									=PatchEmbed(
               height=_UpperCAmelCase							, width=_UpperCAmelCase							, patch_size=_UpperCAmelCase							, in_channels=_UpperCAmelCase							, embed_dim=_UpperCAmelCase							, )

        # 3. Define transformers blocks
        lowerCamelCase									=nn.ModuleList(
            [
                BasicTransformerBlock(
                    _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, dropout=_UpperCAmelCase							, cross_attention_dim=_UpperCAmelCase							, activation_fn=_UpperCAmelCase							, num_embeds_ada_norm=_UpperCAmelCase							, attention_bias=_UpperCAmelCase							, only_cross_attention=_UpperCAmelCase							, upcast_attention=_UpperCAmelCase							, norm_type=_UpperCAmelCase							, norm_elementwise_affine=_UpperCAmelCase							, )
                for d in range(_UpperCAmelCase  )
            ]  )

        # 4. Define output layers
        lowerCamelCase									=in_channels if out_channels is None else out_channels
        if self.is_input_continuous:
           # TODO: should use out_channels for continuous projections
           if use_linear_projection:
              lowerCamelCase									=nn.Linear(_UpperCAmelCase							, _UpperCAmelCase  )
           else:
              lowerCamelCase									=nn.Convad(_UpperCAmelCase							, _UpperCAmelCase							, kernel_size=1							, stride=1							, padding=0  )
        elif self.is_input_vectorized:
           lowerCamelCase									=nn.LayerNorm(_UpperCAmelCase  )
           lowerCamelCase									=nn.Linear(_UpperCAmelCase							, self.num_vector_embeds - 1  )
        elif self.is_input_patches:
           lowerCamelCase									=nn.LayerNorm(_UpperCAmelCase							, elementwise_affine=_UpperCAmelCase							, eps=1E-6  )
           lowerCamelCase									=nn.Linear(_UpperCAmelCase							, 2 * inner_dim  )
           lowerCamelCase									=nn.Linear(_UpperCAmelCase							, patch_size * patch_size * self.out_channels  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = None							, _UpperCAmelCase = True							, )		->				Tuple:
        # ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
        #   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
        #   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
        # expects mask of shape:
        #   [batch, key_tokens]
        # adds singleton query_tokens dimension:
        #   [batch,                    1, key_tokens]
        # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
        #   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
        #   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
        if attention_mask is not None and attention_mask.ndim == 2:
           # assume that mask is expressed as:
           #   (1 = keep,      0 = discard)
           # convert mask into a bias that can be added to attention scores:
           #       (keep = +0,     discard = -10000.0)
           lowerCamelCase									=(1 - attention_mask.to(hidden_states.dtype  )) * -10000.0
           lowerCamelCase									=attention_mask.unsqueeze(1  )

        # convert encoder_attention_mask to a bias the same way we do for attention_mask
        if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
           lowerCamelCase									=(1 - encoder_attention_mask.to(hidden_states.dtype  )) * -10000.0
           lowerCamelCase									=encoder_attention_mask.unsqueeze(1  )

        # 1. Input
        if self.is_input_continuous:
           lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase				,       lowerCamelCase									=hidden_states.shape
           lowerCamelCase									=hidden_states

           lowerCamelCase									=self.norm(_UpperCAmelCase  )
           if not self.use_linear_projection:
              lowerCamelCase									=self.proj_in(_UpperCAmelCase  )
              lowerCamelCase									=hidden_states.shape[1]
              lowerCamelCase									=hidden_states.permute(0							, 2							, 3							, 1  ).reshape(_UpperCAmelCase							, height * width							, _UpperCAmelCase  )
           else:
              lowerCamelCase									=hidden_states.shape[1]
              lowerCamelCase									=hidden_states.permute(0							, 2							, 3							, 1  ).reshape(_UpperCAmelCase							, height * width							, _UpperCAmelCase  )
              lowerCamelCase									=self.proj_in(_UpperCAmelCase  )
        elif self.is_input_vectorized:
           lowerCamelCase									=self.latent_image_embedding(_UpperCAmelCase  )
        elif self.is_input_patches:
           lowerCamelCase									=self.pos_embed(_UpperCAmelCase  )

        # 2. Blocks
        for block in self.transformer_blocks:
           lowerCamelCase									=block(
               _UpperCAmelCase							, attention_mask=_UpperCAmelCase							, encoder_hidden_states=_UpperCAmelCase							, encoder_attention_mask=_UpperCAmelCase							, timestep=_UpperCAmelCase							, cross_attention_kwargs=_UpperCAmelCase							, class_labels=_UpperCAmelCase							, )

        # 3. Output
        if self.is_input_continuous:
           if not self.use_linear_projection:
              lowerCamelCase									=hidden_states.reshape(_UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  ).permute(0							, 3							, 1							, 2  ).contiguous()
              lowerCamelCase									=self.proj_out(_UpperCAmelCase  )
           else:
              lowerCamelCase									=self.proj_out(_UpperCAmelCase  )
              lowerCamelCase									=hidden_states.reshape(_UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase  ).permute(0							, 3							, 1							, 2  ).contiguous()

           lowerCamelCase									=hidden_states + residual
        elif self.is_input_vectorized:
           lowerCamelCase									=self.norm_out(_UpperCAmelCase  )
           lowerCamelCase									=self.out(_UpperCAmelCase  )
           # (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
           lowerCamelCase									=logits.permute(0							, 2							, 1  )

           # log(p(x_0))
           lowerCamelCase									=F.log_softmax(logits.double()							, dim=1  ).float()
        elif self.is_input_patches:
           # TODO: cleanup!
           lowerCamelCase									=self.transformer_blocks[0].norma.emb(
               _UpperCAmelCase							, _UpperCAmelCase							, hidden_dtype=hidden_states.dtype  )
           lowerCamelCase				,       lowerCamelCase									=self.proj_out_a(F.silu(_UpperCAmelCase  )  ).chunk(2							, dim=1  )
           lowerCamelCase									=self.norm_out(_UpperCAmelCase  ) * (1 + scale[:, None]) + shift[:, None]
           lowerCamelCase									=self.proj_out_a(_UpperCAmelCase  )

           # unpatchify
           lowerCamelCase									=lowerCamelCase									=int(hidden_states.shape[1] ** 0.5  )
           lowerCamelCase									=hidden_states.reshape(
               shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)  )
           lowerCamelCase									=torch.einsum("""nhwpqc->nchpwq"""							, _UpperCAmelCase  )
           lowerCamelCase									=hidden_states.reshape(
               shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)  )

        if not return_dict:
           return (output,)

        return TransformeraDModelOutput(sample=_UpperCAmelCase  )







import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase					:						Dict         =     logging.get_logger(__name__)

_UpperCAmelCase					:						int         =     {
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

_UpperCAmelCase					:						Optional[int]         =     {
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

_UpperCAmelCase					:						Optional[int]         =     {'stas/tiny-wmt19-en-de': 1024}
_UpperCAmelCase					:						List[Any]         =     {
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 1024,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}






def  UpperCAmelCase__						(    lowerCamelCase	:			Dict    ):
   lowerCamelCase									=set()
   lowerCamelCase									=word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)    )
      lowerCamelCase									=char
   return pairs






def  UpperCAmelCase__						(    lowerCamelCase	:			Optional[Any]    ):
   lowerCamelCase									=text.replace("""，"""			,""","""    )
   lowerCamelCase									=re.sub(R"""。\s*"""			,""". """			,lowerCamelCase    )
   lowerCamelCase									=text.replace("""、"""			,""","""    )
   lowerCamelCase									=text.replace("""”"""			,"""\""""    )
   lowerCamelCase									=text.replace("""“"""			,"""\""""    )
   lowerCamelCase									=text.replace("""∶"""			,""":"""    )
   lowerCamelCase									=text.replace("""："""			,""":"""    )
   lowerCamelCase									=text.replace("""？"""			,"""?"""    )
   lowerCamelCase									=text.replace("""《"""			,"""\""""    )
   lowerCamelCase									=text.replace("""》"""			,"""\""""    )
   lowerCamelCase									=text.replace("""）"""			,""")"""    )
   lowerCamelCase									=text.replace("""！"""			,"""!"""    )
   lowerCamelCase									=text.replace("""（"""			,"""("""    )
   lowerCamelCase									=text.replace("""；"""			,""";"""    )
   lowerCamelCase									=text.replace("""１"""			,"""1"""    )
   lowerCamelCase									=text.replace("""」"""			,"""\""""    )
   lowerCamelCase									=text.replace("""「"""			,"""\""""    )
   lowerCamelCase									=text.replace("""０"""			,"""0"""    )
   lowerCamelCase									=text.replace("""３"""			,"""3"""    )
   lowerCamelCase									=text.replace("""２"""			,"""2"""    )
   lowerCamelCase									=text.replace("""５"""			,"""5"""    )
   lowerCamelCase									=text.replace("""６"""			,"""6"""    )
   lowerCamelCase									=text.replace("""９"""			,"""9"""    )
   lowerCamelCase									=text.replace("""７"""			,"""7"""    )
   lowerCamelCase									=text.replace("""８"""			,"""8"""    )
   lowerCamelCase									=text.replace("""４"""			,"""4"""    )
   lowerCamelCase									=re.sub(R"""．\s*"""			,""". """			,lowerCamelCase    )
   lowerCamelCase									=text.replace("""～"""			,"""~"""    )
   lowerCamelCase									=text.replace("""’"""			,"""'"""    )
   lowerCamelCase									=text.replace("""…"""			,"""..."""    )
   lowerCamelCase									=text.replace("""━"""			,"""-"""    )
   lowerCamelCase									=text.replace("""〈"""			,"""<"""    )
   lowerCamelCase									=text.replace("""〉"""			,""">"""    )
   lowerCamelCase									=text.replace("""【"""			,"""["""    )
   lowerCamelCase									=text.replace("""】"""			,"""]"""    )
   lowerCamelCase									=text.replace("""％"""			,"""%"""    )
   return text






def  UpperCAmelCase__						(    lowerCamelCase	:			Optional[int]    ):
   lowerCamelCase									=[]
   for char in text:
      lowerCamelCase									=unicodedata.category(lowerCamelCase    )
      if cat.startswith("""C"""    ):
         continue
      output.append(lowerCamelCase    )
   return "".join(lowerCamelCase    )







class 		lowerCAmelCase_				(  UpperCamelCase ):
     __lowerCamelCase       :    Optional[int] 	=     VOCAB_FILES_NAMES
     __lowerCamelCase       :    List[Any] 	=     PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase       :    List[Any] 	=     PRETRAINED_INIT_CONFIGURATION
     __lowerCamelCase       :    Tuple 	=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase       :    Optional[int] 	=     ['''input_ids''', '''attention_mask''']





     def __init__(	self							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=None							, _UpperCAmelCase=False							, _UpperCAmelCase="<unk>"							, _UpperCAmelCase="<s>"							, _UpperCAmelCase="</s>"							, _UpperCAmelCase="<pad>"							, **_UpperCAmelCase							, )		->				Tuple:
        super().__init__(
            langs=_UpperCAmelCase							, src_vocab_file=_UpperCAmelCase							, tgt_vocab_file=_UpperCAmelCase							, merges_file=_UpperCAmelCase							, do_lower_case=_UpperCAmelCase							, unk_token=_UpperCAmelCase							, bos_token=_UpperCAmelCase							, sep_token=_UpperCAmelCase							, pad_token=_UpperCAmelCase							, **_UpperCAmelCase							, )

        try:
           import sacremoses
        except ImportError:
           raise ImportError(
               """You need to install sacremoses to use XLMTokenizer. """
               """See https://pypi.org/project/sacremoses/ for installation."""  )

        lowerCamelCase									=sacremoses

        lowerCamelCase									=src_vocab_file
        lowerCamelCase									=tgt_vocab_file
        lowerCamelCase									=merges_file
        lowerCamelCase									=do_lower_case

        # cache of sm.MosesPunctNormalizer instance
        lowerCamelCase									={}
        # cache of sm.MosesTokenizer instance
        lowerCamelCase									={}
        lowerCamelCase									={}

        if langs and len(_UpperCAmelCase  ) == 2:
           lowerCamelCase				,       lowerCamelCase									=langs
        else:
           raise ValueError(
               F'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
               """Usually that means that tokenizer can't find a mapping for the given model path """
               """in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."""  )

        with open(_UpperCAmelCase							, encoding="""utf-8"""  ) as src_vocab_handle:
           lowerCamelCase									=json.load(_UpperCAmelCase  )
        with open(_UpperCAmelCase							, encoding="""utf-8"""  ) as tgt_vocab_handle:
           lowerCamelCase									=json.load(_UpperCAmelCase  )
           lowerCamelCase									={v: k for k, v in tgt_vocab.items()}
        with open(_UpperCAmelCase							, encoding="""utf-8"""  ) as merges_handle:
           lowerCamelCase									=merges_handle.read().split("""\n"""  )[:-1]
        lowerCamelCase									=[tuple(merge.split()[:2]  ) for merge in merges]
        lowerCamelCase									=dict(zip(_UpperCAmelCase							, range(len(_UpperCAmelCase  )  )  )  )
        lowerCamelCase									={}





     def 						lowerCamelCase							(	self  )		->				Dict[str, int]:
        return self.get_src_vocab()





     @property
     def 						lowerCamelCase							(	self  )		->				int:
        return self.src_vocab_size





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase  )		->				int:
        if lang not in self.cache_moses_punct_normalizer:
           lowerCamelCase									=self.sm.MosesPunctNormalizer(lang=_UpperCAmelCase  )
           lowerCamelCase									=punct_normalizer
        return self.cache_moses_punct_normalizer[lang].normalize(_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase  )		->				str:
        if lang not in self.cache_moses_tokenizer:
           lowerCamelCase									=self.sm.MosesTokenizer(lang=_UpperCAmelCase  )
           lowerCamelCase									=moses_tokenizer
        return self.cache_moses_tokenizer[lang].tokenize(
            _UpperCAmelCase							, aggressive_dash_splits=_UpperCAmelCase							, return_str=_UpperCAmelCase							, escape=_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase  )		->				Dict:
        if lang not in self.cache_moses_detokenizer:
           lowerCamelCase									=self.sm.MosesDetokenizer(lang=_UpperCAmelCase  )
           lowerCamelCase									=moses_detokenizer
        return self.cache_moses_detokenizer[lang].detokenize(_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase  )		->				List[str]:
        lowerCamelCase									=replace_unicode_punct(_UpperCAmelCase  )
        lowerCamelCase									=self.moses_punct_norm(_UpperCAmelCase							, _UpperCAmelCase  )
        lowerCamelCase									=remove_non_printing_char(_UpperCAmelCase  )
        return text





     @property
     def 						lowerCamelCase							(	self  )		->				Dict:
        return len(self.encoder  )





     @property
     def 						lowerCamelCase							(	self  )		->				Union[str, Any]:
        return len(self.decoder  )





     def 						lowerCamelCase							(	self  )		->				Optional[Any]:
        return dict(self.encoder							, **self.added_tokens_encoder  )





     def 						lowerCamelCase							(	self  )		->				Union[str, Any]:
        return dict(self.decoder							, **self.added_tokens_decoder  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Union[str, Any]:
        lowerCamelCase									=tuple(token[:-1]  ) + (token[-1] + """</w>""",)
        if token in self.cache:
           return self.cache[token]
        lowerCamelCase									=get_pairs(_UpperCAmelCase  )

        if not pairs:
           return token + "</w>"

        while True:
           lowerCamelCase									=min(_UpperCAmelCase							, key=lambda _UpperCAmelCase  : self.bpe_ranks.get(_UpperCAmelCase							, float("""inf"""  )  )  )
           if bigram not in self.bpe_ranks:
              break
           lowerCamelCase				,       lowerCamelCase									=bigram
           lowerCamelCase									=[]
           lowerCamelCase									=0
           while i < len(_UpperCAmelCase  ):
              try:
                 lowerCamelCase									=word.index(_UpperCAmelCase							, _UpperCAmelCase  )
              except ValueError:
                 new_word.extend(word[i:]  )
                 break
              else:
                 new_word.extend(word[i:j]  )
                 lowerCamelCase									=j

              if word[i] == first and i < len(_UpperCAmelCase  ) - 1 and word[i + 1] == second:
                 new_word.append(first + second  )
                 i += 2
              else:
                 new_word.append(word[i]  )
                 i += 1
           lowerCamelCase									=tuple(_UpperCAmelCase  )
           lowerCamelCase									=new_word
           if len(_UpperCAmelCase  ) == 1:
              break
           else:
              lowerCamelCase									=get_pairs(_UpperCAmelCase  )
        lowerCamelCase									=""" """.join(_UpperCAmelCase  )
        if word == "\n  </w>":
           lowerCamelCase									="""\n</w>"""
        lowerCamelCase									=word
        return word





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase="en"							, _UpperCAmelCase=False  )		->				str:
        lowerCamelCase									=self.src_lang

        if self.do_lower_case:
           lowerCamelCase									=text.lower()

        if bypass_tokenizer:
           lowerCamelCase									=text.split()
        else:
           lowerCamelCase									=self.moses_pipeline(_UpperCAmelCase							, lang=_UpperCAmelCase  )
           lowerCamelCase									=self.moses_tokenize(_UpperCAmelCase							, lang=_UpperCAmelCase  )

        lowerCamelCase									=[]
        for token in text:
           if token:
              split_tokens.extend(list(self.bpe(_UpperCAmelCase  ).split(""" """  )  )  )

        return split_tokens





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				List[str]:
        return self.encoder.get(_UpperCAmelCase							, self.encoder.get(self.unk_token  )  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Tuple:
        return self.decoder.get(_UpperCAmelCase							, self.unk_token  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				List[Any]:
        lowerCamelCase									=[t.replace(""" """							, """"""  ).replace("""</w>"""							, """ """  ) for t in tokens]
        lowerCamelCase									="""""".join(_UpperCAmelCase  ).split()
        # detokenize
        lowerCamelCase									=self.moses_detokenize(_UpperCAmelCase							, self.tgt_lang  )
        return text





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None  )		->				List[int]:
        lowerCamelCase									=[self.sep_token_id]

        # no bos used in fairseq
        if token_ids_a is None:
           return token_ids_a + sep
        return token_ids_a + sep + token_ids_a + sep





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None							, _UpperCAmelCase = False  )		->				List[int]:

        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=_UpperCAmelCase							, token_ids_a=_UpperCAmelCase							, already_has_special_tokens=_UpperCAmelCase  )
        # no bos used in fairseq
        if token_ids_a is not None:
           return ([0] * len(_UpperCAmelCase  )) + [1] + ([0] * len(_UpperCAmelCase  )) + [1]
        return ([0] * len(_UpperCAmelCase  )) + [1]





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None  )		->				List[int]:
        lowerCamelCase									=[self.sep_token_id]

        # no bos used in fairseq
        if token_ids_a is None:
           return len(token_ids_a + sep  ) * [0]
        return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None  )		->				Tuple[str]:
        if not os.path.isdir(_UpperCAmelCase  ):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
           return

        lowerCamelCase									=os.path.join(
            _UpperCAmelCase							, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""src_vocab_file"""]  )
        lowerCamelCase									=os.path.join(
            _UpperCAmelCase							, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""tgt_vocab_file"""]  )
        lowerCamelCase									=os.path.join(
            _UpperCAmelCase							, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

        with open(_UpperCAmelCase							, """w"""							, encoding="""utf-8"""  ) as f:
           f.write(json.dumps(self.encoder							, indent=2							, sort_keys=_UpperCAmelCase							, ensure_ascii=_UpperCAmelCase  ) + """\n"""  )

        with open(_UpperCAmelCase							, """w"""							, encoding="""utf-8"""  ) as f:
           lowerCamelCase									={v: k for k, v in self.decoder.items()}
           f.write(json.dumps(_UpperCAmelCase							, indent=2							, sort_keys=_UpperCAmelCase							, ensure_ascii=_UpperCAmelCase  ) + """\n"""  )

        lowerCamelCase									=0
        with open(_UpperCAmelCase							, """w"""							, encoding="""utf-8"""  ) as writer:
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							, key=lambda _UpperCAmelCase  : kv[1]  ):
              if index != token_index:
                 logger.warning(
                     F'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
                     """ Please check that the tokenizer is not corrupted!"""  )
                 lowerCamelCase									=token_index
              writer.write(""" """.join(_UpperCAmelCase  ) + """\n"""  )
              index += 1

        return src_vocab_file, tgt_vocab_file, merges_file





     def __getstate__(	self  )		->				List[Any]:
        lowerCamelCase									=self.__dict__.copy()
        lowerCamelCase									=None
        return state





     def __setstate__(	self							, _UpperCAmelCase  )		->				Union[str, Any]:
        lowerCamelCase									=d

        try:
           import sacremoses
        except ImportError:
           raise ImportError(
               """You need to install sacremoses to use XLMTokenizer. """
               """See https://pypi.org/project/sacremoses/ for installation."""  )

        lowerCamelCase									=sacremoses