






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a      	=  logging.get_logger(__name__)

_a      	=  '''▁'''

_a      	=  {'''vocab_file''': '''sentencepiece.bpe.model'''}

_a      	=  {
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

_a      	=  {
    '''facebook/xglm-564M''': 2048,
}





class __lowerCamelCase (      snake_case__):





	"""simple docstring"""






	UpperCamelCase__								=      VOCAB_FILES_NAMES
	UpperCamelCase__								=      PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase__								=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase__								=      ["input_ids", "attention_mask"]

	def __init__(							self       ,    UpperCAmelCase       ,    UpperCAmelCase="<s>"       ,    UpperCAmelCase="</s>"       ,    UpperCAmelCase="</s>"       ,    UpperCAmelCase="<s>"       ,    UpperCAmelCase="<unk>"       ,    UpperCAmelCase="<pad>"       ,    UpperCAmelCase = None       ,    **UpperCAmelCase       ,    ):
							"""simple docstring"""







							_UpperCAmelCase									=			{} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							_UpperCAmelCase									=			7
							_UpperCAmelCase									=			[F"""<madeupword{i}>""" for i in range(self.num_madeup_words    )]

							_UpperCAmelCase									=			kwargs.get('additional_special_tokens'       ,    []    )
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=UpperCAmelCase       ,    eos_token=UpperCAmelCase       ,    unk_token=UpperCAmelCase       ,    sep_token=UpperCAmelCase       ,    cls_token=UpperCAmelCase       ,    pad_token=UpperCAmelCase       ,    sp_model_kwargs=self.sp_model_kwargs       ,    **UpperCAmelCase       ,    )

							_UpperCAmelCase									=			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.Load(str(UpperCAmelCase    )    )
							_UpperCAmelCase									=			vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_UpperCAmelCase									=			1

							# Mimic fairseq token-to-id alignment for the first 4 token
							_UpperCAmelCase									=			{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							_UpperCAmelCase									=			len(self.sp_model    )
							_UpperCAmelCase									=			{F"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words    )}
							self.fairseq_tokens_to_ids.update(UpperCAmelCase    )

							_UpperCAmelCase									=			{v: k for k, v in self.fairseq_tokens_to_ids.items()}

	def __getstate__(							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			self.__dict__.copy()
							_UpperCAmelCase									=			None
							_UpperCAmelCase									=			self.sp_model.serialized_model_proto()
							return state

	def __setstate__(							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			d

							# for backward compatibility
							if not hasattr(self       ,    'sp_model_kwargs'    ):
													_UpperCAmelCase									=			{}

							_UpperCAmelCase									=			spm.SentencePieceProcessor(**self.sp_model_kwargs    )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase = None    ):
							"""simple docstring"""







							if token_ids_a is None:
													return [self.sep_token_id] + token_ids_a
							_UpperCAmelCase									=			[self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase = None       ,    UpperCAmelCase = False    ):
							"""simple docstring"""







							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=UpperCAmelCase       ,    token_ids_a=UpperCAmelCase       ,    already_has_special_tokens=UpperCAmelCase    )

							if token_ids_a is None:
													return [1] + ([0] * len(UpperCAmelCase    ))
							return [1] + ([0] * len(UpperCAmelCase    )) + [1, 1] + ([0] * len(UpperCAmelCase    ))

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase = None    ):
							"""simple docstring"""







							_UpperCAmelCase									=			[self.sep_token_id]

							if token_ids_a is None:
													return len(sep + token_ids_a    ) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a    ) * [0]

	@property
	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							return len(self.sp_model    ) + self.fairseq_offset + self.num_madeup_words

	def        UpperCamelCase    (							self    ):
							"""simple docstring"""







							_UpperCAmelCase									=			{self.convert_ids_to_tokens(UpperCAmelCase    ): i for i in range(self.vocab_size    )}
							vocab.update(self.added_tokens_encoder    )
							return vocab

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							return self.sp_model.encode(UpperCAmelCase       ,    out_type=UpperCAmelCase    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
							_UpperCAmelCase									=			self.sp_model.PieceToId(UpperCAmelCase    )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							if index in self.fairseq_ids_to_tokens:
													return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset    )

	def        UpperCamelCase    (							self       ,    UpperCAmelCase    ):
							"""simple docstring"""







							_UpperCAmelCase									=			''.join(UpperCAmelCase    ).replace(UpperCAmelCase       ,    ' '    ).strip()
							return out_string

	def        UpperCamelCase    (							self       ,    UpperCAmelCase       ,    UpperCAmelCase = None    ):
							"""simple docstring"""







							if not os.path.isdir(UpperCAmelCase    ):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
													return
							_UpperCAmelCase									=			os.path.join(
							    UpperCAmelCase       ,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

							if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCAmelCase    ) and os.path.isfile(self.vocab_file    ):
													copyfile(self.vocab_file       ,    UpperCAmelCase    )
							elif not os.path.isfile(self.vocab_file    ):
													with open(UpperCAmelCase       ,    'wb'    ) as fi:
																			_UpperCAmelCase									=			self.sp_model.serialized_model_proto()
																			fi.write(UpperCAmelCase    )

							return (out_vocab_file,)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a 						=      {
    "configuration_lxmert": ["LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "LxmertConfig"],
    "tokenization_lxmert": ["LxmertTokenizer"],
}

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      ["LxmertTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      [
			    "LxmertEncoder",
			    "LxmertForPreTraining",
			    "LxmertForQuestionAnswering",
			    "LxmertModel",
			    "LxmertPreTrainedModel",
			    "LxmertVisualFeatureEncoder",
			    "LxmertXLayer",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_a 						=      [
			    "TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFLxmertForPreTraining",
			    "TFLxmertMainLayer",
			    "TFLxmertModel",
			    "TFLxmertPreTrainedModel",
			    "TFLxmertVisualFeatureEncoder",
			]


if TYPE_CHECKING:
			from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
			from .tokenization_lxmert import LxmertTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_lxmert_fast import LxmertTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_lxmert import (
						    LxmertEncoder,
						    LxmertForPreTraining,
						    LxmertForQuestionAnswering,
						    LxmertModel,
						    LxmertPreTrainedModel,
						    LxmertVisualFeatureEncoder,
						    LxmertXLayer,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_lxmert import (
						    TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFLxmertForPreTraining,
						    TFLxmertMainLayer,
						    TFLxmertModel,
						    TFLxmertPreTrainedModel,
						    TFLxmertVisualFeatureEncoder,
						)

else:
			import sys

			_a 						=      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



