


'''simple docstring'''



from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_openai import OpenAIGPTTokenizer


A__	:      Tuple				=						logging.get_logger(__name__)

A__	:      str				=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A__	:      Union[str, Any]				=						{
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
    """tokenizer_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/tokenizer.json"""},
}

A__	:      Dict				=						{
    """openai-gpt""": 512,
}





class      UpperCAmelCase_						(_UpperCAmelCase						):





    """simple docstring"""






    lowerCamelCase						:      List[Any]							  =    VOCAB_FILES_NAMES
    lowerCamelCase						:      Optional[int]							  =    PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase						:      Tuple							  =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase						:      Optional[Any]							  =    ['input_ids', 'attention_mask']
    lowerCamelCase						:      Dict							  =    OpenAIGPTTokenizer







    def __init__(							self    ,  SCREAMING_SNAKE_CASE_=None    ,  SCREAMING_SNAKE_CASE_=None    ,  SCREAMING_SNAKE_CASE_=None    ,  SCREAMING_SNAKE_CASE_="<unk>"    ,  **SCREAMING_SNAKE_CASE_			)       ->  Dict:
         super().__init__(SCREAMING_SNAKE_CASE_    ,  SCREAMING_SNAKE_CASE_    ,  tokenizer_file=SCREAMING_SNAKE_CASE_    ,  unk_token=SCREAMING_SNAKE_CASE_    ,  **SCREAMING_SNAKE_CASE_			)







    @property
    def    lowercase_       (							self			)       ->  List[str]:
         return True







    def    lowercase_       (							self    ,  SCREAMING_SNAKE_CASE_    ,  SCREAMING_SNAKE_CASE_ = None			)       ->  Tuple[str]:
         __lowerCamelCase     :			List[str]              =	self._tokenizer.model.save(SCREAMING_SNAKE_CASE_    ,  name=SCREAMING_SNAKE_CASE_			)
         return tuple(SCREAMING_SNAKE_CASE_			)










'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


A__	:      Optional[Any]				=						{
    """configuration_xlm_prophetnet""": ["""XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XLMProphetNetConfig"""],
}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__	:      List[Any]				=						["""XLMProphetNetTokenizer"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__	:      Optional[Any]				=						[
          """XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """XLMProphetNetDecoder""",
          """XLMProphetNetEncoder""",
          """XLMProphetNetForCausalLM""",
          """XLMProphetNetForConditionalGeneration""",
          """XLMProphetNetModel""",
          """XLMProphetNetPreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_xlm_prophetnet import (
                XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                XLMProphetNetDecoder,
                XLMProphetNetEncoder,
                XLMProphetNetForCausalLM,
                XLMProphetNetForConditionalGeneration,
                XLMProphetNetModel,
                XLMProphetNetPreTrainedModel,
            )

else:
      import sys

      A__	:      List[str]				=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






