

# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


lowerCamelCase     =      logging.get_logger(__name__)


if is_torch_available():
       import torch
       from torch import nn
       from torch.utils.data import Dataset

       from transformers import Trainer







       class 		__magic_name__    (							lowerCamelCase__							):

          '''simple docstring'''




          def __init__(       self,					lowercase_ = 101 )							->					List[Any]:






            """simple docstring"""






            a__						=length



          def __len__(       self )							->					Dict:






            """simple docstring"""






            return self.length



          def __getitem__(       self,					lowercase_ )							->					int:






            """simple docstring"""






            return i







       class 		__magic_name__    :

          '''simple docstring'''




          def __call__(       self,					lowercase_ )							->					List[str]:






            """simple docstring"""






            return {"input_ids": torch.tensor(lowercase_ ), "labels": torch.tensor(lowercase_ )}







       class 		__magic_name__    (							nn.Module							):

          '''simple docstring'''




          def __init__(       self )							->					Union[str, Any]:






            """simple docstring"""






            super().__init__()
            # Add some (unused) params otherwise DDP will complain.
            a__						=nn.Linear(120,					80 )



          def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_=None )							->					List[Any]:






            """simple docstring"""






            if labels is not None:
              return torch.tensor(0.0,					device=input_ids.device ), input_ids
            else:
              return input_ids




def     UpperCAmelCase__       (    ):





  '''simple docstring'''







  a__						=HfArgumentParser((TrainingArguments,)							)
  sys.argv += ["--output_dir", "./examples"]
  a__						=parser.parse_args_into_dataclasses()[0]

  logger.warning(
      F"""Process rank: {training_args.local_rank}, device: {training_args.device}, """
      F"""tpu_num_cores: {training_args.tpu_num_cores}"""			,   )

  # Essentially, what we want to verify in the distributed case is
  # that we get all samples back, in the right order.
  # (this is crucial for prediction for instance)
  for dataset_length in [10_01, 2_56, 15]:
    a__						=DummyDataset(_A							)

    def compute_metrics(_A   :				EvalPrediction							) -> Dict:
      a__						=list(range(len(_A							)							)							)
      a__						=p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
      return {"success": success}

    a__						=Trainer(
        model=DummyModel()			,   args=_A			,   data_collator=DummyDataCollator()			,   eval_dataset=_A			,   compute_metrics=_A			,   )
    a__						=trainer.evaluate()
    logger.info(_A							)
    if metrics["eval_success"] is not True:
      logger.error(_A							)
      exit(1							)

    a__						=trainer.predict(_A							)
    logger.info(p.metrics							)
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics							)
      exit(1							)

    a__						=2

    a__						=trainer.evaluate()
    logger.info(_A							)
    if metrics["eval_success"] is not True:
      logger.error(_A							)
      exit(1							)

    a__						=trainer.predict(_A							)
    logger.info(p.metrics							)
    if p.metrics["test_success"] is not True:
      logger.error(p.metrics							)
      exit(1							)

    a__						=None

  logger.info('''🔥 All distributed tests successful'''							)




def     UpperCAmelCase__       (    _A   :				List[Any]							):





  '''simple docstring'''







  main()


if __name__ == "__main__":
       main()



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_xglm import XGLMTokenizer
else:
       lowerCamelCase     =      None


lowerCamelCase     =      logging.get_logger(__name__)

lowerCamelCase     =      {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase     =      {
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    },
    '''tokenizer_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json''',
    },
}

lowerCamelCase     =      {
    '''facebook/xglm-564M''': 2048,
}







class 		__magic_name__    (							lowerCamelCase__							):

   '''simple docstring'''




   lowerCamelCase__				:			int												=     VOCAB_FILES_NAMES
   lowerCamelCase__				:			List[str]												=     PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				:			Dict												=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				:			Any												=     ['input_ids', 'attention_mask']
   lowerCamelCase__				:			Any												=     XGLMTokenizer



   def __init__(       self,					lowercase_=None,					lowercase_=None,					lowercase_="<s>",					lowercase_="</s>",					lowercase_="</s>",					lowercase_="<s>",					lowercase_="<unk>",					lowercase_="<pad>",					**lowercase_,					)							->					Any:






     """simple docstring"""






     a__						=7
     a__						=[F"""<madeupword{i}>""" for i in range(self.num_madeup_words )]

     a__						=kwargs.get('''additional_special_tokens''',					[] )
     kwargs["additional_special_tokens"] += [
         word for word in madeup_words if word not in kwargs["additional_special_tokens"]
     ]

     super().__init__(
         lowercase_,					tokenizer_file=lowercase_,					bos_token=lowercase_,					eos_token=lowercase_,					sep_token=lowercase_,					cls_token=lowercase_,					unk_token=lowercase_,					pad_token=lowercase_,					**lowercase_,					)

     a__						=vocab_file
     a__						=False if not self.vocab_file else True



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					List[int]:






     """simple docstring"""






     if token_ids_a is None:
       return [self.sep_token_id] + token_ids_a
     a__						=[self.sep_token_id]
     return sep + token_ids_a + sep + sep + token_ids_a



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					List[int]:






     """simple docstring"""






     a__						=[self.sep_token_id]

     if token_ids_a is None:
       return len(sep + token_ids_a ) * [0]
     return len(sep + token_ids_a + sep + sep + token_ids_a ) * [0]



   def 				_UpperCAmelCase				(       self,					lowercase_,					lowercase_ = None )							->					Tuple[str]:






     """simple docstring"""






     if not self.can_save_slow_tokenizer:
       raise ValueError(
           '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
           '''tokenizer.''' )

     if not os.path.isdir(lowercase_ ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory.""" )
       return
     a__						=os.path.join(
         lowercase_,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

     if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase_ ):
       copyfile(self.vocab_file,					lowercase_ )

     return (out_vocab_file,)
