


"""simple docstring"""







import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment
def 					a__			(				SCREAMING_SNAKE_CASE    :    Any				,      SCREAMING_SNAKE_CASE    :    Tuple=()				,      SCREAMING_SNAKE_CASE    :    Union[str, Any]=None				,      SCREAMING_SNAKE_CASE    :    Tuple="no"				,      SCREAMING_SNAKE_CASE    :    Dict="29500"				):


  '''simple docstring'''







  lowerCAmelCase     :     List[Any]								=							False
  lowerCAmelCase     :     Optional[int]								=							False
  if any(key.startswith("KAGGLE"				) for key in os.environ.keys()				):
    lowerCAmelCase     :     int								=							True
  elif "IPython" in sys.modules:
    lowerCAmelCase     :     List[str]								=							"google.colab" in str(sys.modules["IPython"].get_ipython()				)

  try:
    lowerCAmelCase     :     str								=							PrecisionType(mixed_precision.lower()				)
  except ValueError:
    raise ValueError(
        f"""Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}."""				)

  if (in_colab or in_kaggle) and (os.environ.get("TPU_NAME"				,      SCREAMING_SNAKE_CASE				) is not None):
    # TPU launch
    import torch_xla.distributed.xla_multiprocessing as xmp

    if len(AcceleratorState._shared_state				) > 0:
      raise ValueError(
          "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
          "your training function. Restart your notebook and make sure no cells initializes an "
          "`Accelerator`."				)
    if num_processes is None:
      lowerCAmelCase     :     Optional[Any]								=							8

    lowerCAmelCase     :     Any								=							PrepareForLaunch(SCREAMING_SNAKE_CASE				,      distributed_type="TPU"				)
    print(f"""Launching a training on {num_processes} TPU cores."""				)
    xmp.spawn(SCREAMING_SNAKE_CASE				,      args=SCREAMING_SNAKE_CASE				,      nprocs=SCREAMING_SNAKE_CASE				,      start_method="fork"				)
  elif in_colab:
    # No need for a distributed launch otherwise as it's either CPU or one GPU.
    if torch.cuda.is_available():
      print("Launching training on one GPU."				)
    else:
      print("Launching training on one CPU."				)
    function(*SCREAMING_SNAKE_CASE				)
  else:
    if num_processes is None:
      raise ValueError(
          "You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."				)

    if num_processes > 1:
      # Multi-GPU launch
      from torch.multiprocessing import start_processes
      from torch.multiprocessing.spawn import ProcessRaisedException

      if len(AcceleratorState._shared_state				) > 0:
        raise ValueError(
            "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
            "inside your training function. Restart your notebook and make sure no cells initializes an "
            "`Accelerator`."				)

      if torch.cuda.is_initialized():
        raise ValueError(
            "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
            "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
            "function."				)

      # torch.distributed will expect a few environment variable to be here. We set the ones common to each
      # process here (the other ones will be set be the launcher).
      with patch_environment(
          world_size=SCREAMING_SNAKE_CASE				,      master_addr="127.0.01"				,      master_port=SCREAMING_SNAKE_CASE				,      mixed_precision=SCREAMING_SNAKE_CASE				):
        lowerCAmelCase     :     Any								=							PrepareForLaunch(SCREAMING_SNAKE_CASE				,      distributed_type="MULTI_GPU"				)
        print(f"""Launching training on {num_processes} GPUs."""				)
        try:
          start_processes(SCREAMING_SNAKE_CASE				,      args=SCREAMING_SNAKE_CASE				,      nprocs=SCREAMING_SNAKE_CASE				,      start_method="fork"				)
        except ProcessRaisedException as e:
          if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
            raise RuntimeError(
                "CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. "
                "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
                "Please review your imports and test them when running the `notebook_launcher()` to identify "
                "which one is problematic."				) from e

    else:
      # No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
      if is_mps_available():
        lowerCAmelCase     :     Tuple								=							"1"
        print("Launching training on MPS."				)
      elif torch.cuda.is_available():
        print("Launching training on one GPU."				)
      else:
        print("Launching training on CPU."				)
      function(*SCREAMING_SNAKE_CASE				)





def 					a__			(				SCREAMING_SNAKE_CASE    :    Dict				,      SCREAMING_SNAKE_CASE    :    Dict=()				,      SCREAMING_SNAKE_CASE    :    Any=2				):


  '''simple docstring'''







  from torch.multiprocessing import start_processes

  with tempfile.NamedTemporaryFile() as tmp_file:
    # torch.distributed will expect a few environment variable to be here. We set the ones common to each
    # process here (the other ones will be set be the launcher).
    with patch_environment(
        world_size=SCREAMING_SNAKE_CASE				,      master_addr="127.0.01"				,      master_port="29500"				,      accelerate_mixed_precision="no"				,      accelerate_debug_rdv_file=tmp_file.name				,      accelerate_use_cpu="yes"				,      ):
      lowerCAmelCase     :     Optional[Any]								=							PrepareForLaunch(SCREAMING_SNAKE_CASE				,      debug=SCREAMING_SNAKE_CASE				)
      start_processes(SCREAMING_SNAKE_CASE				,      args=SCREAMING_SNAKE_CASE				,      nprocs=SCREAMING_SNAKE_CASE				,      start_method="fork"				)













import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase       :       Any        =				get_tests_dir('''fixtures/test_sentencepiece.model''')
lowerCamelCase       :       Any        =				get_tests_dir('''fixtures/test_sentencepiece_bpe.model''')

lowerCamelCase       :       Optional[int]        =				'''pt''' if is_torch_available() else '''tf'''

@require_sentencepiece
@require_tokenizers
class 						lowerCAmelCase   (       __a					,				unittest.TestCase	):

					'''simple docstring'''

					_A  :	Dict     =					CamembertTokenizer
					_A  :	Union[str, Any]     =					CamembertTokenizerFast
					_A  :	Union[str, Any]     =					True
					_A  :	Tuple     =					True



					def 					lowerCAmelCase (     self :		Optional[int]							)	->     Optional[int]:


												"""simple docstring"""

												super().setUp()

												# We have a SentencePiece fixture for testing
												__lowercase					:							Union[str, Any]			=   CamembertTokenizer(__a							)
												tokenizer.save_pretrained(self.tmpdirname							)



					def 					lowerCAmelCase (     self :		Optional[int]							)	->     Optional[Any]:


												"""simple docstring"""

												__lowercase					:							Dict			=   """<pad>"""
												__lowercase					:							Dict			=   1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a							)						,   __a							)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a							)						,   __a							)



					def 					lowerCAmelCase (     self :		Optional[Any]							)	->     Tuple:


												"""simple docstring"""

												__lowercase					:							int			=   list(self.get_tokenizer().get_vocab().keys()							)

												self.assertEqual(vocab_keys[0]						,   """<s>NOTUSED"""							)
												self.assertEqual(vocab_keys[1]						,   """<pad>"""							)
												self.assertEqual(vocab_keys[-1]						,   """<mask>"""							)
												self.assertEqual(len(__a							)						,   1004							)



					def 					lowerCAmelCase (     self :		str							)	->     List[Any]:


												"""simple docstring"""

												self.assertEqual(self.get_tokenizer().vocab_size						,   1005							)



					def 					lowerCAmelCase (     self :		Any							)	->     Optional[Any]:


												"""simple docstring"""

												__lowercase					:							Tuple			=   CamembertTokenizer(__a							)
												tokenizer.save_pretrained(self.tmpdirname							)
												__lowercase					:							Tuple			=   CamembertTokenizerFast.from_pretrained(self.tmpdirname							)

												__lowercase					:							List[str]			=   """I was born in 92000, and this is falsé."""

												__lowercase					:							Optional[Any]			=   tokenizer.encode(__a							)
												__lowercase					:							List[Any]			=   rust_tokenizer.encode(__a							)
												self.assertListEqual(__a						,   __a							)

												__lowercase					:							Tuple			=   tokenizer.encode(__a						,   add_special_tokens=__a							)
												__lowercase					:							Union[str, Any]			=   rust_tokenizer.encode(__a						,   add_special_tokens=__a							)
												self.assertListEqual(__a						,   __a							)

												# <unk> tokens are not the same for `rust` than for `slow`.
												# Because spm gives back raw token instead of `unk` in EncodeAsPieces
												# tokens = tokenizer.tokenize(sequence)
												__lowercase					:							Dict			=   tokenizer.convert_ids_to_tokens(__a							)
												__lowercase					:							Any			=   rust_tokenizer.tokenize(__a							)
												self.assertListEqual(__a						,   __a							)



					def 					lowerCAmelCase (     self :		Any							)	->     Dict:


												"""simple docstring"""

												if not self.test_rust_tokenizer:
																			return

												__lowercase					:							List[str]			=   self.get_tokenizer()
												__lowercase					:							Any			=   self.get_rust_tokenizer()

												__lowercase					:							Any			=   """I was born in 92000, and this is falsé."""

												__lowercase					:							Tuple			=   tokenizer.tokenize(__a							)
												__lowercase					:							Optional[Any]			=   rust_tokenizer.tokenize(__a							)
												self.assertListEqual(__a						,   __a							)

												__lowercase					:							Dict			=   tokenizer.encode(__a						,   add_special_tokens=__a							)
												__lowercase					:							Dict			=   rust_tokenizer.encode(__a						,   add_special_tokens=__a							)
												self.assertListEqual(__a						,   __a							)

												__lowercase					:							Any			=   self.get_rust_tokenizer()
												__lowercase					:							str			=   tokenizer.encode(__a							)
												__lowercase					:							Union[str, Any]			=   rust_tokenizer.encode(__a							)
												self.assertListEqual(__a						,   __a							)



					@slow
					def 					lowerCAmelCase (     self :		Optional[int]							)	->     List[Any]:


												"""simple docstring"""

												__lowercase					:							str			=   {"""input_ids""": [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 27575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 22804, 18818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 10326, 24, 2267, 20, 416, 5072, 15612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
												# fmt: on

												# camembert is a french model. So we also use french texts.
												__lowercase					:							List[str]			=   [
												    """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
												    """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
												    """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
												    """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
												    """telles que la traduction et la synthèse de texte.""",
												]

												self.tokenizer_integration_test_util(
												    expected_encoding=__a						,   model_name="""camembert-base"""						,   revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""						,   sequences=__a						,   )