





"""simple docstring"""






import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases



class 						lowerCAmelCase       (   snake_case				):






       def 				__A			( self     ,	a__     ,	a__=False  ):
              _UpperCAmelCase     ,				_UpperCAmelCase     ,				_UpperCAmelCase			     =  (
                  filename_skip_take['filename'],
                  filename_skip_take['skip'] if 'skip' in filename_skip_take else None,
                  filename_skip_take['take'] if 'take' in filename_skip_take else None,
              )
              open(os.path.join(a__  )     ,	'wb'  ).close()
              _UpperCAmelCase			     =  pa.Table.from_pydict({'filename': [Path(a__  ).name] * 1_00}  )
              if take == -1:
                     _UpperCAmelCase			     =  len(a__  ) - skip
              if skip is not None and take is not None:
                     _UpperCAmelCase			     =  pa_table.slice(a__     ,	a__  )
              return pa_table



class 						lowerCAmelCase       (   snake_case				):






       def 				__A			( self  ):
              _UpperCAmelCase			     =  'my_name'
              _UpperCAmelCase			     =  SplitInfo(name='train'     ,	num_examples=1_00  )
              _UpperCAmelCase			     =  SplitInfo(name='test'     ,	num_examples=1_00  )
              _UpperCAmelCase			     =  [train_info, test_info]
              _UpperCAmelCase			     =  SplitDict()
              split_dict.add(a__  )
              split_dict.add(a__  )
              _UpperCAmelCase			     =  DatasetInfo(splits=a__  )

              with tempfile.TemporaryDirectory() as tmp_dir:
                     _UpperCAmelCase			     =  ReaderTest(a__     ,	a__  )

                     _UpperCAmelCase			     =  'test[:33%]'
                     _UpperCAmelCase			     =  Dataset(**reader.read(a__     ,	a__     ,	a__  )  )
                     self.assertEqual(dset['filename'][0]     ,	f"""{name}-test"""  )
                     self.assertEqual(dset.num_rows     ,	33  )
                     self.assertEqual(dset.num_columns     ,	1  )

                     _UpperCAmelCase			     =  ['train', 'test[:33%]']
                     _UpperCAmelCase			     =  [Split.TRAIN, ReadInstruction.from_spec('test[:33%]'  )]
                     for instructions in [instructionsa, instructionsa]:
                            _UpperCAmelCase			     =  [reader.read(a__     ,	a__     ,	a__  ) for instr in instructions]
                            _UpperCAmelCase     ,				_UpperCAmelCase			     =  (Dataset(**a__  ) for dataset_kwargs in datasets_kwargs)
                            self.assertEqual(train_dset['filename'][0]     ,	f"""{name}-train"""  )
                            self.assertEqual(train_dset.num_rows     ,	1_00  )
                            self.assertEqual(train_dset.num_columns     ,	1  )
                            self.assertIsInstance(train_dset.split     ,	a__  )
                            self.assertEqual(str(train_dset.split  )     ,	'train'  )
                            self.assertEqual(test_dset['filename'][0]     ,	f"""{name}-test"""  )
                            self.assertEqual(test_dset.num_rows     ,	33  )
                            self.assertEqual(test_dset.num_columns     ,	1  )
                            self.assertIsInstance(test_dset.split     ,	a__  )
                            self.assertEqual(str(test_dset.split  )     ,	'test[:33%]'  )
                            del train_dset, test_dset






       def 				__A			( self  ):
              _UpperCAmelCase			     =  'my_name'
              _UpperCAmelCase			     =  SplitInfo(name='train'     ,	num_examples=10_00     ,	shard_lengths=[1_00] * 10  )
              _UpperCAmelCase			     =  [train_info]
              _UpperCAmelCase			     =  SplitDict()
              split_dict.add(a__  )
              _UpperCAmelCase			     =  DatasetInfo(splits=a__  )

              with tempfile.TemporaryDirectory() as tmp_dir:
                     _UpperCAmelCase			     =  ReaderTest(a__     ,	a__  )

                     _UpperCAmelCase			     =  'train[:33%]'
                     _UpperCAmelCase			     =  Dataset(**reader.read(a__     ,	a__     ,	a__  )  )
                     self.assertEqual(dset['filename'][0]     ,	f"""{name}-train-00000-of-00010"""  )
                     self.assertEqual(dset['filename'][-1]     ,	f"""{name}-train-00003-of-00010"""  )
                     self.assertEqual(dset.num_rows     ,	3_30  )
                     self.assertEqual(dset.num_columns     ,	1  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  SplitInfo(name='train'     ,	num_examples=1_00  )
              _UpperCAmelCase			     =  SplitInfo(name='test'     ,	num_examples=1_00  )
              _UpperCAmelCase			     =  SplitDict()
              split_dict.add(a__  )
              split_dict.add(a__  )
              _UpperCAmelCase			     =  DatasetInfo(splits=a__  )

              with tempfile.TemporaryDirectory() as tmp_dir:
                     _UpperCAmelCase			     =  ReaderTest(a__     ,	a__  )

                     _UpperCAmelCase			     =  [
                         {'filename': os.path.join(a__     ,	'train'  )},
                         {'filename': os.path.join(a__     ,	'test'  ), 'skip': 10, 'take': 10},
                     ]
                     _UpperCAmelCase			     =  Dataset(**reader.read_files(a__     ,	original_instructions='train+test[10:20]'  )  )
                     self.assertEqual(dset.num_rows     ,	1_10  )
                     self.assertEqual(dset.num_columns     ,	1  )
                     del dset


@pytest.mark.parametrize('in_memory',[False, True]				)
def    __lowerCamelCase     (							SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)		->							List[Any]:






       """simple docstring"""




       _UpperCAmelCase			     =  arrow_file
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              _UpperCAmelCase			     =  ArrowReader.read_table(SCREAMING_SNAKE_CASE,in_memory=SCREAMING_SNAKE_CASE				)
       assert table.shape == dataset.data.shape
       assert set(table.column_names				) == set(dataset.data.column_names				)
       assert dict(table.to_pydict()				) == dict(dataset.data.to_pydict()				)  # to_pydict returns OrderedDict

@pytest.mark.parametrize('in_memory',[False, True]				)
def    __lowerCamelCase     (							SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)		->							Union[str, Any]:






       """simple docstring"""




       _UpperCAmelCase			     =  arrow_file
       _UpperCAmelCase			     =  ArrowReader('',SCREAMING_SNAKE_CASE				)
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              _UpperCAmelCase			     =  reader.read_files([{'filename': filename}],in_memory=SCREAMING_SNAKE_CASE				)
       assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
       _UpperCAmelCase			     =  dataset_kwargs['arrow_table']
       assert table.shape == dataset.data.shape
       assert set(table.column_names				) == set(dataset.data.column_names				)
       assert dict(table.to_pydict()				) == dict(dataset.data.to_pydict()				)  # to_pydict returns OrderedDict

def    __lowerCamelCase     (							)		->							Optional[int]:






       """simple docstring"""




       assert ReadInstruction('train',to=10,unit='abs'				).to_spec() == "train[:10]"
       assert ReadInstruction('train',from_=-80,to=10,unit='%'				).to_spec() == "train[-80%:10%]"

       _UpperCAmelCase			     =  'train+test'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == spec_train_test

       _UpperCAmelCase			     =  'train[2:10]'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == spec_train_abs

       _UpperCAmelCase			     =  'train[15%:-20%]'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == spec_train_pct

       _UpperCAmelCase			     =  'train[:10%](closest)'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == "train[:10%]"

       _UpperCAmelCase			     =  'train[:10%](pct1_dropremainder)'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == spec_train_pct_rounding

       _UpperCAmelCase			     =  'train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)'
       assert ReadInstruction.from_spec(SCREAMING_SNAKE_CASE				).to_spec() == spec_train_test_pct_rounding

def    __lowerCamelCase     (							)		->							Dict:






       """simple docstring"""




       _UpperCAmelCase			     =  'dummy'
       _UpperCAmelCase			     =  [SplitInfo(name='train',num_examples=100				)]
       _UpperCAmelCase			     =  'train[:33%]'
       _UpperCAmelCase			     =  'arrow'
       _UpperCAmelCase			     =  'prefix'

       _UpperCAmelCase			     =  make_file_instructions(SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)
       assert isinstance(SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)
       assert file_instructions.num_examples == 33
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(SCREAMING_SNAKE_CASE,F"""{name}-train.arrow"""				), "skip": 0, "take": 33}
       ]

       _UpperCAmelCase			     =  [SplitInfo(name='train',num_examples=100,shard_lengths=[10] * 10				)]
       _UpperCAmelCase			     =  make_file_instructions(SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)
       assert isinstance(SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)
       assert file_instructions.num_examples == 33
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(SCREAMING_SNAKE_CASE,F"""{name}-train-00000-of-00010.arrow"""				), "skip": 0, "take": -1},
           {"filename": os.path.join(SCREAMING_SNAKE_CASE,F"""{name}-train-00001-of-00010.arrow"""				), "skip": 0, "take": -1},
           {"filename": os.path.join(SCREAMING_SNAKE_CASE,F"""{name}-train-00002-of-00010.arrow"""				), "skip": 0, "take": -1},
           {"filename": os.path.join(SCREAMING_SNAKE_CASE,F"""{name}-train-00003-of-00010.arrow"""				), "skip": 0, "take": 3},
       ]


@pytest.mark.parametrize('name, expected_exception',[(None, TypeError), ('', ValueError)]				)
def    __lowerCamelCase     (							SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)		->							Any:






       """simple docstring"""




       _UpperCAmelCase			     =  [SplitInfo(name='train',num_examples=100				)]
       _UpperCAmelCase			     =  'train'
       _UpperCAmelCase			     =  'arrow'
       _UpperCAmelCase			     =  'prefix_path'
       with pytest.raises(SCREAMING_SNAKE_CASE				):
              _UpperCAmelCase			     =  make_file_instructions(SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE				)













"""simple docstring"""






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMModel,
        TFXLMWithLMHeadModel,
        XLMConfig,
    )



class 						lowerCAmelCase       :






       def __init__( self     ,	a__     ,	):
              _UpperCAmelCase			     =  parent
              _UpperCAmelCase			     =  13
              _UpperCAmelCase			     =  7
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  False
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  99
              _UpperCAmelCase			     =  0
              _UpperCAmelCase			     =  32
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  4
              _UpperCAmelCase			     =  0.1
              _UpperCAmelCase			     =  0.1
              _UpperCAmelCase			     =  5_12
              _UpperCAmelCase			     =  16
              _UpperCAmelCase			     =  2
              _UpperCAmelCase			     =  0.02
              _UpperCAmelCase			     =  3
              _UpperCAmelCase			     =  4
              _UpperCAmelCase			     =  'last'
              _UpperCAmelCase			     =  True
              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  0






       def 				__A			( self  ):
              _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size  )
              _UpperCAmelCase			     =  random_attention_mask([self.batch_size, self.seq_length]     ,	dtype=tf.floataa  )

              _UpperCAmelCase			     =  None
              if self.use_input_lengths:
                     _UpperCAmelCase			     =  (
                         ids_tensor([self.batch_size]     ,	vocab_size=2  ) + self.seq_length - 2
                     )  # small variation of seq_length

              _UpperCAmelCase			     =  None
              if self.use_token_type_ids:
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.n_langs  )

              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  None
              _UpperCAmelCase			     =  None
              if self.use_labels:
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	self.type_sequence_label_size  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	2     ,	dtype=tf.floataa  )
                     _UpperCAmelCase			     =  ids_tensor([self.batch_size]     ,	self.num_choices  )

              _UpperCAmelCase			     =  XLMConfig(
                  vocab_size=self.vocab_size     ,	n_special=self.n_special     ,	emb_dim=self.hidden_size     ,	n_layers=self.num_hidden_layers     ,	n_heads=self.num_attention_heads     ,	dropout=self.hidden_dropout_prob     ,	attention_dropout=self.attention_probs_dropout_prob     ,	gelu_activation=self.gelu_activation     ,	sinusoidal_embeddings=self.sinusoidal_embeddings     ,	asm=self.asm     ,	causal=self.causal     ,	n_langs=self.n_langs     ,	max_position_embeddings=self.max_position_embeddings     ,	initializer_range=self.initializer_range     ,	summary_type=self.summary_type     ,	use_proj=self.use_proj     ,	bos_token_id=self.bos_token_id     ,	)

              return (
                  config,
                  input_ids,
                  token_type_ids,
                  input_lengths,
                  sequence_labels,
                  token_labels,
                  is_impossible_labels,
                  choice_labels,
                  input_mask,
              )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMModel(config=a__  )
              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )

              _UpperCAmelCase			     =  [input_ids, input_mask]
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMWithLMHeadModel(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )

              _UpperCAmelCase			     =  outputs

              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMForQuestionAnsweringSimple(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths}

              _UpperCAmelCase			     =  model(a__  )

              self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length)  )
              self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  TFXLMForSequenceClassification(a__  )

              _UpperCAmelCase			     =  {'input_ids': input_ids, 'lengths': input_lengths}

              _UpperCAmelCase			     =  model(a__  )

              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.type_sequence_label_size)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  self.num_labels
              _UpperCAmelCase			     =  TFXLMForTokenClassification(config=a__  )
              _UpperCAmelCase			     =  {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels)  )






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__     ,	):
              _UpperCAmelCase			     =  self.num_choices
              _UpperCAmelCase			     =  TFXLMForMultipleChoice(config=a__  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  tf.tile(tf.expand_dims(a__     ,	1  )     ,	(1, self.num_choices, 1)  )
              _UpperCAmelCase			     =  {
                  'input_ids': multiple_choice_inputs_ids,
                  'attention_mask': multiple_choice_input_mask,
                  'token_type_ids': multiple_choice_token_type_ids,
              }
              _UpperCAmelCase			     =  model(a__  )
              self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices)  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.prepare_config_and_inputs()
              (
                  (
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				(
                  _UpperCAmelCase
              )     ,				
              )			     =  config_and_inputs
              _UpperCAmelCase			     =  {
                  'input_ids': input_ids,
                  'token_type_ids': token_type_ids,
                  'langs': token_type_ids,
                  'lengths': input_lengths,
              }
              return config, inputs_dict



@require_tf
class 						lowerCAmelCase       (   snake_case      ,						snake_case      ,						unittest.TestCase				):
       lowerCAmelCase__    			=	(
           (
               TFXLMModel,
               TFXLMWithLMHeadModel,
               TFXLMForSequenceClassification,
               TFXLMForQuestionAnsweringSimple,
               TFXLMForTokenClassification,
               TFXLMForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       lowerCAmelCase__    			=	(
           (TFXLMWithLMHeadModel,) if is_tf_available() else ()
       )  # TODO (PVP): Check other models whether language generation is also applicable
       lowerCAmelCase__    			=	(
           {
               """feature-extraction""": TFXLMModel,
               """fill-mask""": TFXLMWithLMHeadModel,
               """question-answering""": TFXLMForQuestionAnsweringSimple,
               """text-classification""": TFXLMForSequenceClassification,
               """text-generation""": TFXLMWithLMHeadModel,
               """token-classification""": TFXLMForTokenClassification,
               """zero-shot""": TFXLMForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )
       lowerCAmelCase__    			=	False
       lowerCAmelCase__    			=	False






       def 				__A			( self     ,	a__     ,	a__     ,	a__     ,	a__     ,	a__  ):
              if (
                  pipeline_test_casse_name == "QAPipelineTests"
                  and tokenizer_name is not None
                  and not tokenizer_name.endswith('Fast'  )
              ):
                     # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
                     # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
                     # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
                     return True

              return False






       def 				__A			( self  ):
              _UpperCAmelCase			     =  TFXLMModelTester(self  )
              _UpperCAmelCase			     =  ConfigTester(self     ,	config_class=a__     ,	emb_dim=37  )






       def 				__A			( self  ):
              self.config_tester.run_common_tests()






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_model(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_lm_head(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_qa(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_sequence_classif(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_for_token_classification(*a__  )






       def 				__A			( self  ):
              _UpperCAmelCase			     =  self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_xlm_for_multiple_choice(*a__  )






       @slow
       def 				__A			( self  ):
              for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     _UpperCAmelCase			     =  TFXLMModel.from_pretrained(a__  )
                     self.assertIsNotNone(a__  )



@require_tf
class 						lowerCAmelCase       (   unittest.TestCase				):






       @slow
       def 				__A			( self  ):
              _UpperCAmelCase			     =  TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048'  )
              _UpperCAmelCase			     =  tf.convert_to_tensor([[14, 4_47]]     ,	dtype=tf.intaa  )  # the president
              _UpperCAmelCase			     =  [
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
                  14,
                  4_47,
              ]  # the president the president the president the president the president the president the president the president the president the president
              # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
              _UpperCAmelCase			     =  model.generate(a__     ,	do_sample=a__  )
              self.assertListEqual(output_ids[0].numpy().tolist()     ,	a__  )






