

import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_		:			int										=   datasets.utils.logging.get_logger(__name__)
@dataclass
class 					A_   (      datasets.BuilderConfig						):


    '''simple docstring'''






    a__     							=       None
    a__     							=       "utf-8"
    a__     							=       None
    a__     							=       10 << 20  # 10MB
    a__     							=       False
    a__     							=       "line"






class 					A_   (      datasets.ArrowBasedBuilder						):


    '''simple docstring'''






    a__     							=       TextConfig


    def      lowerCAmelCase_						(self  )     ->      List[Any]:
        return datasets.DatasetInfo(features=self.config.features  )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Tuple:
        if not self.config.data_files:
            raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
        __UpperCAmelCase									=			dl_manager.download_and_extract(self.config.data_files  )
        if isinstance(lowercase__     ,      (str, list, tuple)  ):
            __UpperCAmelCase									=			data_files
            if isinstance(lowercase__     ,      lowercase__  ):
                __UpperCAmelCase									=			[files]
            __UpperCAmelCase									=			[dl_manager.iter_files(lowercase__  ) for file in files]
            return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,      gen_kwargs={'''files''': files}  )]
        __UpperCAmelCase									=			[]
        for split_name, files in data_files.items():
            if isinstance(lowercase__     ,      lowercase__  ):
                __UpperCAmelCase									=			[files]
            __UpperCAmelCase									=			[dl_manager.iter_files(lowercase__  ) for file in files]
            splits.append(datasets.SplitGenerator(name=lowercase__     ,      gen_kwargs={'''files''': files}  )  )
        return splits


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      pa.Table:
        if self.config.features is not None:
            __UpperCAmelCase									=			self.config.features.arrow_schema
            if all(not require_storage_cast(lowercase__  ) for feature in self.config.features.values()  ):
                # cheaper cast
                __UpperCAmelCase									=			pa_table.cast(lowercase__  )
            else:
                # more expensive cast; allows str <-> int/float or str to Audio for example
                __UpperCAmelCase									=			table_cast(lowercase__     ,      lowercase__  )
            return pa_table
        else:
            return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )






    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Optional[Any]:
        __UpperCAmelCase									=			list(self.config.features  ) if self.config.features is not None else ['''text''']
        for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase__  )  ):
            # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
            with open(lowercase__     ,      encoding=self.config.encoding     ,      errors=self.config.errors  ) as f:
                if self.config.sample_by == "line":
                    __UpperCAmelCase									=			0
                    while True:
                        __UpperCAmelCase									=			f.read(self.config.chunksize  )
                        if not batch:
                            break
                        batch += f.readline()  # finish current line
                        # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                        __UpperCAmelCase									=			StringIO(lowercase__  ).readlines()
                        if not self.config.keep_linebreaks:
                            __UpperCAmelCase									=			[line.rstrip('''\n'''  ) for line in batch]
                        __UpperCAmelCase									=			pa.Table.from_arrays([pa.array(lowercase__  )]     ,      names=lowercase__  )
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield (file_idx, batch_idx), self._cast_table(lowercase__  )
                        batch_idx += 1
                elif self.config.sample_by == "paragraph":
                    __UpperCAmelCase									=			0
                    __UpperCAmelCase									=			''''''
                    while True:
                        __UpperCAmelCase									=			f.read(self.config.chunksize  )
                        if not new_batch:
                            break
                        batch += new_batch
                        batch += f.readline()  # finish current line
                        __UpperCAmelCase									=			batch.split('''\n\n'''  )
                        __UpperCAmelCase									=			pa.Table.from_arrays(
                            [pa.array([example for example in batch[:-1] if example]  )]     ,      names=lowercase__  )
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                        yield (file_idx, batch_idx), self._cast_table(lowercase__  )
                        batch_idx += 1
                        __UpperCAmelCase									=			batch[-1]
                    if batch:
                        __UpperCAmelCase									=			pa.Table.from_arrays([pa.array([batch]  )]     ,      names=lowercase__  )
                        yield (file_idx, batch_idx), self._cast_table(lowercase__  )
                elif self.config.sample_by == "document":
                    __UpperCAmelCase									=			f.read()
                    __UpperCAmelCase									=			pa.Table.from_arrays([pa.array([text]  )]     ,      names=lowercase__  )
                    yield file_idx, self._cast_table(lowercase__  )







from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import (
           TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
           TFXLMForMultipleChoice,
           TFXLMForQuestionAnsweringSimple,
           TFXLMForSequenceClassification,
           TFXLMForTokenClassification,
           TFXLMModel,
           TFXLMWithLMHeadModel,
           XLMConfig,
       )
class 					A_   :


    '''simple docstring'''






    def __init__(self     ,      lowercase__     ,      )     ->      Tuple:
        __UpperCAmelCase									=			parent
        __UpperCAmelCase									=			13
        __UpperCAmelCase									=			7
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			False
        __UpperCAmelCase									=			False
        __UpperCAmelCase									=			False
        __UpperCAmelCase									=			2
        __UpperCAmelCase									=			99
        __UpperCAmelCase									=			0
        __UpperCAmelCase									=			32
        __UpperCAmelCase									=			2
        __UpperCAmelCase									=			4
        __UpperCAmelCase									=			0.1
        __UpperCAmelCase									=			0.1
        __UpperCAmelCase									=			512
        __UpperCAmelCase									=			16
        __UpperCAmelCase									=			2
        __UpperCAmelCase									=			0.02
        __UpperCAmelCase									=			3
        __UpperCAmelCase									=			4
        __UpperCAmelCase									=			'''last'''
        __UpperCAmelCase									=			True
        __UpperCAmelCase									=			None
        __UpperCAmelCase									=			0


    def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
        __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size  )
        __UpperCAmelCase									=			random_attention_mask([self.batch_size, self.seq_length]     ,      dtype=tf.floataa  )

        __UpperCAmelCase									=			None
        if self.use_input_lengths:
            __UpperCAmelCase									=			(
                ids_tensor([self.batch_size]     ,      vocab_size=2  ) + self.seq_length - 2
            )  # small variation of seq_length

        __UpperCAmelCase									=			None
        if self.use_token_type_ids:
            __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.n_langs  )

        __UpperCAmelCase									=			None
        __UpperCAmelCase									=			None
        __UpperCAmelCase									=			None
        if self.use_labels:
            __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.type_sequence_label_size  )
            __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels  )
            __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      2     ,      dtype=tf.floataa  )
            __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.num_choices  )

        __UpperCAmelCase									=			XLMConfig(
            vocab_size=self.vocab_size     ,      n_special=self.n_special     ,      emb_dim=self.hidden_size     ,      n_layers=self.num_hidden_layers     ,      n_heads=self.num_attention_heads     ,      dropout=self.hidden_dropout_prob     ,      attention_dropout=self.attention_probs_dropout_prob     ,      gelu_activation=self.gelu_activation     ,      sinusoidal_embeddings=self.sinusoidal_embeddings     ,      asm=self.asm     ,      causal=self.causal     ,      n_langs=self.n_langs     ,      max_position_embeddings=self.max_position_embeddings     ,      initializer_range=self.initializer_range     ,      summary_type=self.summary_type     ,      use_proj=self.use_proj     ,      bos_token_id=self.bos_token_id     ,      )

        return (
            config,
            input_ids,
            token_type_ids,
            input_lengths,
            sequence_labels,
            token_labels,
            is_impossible_labels,
            choice_labels,
            input_mask,
        )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      List[Any]:
        __UpperCAmelCase									=			TFXLMModel(config=lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCAmelCase									=			model(lowercase__  )

        __UpperCAmelCase									=			[input_ids, input_mask]
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      List[str]:
        __UpperCAmelCase									=			TFXLMWithLMHeadModel(lowercase__  )

        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
        __UpperCAmelCase									=			model(lowercase__  )

        __UpperCAmelCase									=			outputs

        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Optional[Any]:
        __UpperCAmelCase									=			TFXLMForQuestionAnsweringSimple(lowercase__  )

        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCAmelCase									=			model(lowercase__  )

        self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)  )
        self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Dict:
        __UpperCAmelCase									=			TFXLMForSequenceClassification(lowercase__  )

        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths}

        __UpperCAmelCase									=			model(lowercase__  )

        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.type_sequence_label_size)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      int:
        __UpperCAmelCase									=			self.num_labels
        __UpperCAmelCase									=			TFXLMForTokenClassification(config=lowercase__  )
        __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)  )


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Tuple:
        __UpperCAmelCase									=			self.num_choices
        __UpperCAmelCase									=			TFXLMForMultipleChoice(config=lowercase__  )
        __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
        __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
        __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
        __UpperCAmelCase									=			{
            '''input_ids''': multiple_choice_inputs_ids,
            '''attention_mask''': multiple_choice_input_mask,
            '''token_type_ids''': multiple_choice_token_type_ids,
        }
        __UpperCAmelCase									=			model(lowercase__  )
        self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)  )






    def      lowerCAmelCase_						(self  )     ->      str:
        __UpperCAmelCase									=			self.prepare_config_and_inputs()
        (
            (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     (
            __UpperCAmelCase
        )					,     
        )									=			config_and_inputs
        __UpperCAmelCase									=			{
            '''input_ids''': input_ids,
            '''token_type_ids''': token_type_ids,
            '''langs''': token_type_ids,
            '''lengths''': input_lengths,
        }
        return config, inputs_dict






@require_tf
class 					A_   (      _a ,     _a ,     unittest.TestCase						):


    '''simple docstring'''






    a__     							=       (
        (
            TFXLMModel,
            TFXLMWithLMHeadModel,
            TFXLMForSequenceClassification,
            TFXLMForQuestionAnsweringSimple,
            TFXLMForTokenClassification,
            TFXLMForMultipleChoice,
        )
        if is_tf_available()
        else ()
    )
    a__     							=       (
        (TFXLMWithLMHeadModel,) if is_tf_available() else ()
    )  # TODO (PVP): Check other models whether language generation is also applicable
    a__     							=       (
        {
            "feature-extraction": TFXLMModel,
            "fill-mask": TFXLMWithLMHeadModel,
            "question-answering": TFXLMForQuestionAnsweringSimple,
            "text-classification": TFXLMForSequenceClassification,
            "text-generation": TFXLMWithLMHeadModel,
            "token-classification": TFXLMForTokenClassification,
            "zero-shot": TFXLMForSequenceClassification,
        }
        if is_tf_available()
        else {}
    )
    a__     							=       False
    a__     							=       False


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      Tuple:
        if (
            pipeline_test_casse_name == "QAPipelineTests"
            and tokenizer_name is not None
            and not tokenizer_name.endswith('''Fast'''  )
        ):
            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
            return True

        return False


    def      lowerCAmelCase_						(self  )     ->      Tuple:
        __UpperCAmelCase									=			TFXLMModelTester(self  )
        __UpperCAmelCase									=			ConfigTester(self     ,      config_class=lowercase__     ,      emb_dim=37  )


    def      lowerCAmelCase_						(self  )     ->      str:
        self.config_tester.run_common_tests()


    def      lowerCAmelCase_						(self  )     ->      Optional[int]:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_model(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Any:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_lm_head(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Dict:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_qa(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      int:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_sequence_classif(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_for_token_classification(*lowercase__  )


    def      lowerCAmelCase_						(self  )     ->      Tuple:
        __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_xlm_for_multiple_choice(*lowercase__  )






    @slow
    def      lowerCAmelCase_						(self  )     ->      int:
        for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            __UpperCAmelCase									=			TFXLMModel.from_pretrained(lowercase__  )
            self.assertIsNotNone(lowercase__  )






@require_tf
class 					A_   (      unittest.TestCase						):


    '''simple docstring'''






    @slow
    def      lowerCAmelCase_						(self  )     ->      Tuple:
        __UpperCAmelCase									=			TFXLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''  )
        __UpperCAmelCase									=			tf.convert_to_tensor([[14, 447]]     ,      dtype=tf.intaa  )  # the president
        __UpperCAmelCase									=			[
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
            14,
            447,
        ]  # the president the president the president the president the president the president the president the president the president the president
        # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
        __UpperCAmelCase									=			model.generate(lowercase__     ,      do_sample=lowercase__  )
        self.assertListEqual(output_ids[0].numpy().tolist()     ,      lowercase__  )




