






import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer


def   _A   (				SCREAMING_SNAKE_CASE__  :	Dict     ,  SCREAMING_SNAKE_CASE__  :	Dict     ,  SCREAMING_SNAKE_CASE__  :	Union[str, Any]     ,  SCREAMING_SNAKE_CASE__  :	int     ,  SCREAMING_SNAKE_CASE__  :	Optional[Any]=True     ,  SCREAMING_SNAKE_CASE__  :	Any="pt"       ):
    UpperCamelCase       :Dict								=   {'''add_prefix_space''': True} if isinstance(__UpperCAmelCase     ,  __UpperCAmelCase       ) and not line.startswith(''' '''       ) else {}
    UpperCamelCase       :List[str]								=   padding_side
    return tokenizer(
        [line]     ,  max_length=__UpperCAmelCase     ,  padding='''max_length''' if pad_to_max_length else None     ,  truncation=__UpperCAmelCase     ,  return_tensors=__UpperCAmelCase     ,  add_special_tokens=__UpperCAmelCase     ,  **__UpperCAmelCase     ,  )


def   _A   (				SCREAMING_SNAKE_CASE__  :	Union[str, Any]     ,  SCREAMING_SNAKE_CASE__  :	Optional[Any]     ,  SCREAMING_SNAKE_CASE__  :	Dict=None     ,  ):
    UpperCamelCase       :List[Any]								=   input_ids.ne(__UpperCAmelCase       ).any(dim=0       )
    if attention_mask is None:
        return input_ids[:, keep_column_mask]
    else:
        return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])







class 			UpperCAmelCase_						(					_SCREAMING_SNAKE_CASE ):



      """simple docstring"""
      def __init__(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_="train"    ,				SCREAMING_SNAKE_CASE_=None    ,				SCREAMING_SNAKE_CASE_=None    ,				SCREAMING_SNAKE_CASE_=None    ,				SCREAMING_SNAKE_CASE_=""    ,				)				->      Union[str, Any]:
          super().__init__()
          UpperCamelCase       :List[Any]								=   Path(_lowerCAmelCase ).joinpath(type_path + '''.source''' )
          UpperCamelCase       :Any								=   Path(_lowerCAmelCase ).joinpath(type_path + '''.target''' )
          UpperCamelCase       :str								=   self.get_char_lens(self.src_file )
          UpperCamelCase       :Any								=   max_source_length
          UpperCamelCase       :Union[str, Any]								=   max_target_length
          assert min(self.src_lens ) > 0, F'''found empty line in {self.src_file}'''
          UpperCamelCase       :List[Any]								=   tokenizer
          UpperCamelCase       :List[str]								=   prefix
          if n_obs is not None:
              UpperCamelCase       :str								=   self.src_lens[:n_obs]
          UpperCamelCase       :Optional[int]								=   src_lang
          UpperCamelCase       :Union[str, Any]								=   tgt_lang
      def __len__(				self )				->      Any:
          return len(self.src_lens )
      def __getitem__(				self    ,				SCREAMING_SNAKE_CASE_ )				->      Union[str, Any]:
          UpperCamelCase       :List[str]								=   index + 1  # linecache starts at 1
          UpperCamelCase       :str								=   self.prefix + linecache.getline(str(self.src_file )    ,				_lowerCAmelCase ).rstrip('''\n''' )
          UpperCamelCase       :Dict								=   linecache.getline(str(self.tgt_file )    ,				_lowerCAmelCase ).rstrip('''\n''' )
          assert source_line, F'''empty source line for index {index}'''
          assert tgt_line, F'''empty tgt line for index {index}'''

          # Need to add eos token manually for T5
          if isinstance(self.tokenizer    ,				_lowerCAmelCase ):
              source_line += self.tokenizer.eos_token
              tgt_line += self.tokenizer.eos_token

          # Pad source and target to the right
          UpperCamelCase       :Union[str, Any]								=   (
              self.tokenizer.question_encoder if isinstance(self.tokenizer    ,				_lowerCAmelCase ) else self.tokenizer
          )
          UpperCamelCase       :str								=   self.tokenizer.generator if isinstance(self.tokenizer    ,				_lowerCAmelCase ) else self.tokenizer

          UpperCamelCase       :Optional[Any]								=   encode_line(_lowerCAmelCase    ,				_lowerCAmelCase    ,				self.max_source_length    ,				'''right''' )
          UpperCamelCase       :Dict								=   encode_line(_lowerCAmelCase    ,				_lowerCAmelCase    ,				self.max_target_length    ,				'''right''' )

          UpperCamelCase       :Union[str, Any]								=   source_inputs['''input_ids'''].squeeze()
          UpperCamelCase       :List[str]								=   target_inputs['''input_ids'''].squeeze()
          UpperCamelCase       :List[str]								=   source_inputs['''attention_mask'''].squeeze()
          return {
              "input_ids": source_ids,
              "attention_mask": src_mask,
              "decoder_input_ids": target_ids,
          }
      @staticmethod
      def     UpperCAmelCase						(				SCREAMING_SNAKE_CASE_ )				->      Optional[int]:
          return [len(_lowerCAmelCase ) for x in Path(_lowerCAmelCase ).open().readlines()]




      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_ )				->      Union[str, Any]:
          UpperCamelCase       :List[str]								=   torch.stack([x['''input_ids'''] for x in batch] )
          UpperCamelCase       :Tuple								=   torch.stack([x['''attention_mask'''] for x in batch] )
          UpperCamelCase       :str								=   torch.stack([x['''decoder_input_ids'''] for x in batch] )
          UpperCamelCase       :Optional[int]								=   (
              self.tokenizer.generator.pad_token_id
              if isinstance(self.tokenizer    ,				_lowerCAmelCase )
              else self.tokenizer.pad_token_id
          )
          UpperCamelCase       :List[Any]								=   (
              self.tokenizer.question_encoder.pad_token_id
              if isinstance(self.tokenizer    ,				_lowerCAmelCase )
              else self.tokenizer.pad_token_id
          )
          UpperCamelCase       :Union[str, Any]								=   trim_batch(_lowerCAmelCase    ,				_lowerCAmelCase )
          UpperCamelCase							,   UpperCamelCase       :Any								=   trim_batch(_lowerCAmelCase    ,				_lowerCAmelCase    ,				attention_mask=_lowerCAmelCase )
          UpperCamelCase       :Union[str, Any]								=   {
              '''input_ids''': source_ids,
              '''attention_mask''': source_mask,
              '''decoder_input_ids''': y,
          }
          return batch







__snake_case												=  getLogger(__name__)


def   _A   (				SCREAMING_SNAKE_CASE__  :	List[List]       ):
    return list(itertools.chain.from_iterable(__UpperCAmelCase       )       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	str       ):
    UpperCamelCase       :List[str]								=   get_git_info()
    save_json(__UpperCAmelCase     ,  os.path.join(__UpperCAmelCase     ,  '''git_log.json'''       )       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	Dict     ,  SCREAMING_SNAKE_CASE__  :	Tuple     ,  SCREAMING_SNAKE_CASE__  :	Optional[int]=4     ,  **SCREAMING_SNAKE_CASE__  :	Tuple       ):
    with open(__UpperCAmelCase     ,  '''w'''       ) as f:
        json.dump(__UpperCAmelCase     ,  __UpperCAmelCase     ,  indent=__UpperCAmelCase     ,  **__UpperCAmelCase       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	List[str]       ):
    with open(__UpperCAmelCase       ) as f:
        return json.load(__UpperCAmelCase       )


def   _A   (				):
    UpperCamelCase       :int								=   git.Repo(search_parent_directories=__UpperCAmelCase       )
    UpperCamelCase       :List[Any]								=   {
        '''repo_id''': str(__UpperCAmelCase       ),
        '''repo_sha''': str(repo.head.object.hexsha       ),
        '''repo_branch''': str(repo.active_branch       ),
        '''hostname''': str(socket.gethostname()       ),
    }
    return repo_infos


def   _A   (				SCREAMING_SNAKE_CASE__  :	Callable     ,  SCREAMING_SNAKE_CASE__  :	Iterable       ):
    return list(map(__UpperCAmelCase     ,  __UpperCAmelCase       )       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	str     ,  SCREAMING_SNAKE_CASE__  :	Dict       ):
    with open(__UpperCAmelCase     ,  '''wb'''       ) as f:
        return pickle.dump(__UpperCAmelCase     ,  __UpperCAmelCase       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	List[Any]       ):

    def remove_articles(SCREAMING_SNAKE_CASE__  :	Any       ):
        return re.sub(R'''\b(a|an|the)\b'''     ,  ''' '''     ,  __UpperCAmelCase       )

    def white_space_fix(SCREAMING_SNAKE_CASE__  :	List[str]       ):
        return " ".join(text.split()       )

    def remove_punc(SCREAMING_SNAKE_CASE__  :	List[Any]       ):
        UpperCamelCase       :List[Any]								=   set(string.punctuation       )
        return "".join(ch for ch in text if ch not in exclude       )

    def lower(SCREAMING_SNAKE_CASE__  :	Union[str, Any]       ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(__UpperCAmelCase       )       )       )       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	int     ,  SCREAMING_SNAKE_CASE__  :	Any       ):
    UpperCamelCase       :Optional[Any]								=   normalize_answer(__UpperCAmelCase       ).split()
    UpperCamelCase       :Tuple								=   normalize_answer(__UpperCAmelCase       ).split()
    UpperCamelCase       :int								=   Counter(__UpperCAmelCase       ) & Counter(__UpperCAmelCase       )
    UpperCamelCase       :Any								=   sum(common.values()       )
    if num_same == 0:
        return 0
    UpperCamelCase       :List[str]								=   1.0 * num_same / len(__UpperCAmelCase       )
    UpperCamelCase       :List[str]								=   1.0 * num_same / len(__UpperCAmelCase       )
    UpperCamelCase       :Optional[Any]								=   (2 * precision * recall) / (precision + recall)
    return fa


def   _A   (				SCREAMING_SNAKE_CASE__  :	str     ,  SCREAMING_SNAKE_CASE__  :	Optional[int]       ):
    return normalize_answer(__UpperCAmelCase       ) == normalize_answer(__UpperCAmelCase       )


def   _A   (				SCREAMING_SNAKE_CASE__  :	List[str]     ,  SCREAMING_SNAKE_CASE__  :	List[str]       ):
    assert len(__UpperCAmelCase       ) == len(__UpperCAmelCase       )
    UpperCamelCase       :Dict								=   0
    for hypo, pred in zip(__UpperCAmelCase     ,  __UpperCAmelCase       ):
        em += exact_match_score(__UpperCAmelCase     ,  __UpperCAmelCase       )
    if len(__UpperCAmelCase       ) > 0:
        em /= len(__UpperCAmelCase       )
    return {"em": em}


def   _A   (				SCREAMING_SNAKE_CASE__  :	Union[str, Any]       ):
    return model_prefix.startswith('''rag'''       )



def   _A   (				SCREAMING_SNAKE_CASE__  :	Optional[Any]     ,  SCREAMING_SNAKE_CASE__  :	Any     ,  SCREAMING_SNAKE_CASE__  :	List[str]       ):
    UpperCamelCase       :Optional[int]								=   {p: p for p in extra_params}
    # T5 models don't have `dropout` param, they have `dropout_rate` instead
    UpperCamelCase       :List[Any]								=   '''dropout_rate'''
    for p in extra_params:
        if getattr(__UpperCAmelCase     ,  __UpperCAmelCase     ,  __UpperCAmelCase       ):
            if not hasattr(__UpperCAmelCase     ,  __UpperCAmelCase       ) and not hasattr(__UpperCAmelCase     ,  equivalent_param[p]       ):
                logger.info('''config doesn\'t have a `{}` attribute'''.format(__UpperCAmelCase       )       )
                delattr(__UpperCAmelCase     ,  __UpperCAmelCase       )
                continue
            UpperCamelCase       :Optional[Any]								=   p if hasattr(__UpperCAmelCase     ,  __UpperCAmelCase       ) else equivalent_param[p]
            setattr(__UpperCAmelCase     ,  __UpperCAmelCase     ,  getattr(__UpperCAmelCase     ,  __UpperCAmelCase       )       )
            delattr(__UpperCAmelCase     ,  __UpperCAmelCase       )
    return hparams, config











import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      )					-> str:
      SCREAMING_SNAKE_CASE_							=			10
      SCREAMING_SNAKE_CASE_							=			datasets.Features(
          {
              'tokens': datasets.Sequence(datasets.Value('string'							)							),
              'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']							)							),
              'answers': datasets.Sequence(
                  {
                      'text': datasets.Value('string'							),
                      'answer_start': datasets.Value('int32'							),
                  }							),
              'id': datasets.Value('int64'							),
          }							)
      SCREAMING_SNAKE_CASE_							=			datasets.Dataset.from_dict(
          {
              'tokens': [['foo'] * 5] * n,
              'labels': [[1] * 5] * n,
              'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
              'id': list(range(__UpperCAmelCase							)							),
          }			,			features=__UpperCAmelCase			,			)
      return dataset

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Tuple			,			__UpperCAmelCase						:	str							)					-> int:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'file.arrow'							)
      dataset.map(cache_file_name=__UpperCAmelCase							)
      return filename


# FILE_CONTENT + files


lowerCamelCase__ :   List[Any]					=							'\\n    Text data.\n    Second line of data.'

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Any							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt'
      SCREAMING_SNAKE_CASE_							=			FILE_CONTENT
      with open(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase							)
      return filename

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict							)					-> List[str]:
      import bza

      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.bz2'
      SCREAMING_SNAKE_CASE_							=			bytes(__UpperCAmelCase			,			'utf-8'							)
      with bza.open(__UpperCAmelCase			,			'wb'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]							)					-> Any:
      import gzip

      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'file.txt.gz'							)
      SCREAMING_SNAKE_CASE_							=			bytes(__UpperCAmelCase			,			'utf-8'							)
      with gzip.open(__UpperCAmelCase			,			'wb'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[str]							)					-> int:
      if datasets.config.LZ4_AVAILABLE:
            import lza.frame

            SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.lz4'
            SCREAMING_SNAKE_CASE_							=			bytes(__UpperCAmelCase			,			'utf-8'							)
            with lza.frame.open(__UpperCAmelCase			,			'wb'							) as f:
                  f.write(__UpperCAmelCase							)
            return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]			,			__UpperCAmelCase						:	Union[str, Any]							)					-> Any:
      if datasets.config.PY7ZR_AVAILABLE:
            import pyazr

            SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.7z'
            with pyazr.SevenZipFile(__UpperCAmelCase			,			'w'							) as archive:
                  archive.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	str							)					-> str:
      import tarfile

      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.tar'
      with tarfile.TarFile(__UpperCAmelCase			,			'w'							) as f:
            f.add(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	str							)					-> List[Any]:
      import lzma

      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.xz'
      SCREAMING_SNAKE_CASE_							=			bytes(__UpperCAmelCase			,			'utf-8'							)
      with lzma.open(__UpperCAmelCase			,			'wb'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	Dict							)					-> str:
      import zipfile

      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict							)					-> Any:
      if datasets.config.ZSTANDARD_AVAILABLE:
            import zstandard as zstd

            SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.txt.zst'
            SCREAMING_SNAKE_CASE_							=			bytes(__UpperCAmelCase			,			'utf-8'							)
            with zstd.open(__UpperCAmelCase			,			'wb'							) as f:
                  f.write(__UpperCAmelCase							)
            return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int							)					-> List[Any]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'file.xml'
      SCREAMING_SNAKE_CASE_							=			textwrap.dedent(
          '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase							)
      return filename


lowerCamelCase__ :   Optional[Any]					=							[
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
lowerCamelCase__ :   Dict					=							[
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
lowerCamelCase__ :   Optional[int]					=							{
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

lowerCamelCase__ :   List[Any]					=							[
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

lowerCamelCase__ :   List[Any]					=							[
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      )					-> Tuple:
      return DATA_DICT_OF_LISTS

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[Any]							)					-> Union[str, Any]:
      SCREAMING_SNAKE_CASE_							=			datasets.Dataset.from_dict(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.arrow'							)
      dataset.map(cache_file_name=__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[str]							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.sqlite'							)
      with contextlib.closing(sqlitea.connect(__UpperCAmelCase							)							) as con:
            SCREAMING_SNAKE_CASE_							=			con.cursor()
            cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'							)
            for item in DATA:
                  cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'			,			tuple(item.values()							)							)
            con.commit()
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[int]							)					-> str:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.csv'							)
      with open(__UpperCAmelCase			,			'w'			,			newline=''							) as f:
            SCREAMING_SNAKE_CASE_							=			csv.DictWriter(__UpperCAmelCase			,			fieldnames=['col_1', 'col_2', 'col_3']							)
            writer.writeheader()
            for item in DATA:
                  writer.writerow(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Tuple							)					-> Optional[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset2.csv'							)
      with open(__UpperCAmelCase			,			'w'			,			newline=''							) as f:
            SCREAMING_SNAKE_CASE_							=			csv.DictWriter(__UpperCAmelCase			,			fieldnames=['col_1', 'col_2', 'col_3']							)
            writer.writeheader()
            for item in DATA:
                  writer.writerow(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	str			,			__UpperCAmelCase						:	Tuple							)					-> str:
      import bza

      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.csv.bz2'
      with open(__UpperCAmelCase			,			'rb'							) as f:
            SCREAMING_SNAKE_CASE_							=			f.read()
      # data = bytes(FILE_CONTENT, "utf-8")
      with bza.open(__UpperCAmelCase			,			'wb'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	List[str]			,			__UpperCAmelCase						:	str							)					-> Optional[int]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.csv.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]			,			__UpperCAmelCase						:	int			,			__UpperCAmelCase						:	str							)					-> Any:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.csv.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(csv_path.replace('.csv'			,			'.CSV'							)							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(csva_path.replace('.csv'			,			'.CSV'							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	int			,			__UpperCAmelCase						:	Tuple			,			__UpperCAmelCase						:	List[Any]							)					-> Union[str, Any]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset_with_dir.csv.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[str]							)					-> Optional[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.parquet'							)
      SCREAMING_SNAKE_CASE_							=			pa.schema(
          {
              'col_1': pa.string(),
              'col_2': pa.intaa(),
              'col_3': pa.floataa(),
          }							)
      with open(__UpperCAmelCase			,			'wb'							) as f:
            SCREAMING_SNAKE_CASE_							=			pq.ParquetWriter(__UpperCAmelCase			,			schema=__UpperCAmelCase							)
            SCREAMING_SNAKE_CASE_							=			pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__UpperCAmelCase							)							)] for k in DATA[0]}			,			schema=__UpperCAmelCase							)
            writer.write_table(__UpperCAmelCase							)
            writer.close()
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Tuple							)					-> Any:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.json'							)
      SCREAMING_SNAKE_CASE_							=			{'data': DATA}
      with open(__UpperCAmelCase			,			'w'							) as f:
            json.dump(__UpperCAmelCase			,			__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]							)					-> Optional[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.json'							)
      SCREAMING_SNAKE_CASE_							=			{'data': DATA_DICT_OF_LISTS}
      with open(__UpperCAmelCase			,			'w'							) as f:
            json.dump(__UpperCAmelCase			,			__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.jsonl'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in DATA:
                  f.write(json.dumps(__UpperCAmelCase							) + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[Any]							)					-> Optional[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset2.jsonl'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in DATA:
                  f.write(json.dumps(__UpperCAmelCase							) + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]							)					-> List[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset_312.jsonl'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in DATA_312:
                  f.write(json.dumps(__UpperCAmelCase							) + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[Any]							)					-> List[Any]:
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset-str.jsonl'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in DATA_STR:
                  f.write(json.dumps(__UpperCAmelCase							) + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]			,			__UpperCAmelCase						:	List[str]							)					-> Union[str, Any]:
      import gzip

      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.txt.gz'							)
      with open(__UpperCAmelCase			,			'rb'							) as orig_file:
            with gzip.open(__UpperCAmelCase			,			'wb'							) as zipped_file:
                  zipped_file.writelines(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	str			,			__UpperCAmelCase						:	Optional[Any]							)					-> List[str]:
      import gzip

      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.jsonl.gz'							)
      with open(__UpperCAmelCase			,			'rb'							) as orig_file:
            with gzip.open(__UpperCAmelCase			,			'wb'							) as zipped_file:
                  zipped_file.writelines(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]			,			__UpperCAmelCase						:	Optional[Any]			,			__UpperCAmelCase						:	Dict							)					-> Optional[int]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.jsonl.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[Any]			,			__UpperCAmelCase						:	Any			,			__UpperCAmelCase						:	List[str]			,			__UpperCAmelCase						:	List[str]							)					-> Optional[int]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset_nested.jsonl.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.join('nested'			,			os.path.basename(__UpperCAmelCase							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[str]			,			__UpperCAmelCase						:	Optional[Any]			,			__UpperCAmelCase						:	str							)					-> Any:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset_with_dir.jsonl.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]			,			__UpperCAmelCase						:	List[Any]			,			__UpperCAmelCase						:	Tuple							)					-> Dict:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.jsonl.tar'
      with tarfile.TarFile(__UpperCAmelCase			,			'w'							) as f:
            f.add(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            f.add(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[int]			,			__UpperCAmelCase						:	Optional[int]			,			__UpperCAmelCase						:	Optional[Any]			,			__UpperCAmelCase						:	List[Any]							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset_nested.jsonl.tar'
      with tarfile.TarFile(__UpperCAmelCase			,			'w'							) as f:
            f.add(__UpperCAmelCase			,			arcname=os.path.join('nested'			,			os.path.basename(__UpperCAmelCase							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[Any]							)					-> Optional[Any]:
      SCREAMING_SNAKE_CASE_							=			['0', '1', '2', '3']
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset.txt'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in data:
                  f.write(item + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict							)					-> Any:
      SCREAMING_SNAKE_CASE_							=			['0', '1', '2', '3']
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset2.txt'							)
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in data:
                  f.write(item + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]							)					-> int:
      SCREAMING_SNAKE_CASE_							=			['0', '1', '2', '3']
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.abc'
      with open(__UpperCAmelCase			,			'w'							) as f:
            for item in data:
                  f.write(item + '\n'							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Any			,			__UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	Dict							)					-> Union[str, Any]:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.text.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Tuple			,			__UpperCAmelCase						:	int			,			__UpperCAmelCase						:	Optional[int]							)					-> Any:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset_with_dir.text.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.join('main_dir'			,			os.path.basename(__UpperCAmelCase							)							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]			,			__UpperCAmelCase						:	Any			,			__UpperCAmelCase						:	Union[str, Any]							)					-> Dict:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.ext.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename('unsupported.ext'							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename('unsupported_2.ext'							)							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]							)					-> Dict:
      SCREAMING_SNAKE_CASE_							=			'\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']							)
      SCREAMING_SNAKE_CASE_							=			str(tmp_path_factory.mktemp('data'							) / 'dataset_with_unicode_new_lines.txt'							)
      with open(__UpperCAmelCase			,			'w'			,			encoding='utf-8'							) as f:
            f.write(__UpperCAmelCase							)
      return path

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      )					-> List[Any]:
      return os.path.join('tests'			,			'features'			,			'data'			,			'test_image_rgb.jpg'							)

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      )					-> Tuple:
      return os.path.join('tests'			,			'features'			,			'data'			,			'test_audio_44100.wav'							)

@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	Dict							)					-> int:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data'							) / 'dataset.img.zip'
      with zipfile.ZipFile(__UpperCAmelCase			,			'w'							) as f:
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							)							)
            f.write(__UpperCAmelCase			,			arcname=os.path.basename(__UpperCAmelCase							).replace('.jpg'			,			'2.jpg'							)							)
      return path





@pytest.fixture(scope='session'							)
def 					UpperCAmelCase_							(      __UpperCAmelCase						:	str							)					-> Dict:
      SCREAMING_SNAKE_CASE_							=			tmp_path_factory.mktemp('data_dir'							)

      (data_dir / "subdir").mkdir()
      with open(data_dir / 'subdir' / 'train.txt'			,			'w'							) as f:
            f.write('foo\n' * 10							)
      with open(data_dir / 'subdir' / 'test.txt'			,			'w'							) as f:
            f.write('bar\n' * 10							)
      # hidden file
      with open(data_dir / 'subdir' / '.test.txt'			,			'w'							) as f:
            f.write('bar\n' * 10							)

      # hidden directory
      (data_dir / ".subdir").mkdir()
      with open(data_dir / '.subdir' / 'train.txt'			,			'w'							) as f:
            f.write('foo\n' * 10							)
      with open(data_dir / '.subdir' / 'test.txt'			,			'w'							) as f:
            f.write('bar\n' * 10							)

      return data_dir