


'''simple docstring'''


import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       ():
		_UpperCAmelCase						:				Tuple	    =					10
		_UpperCAmelCase						:				Dict	    =					datasets.Features(
		    {
		        "tokens": datasets.Sequence(datasets.Value("string"      )      ),
		        "labels": datasets.Sequence(datasets.ClassLabel(names=["negative", "positive"]      )      ),
		        "answers": datasets.Sequence(
		            {
		                "text": datasets.Value("string"      ),
		                "answer_start": datasets.Value("int32"      ),
		            }      ),
		        "id": datasets.Value("int64"      ),
		    }      )
		_UpperCAmelCase						:				Optional[int]	    =					datasets.Dataset.from_dict(
		    {
		        "tokens": [["foo"] * 5] * n,
		        "labels": [[1] * 5] * n,
		        "answers": [{"answer_start": [97], "text": ["1976"]}] * 10,
		        "id": list(range(__lowerCAmelCase      )      ),
		    }					,       features=__lowerCAmelCase					,       )
		return dataset






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "file.arrow"      )
		dataset.map(cache_file_name=__lowerCAmelCase      )
		return filename


# FILE_CONTENT + files


lowerCamelCase__					=   '\\n    Text data.\n    Second line of data.'






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "file.txt"
		_UpperCAmelCase						:				List[str]	    =					FILE_CONTENT
		with open(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase      )
		return filename






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		import bza

		_UpperCAmelCase						:				Any	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.bz2"
		_UpperCAmelCase						:				Tuple	    =					bytes(__lowerCAmelCase					,       "utf-8"      )
		with bza.open(__lowerCAmelCase					,       "wb"      ) as f:
				f.write(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		import gzip

		_UpperCAmelCase						:				List[str]	    =					str(tmp_path_factory.mktemp("data"      ) / "file.txt.gz"      )
		_UpperCAmelCase						:				List[str]	    =					bytes(__lowerCAmelCase					,       "utf-8"      )
		with gzip.open(__lowerCAmelCase					,       "wb"      ) as f:
				f.write(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		if datasets.config.LZ4_AVAILABLE:
				import lza.frame

				_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.lz4"
				_UpperCAmelCase						:				Tuple	    =					bytes(__lowerCAmelCase					,       "utf-8"      )
				with lza.frame.open(__lowerCAmelCase					,       "wb"      ) as f:
						f.write(__lowerCAmelCase      )
				return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		if datasets.config.PY7ZR_AVAILABLE:
				import pyazr

				_UpperCAmelCase						:				Any	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.7z"
				with pyazr.SevenZipFile(__lowerCAmelCase					,       "w"      ) as archive:
						archive.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		import tarfile

		_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.tar"
		with tarfile.TarFile(__lowerCAmelCase					,       "w"      ) as f:
				f.add(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		import lzma

		_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.xz"
		_UpperCAmelCase						:				Dict	    =					bytes(__lowerCAmelCase					,       "utf-8"      )
		with lzma.open(__lowerCAmelCase					,       "wb"      ) as f:
				f.write(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		import zipfile

		_UpperCAmelCase						:				Dict	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		if datasets.config.ZSTANDARD_AVAILABLE:
				import zstandard as zstd

				_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "file.txt.zst"
				_UpperCAmelCase						:				List[Any]	    =					bytes(__lowerCAmelCase					,       "utf-8"      )
				with zstd.open(__lowerCAmelCase					,       "wb"      ) as f:
						f.write(__lowerCAmelCase      )
				return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				int	    =					tmp_path_factory.mktemp("data"      ) / "file.xml"
		_UpperCAmelCase						:				Dict	    =					textwrap.dedent(
		    "\\n    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n    <tmx version=\"1.4\">\n      <header segtype=\"sentence\" srclang=\"ca\" />\n      <body>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase      )
		return filename


lowerCamelCase__					=   [
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
lowerCamelCase__					=   [
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
lowerCamelCase__					=   {
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

lowerCamelCase__					=   [
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

lowerCamelCase__					=   [
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       ():
		return DATA_DICT_OF_LISTS






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Any	    =					datasets.Dataset.from_dict(__lowerCAmelCase      )
		_UpperCAmelCase						:				str	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.arrow"      )
		dataset.map(cache_file_name=__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.sqlite"      )
		with contextlib.closing(sqlitea.connect(__lowerCAmelCase      )      ) as con:
				_UpperCAmelCase						:				Optional[int]	    =					con.cursor()
				cur.execute("CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)"      )
				for item in DATA:
						cur.execute("INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)"					,       tuple(item.values()      )      )
				con.commit()
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				str	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.csv"      )
		with open(__lowerCAmelCase					,       "w"					,       newline=""      ) as f:
				_UpperCAmelCase						:				str	    =					csv.DictWriter(__lowerCAmelCase					,       fieldnames=["col_1", "col_2", "col_3"]      )
				writer.writeheader()
				for item in DATA:
						writer.writerow(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset2.csv"      )
		with open(__lowerCAmelCase					,       "w"					,       newline=""      ) as f:
				_UpperCAmelCase						:				List[str]	    =					csv.DictWriter(__lowerCAmelCase					,       fieldnames=["col_1", "col_2", "col_3"]      )
				writer.writeheader()
				for item in DATA:
						writer.writerow(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		import bza

		_UpperCAmelCase						:				Any	    =					tmp_path_factory.mktemp("data"      ) / "dataset.csv.bz2"
		with open(__lowerCAmelCase					,       "rb"      ) as f:
				_UpperCAmelCase						:				Union[str, Any]	    =					f.read()
		# data = bytes(FILE_CONTENT, "utf-8")
		with bza.open(__lowerCAmelCase					,       "wb"      ) as f:
				f.write(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				List[str]	    =					tmp_path_factory.mktemp("data"      ) / "dataset.csv.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[Any]	    =					tmp_path_factory.mktemp("data"      ) / "dataset.csv.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(csv_path.replace(".csv"					,       ".CSV"      )      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(csva_path.replace(".csv"					,       ".CSV"      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				int	    =					tmp_path_factory.mktemp("data"      ) / "dataset_with_dir.csv.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[int]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.parquet"      )
		_UpperCAmelCase						:				List[str]	    =					pa.schema(
		    {
		        "col_1": pa.string(),
		        "col_2": pa.intaa(),
		        "col_3": pa.floataa(),
		    }      )
		with open(__lowerCAmelCase					,       "wb"      ) as f:
				_UpperCAmelCase						:				Any	    =					pq.ParquetWriter(__lowerCAmelCase					,       schema=__lowerCAmelCase      )
				_UpperCAmelCase						:				Any	    =					pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__lowerCAmelCase      )      )] for k in DATA[0]}					,       schema=__lowerCAmelCase      )
				writer.write_table(__lowerCAmelCase      )
				writer.close()
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.json"      )
		_UpperCAmelCase						:				List[Any]	    =					{"data": DATA}
		with open(__lowerCAmelCase					,       "w"      ) as f:
				json.dump(__lowerCAmelCase					,       __lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Dict	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.json"      )
		_UpperCAmelCase						:				Optional[int]	    =					{"data": DATA_DICT_OF_LISTS}
		with open(__lowerCAmelCase					,       "w"      ) as f:
				json.dump(__lowerCAmelCase					,       __lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.jsonl"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in DATA:
						f.write(json.dumps(__lowerCAmelCase      ) + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				List[Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset2.jsonl"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in DATA:
						f.write(json.dumps(__lowerCAmelCase      ) + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[int]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset_312.jsonl"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in DATA_312:
						f.write(json.dumps(__lowerCAmelCase      ) + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				int	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset-str.jsonl"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in DATA_STR:
						f.write(json.dumps(__lowerCAmelCase      ) + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		import gzip

		_UpperCAmelCase						:				int	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.txt.gz"      )
		with open(__lowerCAmelCase					,       "rb"      ) as orig_file:
				with gzip.open(__lowerCAmelCase					,       "wb"      ) as zipped_file:
						zipped_file.writelines(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		import gzip

		_UpperCAmelCase						:				int	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.jsonl.gz"      )
		with open(__lowerCAmelCase					,       "rb"      ) as orig_file:
				with gzip.open(__lowerCAmelCase					,       "wb"      ) as zipped_file:
						zipped_file.writelines(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				int	    =					tmp_path_factory.mktemp("data"      ) / "dataset.jsonl.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Tuple	    =					tmp_path_factory.mktemp("data"      ) / "dataset_nested.jsonl.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.join("nested"					,       os.path.basename(__lowerCAmelCase      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[int]	    =					tmp_path_factory.mktemp("data"      ) / "dataset_with_dir.jsonl.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Dict	    =					tmp_path_factory.mktemp("data"      ) / "dataset.jsonl.tar"
		with tarfile.TarFile(__lowerCAmelCase					,       "w"      ) as f:
				f.add(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				f.add(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Tuple	    =					tmp_path_factory.mktemp("data"      ) / "dataset_nested.jsonl.tar"
		with tarfile.TarFile(__lowerCAmelCase					,       "w"      ) as f:
				f.add(__lowerCAmelCase					,       arcname=os.path.join("nested"					,       os.path.basename(__lowerCAmelCase      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Any	    =					["0", "1", "2", "3"]
		_UpperCAmelCase						:				List[Any]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset.txt"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in data:
						f.write(item + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					["0", "1", "2", "3"]
		_UpperCAmelCase						:				Optional[int]	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset2.txt"      )
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in data:
						f.write(item + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[int]	    =					["0", "1", "2", "3"]
		_UpperCAmelCase						:				int	    =					tmp_path_factory.mktemp("data"      ) / "dataset.abc"
		with open(__lowerCAmelCase					,       "w"      ) as f:
				for item in data:
						f.write(item + "\n"      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				str	    =					tmp_path_factory.mktemp("data"      ) / "dataset.text.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Tuple	    =					tmp_path_factory.mktemp("data"      ) / "dataset_with_dir.text.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.join("main_dir"					,       os.path.basename(__lowerCAmelCase      )      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[Any]	    =					tmp_path_factory.mktemp("data"      ) / "dataset.ext.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename("unsupported.ext"      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename("unsupported_2.ext"      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Optional[Any]	    =					"\n".join(["First", "Second\u2029with Unicode new line", "Third"]      )
		_UpperCAmelCase						:				Any	    =					str(tmp_path_factory.mktemp("data"      ) / "dataset_with_unicode_new_lines.txt"      )
		with open(__lowerCAmelCase					,       "w"					,       encoding="utf-8"      ) as f:
				f.write(__lowerCAmelCase      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       ():
		return os.path.join("tests"					,       "features"					,       "data"					,       "test_image_rgb.jpg"      )






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       ():
		return os.path.join("tests"					,       "features"					,       "data"					,       "test_audio_44100.wav"      )






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase					,       __lowerCAmelCase      ):
		_UpperCAmelCase						:				Any	    =					tmp_path_factory.mktemp("data"      ) / "dataset.img.zip"
		with zipfile.ZipFile(__lowerCAmelCase					,       "w"      ) as f:
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      )      )
				f.write(__lowerCAmelCase					,       arcname=os.path.basename(__lowerCAmelCase      ).replace(".jpg"					,       "2.jpg"      )      )
		return path






@pytest.fixture(scope="session"      )
def        __lowerCAmelCase       (__lowerCAmelCase      ):
		_UpperCAmelCase						:				Union[str, Any]	    =					tmp_path_factory.mktemp("data_dir"      )

		(data_dir / "subdir").mkdir()
		with open(data_dir / "subdir" / "train.txt"					,       "w"      ) as f:
				f.write("foo\n" * 10      )
		with open(data_dir / "subdir" / "test.txt"					,       "w"      ) as f:
				f.write("bar\n" * 10      )
		# hidden file
		with open(data_dir / "subdir" / ".test.txt"					,       "w"      ) as f:
				f.write("bar\n" * 10      )

		# hidden directory
		(data_dir / ".subdir").mkdir()
		with open(data_dir / ".subdir" / "train.txt"					,       "w"      ) as f:
				f.write("foo\n" * 10      )
		with open(data_dir / ".subdir" / "test.txt"					,       "w"      ) as f:
				f.write("bar\n" * 10      )

		return data_dir







'''simple docstring'''


#
# This a `torch.distributed` diagnostics script that checks that all GPUs in the cluster (one or
# many nodes) can talk to each other via nccl and allocate gpu memory.
#
# To run first adjust the number of processes and nodes:
#
# python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# You may need to add --master_addr $MASTER_ADDR --master_port $MASTER_PORT if using a custom addr:port
#
# You can also use the rdzv API: --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT --rdzv_backend c10d
#
# use torch.distributed.launch instead of torch.distributed.run for torch < 1.9
#
# If you get a hanging in `barrier` calls you have some network issues, you may try to debug this with:
#
# NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py
#
# which should tell you what's going on behind the scenes.
#
#
# This script can be run via `srun` in the SLURM environment as well. Here is a SLURM script that
# runs on 2 nodes of 4 gpus per node:
#
# #SBATCH --job-name=test-nodes        # name
# #SBATCH --nodes=2                    # nodes
# #SBATCH --ntasks-per-node=1          # crucial - only 1 task per dist per node!
# #SBATCH --cpus-per-task=10           # number of cores per tasks
# #SBATCH --gres=gpu:4                 # number of gpus
# #SBATCH --time 0:05:00               # maximum execution time (HH:MM:SS)
# #SBATCH --output=%x-%j.out           # output file name
#
# GPUS_PER_NODE=4
# MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
# MASTER_PORT=6000
#
# srun --jobid $SLURM_JOBID bash -c 'python -m torch.distributed.run \
# --nproc_per_node $GPUS_PER_NODE --nnodes $SLURM_NNODES --node_rank $SLURM_PROCID \
# --master_addr $MASTER_ADDR --master_port $MASTER_PORT \
# torch-distributed-gpu-test.py'
#

import fcntl
import os
import socket

import torch
import torch.distributed as dist






def        __lowerCAmelCase       (*__lowerCAmelCase      ):
		with open(__lowerCAmelCase					,       "r"      ) as fh:
				fcntl.flock(__lowerCAmelCase					,       fcntl.LOCK_EX      )
				try:
						print(*__lowerCAmelCase      )
				finally:
						fcntl.flock(__lowerCAmelCase					,       fcntl.LOCK_UN      )


lowerCamelCase__					=   int(os.environ['LOCAL_RANK'])
torch.cuda.set_device(local_rank)
lowerCamelCase__					=   torch.device('cuda', local_rank)
lowerCamelCase__					=   socket.gethostname()

lowerCamelCase__					=   F'''[{hostname}-{local_rank}]'''

try:
						# test distributed
						dist.init_process_group('nccl')
						dist.all_reduce(torch.ones(1).to(device), op=dist.ReduceOp.SUM)
						dist.barrier()

						# test cuda is available and can allocate memory
						torch.cuda.is_available()
						torch.ones(1).cuda(local_rank)

						# global rank
						lowerCamelCase__					=   dist.get_rank()
						lowerCamelCase__					=   dist.get_world_size()

						printflock(F'''{gpu} is OK (global rank: {rank}/{world_size})''')

						dist.barrier()
						if rank == 0:
												printflock(F'''pt={torch.__version__}, cuda={torch.version.cuda}, nccl={torch.cuda.nccl.version()}''')

except Exception:
						printflock(F'''{gpu} is broken''')
						raise



