


import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config






@pytest.fixture(scope='session'   )
def 							__lowercase   ( ):
							a__             =     1_0
							a__             =     datasets.Features(
							    {
							        'tokens': datasets.Sequence(datasets.Value('string'   )   ),
							        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']   )   ),
							        'answers': datasets.Sequence(
							            {
							                'text': datasets.Value('string'   ),
							                'answer_start': datasets.Value('int32'   ),
							            }   ),
							        'id': datasets.Value('int64'   ),
							    }   )
							a__             =     datasets.Dataset.from_dict(
							    {
							        'tokens': [['foo'] * 5] * n,
							        'labels': [[1] * 5] * n,
							        'answers': [{'answer_start': [9_7], 'text': ['1976']}] * 1_0,
							        'id': list(range(__lowerCAmelCase   )   ),
							    }					,    features=__lowerCAmelCase					,    )
							return dataset






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]					,    __lowerCAmelCase							:				Optional[int]   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'file.arrow'   )
							dataset.map(cache_file_name=__lowerCAmelCase   )
							return filename


# FILE_CONTENT + files


snake_case							:       Tuple     			=      '''\
    Text data.
    Second line of data.'''






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt'
							a__             =     FILE_CONTENT
							with open(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase   )
							return filename






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]   ):
							import bza

							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.bz2'
							a__             =     bytes(__lowerCAmelCase					,    'utf-8'   )
							with bza.open(__lowerCAmelCase					,    'wb'   ) as f:
														f.write(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]   ):
							import gzip

							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'file.txt.gz'   )
							a__             =     bytes(__lowerCAmelCase					,    'utf-8'   )
							with gzip.open(__lowerCAmelCase					,    'wb'   ) as f:
														f.write(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Any   ):
							if datasets.config.LZ4_AVAILABLE:
														import lza.frame

														a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.lz4'
														a__             =     bytes(__lowerCAmelCase					,    'utf-8'   )
														with lza.frame.open(__lowerCAmelCase					,    'wb'   ) as f:
																					f.write(__lowerCAmelCase   )
														return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]					,    __lowerCAmelCase							:				Dict   ):
							if datasets.config.PY7ZR_AVAILABLE:
														import pyazr

														a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.7z'
														with pyazr.SevenZipFile(__lowerCAmelCase					,    'w'   ) as archive:
																					archive.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				Tuple   ):
							import tarfile

							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.tar'
							with tarfile.TarFile(__lowerCAmelCase					,    'w'   ) as f:
														f.add(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict   ):
							import lzma

							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.xz'
							a__             =     bytes(__lowerCAmelCase					,    'utf-8'   )
							with lzma.open(__lowerCAmelCase					,    'wb'   ) as f:
														f.write(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				List[str]   ):
							import zipfile

							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[int]   ):
							if datasets.config.ZSTANDARD_AVAILABLE:
														import zstandard as zstd

														a__             =     tmp_path_factory.mktemp('data'   ) / 'file.txt.zst'
														a__             =     bytes(__lowerCAmelCase					,    'utf-8'   )
														with zstd.open(__lowerCAmelCase					,    'wb'   ) as f:
																					f.write(__lowerCAmelCase   )
														return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'file.xml'
							a__             =     textwrap.dedent(
							    '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase   )
							return filename


snake_case							:       int     			=      [
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
snake_case							:       Optional[int]     			=      [
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
snake_case							:       List[str]     			=      {
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

snake_case							:       int     			=      [
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

snake_case							:       Tuple     			=      [
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]






@pytest.fixture(scope='session'   )
def 							__lowercase   ( ):
							return DATA_DICT_OF_LISTS






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[int]   ):
							a__             =     datasets.Dataset.from_dict(__lowerCAmelCase   )
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.arrow'   )
							dataset.map(cache_file_name=__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.sqlite'   )
							with contextlib.closing(sqlitea.connect(__lowerCAmelCase   )   ) as con:
														a__             =     con.cursor()
														cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'   )
														for item in DATA:
																					cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'					,    tuple(item.values()   )   )
														con.commit()
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Any   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.csv'   )
							with open(__lowerCAmelCase					,    'w'					,    newline=''   ) as f:
														a__             =     csv.DictWriter(__lowerCAmelCase					,    fieldnames=['col_1', 'col_2', 'col_3']   )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset2.csv'   )
							with open(__lowerCAmelCase					,    'w'					,    newline=''   ) as f:
														a__             =     csv.DictWriter(__lowerCAmelCase					,    fieldnames=['col_1', 'col_2', 'col_3']   )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				Tuple   ):
							import bza

							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.csv.bz2'
							with open(__lowerCAmelCase					,    'rb'   ) as f:
														a__             =     f.read()
							# data = bytes(FILE_CONTENT, "utf-8")
							with bza.open(__lowerCAmelCase					,    'wb'   ) as f:
														f.write(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]					,    __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Optional[int]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.csv.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Optional[int]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.csv.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(csv_path.replace('.csv'					,    '.CSV'   )   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(csva_path.replace('.csv'					,    '.CSV'   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[Any]					,    __lowerCAmelCase							:				List[Any]					,    __lowerCAmelCase							:				Any   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.csv.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				int   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.parquet'   )
							a__             =     pa.schema(
							    {
							        'col_1': pa.string(),
							        'col_2': pa.intaa(),
							        'col_3': pa.floataa(),
							    }   )
							with open(__lowerCAmelCase					,    'wb'   ) as f:
														a__             =     pq.ParquetWriter(__lowerCAmelCase					,    schema=__lowerCAmelCase   )
														a__             =     pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__lowerCAmelCase   )   )] for k in DATA[0]}					,    schema=__lowerCAmelCase   )
														writer.write_table(__lowerCAmelCase   )
														writer.close()
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.json'   )
							a__             =     {'data': DATA}
							with open(__lowerCAmelCase					,    'w'   ) as f:
														json.dump(__lowerCAmelCase					,    __lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				int   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.json'   )
							a__             =     {'data': DATA_DICT_OF_LISTS}
							with open(__lowerCAmelCase					,    'w'   ) as f:
														json.dump(__lowerCAmelCase					,    __lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Tuple   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in DATA:
																					f.write(json.dumps(__lowerCAmelCase   ) + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				str   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset2.jsonl'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in DATA:
																					f.write(json.dumps(__lowerCAmelCase   ) + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				int   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset_312.jsonl'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in DATA_312:
																					f.write(json.dumps(__lowerCAmelCase   ) + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]   ):
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset-str.jsonl'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in DATA_STR:
																					f.write(json.dumps(__lowerCAmelCase   ) + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[Any]					,    __lowerCAmelCase							:				List[str]   ):
							import gzip

							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.txt.gz'   )
							with open(__lowerCAmelCase					,    'rb'   ) as orig_file:
														with gzip.open(__lowerCAmelCase					,    'wb'   ) as zipped_file:
																					zipped_file.writelines(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Any					,    __lowerCAmelCase							:				int   ):
							import gzip

							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.gz'   )
							with open(__lowerCAmelCase					,    'rb'   ) as orig_file:
														with gzip.open(__lowerCAmelCase					,    'wb'   ) as zipped_file:
																					zipped_file.writelines(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Tuple					,    __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				List[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				int					,    __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Optional[Any]					,    __lowerCAmelCase							:				List[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset_nested.jsonl.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.join('nested'					,    os.path.basename(__lowerCAmelCase   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]					,    __lowerCAmelCase							:				Tuple					,    __lowerCAmelCase							:				Dict   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.jsonl.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				int					,    __lowerCAmelCase							:				Any   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.jsonl.tar'
							with tarfile.TarFile(__lowerCAmelCase					,    'w'   ) as f:
														f.add(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														f.add(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]					,    __lowerCAmelCase							:				List[Any]					,    __lowerCAmelCase							:				int					,    __lowerCAmelCase							:				List[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset_nested.jsonl.tar'
							with tarfile.TarFile(__lowerCAmelCase					,    'w'   ) as f:
														f.add(__lowerCAmelCase					,    arcname=os.path.join('nested'					,    os.path.basename(__lowerCAmelCase   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]   ):
							a__             =     ['0', '1', '2', '3']
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset.txt'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in data:
																					f.write(item + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]   ):
							a__             =     ['0', '1', '2', '3']
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset2.txt'   )
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in data:
																					f.write(item + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				List[str]   ):
							a__             =     ['0', '1', '2', '3']
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.abc'
							with open(__lowerCAmelCase					,    'w'   ) as f:
														for item in data:
																					f.write(item + '\n'   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]					,    __lowerCAmelCase							:				List[Any]					,    __lowerCAmelCase							:				Union[str, Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.text.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Dict					,    __lowerCAmelCase							:				List[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset_with_dir.text.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.join('main_dir'					,    os.path.basename(__lowerCAmelCase   )   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]					,    __lowerCAmelCase							:				Optional[Any]					,    __lowerCAmelCase							:				Optional[Any]   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.ext.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename('unsupported.ext'   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename('unsupported_2.ext'   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]   ):
							a__             =     '\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']   )
							a__             =     str(tmp_path_factory.mktemp('data'   ) / 'dataset_with_unicode_new_lines.txt'   )
							with open(__lowerCAmelCase					,    'w'					,    encoding='utf-8'   ) as f:
														f.write(__lowerCAmelCase   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( ):
							return os.path.join('tests'					,    'features'					,    'data'					,    'test_image_rgb.jpg'   )






@pytest.fixture(scope='session'   )
def 							__lowercase   ( ):
							return os.path.join('tests'					,    'features'					,    'data'					,    'test_audio_44100.wav'   )






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Any   ):
							a__             =     tmp_path_factory.mktemp('data'   ) / 'dataset.img.zip'
							with zipfile.ZipFile(__lowerCAmelCase					,    'w'   ) as f:
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   )   )
														f.write(__lowerCAmelCase					,    arcname=os.path.basename(__lowerCAmelCase   ).replace('.jpg'					,    '2.jpg'   )   )
							return path






@pytest.fixture(scope='session'   )
def 							__lowercase   ( __lowerCAmelCase							:				Union[str, Any]   ):
							a__             =     tmp_path_factory.mktemp('data_dir'   )

							(data_dir / "subdir").mkdir()
							with open(data_dir / 'subdir' / 'train.txt'					,    'w'   ) as f:
														f.write('foo\n' * 1_0   )
							with open(data_dir / 'subdir' / 'test.txt'					,    'w'   ) as f:
														f.write('bar\n' * 1_0   )
							# hidden file
							with open(data_dir / 'subdir' / '.test.txt'					,    'w'   ) as f:
														f.write('bar\n' * 1_0   )

							# hidden directory
							(data_dir / ".subdir").mkdir()
							with open(data_dir / '.subdir' / 'train.txt'					,    'w'   ) as f:
														f.write('foo\n' * 1_0   )
							with open(data_dir / '.subdir' / 'test.txt'					,    'w'   ) as f:
														f.write('bar\n' * 1_0   )

							return data_dir








import argparse

import torch

from transformers import GPTaConfig, GPTaModel, load_tf_weights_in_gpta
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def 							__lowercase   ( __lowerCAmelCase							:				str					,    __lowerCAmelCase							:				Any					,    __lowerCAmelCase							:				int   ):
							# Construct model
							if gpta_config_file == "":
														a__             =     GPTaConfig()
							else:
														a__             =     GPTaConfig.from_json_file(__lowerCAmelCase   )
							a__             =     GPTaModel(__lowerCAmelCase   )

							# Load weights from numpy
							load_tf_weights_in_gpta(__lowerCAmelCase					,    __lowerCAmelCase					,    __lowerCAmelCase   )

							# Save pytorch-model
							a__             =     pytorch_dump_folder_path + '/' + WEIGHTS_NAME
							a__             =     pytorch_dump_folder_path + '/' + CONFIG_NAME
							print(F'Save PyTorch model to {pytorch_weights_dump_path}'   )
							torch.save(model.state_dict()					,    __lowerCAmelCase   )
							print(F'Save configuration file to {pytorch_config_dump_path}'   )
							with open(__lowerCAmelCase					,    'w'					,    encoding='utf-8'   ) as f:
														f.write(config.to_json_string()   )


if __name__ == "__main__":
						snake_case							:       List[Any]     			=      argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--gpt2_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
						)
						parser.add_argument(
						    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
						)
						parser.add_argument(
						    '''--gpt2_config_file''',
						    default='''''',
						    type=str,
						    help=(
						        '''An optional config json file corresponding to the pre-trained OpenAI model. \n'''
						        '''This specifies the model architecture.'''
						    ),
						)
						snake_case							:       Any     			=      parser.parse_args()
						convert_gpta_checkpoint_to_pytorch(args.gpta_checkpoint_path, args.gpta_config_file, args.pytorch_dump_folder_path)




