






"""simple docstring"""


import os
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
       import torch

       from transformers import GPTaLMHeadModel


@require_torch
@require_sentencepiece
@require_tokenizers
class    snake_case						(				unittest.TestCase		):


   '''simple docstring'''

   @slow
   @unittest.skip('''Model is not available.'''    )
   def 							_SCREAMING_SNAKE_CASE							(      self					:	int    ):

     '''simple docstring'''
     __A		     =					'''nvidia/megatron-gpt2-345m/'''
     if "MYDIR" in os.environ:
       __A		     =					os.path.join(os.environ['''MYDIR'''],			_lowerCamelCase    )
     __A		     =					GPTaLMHeadModel.from_pretrained(_lowerCamelCase    )
     model.to(_lowerCamelCase    )
     model.half()

     __A		     =					torch.tensor(
         [[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]],			device=_lowerCamelCase,			dtype=torch.long,			)

     with torch.no_grad():
       __A		     =					model(_lowerCamelCase    ).logits

     __A		     =					torch.Size((1, 9, 5_02_57)    )
     self.assertEqual(output.shape,			_lowerCamelCase    )

     __A		     =					torch.tensor(
         [
             4.94_14,
             -0.29_20,
             -1.21_48,
             -4.02_73,
             -0.51_61,
             -5.21_09,
             -1.24_12,
             -1.83_01,
             -1.77_34,
             -4.71_48,
             -0.23_17,
             -1.08_11,
             -2.17_77,
             0.41_41,
             -3.79_69,
             -4.05_86,
             -2.53_32,
             -3.38_09,
             4.38_67,
         ],			device=_lowerCamelCase,			dtype=torch.half,			)

     for i in range(19    ):
       __A     ,       __A		     =					8 * i // 17, 27_92 * i  # along the diagonal
       __A     ,       __A		     =					output[0, r, c], expected_diag[i]
       __A		     =					f'row={r} col={c} computed={computed} expected={expected}'
       self.assertAlmostEqual(_lowerCamelCase,			_lowerCamelCase,			delta=1e-4,			msg=_lowerCamelCase    )








'''simple docstring'''


import os
import sys
from pathlib import Path

import pytest

from datasets import Dataset, IterableDataset
from datasets.distributed import split_dataset_by_node

from .utils import execute_subprocess_async, get_torch_dist_unique_port, require_torch






def 			__lowerCAmelCase			():
		_UpperCAmelCase		:							Optional[int]             =   Dataset.from_dict({"i": range(17 )} )
		_UpperCAmelCase		:							int             =   len(__lowerCAmelCase )
		_UpperCAmelCase		:							Dict             =   3
		_UpperCAmelCase		:							str             =   [
		    split_dataset_by_node(__lowerCAmelCase   ,			rank=__lowerCAmelCase   ,			world_size=__lowerCAmelCase ) for rank in range(__lowerCAmelCase )
		]
		assert sum(len(__lowerCAmelCase ) for ds in datasets_per_rank ) == full_size
		assert len({tuple(x.values() ) for ds in datasets_per_rank for x in ds} ) == full_size






def 			__lowerCAmelCase			():
		def gen():
				return ({"i": i} for i in range(17 ))

		_UpperCAmelCase		:							Any             =   3
		_UpperCAmelCase		:							Any             =   IterableDataset.from_generator(__lowerCAmelCase )
		_UpperCAmelCase		:							str             =   len(list(__lowerCAmelCase ) )
		_UpperCAmelCase		:							Tuple             =   [
		    split_dataset_by_node(__lowerCAmelCase   ,			rank=__lowerCAmelCase   ,			world_size=__lowerCAmelCase ) for rank in range(__lowerCAmelCase )
		]
		assert sum(len(list(__lowerCAmelCase ) ) for ds in datasets_per_rank ) == full_size
		assert len({tuple(x.values() ) for ds in datasets_per_rank for x in ds} ) == full_size






@pytest.mark.parametrize("shards_per_node"   ,			[1, 2, 3] )
def 			__lowerCAmelCase			(__lowerCAmelCase ):
		def gen(__lowerCAmelCase ):
				for shard in shards:
						yield from ({"i": i, "shard": shard} for i in range(17 ))

		_UpperCAmelCase		:							Tuple             =   3
		_UpperCAmelCase		:							Tuple             =   shards_per_node * world_size
		_UpperCAmelCase		:							Optional[int]             =   {"shards": [F"""shard_{shard_idx}.txt""" for shard_idx in range(__lowerCAmelCase )]}
		_UpperCAmelCase		:							int             =   IterableDataset.from_generator(__lowerCAmelCase   ,			gen_kwargs=__lowerCAmelCase )
		_UpperCAmelCase		:							Any             =   len(list(__lowerCAmelCase ) )
		assert full_ds.n_shards == world_size * shards_per_node
		_UpperCAmelCase		:							Optional[Any]             =   [
		    split_dataset_by_node(__lowerCAmelCase   ,			rank=__lowerCAmelCase   ,			world_size=__lowerCAmelCase ) for rank in range(__lowerCAmelCase )
		]
		assert [ds.n_shards for ds in datasets_per_rank] == [shards_per_node] * world_size
		assert sum(len(list(__lowerCAmelCase ) ) for ds in datasets_per_rank ) == full_size
		assert len({tuple(x.values() ) for ds in datasets_per_rank for x in ds} ) == full_size






def 			__lowerCAmelCase			():
		def gen():
				return ({"i": i} for i in range(17 ))

		_UpperCAmelCase		:							List[str]             =   2
		_UpperCAmelCase		:							int             =   IterableDataset.from_generator(__lowerCAmelCase )
		_UpperCAmelCase		:							Union[str, Any]             =   len(list(__lowerCAmelCase ) )

		_UpperCAmelCase		:							int             =   split_dataset_by_node(__lowerCAmelCase   ,			rank=0   ,			world_size=__lowerCAmelCase ).shuffle(seed=42 )
		assert len(list(__lowerCAmelCase ) ) == 1 + full_size // world_size
		with pytest.raises(__lowerCAmelCase ):
				split_dataset_by_node(__lowerCAmelCase   ,			rank=0   ,			world_size=__lowerCAmelCase ).shuffle()

		_UpperCAmelCase		:							Dict             =   split_dataset_by_node(full_ds.shuffle(seed=42 )   ,			rank=0   ,			world_size=__lowerCAmelCase )
		assert len(list(__lowerCAmelCase ) ) == 1 + full_size // world_size
		with pytest.raises(__lowerCAmelCase ):
				split_dataset_by_node(full_ds.shuffle()   ,			rank=0   ,			world_size=__lowerCAmelCase )






@pytest.mark.parametrize("streaming"   ,			[False, True] )
@require_torch
@pytest.mark.skipif(os.name == "nt"   ,			reason="execute_subprocess_async doesn't support windows" )
@pytest.mark.integration
def 			__lowerCAmelCase			(__lowerCAmelCase ):
		_UpperCAmelCase		:							Union[str, Any]             =   2
		_UpperCAmelCase		:							Dict             =   get_torch_dist_unique_port()
		_UpperCAmelCase		:							int             =   Path(__lowerCAmelCase ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
		_UpperCAmelCase		:							List[Any]             =   F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
		_UpperCAmelCase		:							List[str]             =   F"""
        --streaming={streaming}
    """.split()
		_UpperCAmelCase		:							Tuple             =   [sys.executable] + distributed_args + args
		execute_subprocess_async(__lowerCAmelCase   ,			env=os.environ.copy() )






@pytest.mark.parametrize(
    "nproc_per_node, num_workers"   ,			[
        (2, 2),  # each node has 2 shards and each worker has 1 shards
        (3, 2),  # each node uses all the shards but skips examples, and each worker has 2 shards
    ]   ,			)
@require_torch
@pytest.mark.skipif(os.name == "nt"   ,			reason="execute_subprocess_async doesn't support windows" )
@pytest.mark.integration
def 			__lowerCAmelCase			(__lowerCAmelCase   ,			__lowerCAmelCase ):
		_UpperCAmelCase		:							Tuple             =   True
		_UpperCAmelCase		:							int             =   get_torch_dist_unique_port()
		_UpperCAmelCase		:							Optional[int]             =   Path(__lowerCAmelCase ).resolve().parent / "distributed_scripts" / "launch_torch_distributed.py"
		_UpperCAmelCase		:							List[str]             =   F"""
        -m torch.distributed.launch
        --nproc_per_node={nproc_per_node}
        --master_port={master_port}
        {test_script}
    """.split()
		_UpperCAmelCase		:							int             =   F"""
        --streaming={streaming}
        --num_workers={num_workers}
    """.split()
		_UpperCAmelCase		:							Optional[int]             =   [sys.executable] + distributed_args + args
		execute_subprocess_async(__lowerCAmelCase   ,			env=os.environ.copy() )



