





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam


class    __SCREAMING_SNAKE_CASE			( datasets.BeamBasedBuilder					):







       def 						__lowerCamelCase    (      self     :	List[Any]			)   ->Optional[int]:
          return datasets.DatasetInfo(
              features=datasets.Features({'''content''': datasets.Value('''string'''			)}			)		,					supervised_keys=A		,					)







       def 						__lowerCamelCase    (      self     :	Optional[Any]		,					A     :	int		,					A     :	Tuple			)   ->Dict:
          return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,					gen_kwargs={'''examples''': get_test_dummy_examples()}			)]







       def 						__lowerCamelCase    (      self     :	Any		,					A     :	Dict		,					A     :	Union[str, Any]			)   ->Optional[int]:
          import apache_beam as beam

          return pipeline | "Load Examples" >> beam.Create(A			)


class    __SCREAMING_SNAKE_CASE			( datasets.BeamBasedBuilder					):







       def 						__lowerCamelCase    (      self     :	str			)   ->Optional[int]:
          return datasets.DatasetInfo(
              features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''			)}			)}			)		,					supervised_keys=A		,					)







       def 						__lowerCamelCase    (      self     :	Optional[int]		,					A     :	str		,					A     :	Optional[Any]			)   ->int:
          return [
              datasets.SplitGenerator(name=datasets.Split.TRAIN		,					gen_kwargs={'''examples''': get_test_nested_examples()}			)
          ]







       def 						__lowerCamelCase    (      self     :	Union[str, Any]		,					A     :	Dict		,					A     :	int			)   ->Optional[Any]:
          import apache_beam as beam

          return pipeline | "Load Examples" >> beam.Create(A			)



def        _a						(    )				->				Any:







   """simple docstring"""
   return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']     )]



def        _a						(    )				->				List[Any]:







   """simple docstring"""
   return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar''']     )]




class    __SCREAMING_SNAKE_CASE			( lowerCAmelCase_					):







       @require_beam
       def 						__lowerCamelCase    (      self     :	str			)   ->List[Any]:
          lowerCamelCase__   :						Dict					   =							len(get_test_dummy_examples()			)
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
             lowerCamelCase__   :						List[str]					   =							DummyBeamDataset(cache_dir=A		,					beam_runner='''DirectRunner'''			)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					F"{builder.name}-train.arrow"			)			)			)
             self.assertDictEqual(builder.info.features		,					datasets.Features({'''content''': datasets.Value('''string'''			)}			)			)
             lowerCamelCase__   :						Optional[Any]					   =							builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows		,					A			)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,					A			)
             self.assertDictEqual(dset['''train'''][0]		,					get_test_dummy_examples()[0][1]			)
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]		,					get_test_dummy_examples()[expected_num_examples - 1][1]			)
             self.assertTrue(
                 os.path.exists(os.path.join(A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					'''dataset_info.json'''			)			)			)
             del dset







       @require_beam
       def 						__lowerCamelCase    (      self     :	Optional[int]			)   ->int:
          import apache_beam as beam

          lowerCamelCase__   :						str					   =							beam.io.parquetio.WriteToParquet

          lowerCamelCase__   :						List[Any]					   =							len(get_test_dummy_examples()			)
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
             lowerCamelCase__   :						Union[str, Any]					   =							DummyBeamDataset(cache_dir=A		,					beam_runner='''DirectRunner'''			)
             with patch('''apache_beam.io.parquetio.WriteToParquet'''			) as write_parquet_mock:
                lowerCamelCase__   :						int					   =							partial(A		,					num_shards=2			)
                builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					F"{builder.name}-train-00000-of-00002.arrow"			)			)			)
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					F"{builder.name}-train-00000-of-00002.arrow"			)			)			)
             self.assertDictEqual(builder.info.features		,					datasets.Features({'''content''': datasets.Value('''string'''			)}			)			)
             lowerCamelCase__   :						List[Any]					   =							builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows		,					A			)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,					A			)
             # Order is not preserved when sharding, so we just check that all the elements are there
             self.assertListEqual(sorted(dset['''train''']['''content''']			)		,					sorted(['''foo''', '''bar''', '''foobar''']			)			)
             self.assertTrue(
                 os.path.exists(os.path.join(A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					'''dataset_info.json'''			)			)			)
             del dset







       @require_beam
       def 						__lowerCamelCase    (      self     :	Dict			)   ->List[Any]:







          with tempfile.TemporaryDirectory() as tmp_cache_dir:
             lowerCamelCase__   :						Dict					   =							DummyBeamDataset(cache_dir=A			)
             self.assertRaises(datasets.builder.MissingBeamOptions		,					builder.download_and_prepare			)







       @require_beam
       def 						__lowerCamelCase    (      self     :	Optional[Any]			)   ->Optional[int]:
          lowerCamelCase__   :						Tuple					   =							len(get_test_nested_examples()			)
          with tempfile.TemporaryDirectory() as tmp_cache_dir:
             lowerCamelCase__   :						List[str]					   =							NestedBeamDataset(cache_dir=A		,					beam_runner='''DirectRunner'''			)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					F"{builder.name}-train.arrow"			)			)			)
             self.assertDictEqual(
                 builder.info.features		,					datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''			)}			)}			)			)
             lowerCamelCase__   :						Tuple					   =							builder.as_dataset()
             self.assertEqual(dset['''train'''].num_rows		,					A			)
             self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples		,					A			)
             self.assertDictEqual(dset['''train'''][0]		,					get_test_nested_examples()[0][1]			)
             self.assertDictEqual(
                 dset['''train'''][expected_num_examples - 1]		,					get_test_nested_examples()[expected_num_examples - 1][1]			)
             self.assertTrue(
                 os.path.exists(os.path.join(A		,					builder.name		,					'''default'''		,					'''0.0.0'''		,					'''dataset_info.json'''			)			)			)
             del dset









import argparse

import pytorch_lightning as pl
import torch
from torch import nn

from transformers import LongformerForQuestionAnswering, LongformerModel

class 		__lowerCAmelCase	(	pl.LightningModule ):
							def __init__(						self						:Union[str, Any]		,      __magic_name__						:Optional[int]				):







								'''simple docstring'''







								super().__init__()
								a										=							model
								a										=							2
								a										=							nn.Linear(self.model.config.hidden_size		,      self.num_labels				)





							def       lowerCamelCase__							(						self						:Dict				):







								'''simple docstring'''







								pass







def 		__A		(	__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase    )      ->       Optional[int]:
	# load longformer model from model identifier
	a										=							LongformerModel.from_pretrained(__lowerCamelCase    )
	a										=							LightningModel(__lowerCamelCase    )

	a										=							torch.load(__lowerCamelCase							,			map_location=torch.device("""cpu"""    )    )
	lightning_model.load_state_dict(ckpt["""state_dict"""]    )

	# init longformer question answering model
	a										=							LongformerForQuestionAnswering.from_pretrained(__lowerCamelCase    )

	# transfer weights
	longformer_for_qa.longformer.load_state_dict(lightning_model.model.state_dict()    )
	longformer_for_qa.qa_outputs.load_state_dict(lightning_model.qa_outputs.state_dict()    )
	longformer_for_qa.eval()

	# save model
	longformer_for_qa.save_pretrained(__lowerCamelCase    )

	print(f'Conversion successful. Model saved under {pytorch_dump_folder_path}'    )


if __name__ == "__main__":
		__UpperCamelCase					:       List[Any]    					=      argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--longformer_model",
		    default=None,
		    type=str,
		    required=True,
		    help="model identifier of longformer. Should be either `longformer-base-4096` or `longformer-large-4096`.",
		)
		parser.add_argument(
		    "--longformer_question_answering_ckpt_path",
		    default=None,
		    type=str,
		    required=True,
		    help="Path the official PyTorch Lightning Checkpoint.",
		)
		parser.add_argument(
		    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
		)
		__UpperCamelCase					:       List[Any]    					=      parser.parse_args()
		convert_longformer_qa_checkpoint_to_pytorch(
		    args.longformer_model, args.longformer_question_answering_ckpt_path, args.pytorch_dump_folder_path
		)

