
'''simple docstring'''





import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
a__		: Optional[int]  							=							"\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n"

# TODO: Add description of the dataset here
# You can copy an official description
a__		: int  							=							"\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n"

# TODO: Add a link to an official homepage for the dataset here
a__		: Dict  							=							""

# TODO: Add the licence for the dataset here if you can find it
a__		: Dict  							=							""

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
a__		: Optional[int]  							=							{
    "first_domain": "https://huggingface.co/great-new-dataset-first_domain.zip",
    "second_domain": "https://huggingface.co/great-new-dataset-second_domain.zip",
}


class 			UpperCamelCase__	(     datasets.GeneratorBasedBuilder):
 UpperCAmelCase__     :     Tuple    					=       datasets.Version('1.1.0')

 # This is an example of a dataset with multiple configurations.
 # If you don't want/need to define several sub-sets in your dataset,
 # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

 # If you need to make complex sub-parts in the datasets with configurable options
 # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
 # BUILDER_CONFIG_CLASS = MyBuilderConfig

 # You will be able to load one or the other configurations in the following list with
 # data = datasets.load_dataset('my_dataset', 'first_domain')
 # data = datasets.load_dataset('my_dataset', 'second_domain')
 UpperCAmelCase__     :     Optional[int]    					=       [
     datasets.BuilderConfig(name='first_domain'   ,  version=SCREAMING_SNAKE_CASE   ,  description='This part of my dataset covers a first domain'),
     datasets.BuilderConfig(name='second_domain'   ,  version=SCREAMING_SNAKE_CASE   ,  description='This part of my dataset covers a second domain'),
 ]

 UpperCAmelCase__     :     Dict    					=       'first_domain'  # It's not mandatory to have a default configuration. Just use one if it make sense.






 def 						lowercase_    (      self	:str      )					->      Optional[Any]:







       '''simple docstring'''



       if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
             __A													=      datasets.Features(
                 {
                     'sentence': datasets.Value('string'      ),
                     'option1': datasets.Value('string'      ),
                     'answer': datasets.Value('string'      )
                     # These are the features of your dataset like images, labels ...
                 }      )
       else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
             __A													=      datasets.Features(
                 {
                     'sentence': datasets.Value('string'      ),
                     'option2': datasets.Value('string'      ),
                     'second_domain_answer': datasets.Value('string'      )
                     # These are the features of your dataset like images, labels ...
                 }      )
       return datasets.DatasetInfo(
           # This is the description that will appear on the datasets page.
           description=_DESCRIPTION						,					features=_A						,					homepage=_HOMEPAGE						,					license=_LICENSE						,					citation=_CITATION						,					)






 def 						lowercase_    (      self	:Optional[Any]						,					_A	:str      )					->      Union[str, Any]:







       '''simple docstring'''



       __A													=      _URLS[self.config.name]
       __A													=      dl_manager.download_and_extract(_A      )
       return [
           datasets.SplitGenerator(
               name=datasets.Split.TRAIN						,					gen_kwargs={
                   'filepath': os.path.join(_A						,					'train.jsonl'      ),
                   'split': 'train',
               }						,					),
           datasets.SplitGenerator(
               name=datasets.Split.VALIDATION						,					gen_kwargs={
                   'filepath': os.path.join(_A						,					'dev.jsonl'      ),
                   'split': 'dev',
               }						,					),
           datasets.SplitGenerator(
               name=datasets.Split.TEST						,					gen_kwargs={
                   'filepath': os.path.join(_A						,					'test.jsonl'      ),
                   'split': 'test'
               }						,					),
       ]






 def 						lowercase_    (      self	:str						,					_A	:str						,					_A	:Union[str, Any]      )					->      Union[str, Any]:







       '''simple docstring'''



       with open(_A						,					encoding='utf-8'      ) as f:
             for key, row in enumerate(_A      ):
                   __A													=      json.loads(_A      )
                   if self.config.name == "first_domain":
                         # Yields examples as (key, example) tuples
                         yield key, {
                             "sentence": data["sentence"],
                             "option1": data["option1"],
                             "answer": "" if split == "test" else data["answer"],
                         }
                   else:
                         yield key, {
                             "sentence": data["sentence"],
                             "option2": data["option2"],
                             "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
                         }












'''simple docstring'''






import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class    A						(					SCREAMING_SNAKE_CASE__     ,				unittest.TestCase						):




				'''simple docstring'''





				A       =							BertweetTokenizer
				A       =							False
				def 			a_			(self     )      ->			Any:
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							__UpperCamelCase :     str										=  ["I", "m", "V@@", "R@@", "r", "e@@"]
							__UpperCamelCase :     int										=  dict(zip(_UpperCAmelCase					,    range(len(_UpperCAmelCase     )     )     )     )
							__UpperCamelCase :     List[Any]										=  ["#version: 0.2", "a m</w>"]
							__UpperCamelCase :     str										=  {"unk_token": "<unk>"}

							__UpperCamelCase :     Any										=  os.path.join(self.tmpdirname					,    VOCAB_FILES_NAMES["vocab_file"]     )
							__UpperCamelCase :     Tuple										=  os.path.join(self.tmpdirname					,    VOCAB_FILES_NAMES["merges_file"]     )
							with open(self.vocab_file					,    "w"					,    encoding="utf-8"     ) as fp:
										for token in vocab_tokens:
													fp.write(f"{token} {vocab_tokens[token]}\n"     )
							with open(self.merges_file					,    "w"					,    encoding="utf-8"     ) as fp:
										fp.write("\n".join(_UpperCAmelCase     )     )
				def 			a_			(self					,    **_UpperCAmelCase     )      ->			str:
							kwargs.update(self.special_tokens_map     )
							return BertweetTokenizer.from_pretrained(self.tmpdirname					,    **_UpperCAmelCase     )
				def 			a_			(self					,    _UpperCAmelCase     )      ->			Dict:
							__UpperCamelCase :     Dict										=  "I am VinAI Research"
							__UpperCamelCase :     Any										=  "I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
							return input_text, output_text





				def 			a_			(self     )      ->			str:
							__UpperCamelCase :     Tuple										=  BertweetTokenizer(self.vocab_file					,    self.merges_file					,    **self.special_tokens_map     )
							__UpperCamelCase :     Union[str, Any]										=  "I am VinAI Research"
							__UpperCamelCase :     List[str]										=  "I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
							__UpperCamelCase :     Optional[int]										=  tokenizer.tokenize(_UpperCAmelCase     )
							self.assertListEqual(_UpperCAmelCase					,    _UpperCAmelCase     )

							__UpperCamelCase :     Optional[Any]										=  tokens + [tokenizer.unk_token]

							__UpperCamelCase :     Any										=  [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase     )					,    _UpperCAmelCase     )



