code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import csv
  import json
  import os
  
  import datasets
  
  
  # TODO: Add BibTeX citation
  # Find for instance the citation on arxiv or on the dataset repo/website
  _lowerCamelCase				:	List[str]				    =       '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'
  
  # TODO: Add description of the dataset here
  # You can copy an official description
  _lowerCamelCase				:	Any				    =       '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'
  
  # TODO: Add a link to an official homepage for the dataset here
  _lowerCamelCase				:	Union[str, Any]				    =       ''
  
  # TODO: Add the licence for the dataset here if you can find it
  _lowerCamelCase				:	Union[str, Any]				    =       ''
  
  # TODO: Add link to the official dataset URLs here
  # The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
  # This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
  _lowerCamelCase				:	int				    =       {
      'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
      'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
  }
  
  
  
  class 							__UpperCAmelCase  (       datasets.GeneratorBasedBuilder						):
  
  
  
  
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
  
    __lowerCAmelCase 				=    datasets.Version('''1.1.0'''						)
  
    # This is an example of a dataset with multiple configurations.
    # If you don't want/need to define several sub-sets in your dataset,
    # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.
  
    # If you need to make complex sub-parts in the datasets with configurable options
    # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
    # BUILDER_CONFIG_CLASS = MyBuilderConfig
  
    # You will be able to load one or the other configurations in the following list with
    # data = datasets.load_dataset('my_dataset', 'first_domain')
    # data = datasets.load_dataset('my_dataset', 'second_domain')
    __lowerCAmelCase 				=    [
        datasets.BuilderConfig(name='''first_domain'''		,					version=A__		,					description='''This part of my dataset covers a first domain'''						),
        datasets.BuilderConfig(name='''second_domain'''		,					version=A__		,					description='''This part of my dataset covers a second domain'''						),
    ]
  
    __lowerCAmelCase 				=    '''first_domain'''  # It's not mandatory to have a default configuration. Just use one if it make sense.
  
  
  
  
  
  
  
    def        A     (self						:      Optional[Any]      ):
     # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
     if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
      A							  =      datasets.Features(
          {
              """sentence""": datasets.Value("""string"""      ),
              """option1""": datasets.Value("""string"""      ),
              """answer""": datasets.Value("""string"""      )
              # These are the features of your dataset like images, labels ...
          }      )
     else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
      A							  =      datasets.Features(
          {
              """sentence""": datasets.Value("""string"""      ),
              """option2""": datasets.Value("""string"""      ),
              """second_domain_answer""": datasets.Value("""string"""      )
              # These are the features of your dataset like images, labels ...
          }      )
     return datasets.DatasetInfo(
         # This is the description that will appear on the datasets page.
         description=_DESCRIPTION      ,      features=_lowerCAmelCase      ,      homepage=_HOMEPAGE      ,      license=_LICENSE      ,      citation=_CITATION      ,      )
  
  
  
  
  
  
  
    def        A     (self						:      str      ,      _lowerCAmelCase						:      List[Any]      ):
     # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
     # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
  
     # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
     # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
     # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
     A							  =      _URLS[self.config.name]
     A							  =      dl_manager.download_and_extract(_lowerCAmelCase      )
     return [
         datasets.SplitGenerator(
             name=datasets.Split.TRAIN      ,      gen_kwargs={
                 """filepath""": os.path.join(_lowerCAmelCase      ,      """train.jsonl"""      ),
                 """split""": """train""",
             }      ,      ),
         datasets.SplitGenerator(
             name=datasets.Split.VALIDATION      ,      gen_kwargs={
                 """filepath""": os.path.join(_lowerCAmelCase      ,      """dev.jsonl"""      ),
                 """split""": """dev""",
             }      ,      ),
         datasets.SplitGenerator(
             name=datasets.Split.TEST      ,      gen_kwargs={
                 """filepath""": os.path.join(_lowerCAmelCase      ,      """test.jsonl"""      ),
                 """split""": """test"""
             }      ,      ),
     ]
  
  
  
  
  
  
  
    def        A     (self						:      Any      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      List[str]      ):
     # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
     # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
     with open(_lowerCAmelCase      ,      encoding="""utf-8"""      ) as f:
      for key, row in enumerate(_lowerCAmelCase      ):
       A							  =      json.loads(_lowerCAmelCase      )
       if self.config.name == "first_domain":
        # Yields examples as (key, example) tuples
        yield key, {
            "sentence": data["sentence"],
            "option1": data["option1"],
            "answer": "" if split == "test" else data["answer"],
        }
       else:
        yield key, {
            "sentence": data["sentence"],
            "option2": data["option2"],
            "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
        }
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  from typing import Dict, List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  from ...utils import logging
  
  
  _lowerCamelCase				:	int				    =       logging.get_logger(__name__)
  
  
  
  class 							__UpperCAmelCase  (       A__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowerCAmelCase 				=    ['''input_ids''', '''attention_mask''']
  
  
  
  
  
  
  
  		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      str="[PAD]"      ,      _lowerCAmelCase						:      Dict="[BOS]"      ,      _lowerCAmelCase						:      Any="[EOS]"      ,      _lowerCAmelCase						:      List[Any]="[MASK]"      ,      _lowerCAmelCase						:      int="[CLS]"      ,      _lowerCAmelCase						:      Optional[Any]="[SEP]"      ,      _lowerCAmelCase						:      List[str]=2048      ,      **_lowerCAmelCase						:      Dict      ,      ):
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else pad_token
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else bos_token
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else eos_token
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else mask_token
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else cls_token
  			A							  =      AddedToken(_lowerCAmelCase      ,      lstrip=_lowerCAmelCase      ,      rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase      ,      _lowerCAmelCase      ) else sep_token
  
  			super().__init__(
  			    pad_token=_lowerCAmelCase      ,      bos_token=_lowerCAmelCase      ,      eos_token=_lowerCAmelCase      ,      mask_token=_lowerCAmelCase      ,      cls_token=_lowerCAmelCase      ,      sep_token=_lowerCAmelCase      ,      model_max_length=_lowerCAmelCase      ,      **_lowerCAmelCase      ,      )
  
  			A							  =      2**8  # utf is 8 bits
  
  			# define special tokens dict
  			A							  =      {
  			    self.pad_token: 0,
  			    self.bos_token: 1,
  			    self.eos_token: 2,
  			    self.mask_token: 3,
  			    self.cls_token: 4,
  			    self.sep_token: 5,
  			}
  			A							  =      len(self.special_tokens_encoder      )
  			A							  =      {v: k for k, v in self.special_tokens_encoder.items()}
  
  
  
  
  
  
  
  		def        A     (self						:      Tuple      ):
  			A							  =      self.special_tokens_encoder.copy()
  			vocab.update(self.added_tokens_encoder      )
  			for i in range(self._utf_vocab_size      ):
  				A							  =      chr(_lowerCAmelCase      )
  				A							  =      i + len(self.special_tokens_encoder      )
  			return vocab
  
  
  
  
  
  
  
  		@property
  		def        A     (self						:      Tuple      ):
  			return self._utf_vocab_size + self._num_special_tokens
  
  
  
  
  
  
  
  		def        A     (self						:      Union[str, Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ,      _lowerCAmelCase						:      bool = False      ):
  			if already_has_special_tokens:
  				return super().get_special_tokens_mask(
  				    token_ids_a=_lowerCAmelCase      ,      token_ids_a=_lowerCAmelCase      ,      already_has_special_tokens=_lowerCAmelCase      )
  
  			# normal case: some special tokens
  			if token_ids_a is None:
  				return [1] + [0] * len(_lowerCAmelCase      ) + [1]
  			return [1] + ([0] * len(_lowerCAmelCase      )) + [1] + ([0] * len(_lowerCAmelCase      )) + [1]
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      List[int]      ,      _lowerCAmelCase						:      Optional[List[int]] = None      ):
  			if token_ids_a is None:
  				return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  			else:
  				return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      str      ):
  			A							  =      [chr(_lowerCAmelCase      ) for i in text.encode("""utf-8"""      )]
  			return tokens
  
  
  
  
  
  
  
  		def        A     (self						:      int      ,      _lowerCAmelCase						:      List[str]      ):
  			if token in self.special_tokens_encoder:
  				A							  =      self.special_tokens_encoder[token]
  			elif token in self.added_tokens_encoder:
  				A							  =      self.added_tokens_encoder[token]
  			elif len(_lowerCAmelCase      ) != 1:
  				A							  =      self.unk_token_id
  			else:
  				A							  =      ord(_lowerCAmelCase      ) + self._num_special_tokens
  			return token_id
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Any      ):
  			if index in self.special_tokens_decoder:
  				A							  =      self.special_tokens_decoder[index]
  			elif index in self.added_tokens_decoder:
  				A							  =      self.added_tokens_decoder[index]
  			else:
  				A							  =      chr(index - self._num_special_tokens      )
  			return token
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      List[str]      ):
  			A							  =      b""""""
  			for token in tokens:
  				if token in self.special_tokens_decoder:
  					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
  				elif token in self.added_tokens_decoder:
  					A							  =      self.special_tokens_decoder[token].encode("""utf-8"""      )
  				elif token in self.special_tokens_encoder:
  					A							  =      token.encode("""utf-8"""      )
  				elif token in self.added_tokens_encoder:
  					A							  =      token.encode("""utf-8"""      )
  				else:
  					A							  =      bytes([ord(_lowerCAmelCase      )]      )
  				bstring += tok_string
  			A							  =      bstring.decode("""utf-8"""      ,      errors="""replace"""      )
  			return string
  
  
  
  
  
  
  
  		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Optional[str] = None      ):
  			return ()
  
  