code: 
  
  import json
  import os
  from functools import lru_cache
  from typing import TYPE_CHECKING, List, Optional, Tuple, Union
  
  import numpy as np
  import regex as re
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  if TYPE_CHECKING:
         if is_torch_available():
                import torch
         if is_tf_available():
                import tensorflow as tf
  
  from ...tokenization_utils import AddedToken, PreTrainedTokenizer
  
  
  A_		:			Optional[int]										=   logging.get_logger(__name__)
  
  A_		:			Dict										=   {
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  A_		:			List[str]										=   {
      'vocab_file': {
          'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
      },
      'merges_file': {
          'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
      },
  }
  
  A_		:			str										=   {
      'Salesforce/codegen-350M-mono': 2048,
  }
  
  
  @lru_cache()
  def       __a       (   )      ->     Optional[Any]:
  
  
  
  
  
  
      '''simple docstring'''
      __UpperCAmelCase									=			(
          list(range(ord('''!'''      )       ,   ord('''~'''      ) + 1      )      ) + list(range(ord('''¡'''      )       ,   ord('''¬'''      ) + 1      )      ) + list(range(ord('''®'''      )       ,   ord('''ÿ'''      ) + 1      )      )
      )
      __UpperCAmelCase									=			bs[:]
      __UpperCAmelCase									=			0
      for b in range(2**8      ):
          if b not in bs:
              bs.append(lowercase__      )
              cs.append(2**8 + n      )
              n += 1
      __UpperCAmelCase									=			[chr(lowercase__      ) for n in cs]
      return dict(zip(lowercase__       ,   lowercase__      )      )
  
  
  def       __a       (   SCREAMING_SNAKE_CASE      )      ->     Dict:
  
  
  
  
  
  
      '''simple docstring'''
      __UpperCAmelCase									=			set()
      __UpperCAmelCase									=			word[0]
      for char in word[1:]:
          pairs.add((prev_char, char)      )
          __UpperCAmelCase									=			char
      return pairs
  class 					A_   (      A_						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       VOCAB_FILES_NAMES
      a__     							=       PRETRAINED_VOCAB_FILES_MAP
      a__     							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a__     							=       ['''input_ids''', '''attention_mask''']
  
  
      def __init__(self     ,      lowercase__     ,      lowercase__     ,      lowercase__="replace"     ,      lowercase__="<|endoftext|>"     ,      lowercase__="<|endoftext|>"     ,      lowercase__="<|endoftext|>"     ,      lowercase__=None     ,      lowercase__=False     ,      lowercase__=False     ,      **lowercase__     ,      )     ->      Optional[Any]:
          __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else bos_token
          __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else eos_token
          __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else unk_token
          __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else pad_token
          super().__init__(
              errors=lowercase__     ,      unk_token=lowercase__     ,      bos_token=lowercase__     ,      eos_token=lowercase__     ,      pad_token=lowercase__     ,      add_prefix_space=lowercase__     ,      add_bos_token=lowercase__     ,      **lowercase__     ,      )
          __UpperCAmelCase									=			add_bos_token
  
          with open(lowercase__     ,      encoding='''utf-8'''  ) as vocab_handle:
              __UpperCAmelCase									=			json.load(lowercase__  )
          __UpperCAmelCase									=			{v: k for k, v in self.encoder.items()}
          __UpperCAmelCase									=			errors  # how to handle errors in decoding
          __UpperCAmelCase									=			bytes_to_unicode()
          __UpperCAmelCase									=			{v: k for k, v in self.byte_encoder.items()}
          with open(lowercase__     ,      encoding='''utf-8'''  ) as merges_handle:
              __UpperCAmelCase									=			merges_handle.read().split('''\n'''  )[1:-1]
          __UpperCAmelCase									=			[tuple(merge.split()  ) for merge in bpe_merges]
          __UpperCAmelCase									=			dict(zip(lowercase__     ,      range(len(lowercase__  )  )  )  )
          __UpperCAmelCase									=			{}
          __UpperCAmelCase									=			add_prefix_space
  
          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          __UpperCAmelCase									=			re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''  )
  
  
      @property
      def      lowerCAmelCase_						(self  )     ->      List[str]:
          return len(self.encoder  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
          return dict(self.encoder     ,      **self.added_tokens_encoder  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      str:
          if token in self.cache:
              return self.cache[token]
          __UpperCAmelCase									=			tuple(lowercase__  )
          __UpperCAmelCase									=			get_pairs(lowercase__  )
  
          if not pairs:
              return token
  
          while True:
              __UpperCAmelCase									=			min(lowercase__     ,      key=lambda lowercase__  : self.bpe_ranks.get(lowercase__     ,      float('''inf'''  )  )  )
              if bigram not in self.bpe_ranks:
                  break
              __UpperCAmelCase					,     __UpperCAmelCase									=			bigram
              __UpperCAmelCase									=			[]
              __UpperCAmelCase									=			0
              while i < len(lowercase__  ):
                  try:
                      __UpperCAmelCase									=			word.index(lowercase__     ,      lowercase__  )
                  except ValueError:
                      new_word.extend(word[i:]  )
                      break
                  else:
                      new_word.extend(word[i:j]  )
                      __UpperCAmelCase									=			j
  
                  if word[i] == first and i < len(lowercase__  ) - 1 and word[i + 1] == second:
                      new_word.append(first + second  )
                      i += 2
                  else:
                      new_word.append(word[i]  )
                      i += 1
              __UpperCAmelCase									=			tuple(lowercase__  )
              __UpperCAmelCase									=			new_word
              if len(lowercase__  ) == 1:
                  break
              else:
                  __UpperCAmelCase									=			get_pairs(lowercase__  )
          __UpperCAmelCase									=			''' '''.join(lowercase__  )
          __UpperCAmelCase									=			word
          return word
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__=None  )     ->      int:
          if self.add_bos_token:
              __UpperCAmelCase									=			[self.bos_token_id]
          else:
              __UpperCAmelCase									=			[]
  
          __UpperCAmelCase									=			bos_token_ids + token_ids_a
  
          if token_ids_a is None:
              return output
  
          return output + bos_token_ids + token_ids_a
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[str]:
          __UpperCAmelCase									=			[]
          for token in re.findall(self.pat     ,      lowercase__  ):
              __UpperCAmelCase									=			''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8'''  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase__  ).split(''' '''  )  )
          return bpe_tokens
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[Any]:
          return self.encoder.get(lowercase__     ,      self.encoder.get(self.unk_token  )  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Any:
          return self.decoder.get(lowercase__  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      str:
          __UpperCAmelCase									=			''''''.join(lowercase__  )
          __UpperCAmelCase									=			bytearray([self.byte_decoder[c] for c in text]  ).decode('''utf-8'''     ,      errors=self.errors  )
          return text
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__ = None  )     ->      Optional[int]:
          if not os.path.isdir(lowercase__  ):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
              return
          __UpperCAmelCase									=			os.path.join(
              lowercase__     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
          __UpperCAmelCase									=			os.path.join(
              lowercase__     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )
  
          with open(lowercase__     ,      '''w'''     ,      encoding='''utf-8'''  ) as f:
              f.write(json.dumps(self.encoder     ,      indent=2     ,      sort_keys=lowercase__     ,      ensure_ascii=lowercase__  ) + '''\n'''  )
  
          __UpperCAmelCase									=			0
          with open(lowercase__     ,      '''w'''     ,      encoding='''utf-8'''  ) as writer:
              writer.write('''#version: 0.2\n'''  )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,      key=lambda lowercase__  : kv[1]  ):
                  if index != token_index:
                      logger.warning(
                          F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                          ''' Please check that the tokenizer is not corrupted!'''  )
                      __UpperCAmelCase									=			token_index
                  writer.write(''' '''.join(lowercase__  ) + '''\n'''  )
                  index += 1
  
          return vocab_file, merge_file
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__=False     ,      **lowercase__  )     ->      List[str]:
          __UpperCAmelCase									=			kwargs.pop('''add_prefix_space'''     ,      self.add_prefix_space  )
          if is_split_into_words or add_prefix_space:
              __UpperCAmelCase									=			''' ''' + text
          return (text, kwargs)
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__ = False     ,      lowercase__ = None     ,      lowercase__ = None     ,      **lowercase__     ,      )     ->      Union[str, Any]:
          __UpperCAmelCase									=			super()._decode(
              token_ids=lowercase__     ,      skip_special_tokens=lowercase__     ,      clean_up_tokenization_spaces=lowercase__     ,      **lowercase__     ,      )
  
          if truncate_before_pattern is not None and len(lowercase__  ) > 0:
              __UpperCAmelCase									=			self.truncate(lowercase__     ,      lowercase__  )
  
          return decoded_text
  
  
  
  
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__  )     ->      List[Any]:
          def find_re(lowercase__     ,      lowercase__     ,      lowercase__  ):
              __UpperCAmelCase									=			pattern.search(lowercase__     ,      lowercase__  )
              return m.start() if m else -1
  
          __UpperCAmelCase									=			[re.compile(lowercase__     ,      re.MULTILINE  ) for pattern in truncate_before_pattern]
  
          __UpperCAmelCase									=			list(re.finditer('''^print'''     ,      lowercase__     ,      re.MULTILINE  )  )
  
          if len(lowercase__  ) > 1:
              __UpperCAmelCase									=			completion[: prints[1].start()]
  
          __UpperCAmelCase									=			list(re.finditer('''^def'''     ,      lowercase__     ,      re.MULTILINE  )  )
  
          if len(lowercase__  ) > 1:
              __UpperCAmelCase									=			completion[: defs[1].start()]
  
          __UpperCAmelCase									=			0
  
          __UpperCAmelCase									=			[
              pos for pos in [find_re(lowercase__     ,      lowercase__     ,      lowercase__  ) for terminal in terminals] if pos != -1
          ]
  
          if len(lowercase__  ) > 0:
              return completion[: min(lowercase__  )]
          else:
              return completion
  
  
  
  
  
style_context: 
  
  import csv
  import json
  import os
  
  import datasets
  
  
  # TODO: Add BibTeX citation
  # Find for instance the citation on arxiv or on the dataset repo/website
  A_		:			Any										=   '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'
  
  # TODO: Add description of the dataset here
  # You can copy an official description
  A_		:			Optional[Any]										=   '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'
  
  # TODO: Add a link to an official homepage for the dataset here
  A_		:			Dict										=   ''
  
  # TODO: Add the licence for the dataset here if you can find it
  A_		:			Dict										=   ''
  
  # TODO: Add link to the official dataset URLs here
  # The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
  # This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
  A_		:			Union[str, Any]										=   {
      'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
      'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
  }
  class 					A_   (      datasets.GeneratorBasedBuilder						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       datasets.Version("1.1.0"						)
  
      # This is an example of a dataset with multiple configurations.
      # If you don't want/need to define several sub-sets in your dataset,
      # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.
  
      # If you need to make complex sub-parts in the datasets with configurable options
      # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
      # BUILDER_CONFIG_CLASS = MyBuilderConfig
  
      # You will be able to load one or the other configurations in the following list with
      # data = datasets.load_dataset('my_dataset', 'first_domain')
      # data = datasets.load_dataset('my_dataset', 'second_domain')
      a__     							=       [
          datasets.BuilderConfig(name="first_domain" ,     version=_a ,     description="This part of my dataset covers a first domain"						),
          datasets.BuilderConfig(name="second_domain" ,     version=_a ,     description="This part of my dataset covers a second domain"						),
      ]
  
      a__     							=       "first_domain"  # It's not mandatory to have a default configuration. Just use one if it make sense.
  
  
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
          if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
              __UpperCAmelCase									=			datasets.Features(
                  {
                      '''sentence''': datasets.Value('''string'''  ),
                      '''option1''': datasets.Value('''string'''  ),
                      '''answer''': datasets.Value('''string'''  )
                      # These are the features of your dataset like images, labels ...
                  }  )
          else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
              __UpperCAmelCase									=			datasets.Features(
                  {
                      '''sentence''': datasets.Value('''string'''  ),
                      '''option2''': datasets.Value('''string'''  ),
                      '''second_domain_answer''': datasets.Value('''string'''  )
                      # These are the features of your dataset like images, labels ...
                  }  )
          return datasets.DatasetInfo(
              # This is the description that will appear on the datasets page.
              description=_DESCRIPTION     ,      features=lowercase__     ,      homepage=_HOMEPAGE     ,      license=_LICENSE     ,      citation=_CITATION     ,      )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      int:
          # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
          # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name
  
          # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
          # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
          # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
          __UpperCAmelCase									=			_URLS[self.config.name]
          __UpperCAmelCase									=			dl_manager.download_and_extract(lowercase__  )
          return [
              datasets.SplitGenerator(
                  name=datasets.Split.TRAIN     ,      gen_kwargs={
                      '''filepath''': os.path.join(lowercase__     ,      '''train.jsonl'''  ),
                      '''split''': '''train''',
                  }     ,      ),
              datasets.SplitGenerator(
                  name=datasets.Split.VALIDATION     ,      gen_kwargs={
                      '''filepath''': os.path.join(lowercase__     ,      '''dev.jsonl'''  ),
                      '''split''': '''dev''',
                  }     ,      ),
              datasets.SplitGenerator(
                  name=datasets.Split.TEST     ,      gen_kwargs={
                      '''filepath''': os.path.join(lowercase__     ,      '''test.jsonl'''  ),
                      '''split''': '''test'''
                  }     ,      ),
          ]
  
  
  
  
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__  )     ->      Tuple:
          # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
          # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
          with open(lowercase__     ,      encoding='''utf-8'''  ) as f:
              for key, row in enumerate(lowercase__  ):
                  __UpperCAmelCase									=			json.loads(lowercase__  )
                  if self.config.name == "first_domain":
                      # Yields examples as (key, example) tuples
                      yield key, {
                          "sentence": data["sentence"],
                          "option1": data["option1"],
                          "answer": "" if split == "test" else data["answer"],
                      }
                  else:
                      yield key, {
                          "sentence": data["sentence"],
                          "option2": data["option2"],
                          "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
                      }
  
  
  
  
  