

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
       if is_torch_available():
              import torch
       if is_tf_available():
              import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


A_		:			Optional[int]										=   logging.get_logger(__name__)

A_		:			Dict										=   {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A_		:			List[str]										=   {
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

A_		:			str										=   {
    'Salesforce/codegen-350M-mono': 2048,
}


@lru_cache()
def       __a       (   )      ->     Optional[Any]:






    '''simple docstring'''
    __UpperCAmelCase									=			(
        list(range(ord('''!'''      )       ,   ord('''~'''      ) + 1      )      ) + list(range(ord('''¡'''      )       ,   ord('''¬'''      ) + 1      )      ) + list(range(ord('''®'''      )       ,   ord('''ÿ'''      ) + 1      )      )
    )
    __UpperCAmelCase									=			bs[:]
    __UpperCAmelCase									=			0
    for b in range(2**8      ):
        if b not in bs:
            bs.append(lowercase__      )
            cs.append(2**8 + n      )
            n += 1
    __UpperCAmelCase									=			[chr(lowercase__      ) for n in cs]
    return dict(zip(lowercase__       ,   lowercase__      )      )


def       __a       (   SCREAMING_SNAKE_CASE      )      ->     Dict:






    '''simple docstring'''
    __UpperCAmelCase									=			set()
    __UpperCAmelCase									=			word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)      )
        __UpperCAmelCase									=			char
    return pairs
class 					A_   (      A_						):


    '''simple docstring'''






    a__     							=       VOCAB_FILES_NAMES
    a__     							=       PRETRAINED_VOCAB_FILES_MAP
    a__     							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__     							=       ['''input_ids''', '''attention_mask''']


    def __init__(self     ,      lowercase__     ,      lowercase__     ,      lowercase__="replace"     ,      lowercase__="<|endoftext|>"     ,      lowercase__="<|endoftext|>"     ,      lowercase__="<|endoftext|>"     ,      lowercase__=None     ,      lowercase__=False     ,      lowercase__=False     ,      **lowercase__     ,      )     ->      Optional[Any]:
        __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else bos_token
        __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else eos_token
        __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else unk_token
        __UpperCAmelCase									=			AddedToken(lowercase__     ,      lstrip=lowercase__     ,      rstrip=lowercase__  ) if isinstance(lowercase__     ,      lowercase__  ) else pad_token
        super().__init__(
            errors=lowercase__     ,      unk_token=lowercase__     ,      bos_token=lowercase__     ,      eos_token=lowercase__     ,      pad_token=lowercase__     ,      add_prefix_space=lowercase__     ,      add_bos_token=lowercase__     ,      **lowercase__     ,      )
        __UpperCAmelCase									=			add_bos_token

        with open(lowercase__     ,      encoding='''utf-8'''  ) as vocab_handle:
            __UpperCAmelCase									=			json.load(lowercase__  )
        __UpperCAmelCase									=			{v: k for k, v in self.encoder.items()}
        __UpperCAmelCase									=			errors  # how to handle errors in decoding
        __UpperCAmelCase									=			bytes_to_unicode()
        __UpperCAmelCase									=			{v: k for k, v in self.byte_encoder.items()}
        with open(lowercase__     ,      encoding='''utf-8'''  ) as merges_handle:
            __UpperCAmelCase									=			merges_handle.read().split('''\n'''  )[1:-1]
        __UpperCAmelCase									=			[tuple(merge.split()  ) for merge in bpe_merges]
        __UpperCAmelCase									=			dict(zip(lowercase__     ,      range(len(lowercase__  )  )  )  )
        __UpperCAmelCase									=			{}
        __UpperCAmelCase									=			add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        __UpperCAmelCase									=			re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''  )


    @property
    def      lowerCAmelCase_						(self  )     ->      List[str]:
        return len(self.encoder  )


    def      lowerCAmelCase_						(self  )     ->      Union[str, Any]:
        return dict(self.encoder     ,      **self.added_tokens_encoder  )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      str:
        if token in self.cache:
            return self.cache[token]
        __UpperCAmelCase									=			tuple(lowercase__  )
        __UpperCAmelCase									=			get_pairs(lowercase__  )

        if not pairs:
            return token

        while True:
            __UpperCAmelCase									=			min(lowercase__     ,      key=lambda lowercase__  : self.bpe_ranks.get(lowercase__     ,      float('''inf'''  )  )  )
            if bigram not in self.bpe_ranks:
                break
            __UpperCAmelCase					,     __UpperCAmelCase									=			bigram
            __UpperCAmelCase									=			[]
            __UpperCAmelCase									=			0
            while i < len(lowercase__  ):
                try:
                    __UpperCAmelCase									=			word.index(lowercase__     ,      lowercase__  )
                except ValueError:
                    new_word.extend(word[i:]  )
                    break
                else:
                    new_word.extend(word[i:j]  )
                    __UpperCAmelCase									=			j

                if word[i] == first and i < len(lowercase__  ) - 1 and word[i + 1] == second:
                    new_word.append(first + second  )
                    i += 2
                else:
                    new_word.append(word[i]  )
                    i += 1
            __UpperCAmelCase									=			tuple(lowercase__  )
            __UpperCAmelCase									=			new_word
            if len(lowercase__  ) == 1:
                break
            else:
                __UpperCAmelCase									=			get_pairs(lowercase__  )
        __UpperCAmelCase									=			''' '''.join(lowercase__  )
        __UpperCAmelCase									=			word
        return word


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__=None  )     ->      int:
        if self.add_bos_token:
            __UpperCAmelCase									=			[self.bos_token_id]
        else:
            __UpperCAmelCase									=			[]

        __UpperCAmelCase									=			bos_token_ids + token_ids_a

        if token_ids_a is None:
            return output

        return output + bos_token_ids + token_ids_a


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[str]:
        __UpperCAmelCase									=			[]
        for token in re.findall(self.pat     ,      lowercase__  ):
            __UpperCAmelCase									=			''''''.join(
                self.byte_encoder[b] for b in token.encode('''utf-8'''  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase__  ).split(''' '''  )  )
        return bpe_tokens


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      List[Any]:
        return self.encoder.get(lowercase__     ,      self.encoder.get(self.unk_token  )  )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Any:
        return self.decoder.get(lowercase__  )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      str:
        __UpperCAmelCase									=			''''''.join(lowercase__  )
        __UpperCAmelCase									=			bytearray([self.byte_decoder[c] for c in text]  ).decode('''utf-8'''     ,      errors=self.errors  )
        return text


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__ = None  )     ->      Optional[int]:
        if not os.path.isdir(lowercase__  ):
            logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
            return
        __UpperCAmelCase									=			os.path.join(
            lowercase__     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
        __UpperCAmelCase									=			os.path.join(
            lowercase__     ,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

        with open(lowercase__     ,      '''w'''     ,      encoding='''utf-8'''  ) as f:
            f.write(json.dumps(self.encoder     ,      indent=2     ,      sort_keys=lowercase__     ,      ensure_ascii=lowercase__  ) + '''\n'''  )

        __UpperCAmelCase									=			0
        with open(lowercase__     ,      '''w'''     ,      encoding='''utf-8'''  ) as writer:
            writer.write('''#version: 0.2\n'''  )
            for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,      key=lambda lowercase__  : kv[1]  ):
                if index != token_index:
                    logger.warning(
                        F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                        ''' Please check that the tokenizer is not corrupted!'''  )
                    __UpperCAmelCase									=			token_index
                writer.write(''' '''.join(lowercase__  ) + '''\n'''  )
                index += 1

        return vocab_file, merge_file


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__=False     ,      **lowercase__  )     ->      List[str]:
        __UpperCAmelCase									=			kwargs.pop('''add_prefix_space'''     ,      self.add_prefix_space  )
        if is_split_into_words or add_prefix_space:
            __UpperCAmelCase									=			''' ''' + text
        return (text, kwargs)


    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__ = False     ,      lowercase__ = None     ,      lowercase__ = None     ,      **lowercase__     ,      )     ->      Union[str, Any]:
        __UpperCAmelCase									=			super()._decode(
            token_ids=lowercase__     ,      skip_special_tokens=lowercase__     ,      clean_up_tokenization_spaces=lowercase__     ,      **lowercase__     ,      )

        if truncate_before_pattern is not None and len(lowercase__  ) > 0:
            __UpperCAmelCase									=			self.truncate(lowercase__     ,      lowercase__  )

        return decoded_text






    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__  )     ->      List[Any]:
        def find_re(lowercase__     ,      lowercase__     ,      lowercase__  ):
            __UpperCAmelCase									=			pattern.search(lowercase__     ,      lowercase__  )
            return m.start() if m else -1

        __UpperCAmelCase									=			[re.compile(lowercase__     ,      re.MULTILINE  ) for pattern in truncate_before_pattern]

        __UpperCAmelCase									=			list(re.finditer('''^print'''     ,      lowercase__     ,      re.MULTILINE  )  )

        if len(lowercase__  ) > 1:
            __UpperCAmelCase									=			completion[: prints[1].start()]

        __UpperCAmelCase									=			list(re.finditer('''^def'''     ,      lowercase__     ,      re.MULTILINE  )  )

        if len(lowercase__  ) > 1:
            __UpperCAmelCase									=			completion[: defs[1].start()]

        __UpperCAmelCase									=			0

        __UpperCAmelCase									=			[
            pos for pos in [find_re(lowercase__     ,      lowercase__     ,      lowercase__  ) for terminal in terminals] if pos != -1
        ]

        if len(lowercase__  ) > 0:
            return completion[: min(lowercase__  )]
        else:
            return completion







import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
A_		:			Any										=   '\\n@InProceedings{huggingface:dataset,\ntitle = {A great new dataset},\nauthor={huggingface, Inc.\n},\nyear={2020}\n}\n'

# TODO: Add description of the dataset here
# You can copy an official description
A_		:			Optional[Any]										=   '\\nThis new dataset is designed to solve this great NLP task and is crafted with a lot of care.\n'

# TODO: Add a link to an official homepage for the dataset here
A_		:			Dict										=   ''

# TODO: Add the licence for the dataset here if you can find it
A_		:			Dict										=   ''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
A_		:			Union[str, Any]										=   {
    'first_domain': 'https://huggingface.co/great-new-dataset-first_domain.zip',
    'second_domain': 'https://huggingface.co/great-new-dataset-second_domain.zip',
}
class 					A_   (      datasets.GeneratorBasedBuilder						):


    '''simple docstring'''






    a__     							=       datasets.Version("1.1.0"						)

    # This is an example of a dataset with multiple configurations.
    # If you don't want/need to define several sub-sets in your dataset,
    # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

    # If you need to make complex sub-parts in the datasets with configurable options
    # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
    # BUILDER_CONFIG_CLASS = MyBuilderConfig

    # You will be able to load one or the other configurations in the following list with
    # data = datasets.load_dataset('my_dataset', 'first_domain')
    # data = datasets.load_dataset('my_dataset', 'second_domain')
    a__     							=       [
        datasets.BuilderConfig(name="first_domain" ,     version=_a ,     description="This part of my dataset covers a first domain"						),
        datasets.BuilderConfig(name="second_domain" ,     version=_a ,     description="This part of my dataset covers a second domain"						),
    ]

    a__     							=       "first_domain"  # It's not mandatory to have a default configuration. Just use one if it make sense.


    def      lowerCAmelCase_						(self  )     ->      Tuple:
        # TODO: This method specifies the datasets.DatasetInfo object which contains informations and typings for the dataset
        if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
            __UpperCAmelCase									=			datasets.Features(
                {
                    '''sentence''': datasets.Value('''string'''  ),
                    '''option1''': datasets.Value('''string'''  ),
                    '''answer''': datasets.Value('''string'''  )
                    # These are the features of your dataset like images, labels ...
                }  )
        else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
            __UpperCAmelCase									=			datasets.Features(
                {
                    '''sentence''': datasets.Value('''string'''  ),
                    '''option2''': datasets.Value('''string'''  ),
                    '''second_domain_answer''': datasets.Value('''string'''  )
                    # These are the features of your dataset like images, labels ...
                }  )
        return datasets.DatasetInfo(
            # This is the description that will appear on the datasets page.
            description=_DESCRIPTION     ,      features=lowercase__     ,      homepage=_HOMEPAGE     ,      license=_LICENSE     ,      citation=_CITATION     ,      )


    def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      int:
        # TODO: This method is tasked with downloading/extracting the data and defining the splits depending on the configuration
        # If several configurations are possible (listed in BUILDER_CONFIGS), the configuration selected by the user is in self.config.name

        # dl_manager is a datasets.download.DownloadManager that can be used to download and extract URLS
        # It can accept any type or nested list/dict and will give back the same structure with the url replaced with path to local files.
        # By default the archives will be extracted and a path to a cached folder where they are extracted is returned instead of the archive
        __UpperCAmelCase									=			_URLS[self.config.name]
        __UpperCAmelCase									=			dl_manager.download_and_extract(lowercase__  )
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN     ,      gen_kwargs={
                    '''filepath''': os.path.join(lowercase__     ,      '''train.jsonl'''  ),
                    '''split''': '''train''',
                }     ,      ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION     ,      gen_kwargs={
                    '''filepath''': os.path.join(lowercase__     ,      '''dev.jsonl'''  ),
                    '''split''': '''dev''',
                }     ,      ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST     ,      gen_kwargs={
                    '''filepath''': os.path.join(lowercase__     ,      '''test.jsonl'''  ),
                    '''split''': '''test'''
                }     ,      ),
        ]






    def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__  )     ->      Tuple:
        # TODO: This method handles input defined in _split_generators to yield (key, example) tuples from the dataset.
        # The `key` is for legacy reasons (tfds) and is not important in itself, but must be unique for each example.
        with open(lowercase__     ,      encoding='''utf-8'''  ) as f:
            for key, row in enumerate(lowercase__  ):
                __UpperCAmelCase									=			json.loads(lowercase__  )
                if self.config.name == "first_domain":
                    # Yields examples as (key, example) tuples
                    yield key, {
                        "sentence": data["sentence"],
                        "option1": data["option1"],
                        "answer": "" if split == "test" else data["answer"],
                    }
                else:
                    yield key, {
                        "sentence": data["sentence"],
                        "option2": data["option2"],
                        "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
                    }




