code: 
  
  
  
  '''simple docstring'''
  
  import csv
  import dataclasses
  import json
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  from ...utils import is_tf_available, is_torch_available, logging
  
  
  UpperCamelCase_     			=       logging.get_logger(__name__)
  
  
  @dataclass
  class   a_  :
         __lowerCAmelCase    :				Any    =      4_2
         __lowerCAmelCase    :				Union[str, Any]    =      4_2
         __lowerCAmelCase    :				Any    =      None
         __lowerCAmelCase    :				Dict    =      None
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           return json.dumps(dataclasses.asdict(self  )   ,   indent=2  ) + "\n"
  
  
  
  @dataclass(frozen=_a    )
  class   a_  :
         __lowerCAmelCase    :				int    =      4_2
         __lowerCAmelCase    :				List[Any]    =      None
         __lowerCAmelCase    :				List[str]    =      None
         __lowerCAmelCase    :				Tuple    =      None
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           return json.dumps(dataclasses.asdict(self  )  ) + "\n"
  
  
  
  class   a_  :
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           raise NotImplementedError()
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           raise NotImplementedError()
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           raise NotImplementedError()
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           raise NotImplementedError()
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           raise NotImplementedError()
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           if len(self.get_labels()  ) > 1:
             _lowerCAmelCase       :  Dict   				=					self.get_labels()[int(example.label  )]
           return example
  
  
  
  
  
  
  
         @classmethod
         def 							__UpperCamelCase       (     cls   ,   snake_case_   ,   snake_case_=None  ):
           with open(__lowerCamelCase   ,   """r"""   ,   encoding="""utf-8-sig"""  ) as f:
             return list(csv.reader(__lowerCamelCase   ,   delimiter="""\t"""   ,   quotechar=__lowerCamelCase  )  )
  
  
  
  class   a_  (_a    ):
  
  
  
  
         def __init__(     self   ,   snake_case_=None   ,   snake_case_=None   ,   snake_case_="classification"   ,   snake_case_=False  ):
           _lowerCAmelCase       :  Optional[Any]   				=					[] if labels is None else labels
           _lowerCAmelCase       :  Dict   				=					[] if examples is None else examples
           _lowerCAmelCase       :  Any   				=					mode
           _lowerCAmelCase       :  Optional[int]   				=					verbose
  
  
  
  
         def __len__(     self  ):
           return len(self.examples  )
  
  
  
  
         def __getitem__(     self   ,   snake_case_  ):
           if isinstance(__lowerCamelCase   ,   __lowerCamelCase  ):
             return SingleSentenceClassificationProcessor(labels=self.labels   ,   examples=self.examples[idx]  )
           return self.examples[idx]
  
  
  
  
         @classmethod
         def 							__UpperCamelCase       (     cls   ,   snake_case_   ,   snake_case_=""   ,   snake_case_=0   ,   snake_case_=1   ,   snake_case_=None   ,   snake_case_=False   ,   **snake_case_  ):
           _lowerCAmelCase       :  Optional[int]   				=					cls(**__lowerCamelCase  )
           processor.add_examples_from_csv(
               __lowerCamelCase   ,   split_name=__lowerCamelCase   ,   column_label=__lowerCamelCase   ,   column_text=__lowerCamelCase   ,   column_id=__lowerCamelCase   ,   skip_first_row=__lowerCamelCase   ,   overwrite_labels=__lowerCamelCase   ,   overwrite_examples=__lowerCamelCase   ,   )
           return processor
  
  
  
  
         @classmethod
         def 							__UpperCamelCase       (     cls   ,   snake_case_   ,   snake_case_=None   ,   **snake_case_  ):
           _lowerCAmelCase       :  List[Any]   				=					cls(**__lowerCamelCase  )
           processor.add_examples(__lowerCamelCase   ,   labels=__lowerCamelCase  )
           return processor
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_=""   ,   snake_case_=0   ,   snake_case_=1   ,   snake_case_=None   ,   snake_case_=False   ,   snake_case_=False   ,   snake_case_=False   ,   ):
           _lowerCAmelCase       :  int   				=					self._read_tsv(__lowerCamelCase  )
           if skip_first_row:
             _lowerCAmelCase       :  int   				=					lines[1:]
           _lowerCAmelCase       :  Optional[Any]   				=					[]
           _lowerCAmelCase       :  List[str]   				=					[]
           _lowerCAmelCase       :  Dict   				=					[]
           for i, line in enumerate(__lowerCamelCase  ):
             texts.append(line[column_text]  )
             labels.append(line[column_label]  )
             if column_id is not None:
               ids.append(line[column_id]  )
             else:
               _lowerCAmelCase       :  str   				=					f'{split_name}-{i}' if split_name else str(__lowerCamelCase  )
               ids.append(__lowerCamelCase  )
  
           return self.add_examples(
               __lowerCamelCase   ,   __lowerCamelCase   ,   __lowerCamelCase   ,   overwrite_labels=__lowerCamelCase   ,   overwrite_examples=__lowerCamelCase  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_=None   ,   snake_case_=None   ,   snake_case_=False   ,   snake_case_=False  ):
           if labels is not None and len(__lowerCamelCase  ) != len(__lowerCamelCase  ):
             raise ValueError(
                 f'Text and labels have mismatched lengths {len(__lowerCamelCase  )} and {len(__lowerCamelCase  )}'  )
           if ids is not None and len(__lowerCamelCase  ) != len(__lowerCamelCase  ):
             raise ValueError(f'Text and ids have mismatched lengths {len(__lowerCamelCase  )} and {len(__lowerCamelCase  )}'  )
           if ids is None:
             _lowerCAmelCase       :  Optional[Any]   				=					[None] * len(__lowerCamelCase  )
           if labels is None:
             _lowerCAmelCase       :  Union[str, Any]   				=					[None] * len(__lowerCamelCase  )
           _lowerCAmelCase       :  Optional[int]   				=					[]
           _lowerCAmelCase       :  List[Any]   				=					set()
           for text_or_text_and_label, label, guid in zip(__lowerCamelCase   ,   __lowerCamelCase   ,   __lowerCamelCase  ):
             if isinstance(__lowerCamelCase   ,   (tuple, list)  ) and label is None:
               _lowerCAmelCase       :  List[str]   				=					text_or_text_and_label
             else:
               _lowerCAmelCase       :  Union[str, Any]   				=					text_or_text_and_label
             added_labels.add(__lowerCamelCase  )
             examples.append(InputExample(guid=__lowerCamelCase   ,   text_a=__lowerCamelCase   ,   text_b=__lowerCamelCase   ,   label=__lowerCamelCase  )  )
  
           # Update examples
           if overwrite_examples:
             _lowerCAmelCase       :  Optional[Any]   				=					examples
           else:
             self.examples.extend(__lowerCamelCase  )
  
           # Update labels
           if overwrite_labels:
             _lowerCAmelCase       :  Optional[Any]   				=					list(__lowerCamelCase  )
           else:
             _lowerCAmelCase       :  Any   				=					list(set(self.labels  ).union(__lowerCamelCase  )  )
  
           return self.examples
  
  
  
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_   ,   snake_case_=None   ,   snake_case_=False   ,   snake_case_=0   ,   snake_case_=True   ,   snake_case_=None   ,   ):
           if max_length is None:
             _lowerCAmelCase       :  Optional[int]   				=					tokenizer.max_len
  
           _lowerCAmelCase       :  Any   				=					{label: i for i, label in enumerate(self.labels  )}
  
           _lowerCAmelCase       :  Optional[Any]   				=					[]
           for ex_index, example in enumerate(self.examples  ):
             if ex_index % 1_0_0_0_0 == 0:
               logger.info(f'Tokenizing example {ex_index}'  )
  
             _lowerCAmelCase       :  Any   				=					tokenizer.encode(
                 example.text_a   ,   add_special_tokens=__lowerCamelCase   ,   max_length=min(__lowerCamelCase   ,   tokenizer.max_len  )   ,   )
             all_input_ids.append(__lowerCamelCase  )
  
           _lowerCAmelCase       :  List[str]   				=					max(len(__lowerCamelCase  ) for input_ids in all_input_ids  )
  
           _lowerCAmelCase       :  Any   				=					[]
           for ex_index, (input_ids, example) in enumerate(zip(__lowerCamelCase   ,   self.examples  )  ):
             if ex_index % 1_0_0_0_0 == 0:
               logger.info(f'Writing example {ex_index}/{len(self.examples  )}'  )
             # The mask has 1 for real tokens and 0 for padding tokens. Only real
             # tokens are attended to.
             _lowerCAmelCase       :  Optional[int]   				=					[1 if mask_padding_with_zero else 0] * len(__lowerCamelCase  )
  
             # Zero-pad up to the sequence length.
             _lowerCAmelCase       :  Optional[Any]   				=					batch_length - len(__lowerCamelCase  )
             if pad_on_left:
               _lowerCAmelCase       :  List[str]   				=					([pad_token] * padding_length) + input_ids
               _lowerCAmelCase       :  Dict   				=					([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
             else:
               _lowerCAmelCase       :  Dict   				=					input_ids + ([pad_token] * padding_length)
               _lowerCAmelCase       :  int   				=					attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)
  
             if len(__lowerCamelCase  ) != batch_length:
               raise ValueError(f'Error with input length {len(__lowerCamelCase  )} vs {batch_length}'  )
             if len(__lowerCamelCase  ) != batch_length:
               raise ValueError(f'Error with input length {len(__lowerCamelCase  )} vs {batch_length}'  )
  
             if self.mode == "classification":
               _lowerCAmelCase       :  Tuple   				=					label_map[example.label]
             elif self.mode == "regression":
               _lowerCAmelCase       :  Optional[int]   				=					float(example.label  )
             else:
               raise ValueError(self.mode  )
  
             if ex_index < 5 and self.verbose:
               logger.info("""*** Example ***"""  )
               logger.info(f'guid: {example.guid}'  )
               logger.info(f'input_ids: {" ".join([str(__lowerCamelCase  ) for x in input_ids]  )}'  )
               logger.info(f'attention_mask: {" ".join([str(__lowerCamelCase  ) for x in attention_mask]  )}'  )
               logger.info(f'label: {example.label} (id = {label})'  )
  
             features.append(InputFeatures(input_ids=__lowerCamelCase   ,   attention_mask=__lowerCamelCase   ,   label=__lowerCamelCase  )  )
  
           if return_tensors is None:
             return features
           elif return_tensors == "tf":
             if not is_tf_available():
               raise RuntimeError("""return_tensors set to 'tf' but TensorFlow 2.0 can't be imported"""  )
             import tensorflow as tf
  
             def gen():
               for ex in features:
                 yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)
  
             _lowerCAmelCase       :  Tuple   				=					tf.data.Dataset.from_generator(
                 __lowerCamelCase   ,   ({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa)   ,   ({"""input_ids""": tf.TensorShape([None]  ), """attention_mask""": tf.TensorShape([None]  )}, tf.TensorShape([]  ))   ,   )
             return dataset
           elif return_tensors == "pt":
             if not is_torch_available():
               raise RuntimeError("""return_tensors set to 'pt' but PyTorch can't be imported"""  )
             import torch
             from torch.utils.data import TensorDataset
  
             _lowerCAmelCase       :  Dict   				=					torch.tensor([f.input_ids for f in features]   ,   dtype=torch.long  )
             _lowerCAmelCase       :  Any   				=					torch.tensor([f.attention_mask for f in features]   ,   dtype=torch.long  )
             if self.mode == "classification":
               _lowerCAmelCase       :  int   				=					torch.tensor([f.label for f in features]   ,   dtype=torch.long  )
             elif self.mode == "regression":
               _lowerCAmelCase       :  int   				=					torch.tensor([f.label for f in features]   ,   dtype=torch.float  )
  
             _lowerCAmelCase       :  Dict   				=					TensorDataset(__lowerCamelCase   ,   __lowerCamelCase   ,   __lowerCamelCase  )
             return dataset
           else:
             raise ValueError("""return_tensors should be one of 'tf' or 'pt'"""  )
  
  
  
style_context: 
  '''simple docstring'''
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  @require_tokenizers
  class   a_  (_a							,			unittest.TestCase    ):
         __lowerCAmelCase    :				List[Any]    =      GPTNeoXJapaneseTokenizer
         __lowerCAmelCase    :				Any    =      False
         __lowerCAmelCase    :				List[Any]    =      {"""do_clean_text""": False, """add_prefix_space""": False}
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           super().setUp()
  
           _lowerCAmelCase       :  Any   				=					[
               """こん""",
               """こんに""",
               """にちは""",
               """ばんは""",
               """世界,㔺界""",
               """、""",
               """。""",
               """<BR>""",
               """<SP>""",
               """<TAB>""",
               """<URL>""",
               """<EMAIL>""",
               """<TEL>""",
               """<DATE>""",
               """<PRICE>""",
               """<BLOCK>""",
               """<KIGOU>""",
               """<U2000U2BFF>""",
               """<|emoji1|>""",
               """<unk>""",
               """<|startoftext|>""",
               """<|endoftext|>""",
           ]
           _lowerCAmelCase       :  Union[str, Any]   				=					{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
           _lowerCAmelCase       :  Any   				=					{"""unk_token""": """<unk>"""}
  
           _lowerCAmelCase       :  Optional[int]   				=					os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES["""vocab_file"""]  )
           _lowerCAmelCase       :  Any   				=					os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES["""emoji_file"""]  )
           with open(self.vocab_file   ,   """w"""   ,   encoding="""utf-8"""  ) as vocab_writer:
             vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )
           with open(self.emoji_file   ,   """w"""  ) as emoji_writer:
             emoji_writer.write(json.dumps(snake_case_  )  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   **snake_case_  ):
           kwargs.update(self.special_tokens_map  )
           return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname   ,   **snake_case_  )
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase       :  Tuple   				=					"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
           _lowerCAmelCase       :  str   				=					"""こんにちは、世界。 \nこんばんは、世界。😀"""
           return input_text, output_text
  
  
  
  
         def 							__UpperCamelCase       (     self   ,   snake_case_  ):
           _lowerCAmelCase				,					_lowerCAmelCase       :  str   				=					self.get_input_output_texts(snake_case_  )
           _lowerCAmelCase       :  Tuple   				=					tokenizer.encode(snake_case_   ,   add_special_tokens=snake_case_  )
           _lowerCAmelCase       :  Optional[Any]   				=					tokenizer.decode(snake_case_   ,   clean_up_tokenization_spaces=snake_case_  )
           return text, ids
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           pass  # TODO add if relevant
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  List[Any]   				=					self.get_tokenizer()
  
           # Testing tokenization
           _lowerCAmelCase       :  Tuple   				=					"""こんにちは、世界。　こんばんは、㔺界。"""
           _lowerCAmelCase       :  Optional[Any]   				=					["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
           _lowerCAmelCase       :  Tuple   				=					tokenizer.tokenize(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
           # Testing conversion to ids without special tokens
           _lowerCAmelCase       :  Optional[int]   				=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
           _lowerCAmelCase       :  Optional[int]   				=					tokenizer.convert_tokens_to_ids(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
           # Testing conversion to ids with special tokens
           _lowerCAmelCase       :  Optional[Any]   				=					tokens + [tokenizer.unk_token]
           _lowerCAmelCase       :  List[str]   				=					[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
           _lowerCAmelCase       :  Dict   				=					tokenizer.convert_tokens_to_ids(snake_case_  )
           self.assertListEqual(snake_case_   ,   snake_case_  )
  
  
  
  
         @slow
         def 							__UpperCamelCase       (     self  ):
           _lowerCAmelCase       :  int   				=					self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""  )
  
           _lowerCAmelCase       :  List[Any]   				=					tokenizer.encode("""ありがとう。"""   ,   add_special_tokens=snake_case_  )
           _lowerCAmelCase       :  List[str]   				=					tokenizer.encode("""どういたしまして。"""   ,   add_special_tokens=snake_case_  )
  
           _lowerCAmelCase       :  int   				=					tokenizer.build_inputs_with_special_tokens(snake_case_  )
           _lowerCAmelCase       :  Any   				=					tokenizer.build_inputs_with_special_tokens(snake_case_   ,   snake_case_  )
  
           assert encoded_sentence == ids_a
           assert encoded_pair == ids_a + ids_a
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           # Intentionally convert some words to accommodate character fluctuations unique to Japanese
           pass
  
  
  
  
  
  
  
         def 							__UpperCamelCase       (     self  ):
           # tokenizer has no padding token
           pass
  
  
  