
import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


lowercase__		:List[str]   =    logging.get_logger(__name__)

lowercase__		:int   =    {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

lowercase__		:str   =    {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

lowercase__		:Any   =    {
    "abeja/gpt-neox-japanese-2.7b": 2048,
}



def        UpperCamelCase    (							lowerCAmelCase__ ,			lowerCAmelCase__		):
					'''simple docstring'''
					with open(lowerCAmelCase__ ,			'''r''' ,			encoding='''utf-8'''		) as f:
										lowercase							=						json.loads(f.read()		)

					lowercase							=						collections.OrderedDict()
					lowercase							=						collections.OrderedDict()
					lowercase							=						collections.OrderedDict()
					with open(lowerCAmelCase__ ,			'''r''' ,			encoding='''utf-8'''		) as f:
										lowercase							=						f.readlines()
					lowercase							=						[[t.rstrip('''\n'''		)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''		).split(''','''		) for t in token]
					for idx, b in enumerate(lowerCAmelCase__		):
										lowercase							=						b
										lowercase							=						idx
										for wd in b:
															lowercase							=						idx

					return vocab, raw_vocab, ids_to_tokens, emoji







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       List[Any]       					=VOCAB_FILES_NAMES
							lowercase_       :       Any       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       str       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Union[str, Any]       					=['''input_ids''', '''attention_mask''']




							def __init__(				self   ,A__   ,A__   ,A__="<|endoftext|>"   ,A__="<|endoftext|>"   ,A__="<|startoftext|>"   ,A__="<|endoftext|>"   ,A__=False   ,**A__   ,):
												super().__init__(
												    unk_token=A__   ,pad_token=A__   ,bos_token=A__   ,eos_token=A__   ,do_clean_text=A__   ,**A__   ,)
												if not os.path.isfile(A__):
																	raise ValueError(
																	    f'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
																	    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
												if not os.path.isfile(A__):
																	raise ValueError(
																	    f'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
																	    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`''')
												lowercase							=						do_clean_text
												lowercase	,						lowercase	,						lowercase	,						lowercase							=						load_vocab_and_emoji(A__   ,A__)
												lowercase							=						SubWordJapaneseTokenizer(
												    vocab=self.vocab   ,ids_to_tokens=self.ids_to_tokens   ,emoji=self.emoji)




							@property
							def  A__      (				self):
												# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
												return len(self.raw_vocab)




							def  A__      (				self):
												return dict(self.raw_vocab   ,**self.added_tokens_encoder)




							def  A__      (				self   ,A__):
												return self.subword_tokenizer.tokenize(A__   ,clean=self.do_clean_text)




							def  A__      (				self   ,A__):
												return self.vocab.get(A__   ,self.vocab.get(self.unk_token))




							def  A__      (				self   ,A__):
												return self.subword_tokenizer.convert_id_to_token(A__)




							def  A__      (				self   ,A__):
												lowercase							=						''''''.join(A__).strip()
												return out_string




							def  A__      (				self   ,A__):
												lowercase							=						[]
												for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(A__   ,add_special_tokens=A__) + [self.eos_token_id])

												if len(A__) > self.model_max_length:
																	lowercase							=						input_ids[-self.model_max_length :]
												return input_ids





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						0
												if os.path.isdir(A__):
																	lowercase							=						os.path.join(
																	    A__   ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
																	lowercase							=						os.path.join(
																	    A__   ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file'''])
												else:
																	lowercase							=						(
																	    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
																	)
																	lowercase							=						(
																	    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
																	)
												with open(A__   ,'''w'''   ,encoding='''utf-8''') as writer:
																	for token_index, token in self.ids_to_tokens.items():
																						if index != token_index:
																											logger.warning(
																											    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																											    ''' Please check that the vocabulary is not corrupted!''')
																											lowercase							=						token_index
																						writer.write(''','''.join(A__) + '''\n''')
																						index += 1
												with open(A__   ,'''w'''   ,encoding='''utf-8''') as writer:
																	json.dump(self.emoji   ,A__)
												return vocab_file, emoji_file







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):




							def __init__(				self   ,A__   ,A__   ,A__):
												lowercase							=						vocab  # same as swe
												lowercase							=						ids_to_tokens  # same as bpe
												lowercase							=						emoji
												lowercase							=						np.max([len(A__) for w in self.vocab.keys()])
												lowercase							=						re.compile(r'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)''')
												lowercase							=						re.compile(r'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*''')
												lowercase							=						re.compile(r'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}''')
												lowercase							=						re.compile(
												    r'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
												lowercase							=						re.compile(
												    r'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*''')
												lowercase							=						re.compile(
												    r'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*''')
												lowercase							=						'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
												lowercase							=						'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
												lowercase							=						str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks})




							def __len__(				self):
												return len(self.ids_to_tokens)




							def  A__      (				self   ,A__):
												lowercase							=						self.content_repattera.sub('''<URL>'''   ,A__)
												lowercase							=						self.content_repattera.sub('''<EMAIL>'''   ,A__)
												lowercase							=						self.content_repattera.sub('''<TEL>'''   ,A__)
												lowercase							=						self.content_repattera.sub('''<DATE>'''   ,A__)
												lowercase							=						self.content_repattera.sub('''<DATE>'''   ,A__)
												lowercase							=						self.content_repattera.sub('''<PRICE>'''   ,A__)
												lowercase							=						content.translate(self.content_transa)
												while "<BLOCK><BLOCK>" in content:
																	lowercase							=						content.replace('''<BLOCK><BLOCK>'''   ,'''<BLOCK>''')
												return content




							def  A__      (				self   ,A__   ,A__=False):
												lowercase							=						text.replace(''' '''   ,'''<SP>''')
												lowercase							=						text.replace('''　'''   ,'''<SP>''')
												lowercase							=						text.replace('''\r\n'''   ,'''<BR>''')
												lowercase							=						text.replace('''\n'''   ,'''<BR>''')
												lowercase							=						text.replace('''\r'''   ,'''<BR>''')
												lowercase							=						text.replace('''\t'''   ,'''<TAB>''')
												lowercase							=						text.replace('''—'''   ,'''ー''')
												lowercase							=						text.replace('''−'''   ,'''ー''')
												for k, v in self.emoji["emoji"].items():
																	if k in text:
																						lowercase							=						text.replace(A__   ,A__)
												if clean:
																	lowercase							=						self.clean_text(A__)

												def check_simbol(A__):
																	lowercase							=						x.encode()
																	if len(A__) == 1 and len(A__) == 2:
																						lowercase							=						(int(e[0]) << 8) + int(e[1])
																						if (
																						    (c >= 0Xc2_a1 and c <= 0Xc2_bf)
																						    or (c >= 0Xc7_80 and c <= 0Xc7_83)
																						    or (c >= 0Xca_b9 and c <= 0Xcb_bf)
																						    or (c >= 0Xcc_80 and c <= 0Xcd_a2)
																						):
																											return True
																	return False

												def checkuae(A__):
																	lowercase							=						x.encode()
																	if len(A__) == 1 and len(A__) == 3:
																						lowercase							=						(int(e[0]) << 1_6) + (int(e[1]) << 8) + int(e[2])
																						if c >= 0Xe2_80_80 and c <= 0Xe2_b0_7f:
																											return True
																	return False

												lowercase							=						0
												lowercase							=						[]
												while pos < len(A__):
																	lowercase							=						min(len(A__)   ,pos + self.maxlen + 1) if text[pos] == '''<''' else pos + 3
																	lowercase							=						[]  # (token_id, token, pos)
																	for e in range(A__   ,A__   ,-1):
																						lowercase							=						text[pos:e]
																						if wd in self.vocab:
																											if wd[0] == "<" and len(A__) > 2:
																																lowercase							=						[(self.vocab[wd], wd, e)]
																																break
																											else:
																																candidates.append((self.vocab[wd], wd, e))
																	if len(A__) > 0:
																						# the smallest token_id is adopted
																						lowercase	,						lowercase	,						lowercase							=						sorted(A__   ,key=lambda A__: x[0])[0]
																						result.append(A__)
																						lowercase							=						e
																	else:
																						lowercase							=						pos + 1
																						lowercase							=						text[pos:end]
																						if check_simbol(A__):
																											result.append('''<KIGOU>''')
																						elif checkuae(A__):
																											result.append('''<U2000U2BFF>''')
																						else:
																											for i in wd.encode('''utf-8'''):
																																result.append('''<|byte%d|>''' % i)
																						lowercase							=						end
												return result





							def  A__      (				self   ,A__   ,A__="\n"):
												lowercase							=						[]
												lowercase							=						[]
												lowercase							=						self.ids_to_tokens[index][0]
												if word[:6] == "<|byte" and word[-2:] == "|>":
																	byte_tokens.append(int(word[6:-2]))
												else:
																	if len(A__) > 0:
																						words.append(bytearray(A__).decode('''utf-8'''   ,errors='''replace'''))
																						lowercase							=						[]
																	if word[:7] == "<|emoji" and word[-2:] == "|>":
																						words.append(self.emoji['''emoji_inv'''][word])
																	elif word == "<SP>":
																						words.append(''' ''')
																	elif word == "<BR>":
																						words.append(A__)
																	elif word == "<TAB>":
																						words.append('''\t''')
																	elif word == "<BLOCK>":
																						words.append('''▀''')
																	elif word == "<KIGOU>":
																						words.append('''ǀ''')
																	elif word == "<U2000U2BFF>":
																						words.append('''‖''')
																	else:
																						words.append(A__)
												if len(A__) > 0:
																	words.append(bytearray(A__).decode('''utf-8'''   ,errors='''replace'''))
												lowercase							=						''''''.join(A__)
												return text





'''simple docstring'''

import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast







@dataclass
class 				_lowercase     (				datasets.BuilderConfig			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       Optional[datasets.Features]      	=					None







class 				_lowercase     (				datasets.ArrowBasedBuilder			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       Union[str, Any]      	=					PandasConfig
 def a					(      self   :	Union[str, Any]							)  ->    int:
  return datasets.DatasetInfo(features=self.config.features							)
 def a					(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Union[str, Any]							)  ->    str:
  if not self.config.data_files:
   raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
  __lowerCAmelCase											=						dl_manager.download_and_extract(self.config.data_files							)
  if isinstance(SCREAMING_SNAKE_CASE__      ,       (str, list, tuple)							):
   __lowerCAmelCase											=						data_files
   if isinstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							):
    __lowerCAmelCase											=						[files]
   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
   __lowerCAmelCase											=						[dl_manager.iter_files(SCREAMING_SNAKE_CASE__							) for file in files]
   return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,       gen_kwargs={"""files""": files}							)]
  __lowerCAmelCase											=						[]
  for split_name, files in data_files.items():
   if isinstance(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							):
    __lowerCAmelCase											=						[files]
   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
   __lowerCAmelCase											=						[dl_manager.iter_files(SCREAMING_SNAKE_CASE__							) for file in files]
   splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE__      ,       gen_kwargs={"""files""": files}							)							)
  return splits
 def a					(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	pa.Table							)  ->    pa.Table:
  if self.config.features is not None:
   # more expensive cast to support nested features with keys in a different order
   # allows str <-> int/float or str to Audio for example
   __lowerCAmelCase											=						table_cast(SCREAMING_SNAKE_CASE__      ,       self.config.features.arrow_schema							)
  return pa_table





 def a					(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]							)  ->    List[Any]:
  for i, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE__							)							):
   with open(SCREAMING_SNAKE_CASE__      ,       """rb"""							) as f:
    __lowerCAmelCase											=						pa.Table.from_pandas(pd.read_pickle(SCREAMING_SNAKE_CASE__							)							)
    yield i, self._cast_table(SCREAMING_SNAKE_CASE__							)






