





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


_a 						=      logging.get_logger(__name__)

_a 						=      {"vocab_file": "vocab.txt", "emoji_file": "emoji.json"}

_a 						=      {
    "vocab_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt",
    },
    "emoji_file": {
        "abeja/gpt-neox-japanese-2.7b": "https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json",
    },
}

_a 						=      {
    "abeja/gpt-neox-japanese-2.7b": 2_048,
}





def lowerCAmelCase__(__snake_case       ,__snake_case							)				->      str:







	'''simple docstring'''







	with open(__snake_case       ,'''r'''       ,encoding='''utf-8'''							) as f:
		lowerCamelCase__								=		json.loads(f.read()							)

	lowerCamelCase__								=		collections.OrderedDict()
	lowerCamelCase__								=		collections.OrderedDict()
	lowerCamelCase__								=		collections.OrderedDict()
	with open(__snake_case       ,'''r'''       ,encoding='''utf-8'''							) as f:
		lowerCamelCase__								=		f.readlines()
	lowerCamelCase__								=		[[t.rstrip('''\n'''							)] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n'''							).split(''','''							) for t in token]
	for idx, b in enumerate(__snake_case							):
		lowerCamelCase__								=		b
		lowerCamelCase__								=		idx
		for wd in b:
			lowerCamelCase__								=		idx

	return vocab, raw_vocab, ids_to_tokens, emoji






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						VOCAB_FILES_NAMES
			lowerCAmelCase_							     =						PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase_							     =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase_							     =						["""input_ids""", """attention_mask"""]





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase="<|endoftext|>"       ,  __lowerCAmelCase="<|endoftext|>"       ,  __lowerCAmelCase="<|startoftext|>"       ,  __lowerCAmelCase="<|endoftext|>"       ,  __lowerCAmelCase=False       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				super().__init__(
				    unk_token=__lowerCAmelCase       ,  pad_token=__lowerCAmelCase       ,  bos_token=__lowerCAmelCase       ,  eos_token=__lowerCAmelCase       ,  do_clean_text=__lowerCAmelCase       ,  **__lowerCAmelCase       ,  )
				if not os.path.isfile(__lowerCAmelCase							):
					raise ValueError(
					    F'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
					    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''							)
				if not os.path.isfile(__lowerCAmelCase							):
					raise ValueError(
					    F'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
					    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''							)
				lowerCamelCase__								=		do_clean_text
				lowerCamelCase__				,	lowerCamelCase__				,	lowerCamelCase__				,	lowerCamelCase__								=		load_vocab_and_emoji(__lowerCAmelCase       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		SubWordJapaneseTokenizer(
				    vocab=self.vocab       ,  ids_to_tokens=self.ids_to_tokens       ,  emoji=self.emoji							)





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return len(self.raw_vocab							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return dict(self.raw_vocab       ,  **self.added_tokens_encoder							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.subword_tokenizer.tokenize(__lowerCAmelCase       ,  clean=self.do_clean_text							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.vocab.get(__lowerCAmelCase       ,  self.vocab.get(self.unk_token							)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.subword_tokenizer.convert_id_to_token(__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		''''''.join(__lowerCAmelCase							).strip()
				return out_string





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				for is_user, text in conversation.iter_texts():
					input_ids.extend(self.encode(__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase							) + [self.eos_token_id]							)

				if len(__lowerCAmelCase							) > self.model_max_length:
					lowerCamelCase__								=		input_ids[-self.model_max_length :]
				return input_ids





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase = None							):



				'''simple docstring'''







				lowerCamelCase__								=		0
				if os.path.isdir(__lowerCAmelCase							):
					lowerCamelCase__								=		os.path.join(
					    __lowerCAmelCase       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
					lowerCamelCase__								=		os.path.join(
					    __lowerCAmelCase       ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']							)
				else:
					lowerCamelCase__								=		(
					    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
					)
					lowerCamelCase__								=		(
					    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
					)
				with open(__lowerCAmelCase       ,  '''w'''       ,  encoding='''utf-8'''							) as writer:
					for token_index, token in self.ids_to_tokens.items():
						if index != token_index:
							logger.warning(
							    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
							    ''' Please check that the vocabulary is not corrupted!'''							)
							lowerCamelCase__								=		token_index
						writer.write(''','''.join(__lowerCAmelCase							) + '''\n'''							)
						index += 1
				with open(__lowerCAmelCase       ,  '''w'''       ,  encoding='''utf-8'''							) as writer:
					json.dump(self.emoji       ,  __lowerCAmelCase							)
				return vocab_file, emoji_file






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		vocab  # same as swe
				lowerCamelCase__								=		ids_to_tokens  # same as bpe
				lowerCamelCase__								=		emoji
				lowerCamelCase__								=		np.max([len(__lowerCAmelCase							) for w in self.vocab.keys()]							)
				lowerCamelCase__								=		re.compile(r'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''							)
				lowerCamelCase__								=		re.compile(r'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''							)
				lowerCamelCase__								=		re.compile(r'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''							)
				lowerCamelCase__								=		re.compile(
				    r'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''							)
				lowerCamelCase__								=		re.compile(
				    r'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''							)
				lowerCamelCase__								=		re.compile(
				    r'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''							)
				lowerCamelCase__								=		'''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
				lowerCamelCase__								=		'''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
				lowerCamelCase__								=		str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}							)





			def __len__(    self							):



				'''simple docstring'''







				return len(self.ids_to_tokens							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.content_repattera.sub('''<URL>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.content_repattera.sub('''<EMAIL>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.content_repattera.sub('''<TEL>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.content_repattera.sub('''<DATE>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.content_repattera.sub('''<DATE>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.content_repattera.sub('''<PRICE>'''       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		content.translate(self.content_transa							)
				while "<BLOCK><BLOCK>" in content:
					lowerCamelCase__								=		content.replace('''<BLOCK><BLOCK>'''       ,  '''<BLOCK>'''							)
				return content





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=False							):



				'''simple docstring'''







				lowerCamelCase__								=		text.replace(''' '''       ,  '''<SP>'''							)
				lowerCamelCase__								=		text.replace('''　'''       ,  '''<SP>'''							)
				lowerCamelCase__								=		text.replace('''\r\n'''       ,  '''<BR>'''							)
				lowerCamelCase__								=		text.replace('''\n'''       ,  '''<BR>'''							)
				lowerCamelCase__								=		text.replace('''\r'''       ,  '''<BR>'''							)
				lowerCamelCase__								=		text.replace('''\t'''       ,  '''<TAB>'''							)
				lowerCamelCase__								=		text.replace('''—'''       ,  '''ー'''							)
				lowerCamelCase__								=		text.replace('''−'''       ,  '''ー'''							)
				for k, v in self.emoji["emoji"].items():
					if k in text:
						lowerCamelCase__								=		text.replace(__lowerCAmelCase       ,  __lowerCAmelCase							)
				if clean:
					lowerCamelCase__								=		self.clean_text(__lowerCAmelCase							)

				def check_simbol(__lowerCAmelCase							):
					lowerCamelCase__								=		x.encode()
					if len(__lowerCAmelCase							) == 1 and len(__lowerCAmelCase							) == 2:
						lowerCamelCase__								=		(int(e[0]							) << 8) + int(e[1]							)
						if (
						    (c >= 0xC_2_A_1 and c <= 0xC_2_B_F)
						    or (c >= 0xC_7_8_0 and c <= 0xC_7_8_3)
						    or (c >= 0xC_A_B_9 and c <= 0xC_B_B_F)
						    or (c >= 0xC_C_8_0 and c <= 0xC_D_A_2)
						):
							return True
					return False

				def checkuae(__lowerCAmelCase							):
					lowerCamelCase__								=		x.encode()
					if len(__lowerCAmelCase							) == 1 and len(__lowerCAmelCase							) == 3:
						lowerCamelCase__								=		(int(e[0]							) << 1_6) + (int(e[1]							) << 8) + int(e[2]							)
						if c >= 0xE_2_8_0_8_0 and c <= 0xE_2_B_0_7_F:
							return True
					return False

				lowerCamelCase__								=		0
				lowerCamelCase__								=		[]
				while pos < len(__lowerCAmelCase							):
					lowerCamelCase__								=		min(len(__lowerCAmelCase							)       ,  pos + self.maxlen + 1							) if text[pos] == '''<''' else pos + 3
					lowerCamelCase__								=		[]  # (token_id, token, pos)
					for e in range(__lowerCAmelCase       ,  __lowerCAmelCase       ,  -1							):
						lowerCamelCase__								=		text[pos:e]
						if wd in self.vocab:
							if wd[0] == "<" and len(__lowerCAmelCase							) > 2:
								lowerCamelCase__								=		[(self.vocab[wd], wd, e)]
								break
							else:
								candidates.append((self.vocab[wd], wd, e)							)
					if len(__lowerCAmelCase							) > 0:
						# the smallest token_id is adopted
						lowerCamelCase__				,	lowerCamelCase__				,	lowerCamelCase__								=		sorted(__lowerCAmelCase       ,  key=lambda __lowerCAmelCase							: x[0]							)[0]
						result.append(__lowerCAmelCase							)
						lowerCamelCase__								=		e
					else:
						lowerCamelCase__								=		pos + 1
						lowerCamelCase__								=		text[pos:end]
						if check_simbol(__lowerCAmelCase							):
							result.append('''<KIGOU>'''							)
						elif checkuae(__lowerCAmelCase							):
							result.append('''<U2000U2BFF>'''							)
						else:
							for i in wd.encode('''utf-8'''							):
								result.append('''<|byte%d|>''' % i							)
						lowerCamelCase__								=		end
				return result





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase="\n"							):



				'''simple docstring'''







				lowerCamelCase__								=		[]
				lowerCamelCase__								=		[]
				lowerCamelCase__								=		self.ids_to_tokens[index][0]
				if word[:6] == "<|byte" and word[-2:] == "|>":
					byte_tokens.append(int(word[6:-2]							)							)
				else:
					if len(__lowerCAmelCase							) > 0:
						words.append(bytearray(__lowerCAmelCase							).decode('''utf-8'''       ,  errors='''replace'''							)							)
						lowerCamelCase__								=		[]
					if word[:7] == "<|emoji" and word[-2:] == "|>":
						words.append(self.emoji['''emoji_inv'''][word]							)
					elif word == "<SP>":
						words.append(''' '''							)
					elif word == "<BR>":
						words.append(__lowerCAmelCase							)
					elif word == "<TAB>":
						words.append('''\t'''							)
					elif word == "<BLOCK>":
						words.append('''▀'''							)
					elif word == "<KIGOU>":
						words.append('''ǀ'''							)
					elif word == "<U2000U2BFF>":
						words.append('''‖'''							)
					else:
						words.append(__lowerCAmelCase							)
				if len(__lowerCAmelCase							) > 0:
					words.append(bytearray(__lowerCAmelCase							).decode('''utf-8'''       ,  errors='''replace'''							)							)
				lowerCamelCase__								=		''''''.join(__lowerCAmelCase							)
				return text










import numpy as np

from transformers import Pipeline





def lowerCAmelCase__(__snake_case							)				->      Tuple:







	'''simple docstring'''







	lowerCamelCase__								=		np.max(__snake_case       ,axis=-1       ,keepdims=__snake_case							)
	lowerCamelCase__								=		np.exp(outputs - maxes							)
	return shifted_exp / shifted_exp.sum(axis=-1       ,keepdims=__snake_case							)






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			def 		__lowerCamelCase				(    self       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		{}
				if "second_text" in kwargs:
					lowerCamelCase__								=		kwargs['''second_text''']
				return preprocess_kwargs, {}, {}





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=None							):



				'''simple docstring'''







				return self.tokenizer(__lowerCAmelCase       ,  text_pair=__lowerCAmelCase       ,  return_tensors=self.framework							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				return self.model(**__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		model_outputs.logits[0].numpy()
				lowerCamelCase__								=		softmax(__lowerCAmelCase							)

				lowerCamelCase__								=		np.argmax(__lowerCAmelCase							)
				lowerCamelCase__								=		self.model.config.idalabel[best_class]
				lowerCamelCase__								=		probabilities[best_class].item()
				lowerCamelCase__								=		logits.tolist()
				return {"label": label, "score": score, "logits": logits}



