
"""simple docstring"""







import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


lowerCamelCase__   					=		logging.get_logger(__name__)

lowerCamelCase__   					=		{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

lowerCamelCase__   					=		{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

lowerCamelCase__   					=		{
    """abeja/gpt-neox-japanese-2.7b""": 2_048,
}
def     __lowerCAmelCase				(_UpperCamelCase       ,  _UpperCamelCase		):
	with open(_UpperCamelCase       ,  'r'       ,  encoding='utf-8'		) as f:
		__lowerCAmelCase			:						Any									=      json.loads(f.read()		)

	__lowerCAmelCase			:						Optional[int]									=      collections.OrderedDict()
	__lowerCAmelCase			:						Dict									=      collections.OrderedDict()
	__lowerCAmelCase			:						Dict									=      collections.OrderedDict()
	with open(_UpperCamelCase       ,  'r'       ,  encoding='utf-8'		) as f:
		__lowerCAmelCase			:						Tuple									=      f.readlines()
	__lowerCAmelCase			:						List[Any]									=      [[t.rstrip('\n'		)] if (t == ',' or ',' not in t) else t.rstrip('\n'		).split(','		) for t in token]
	for idx, b in enumerate(_UpperCamelCase		):
		__lowerCAmelCase			:						Tuple									=      b
		__lowerCAmelCase			:						Optional[Any]									=      idx
		for wd in b:
			__lowerCAmelCase			:						Any									=      idx

	return vocab, raw_vocab, ids_to_tokens, emoji




class  A__       (	_lowerCamelCase):
		A_						:       str				   =	VOCAB_FILES_NAMES
		A_						:       str				   =	PRETRAINED_VOCAB_FILES_MAP
		A_						:       List[Any]				   =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A_						:       Any				   =	['input_ids', 'attention_mask']

		def __init__(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE="<|endoftext|>"       ,		_SCREAMING_SNAKE_CASE="<|endoftext|>"       ,		_SCREAMING_SNAKE_CASE="<|startoftext|>"       ,		_SCREAMING_SNAKE_CASE="<|endoftext|>"       ,		_SCREAMING_SNAKE_CASE=False       ,		**_SCREAMING_SNAKE_CASE       ,		):
			super().__init__(
			    unk_token=_SCREAMING_SNAKE_CASE       ,		pad_token=_SCREAMING_SNAKE_CASE       ,		bos_token=_SCREAMING_SNAKE_CASE       ,		eos_token=_SCREAMING_SNAKE_CASE       ,		do_clean_text=_SCREAMING_SNAKE_CASE       ,		**_SCREAMING_SNAKE_CASE       ,		)
			if not os.path.isfile(_SCREAMING_SNAKE_CASE ):
				raise ValueError(
				    f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
				    ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' )
			if not os.path.isfile(_SCREAMING_SNAKE_CASE ):
				raise ValueError(
				    f"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
				    ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`' )
			__lowerCAmelCase			:						Optional[int]									=      do_clean_text
			__lowerCAmelCase			,   __lowerCAmelCase			,   __lowerCAmelCase			,   __lowerCAmelCase			:						int									=      load_vocab_and_emoji(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Optional[Any]									=      SubWordJapaneseTokenizer(
			    vocab=self.vocab       ,		ids_to_tokens=self.ids_to_tokens       ,		emoji=self.emoji )

		@property
		def 							__lowerCamelCase							(      self ):
			# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
			return len(self.raw_vocab )

		def 							__lowerCamelCase							(      self ):
			return dict(self.raw_vocab       ,		**self.added_tokens_encoder )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			return self.subword_tokenizer.tokenize(_SCREAMING_SNAKE_CASE       ,		clean=self.do_clean_text )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			return self.vocab.get(_SCREAMING_SNAKE_CASE       ,		self.vocab.get(self.unk_token ) )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			return self.subword_tokenizer.convert_id_to_token(_SCREAMING_SNAKE_CASE )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						Union[str, Any]									=      ''.join(_SCREAMING_SNAKE_CASE ).strip()
			return out_string

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						int									=      []
			for is_user, text in conversation.iter_texts():
				input_ids.extend(self.encode(_SCREAMING_SNAKE_CASE       ,		add_special_tokens=_SCREAMING_SNAKE_CASE ) + [self.eos_token_id] )

			if len(_SCREAMING_SNAKE_CASE ) > self.model_max_length:
				__lowerCAmelCase			:						int									=      input_ids[-self.model_max_length :]
			return input_ids







		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE = None ):
			__lowerCAmelCase			:						str									=      0
			if os.path.isdir(_SCREAMING_SNAKE_CASE ):
				__lowerCAmelCase			:						int									=      os.path.join(
				    _SCREAMING_SNAKE_CASE       ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
				__lowerCAmelCase			:						Dict									=      os.path.join(
				    _SCREAMING_SNAKE_CASE       ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file'] )
			else:
				__lowerCAmelCase			:						int									=      (
				    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
				)
				__lowerCAmelCase			:						List[str]									=      (
				    (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
				)
			with open(_SCREAMING_SNAKE_CASE       ,		'w'       ,		encoding='utf-8' ) as writer:
				for token_index, token in self.ids_to_tokens.items():
					if index != token_index:
						logger.warning(
						    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
						    ' Please check that the vocabulary is not corrupted!' )
						__lowerCAmelCase			:						Tuple									=      token_index
					writer.write(','.join(_SCREAMING_SNAKE_CASE ) + '\n' )
					index += 1
			with open(_SCREAMING_SNAKE_CASE       ,		'w'       ,		encoding='utf-8' ) as writer:
				json.dump(self.emoji       ,		_SCREAMING_SNAKE_CASE )
			return vocab_file, emoji_file




class  A__       (	_lowerCamelCase):

		def __init__(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						List[str]									=      vocab  # same as swe
			__lowerCAmelCase			:						str									=      ids_to_tokens  # same as bpe
			__lowerCAmelCase			:						Tuple									=      emoji
			__lowerCAmelCase			:						str									=      np.max([len(_SCREAMING_SNAKE_CASE ) for w in self.vocab.keys()] )
			__lowerCAmelCase			:						List[str]									=      re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)' )
			__lowerCAmelCase			:						List[Any]									=      re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*' )
			__lowerCAmelCase			:						Tuple									=      re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}' )
			__lowerCAmelCase			:						Any									=      re.compile(
			    R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*' )
			__lowerCAmelCase			:						List[str]									=      re.compile(
			    R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*' )
			__lowerCAmelCase			:						Tuple									=      re.compile(
			    R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*' )
			__lowerCAmelCase			:						Optional[int]									=      '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
			__lowerCAmelCase			:						str									=      '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
			__lowerCAmelCase			:						List[Any]									=      str.maketrans({k: '<BLOCK>' for k in keisen + blocks} )

		def __len__(      self ):
			return len(self.ids_to_tokens )

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE ):
			__lowerCAmelCase			:						List[str]									=      self.content_repattera.sub('<URL>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Optional[Any]									=      self.content_repattera.sub('<EMAIL>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Any									=      self.content_repattera.sub('<TEL>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Union[str, Any]									=      self.content_repattera.sub('<DATE>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Union[str, Any]									=      self.content_repattera.sub('<DATE>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Optional[Any]									=      self.content_repattera.sub('<PRICE>'       ,		_SCREAMING_SNAKE_CASE )
			__lowerCAmelCase			:						Tuple									=      content.translate(self.content_transa )
			while "<BLOCK><BLOCK>" in content:
				__lowerCAmelCase			:						int									=      content.replace('<BLOCK><BLOCK>'       ,		'<BLOCK>' )
			return content

		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE=False ):
			__lowerCAmelCase			:						str									=      text.replace(' '       ,		'<SP>' )
			__lowerCAmelCase			:						int									=      text.replace('　'       ,		'<SP>' )
			__lowerCAmelCase			:						int									=      text.replace('\r\n'       ,		'<BR>' )
			__lowerCAmelCase			:						Tuple									=      text.replace('\n'       ,		'<BR>' )
			__lowerCAmelCase			:						Optional[Any]									=      text.replace('\r'       ,		'<BR>' )
			__lowerCAmelCase			:						Tuple									=      text.replace('\t'       ,		'<TAB>' )
			__lowerCAmelCase			:						List[str]									=      text.replace('—'       ,		'ー' )
			__lowerCAmelCase			:						Optional[Any]									=      text.replace('−'       ,		'ー' )
			for k, v in self.emoji["emoji"].items():
				if k in text:
					__lowerCAmelCase			:						Any									=      text.replace(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE )
			if clean:
				__lowerCAmelCase			:						Optional[int]									=      self.clean_text(_SCREAMING_SNAKE_CASE )

			def check_simbol(_SCREAMING_SNAKE_CASE ):
				__lowerCAmelCase			:						Any									=      x.encode()
				if len(_SCREAMING_SNAKE_CASE ) == 1 and len(_SCREAMING_SNAKE_CASE ) == 2:
					__lowerCAmelCase			:						Any									=      (int(e[0] ) << 8) + int(e[1] )
					if (
					    (c >= 0xc_2_a_1 and c <= 0xc_2_b_f)
					    or (c >= 0xc_7_8_0 and c <= 0xc_7_8_3)
					    or (c >= 0xc_a_b_9 and c <= 0xc_b_b_f)
					    or (c >= 0xc_c_8_0 and c <= 0xc_d_a_2)
					):
						return True
				return False

			def checkuae(_SCREAMING_SNAKE_CASE ):
				__lowerCAmelCase			:						int									=      x.encode()
				if len(_SCREAMING_SNAKE_CASE ) == 1 and len(_SCREAMING_SNAKE_CASE ) == 3:
					__lowerCAmelCase			:						List[Any]									=      (int(e[0] ) << 16) + (int(e[1] ) << 8) + int(e[2] )
					if c >= 0xe_2_8_0_8_0 and c <= 0xe_2_b_0_7_f:
						return True
				return False

			__lowerCAmelCase			:						List[Any]									=      0
			__lowerCAmelCase			:						Any									=      []
			while pos < len(_SCREAMING_SNAKE_CASE ):
				__lowerCAmelCase			:						Any									=      min(len(_SCREAMING_SNAKE_CASE )       ,		pos + self.maxlen + 1 ) if text[pos] == '<' else pos + 3
				__lowerCAmelCase			:						Union[str, Any]									=      []  # (token_id, token, pos)
				for e in range(_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE       ,		-1 ):
					__lowerCAmelCase			:						Optional[Any]									=      text[pos:e]
					if wd in self.vocab:
						if wd[0] == "<" and len(_SCREAMING_SNAKE_CASE ) > 2:
							__lowerCAmelCase			:						Optional[Any]									=      [(self.vocab[wd], wd, e)]
							break
						else:
							candidates.append((self.vocab[wd], wd, e) )
				if len(_SCREAMING_SNAKE_CASE ) > 0:
					# the smallest token_id is adopted
					__lowerCAmelCase			,   __lowerCAmelCase			,   __lowerCAmelCase			:						int									=      sorted(_SCREAMING_SNAKE_CASE       ,		key=lambda _SCREAMING_SNAKE_CASE : x[0] )[0]
					result.append(_SCREAMING_SNAKE_CASE )
					__lowerCAmelCase			:						Union[str, Any]									=      e
				else:
					__lowerCAmelCase			:						List[Any]									=      pos + 1
					__lowerCAmelCase			:						str									=      text[pos:end]
					if check_simbol(_SCREAMING_SNAKE_CASE ):
						result.append('<KIGOU>' )
					elif checkuae(_SCREAMING_SNAKE_CASE ):
						result.append('<U2000U2BFF>' )
					else:
						for i in wd.encode('utf-8' ):
							result.append('<|byte%d|>' % i )
					__lowerCAmelCase			:						List[str]									=      end
			return result







		def 							__lowerCamelCase							(      self       ,		_SCREAMING_SNAKE_CASE       ,		_SCREAMING_SNAKE_CASE="\n" ):
			__lowerCAmelCase			:						Union[str, Any]									=      []
			__lowerCAmelCase			:						List[Any]									=      []
			__lowerCAmelCase			:						Dict									=      self.ids_to_tokens[index][0]
			if word[:6] == "<|byte" and word[-2:] == "|>":
				byte_tokens.append(int(word[6:-2] ) )
			else:
				if len(_SCREAMING_SNAKE_CASE ) > 0:
					words.append(bytearray(_SCREAMING_SNAKE_CASE ).decode('utf-8'       ,		errors='replace' ) )
					__lowerCAmelCase			:						Dict									=      []
				if word[:7] == "<|emoji" and word[-2:] == "|>":
					words.append(self.emoji['emoji_inv'][word] )
				elif word == "<SP>":
					words.append(' ' )
				elif word == "<BR>":
					words.append(_SCREAMING_SNAKE_CASE )
				elif word == "<TAB>":
					words.append('\t' )
				elif word == "<BLOCK>":
					words.append('▀' )
				elif word == "<KIGOU>":
					words.append('ǀ' )
				elif word == "<U2000U2BFF>":
					words.append('‖' )
				else:
					words.append(_SCREAMING_SNAKE_CASE )
			if len(_SCREAMING_SNAKE_CASE ) > 0:
				words.append(bytearray(_SCREAMING_SNAKE_CASE ).decode('utf-8'       ,		errors='replace' ) )
			__lowerCAmelCase			:						List[str]									=      ''.join(_SCREAMING_SNAKE_CASE )
			return text






"""simple docstring"""
def      __lowerCamelCase   (							a_     :		int							)			->      bool:
      if not isinstance(a_ ,   a_							):
            __SCREAMING_SNAKE_CASE   :int						=  f'''Input value of [number={number}] must be an integer'''
            raise TypeError(a_							)
      if number < 0:
            return False
      __SCREAMING_SNAKE_CASE   :Any						=  number * number
      while number > 0:
            if number % 10 != number_square % 10:
                  return False
            number //= 10
            number_square //= 10
      return True


if __name__ == "__main__":
  import doctest

  doctest.testmod()