
'''simple docstring'''
from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('3.8'):
     import importlib_metadata
else:
     import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
A__ :		Tuple					='\\n'

A__ :		Optional[Any]					='\\n    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.\n    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.\n'

A__ :		List[str]					='\nCompute XTREME-S evaluation metric associated to each XTREME-S dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\n    bleu_kwargs: optional dict of keywords to be passed when computing \'bleu\'.\n        Keywords include Dict can be one of \'smooth_method\', \'smooth_value\', \'force\', \'lowercase\',\n        \'tokenize\', \'use_effective_order\'.\n    wer_kwargs: optional dict of keywords to be passed when computing \'wer\' and \'cer\'.\n        Keywords include \'concatenate_texts\'.\nReturns: depending on the XTREME-S task, one or several of:\n    "accuracy": Accuracy - for \'fleurs-lang_id\', \'minds14\'\n    "f1": F1 score - for \'minds14\'\n    "wer": Word error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "cer": Character error rate - for \'mls\', \'fleurs-asr\', \'voxpopuli\', \'babel\'\n    "bleu": BLEU score according to the `sacrebleu` metric - for \'covost2\'\nExamples:\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'mls\')  # \'mls\', \'voxpopuli\', \'fleurs-asr\' or \'babel\'\n    >>> references = ["it is sunny here", "paper and pen are essentials"]\n    >>> predictions = ["it\'s sunny", "paper pen are essential"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'wer\': 0.56, \'cer\': 0.27}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'covost2\')\n    >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"]\n    >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'bleu\': 31.65}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'fleurs-lang_id\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'accuracy\': 0.6}\n\n    >>> xtreme_s_metric = datasets.load_metric(\'xtreme_s\', \'minds14\')\n    >>> references = [0, 1, 0, 0, 1]\n    >>> predictions = [0, 1, 1, 0, 0]\n    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)\n    >>> print({k: round(v, 2) for k, v in results.items()})\n    {\'f1\': 0.58, \'accuracy\': 0.6}\n'

A__ :		List[str]					=['fleurs-asr', 'mls', 'voxpopuli', 'babel', 'covost2', 'fleurs-lang_id', 'minds14']
A__ :		Optional[Any]					=''

try:
     from jiwer import transforms as tr

     A__ :		Any					=True
except ImportError:
     A__ :		Optional[int]					=False

if _jiwer_available and version.parse(importlib_metadata.version('jiwer')) < version.parse('2.3.0'):







     class 			__A					(			tr.AbstractTransform      ):
      def __init__( self      :     Union[str, Any]					, lowerCamelCase      :     str = " "							):



        """simple docstring"""





        __A		:  Union[str, Any]       				=		sentence_delimiter
      def 		lowercase_( self      :     Any					, lowerCamelCase      :     str							):



        """simple docstring"""





        return list(_lowerCAmelCase							)






      def 		lowercase_( self      :     Any					, lowerCamelCase      :     List[str]							):



        """simple docstring"""





        __A		:  str       				=		[]
        for sent_idx, sentence in enumerate(_lowerCAmelCase							):
          chars.extend(self.process_string(_lowerCAmelCase							)							)
          if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(_lowerCAmelCase							) - 1:
            chars.append(self.sentence_delimiter							)
        return chars

     A__ :		Optional[int]					=tr.Compose(
         [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
     )
elif _jiwer_available:
     A__ :		List[Any]					=tr.Compose(
         [
             tr.RemoveMultipleSpaces(),
             tr.Strip(),
             tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
             tr.ReduceToListOfListOfChars(),
         ]
     )
else:
     A__ :		Union[str, Any]					=None







def 							A_  (    __SCREAMING_SNAKE_CASE    :	Dict      ,  __SCREAMING_SNAKE_CASE    :	Optional[Any]			)					->							List[Any]:





  """simple docstring"""

  return float((preds == labels).mean()			)







def 							A_  (    __SCREAMING_SNAKE_CASE    :	Tuple      ,  __SCREAMING_SNAKE_CASE    :	List[Any]			)					->							Dict:





  """simple docstring"""

  return {
      "f1": float(fa_score(y_true=__SCREAMING_SNAKE_CASE      ,  y_pred=__SCREAMING_SNAKE_CASE      ,  average="""macro"""			)			),
      "accuracy": simple_accuracy(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			),
  }







def 							A_  (    __SCREAMING_SNAKE_CASE    :	Tuple      ,  __SCREAMING_SNAKE_CASE    :	Dict      ,  __SCREAMING_SNAKE_CASE    :	Tuple="exp"      ,  __SCREAMING_SNAKE_CASE    :	Tuple=None      ,  __SCREAMING_SNAKE_CASE    :	Dict=False      ,  __SCREAMING_SNAKE_CASE    :	Tuple=False      ,  __SCREAMING_SNAKE_CASE    :	Any=None      ,  __SCREAMING_SNAKE_CASE    :	Optional[Any]=False      ,  )					->							Tuple:





  """simple docstring"""

  __A		:  Dict       				=		[[label] for label in labels]
  __A		:  Tuple       				=		list(__SCREAMING_SNAKE_CASE			)
  try:
    import sacrebleu as scb
  except ImportError:
    raise ValueError(
        """sacrebleu has to be installed in order to apply the bleu metric for covost2."""
        """You can install it via `pip install sacrebleu`."""			)

  if version.parse(scb.__version__			) < version.parse("""1.4.12"""			):
    raise ImportWarning(
        """To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"""
        """You can install it with `pip install \"sacrebleu>=1.4.12\"`."""			)

  __A		:  Optional[Any]       				=		len(labels[0]			)
  if any(len(__SCREAMING_SNAKE_CASE			) != references_per_prediction for refs in labels			):
    raise ValueError("""Sacrebleu requires the same number of references for each prediction"""			)
  __A		:  Dict       				=		[[refs[i] for refs in labels] for i in range(__SCREAMING_SNAKE_CASE			)]
  __A		:  Any       				=		scb.corpus_bleu(
      __SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  smooth_method=__SCREAMING_SNAKE_CASE      ,  smooth_value=__SCREAMING_SNAKE_CASE      ,  force=__SCREAMING_SNAKE_CASE      ,  lowercase=__SCREAMING_SNAKE_CASE      ,  use_effective_order=__SCREAMING_SNAKE_CASE      ,  **({"""tokenize""": tokenize} if tokenize else {})      ,  )
  return {"bleu": output.score}







def 							A_  (    __SCREAMING_SNAKE_CASE    :	Any      ,  __SCREAMING_SNAKE_CASE    :	Dict      ,  __SCREAMING_SNAKE_CASE    :	List[str]      ,  __SCREAMING_SNAKE_CASE    :	List[str]			)					->							List[str]:





  """simple docstring"""

  try:
    from jiwer import compute_measures
  except ImportError:
    raise ValueError(
        F"jiwer has to be installed in order to apply the wer metric for {config_name}."
        """You can install it via `pip install jiwer`."""			)

  if concatenate_texts:
    __A		:  str       				=		compute_measures(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			)["""wer"""]

    __A		:  Tuple       				=		compute_measures(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  truth_transform=__SCREAMING_SNAKE_CASE      ,  hypothesis_transform=__SCREAMING_SNAKE_CASE			)["""wer"""]
    return {"wer": wer, "cer": cer}
  else:

    def compute_score(__SCREAMING_SNAKE_CASE    :	str      ,  __SCREAMING_SNAKE_CASE    :	int      ,  __SCREAMING_SNAKE_CASE    :	Any="wer"			):
      __A		:  Any       				=		0
      __A		:  Tuple       				=		0
      for prediction, reference in zip(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			):
        if score_type == "wer":
          __A		:  Any       				=		compute_measures(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			)
        elif score_type == "cer":
          __A		:  int       				=		compute_measures(
              __SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  truth_transform=__SCREAMING_SNAKE_CASE      ,  hypothesis_transform=__SCREAMING_SNAKE_CASE			)
        incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
        total += measures["substitutions"] + measures["deletions"] + measures["hits"]
      return incorrect / total

    return {"wer": compute_score(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  """wer"""			), "cer": compute_score(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  """cer"""			)}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,    _KWARGS_DESCRIPTION      )
class 			__A					(			datasets.Metric      ):
 def 		lowercase_( self      :     Dict							):



   """simple docstring"""





   if self.config_name not in _CONFIG_NAMES:
     raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}"							)

   __A		:  Union[str, Any]       				=		"""int64""" if self.config_name in ["""fleurs-lang_id""", """minds14"""] else """string"""

   return datasets.MetricInfo(
       description=_DESCRIPTION					, citation=_CITATION					, inputs_description=_KWARGS_DESCRIPTION					, features=datasets.Features(
           {"""predictions""": datasets.Value(_lowerCAmelCase							), """references""": datasets.Value(_lowerCAmelCase							)}							)					, codebase_urls=[]					, reference_urls=[]					, format="""numpy"""					, )






 def 		lowercase_( self      :     Union[str, Any]					, lowerCamelCase      :     Union[str, Any]					, lowerCamelCase      :     Optional[Any]					, lowerCamelCase      :     Optional[int]=None					, lowerCamelCase      :     Any=None							):



   """simple docstring"""





   __A		:  Optional[Any]       				=		bleu_kwargs if bleu_kwargs is not None else {}
   __A		:  Union[str, Any]       				=		wer_kwargs if wer_kwargs is not None else {}

   if self.config_name == "fleurs-lang_id":
     return {"accuracy": simple_accuracy(_lowerCAmelCase					, _lowerCAmelCase							)}
   elif self.config_name == "minds14":
     return fa_and_simple_accuracy(_lowerCAmelCase					, _lowerCAmelCase							)
   elif self.config_name == "covost2":
     __A		:  Tuple       				=		bleu_kwargs.pop("""smooth_method"""					, """exp"""							)
     __A		:  Optional[int]       				=		bleu_kwargs.pop("""smooth_value"""					, _lowerCAmelCase							)
     __A		:  Optional[Any]       				=		bleu_kwargs.pop("""force"""					, _lowerCAmelCase							)
     __A		:  str       				=		bleu_kwargs.pop("""lowercase"""					, _lowerCAmelCase							)
     __A		:  int       				=		bleu_kwargs.pop("""tokenize"""					, _lowerCAmelCase							)
     __A		:  Tuple       				=		bleu_kwargs.pop("""use_effective_order"""					, _lowerCAmelCase							)
     return bleu(
         preds=_lowerCAmelCase					, labels=_lowerCAmelCase					, smooth_method=_lowerCAmelCase					, smooth_value=_lowerCAmelCase					, force=_lowerCAmelCase					, lowercase=_lowerCAmelCase					, tokenize=_lowerCAmelCase					, use_effective_order=_lowerCAmelCase					, )
   elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
     __A		:  Optional[int]       				=		wer_kwargs.pop("""concatenate_texts"""					, _lowerCAmelCase							)
     return wer_and_cer(_lowerCAmelCase					, _lowerCAmelCase					, _lowerCAmelCase					, self.config_name							)
   else:
     raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}"							)










'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCamelCase					:						Dict					     =							logging.get_logger(__name__)

_lowerCamelCase					:						Dict					     =							{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCamelCase					:						str					     =							{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_lowerCamelCase					:						Union[str, Any]					     =							{
    'openai-gpt': 512,
}




def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   text.strip()
	if not text:
		return []
	A					=   text.split()
	return tokens



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		def __init__(self    :   Optional[int]   ,   _lowerCAmelCase    :   int=True   ,   _lowerCAmelCase    :   List[str]=None   ,   _lowerCAmelCase    :   str=True   ,   _lowerCAmelCase    :   int=None   ,   _lowerCAmelCase    :   Optional[Any]=True   ,   ):
			if never_split is None:
				A					=   []
			A					=   do_lower_case
			A					=   set(_lowerCAmelCase   )
			A					=   tokenize_chinese_chars
			A					=   strip_accents
			A					=   do_split_on_punc







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ,   _lowerCAmelCase    :   Any=None   ):
			A					=   self.never_split.union(set(_lowerCAmelCase   )   ) if never_split else self.never_split
			A					=   self._clean_text(_lowerCAmelCase   )

			# This was added on November 1st, 2018 for the multilingual and Chinese
			# models. This is also applied to the English models now, but it doesn't
			# matter since the English models were not trained on any Chinese data
			# and generally don't have any Chinese data in them (there are Chinese
			# characters in the vocabulary because Wikipedia does have some Chinese
			# words in the English Wikipedia.).
			if self.tokenize_chinese_chars:
				A					=   self._tokenize_chinese_chars(_lowerCAmelCase   )
			# prevents treating the same character with different unicode codepoints as different characters
			A					=   unicodedata.normalize("""NFC"""   ,   _lowerCAmelCase   )
			A					=   whitespace_tokenize(_lowerCAmelCase   )
			A					=   []
			for token in orig_tokens:
				if token not in never_split:
					if self.do_lower_case:
						A					=   token.lower()
						if self.strip_accents is not False:
							A					=   self._run_strip_accents(_lowerCAmelCase   )
					elif self.strip_accents:
						A					=   self._run_strip_accents(_lowerCAmelCase   )
				split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase   ,   _lowerCAmelCase   )   )

			A					=   whitespace_tokenize(""" """.join(_lowerCAmelCase   )   )
			return output_tokens







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   unicodedata.normalize("""NFD"""   ,   _lowerCAmelCase   )
			A					=   []
			for char in text:
				A					=   unicodedata.category(_lowerCAmelCase   )
				if cat == "Mn":
					continue
				output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   Dict   ,   _lowerCAmelCase    :   List[str]=None   ):
			if not self.do_split_on_punc or (never_split is not None and text in never_split):
				return [text]
			A					=   list(_lowerCAmelCase   )
			A					=   0
			A					=   True
			A					=   []
			while i < len(_lowerCAmelCase   ):
				A					=   chars[i]
				if _is_punctuation(_lowerCAmelCase   ):
					output.append([char]   )
					A					=   True
				else:
					if start_new_word:
						output.append([]   )
					A					=   False
					output[-1].append(_lowerCAmelCase   )
				i += 1

			return ["".join(_lowerCAmelCase   ) for x in output]







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if self._is_chinese_char(_lowerCAmelCase   ):
					output.append(""" """   )
					output.append(_lowerCAmelCase   )
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Any   ):
			# This defines a "chinese character" as anything in the CJK Unicode block:
			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
			#
			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
			# despite its name. The modern Korean Hangul alphabet is a different block,
			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
			# space-separated words, so they are not treated specially and handled
			# like the all of the other languages.
			if (
			    (cp >= 0X4_e00 and cp <= 0X9_fff)
			    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
			    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
			    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
			    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
			    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
			    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
			    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
			):  #
				return True

			return False







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Any   ):
			A					=   []
			for char in text:
				A					=   ord(_lowerCAmelCase   )
				if cp == 0 or cp == 0Xf_ffd or _is_control(_lowerCAmelCase   ):
					continue
				if _is_whitespace(_lowerCAmelCase   ):
					output.append(""" """   )
				else:
					output.append(_lowerCAmelCase   )
			return "".join(_lowerCAmelCase   )






def  __a			(  UpperCAmelCase     )   ->Union[str, Any]:



	"""simple docstring"""







	A					=   set()
	A					=   word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)     )
		A					=   char
	return pairs




def  __a			(  UpperCAmelCase     )   ->List[str]:



	"""simple docstring"""







	A					=   text.replace("""—"""							,      """-"""     )
	A					=   text.replace("""–"""							,      """-"""     )
	A					=   text.replace("""―"""							,      """-"""     )
	A					=   text.replace("""…"""							,      """..."""     )
	A					=   text.replace("""´"""							,      """'"""     )
	A					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""							,      R""" \1 """							,      UpperCAmelCase     )
	A					=   re.sub(R"""\s*\n\s*"""							,      """ \n """							,      UpperCAmelCase     )
	A					=   re.sub(R"""[^\S\n]+"""							,      """ """							,      UpperCAmelCase     )
	return text.strip()



class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     VOCAB_FILES_NAMES
		__lowerCAmelCase								=     PRETRAINED_VOCAB_FILES_MAP
		__lowerCAmelCase								=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__lowerCAmelCase								=     ['''input_ids''', '''attention_mask''']







		def __init__(self    :   Dict   ,   _lowerCAmelCase    :   Optional[int]   ,   _lowerCAmelCase    :   List[Any]   ,   _lowerCAmelCase    :   Dict="<unk>"   ,   **_lowerCAmelCase    :   Dict   ):
			super().__init__(unk_token=_lowerCAmelCase   ,   **_lowerCAmelCase   )

			try:
				import ftfy
				from spacy.lang.en import English

				A					=   English()
				A					=   _nlp.tokenizer
				A					=   ftfy.fix_text
			except ImportError:
				logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""   )
				A					=   BasicTokenizer(do_lower_case=_lowerCAmelCase   )
				A					=   None

			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as vocab_handle:
				A					=   json.load(_lowerCAmelCase   )
			A					=   {v: k for k, v in self.encoder.items()}
			with open(_lowerCAmelCase   ,   encoding="""utf-8"""   ) as merges_handle:
				A					=   merges_handle.read().split("""\n"""   )[1:-1]
			A					=   [tuple(merge.split()   ) for merge in merges]
			A					=   dict(zip(_lowerCAmelCase   ,   range(len(_lowerCAmelCase   )   )   )   )
			A					=   {}







		@property
		def 							A  (self    :   Dict   ):
			return True







		@property
		def 							A  (self    :   Dict   ):
			return len(self.encoder   )







		def 							A  (self    :   str   ):
			return dict(self.encoder   ,   **self.added_tokens_encoder   )







		def 							A  (self    :   Optional[Any]   ,   _lowerCAmelCase    :   List[Any]   ):
			A					=   tuple(token[:-1]   ) + (token[-1] + """</w>""",)
			if token in self.cache:
				return self.cache[token]
			A					=   get_pairs(_lowerCAmelCase   )

			if not pairs:
				return token + "</w>"

			while True:
				A					=   min(_lowerCAmelCase   ,   key=lambda _lowerCAmelCase   : self.bpe_ranks.get(_lowerCAmelCase   ,   float("""inf"""   )   )   )
				if bigram not in self.bpe_ranks:
					break
				A	,     A					=   bigram
				A					=   []
				A					=   0
				while i < len(_lowerCAmelCase   ):
					try:
						A					=   word.index(_lowerCAmelCase   ,   _lowerCAmelCase   )
					except ValueError:
						new_word.extend(word[i:]   )
						break
					else:
						new_word.extend(word[i:j]   )
						A					=   j

					if word[i] == first and i < len(_lowerCAmelCase   ) - 1 and word[i + 1] == second:
						new_word.append(first + second   )
						i += 2
					else:
						new_word.append(word[i]   )
						i += 1
				A					=   tuple(_lowerCAmelCase   )
				A					=   new_word
				if len(_lowerCAmelCase   ) == 1:
					break
				else:
					A					=   get_pairs(_lowerCAmelCase   )
			A					=   """ """.join(_lowerCAmelCase   )
			if word == "\n  </w>":
				A					=   """\n</w>"""
			A					=   word
			return word







		def 							A  (self    :   List[str]   ,   _lowerCAmelCase    :   Optional[int]   ):
			A					=   []
			if self.fix_text is None:
				# Using BERT's BasicTokenizer
				A					=   self.nlp.tokenize(_lowerCAmelCase   )
				for token in text:
					split_tokens.extend(list(self.bpe(_lowerCAmelCase   ).split(""" """   )   )   )
			else:
				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
				A					=   self.nlp(text_standardize(self.fix_text(_lowerCAmelCase   )   )   )
				for token in text:
					split_tokens.extend(list(self.bpe(token.text.lower()   ).split(""" """   )   )   )
			return split_tokens







		def 							A  (self    :   int   ,   _lowerCAmelCase    :   Dict   ):
			return self.encoder.get(_lowerCAmelCase   ,   self.encoder.get(self.unk_token   )   )







		def 							A  (self    :   Dict   ,   _lowerCAmelCase    :   Tuple   ):
			return self.decoder.get(_lowerCAmelCase   ,   self.unk_token   )







		def 							A  (self    :   Optional[int]   ,   _lowerCAmelCase    :   int   ):
			A					=   """""".join(_lowerCAmelCase   ).replace("""</w>"""   ,   """ """   ).strip()
			return out_string







		def 							A  (self    :   str   ,   _lowerCAmelCase    :   str   ,   _lowerCAmelCase    :   Optional[str] = None   ):
			if not os.path.isdir(_lowerCAmelCase   ):
				logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
				return
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
			A					=   os.path.join(
			    _lowerCAmelCase   ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as f:
				f.write(json.dumps(self.encoder   ,   indent=2   ,   sort_keys=_lowerCAmelCase   ,   ensure_ascii=_lowerCAmelCase   ) + """\n"""   )

			A					=   0
			with open(_lowerCAmelCase   ,   """w"""   ,   encoding="""utf-8"""   ) as writer:
				writer.write("""#version: 0.2\n"""   )
				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,   key=lambda _lowerCAmelCase   : kv[1]   ):
					if index != token_index:
						logger.warning(
						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
						    """ Please check that the tokenizer is not corrupted!"""   )
						A					=   token_index
					writer.write(""" """.join(_lowerCAmelCase   ) + """\n"""   )
					index += 1

			return vocab_file, merge_file

