





"""simple docstring"""




import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    Dict					) ->str:






	'''simple docstring'''






	a      :		Any	=				int(_lowercase					)
	a,       a,       a      :		List[Any]	=				t // 3600, (t // 60) % 60, t % 60
	return F"""{h}:{m:02d}:{s:02d}""" if h != 0 else F"""{m:02d}:{s:02d}"""






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    Any    ,       _lowercase						:    int    ,       _lowercase						:    int    ,       _lowercase						:    Any    ,       _lowercase						:    str=300					) ->Dict:






	'''simple docstring'''






	return F"""
    <div>
      {prefix}
      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>
      {label}
    </div>
    """






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    int					) ->str:






	'''simple docstring'''






	a      :		Dict	=				"<table border=\"1\" class=\"dataframe\">\n"
	html_code += """  <thead>\n <tr style="text-align: left;">\n"""
	for i in items[0]:
		html_code += F"""      <th>{i}</th>\n"""
	html_code += "    </tr>\n  </thead>\n  <tbody>\n"
	for line in items[1:]:
		html_code += "    <tr>\n"
		for elt in line:
			a      :		List[Any]	=				F"""{elt:.6f}""" if isinstance(_lowercase    ,       _lowercase					) else str(_lowercase					)
			html_code += F"""      <td>{elt}</td>\n"""
		html_code += "    </tr>\n"
	html_code += "  </tbody>\n</table><p>"
	return html_code


class        __UpperCamelCase				:
							lowerCamelCase   :					Tuple      					=5
							lowerCamelCase   :					List[Any]      					=0.2






							def __init__(   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      ,					lowerCAmelCase__ = True      ,					lowerCAmelCase__ = None      ,					lowerCAmelCase__ = 300      ,					)	-> int:
								a      :		int	=				total
								a      :		Dict	=				"" if prefix is None else prefix
								a      :		Union[str, Any]	=				leave
								a      :		Tuple	=				parent
								a      :		Any	=				width
								a      :		Optional[int]	=				None
								a      :		Optional[Any]	=				None
								a      :		List[str]	=				None






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = False      ,					lowerCAmelCase__ = None      )	-> str:
								a      :		Optional[int]	=				value
								if comment is not None:
									a      :		List[Any]	=				comment
								if self.last_value is None:
									a      :		Union[str, Any]	=				time.time()
									a      :		Dict	=				value
									a      :		str	=				None
									a      :		Tuple	=				self.warmup
									a      :		Optional[Any]	=				1
									self.update_bar(lowerCAmelCase__      )
								elif value <= self.last_value and not force_update:
									return
								elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for      ,					self.total      ):
									if self.first_calls > 0:
										self.first_calls -= 1
									a      :		Dict	=				time.time()
									a      :		str	=				current_time - self.start_time
									# We could have value = self.start_value if the update is called twixe with the same start value.
									if value > self.start_value:
										a      :		Union[str, Any]	=				self.elapsed_time / (value - self.start_value)
									else:
										a      :		Union[str, Any]	=				None
									if value >= self.total:
										a      :		Union[str, Any]	=				self.total
										a      :		str	=				None
										if not self.leave:
											self.close()
									elif self.average_time_per_item is not None:
										a      :		Any	=				self.average_time_per_item * (self.total - value)
									self.update_bar(lowerCAmelCase__      )
									a      :		Optional[Any]	=				value
									a      :		str	=				current_time
									if self.average_time_per_item is None:
										a      :		int	=				1
									else:
										a      :		Optional[int]	=				max(int(self.update_every / self.average_time_per_item      )      ,					1      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      )	-> List[str]:
								a      :		List[Any]	=				" " * (len(str(self.total      )      ) - len(str(lowerCAmelCase__      )      )) + str(lowerCAmelCase__      )
								if self.elapsed_time is None:
									a      :		Optional[Any]	=				f"""[{spaced_value}/{self.total} : < :"""
								elif self.predicted_remaining is None:
									a      :		Any	=				f"""[{spaced_value}/{self.total} {format_time(self.elapsed_time      )}"""
								else:
									a      :		int	=				(
									    f"""[{spaced_value}/{self.total} {format_time(self.elapsed_time      )} <"""
									    f""" {format_time(self.predicted_remaining      )}"""
									)
									self.label += f""", {1/self.average_time_per_item:.2f} it/s"""
								self.label += "]" if self.comment is None or len(self.comment      ) == 0 else f""", {self.comment}]"""
								self.display()






							def     __a   (   self      )	-> List[str]:
								a      :		int	=				html_progress_bar(self.value      ,					self.total      ,					self.prefix      ,					self.label      ,					self.width      )
								if self.parent is not None:
									# If this is a child bar, the parent will take care of the display.
									self.parent.display()
									return
								if self.output is None:
									a      :		List[Any]	=				disp.display(disp.HTML(self.html_code      )      ,					display_id=lowerCAmelCase__      )
								else:
									self.output.update(disp.HTML(self.html_code      )      )






							def     __a   (   self      )	-> Optional[Any]:
								if self.parent is None and self.output is not None:
									self.output.update(disp.HTML(""      )      )


class        __UpperCamelCase				(						a__						):






							def __init__(   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      )	-> Dict:
								super().__init__(lowerCAmelCase__      )
								a      :		Optional[Any]	=				None if column_names is None else [column_names]
								a      :		str	=				None






							def     __a   (   self      )	-> Dict:
								a      :		List[Any]	=				html_progress_bar(self.value      ,					self.total      ,					self.prefix      ,					self.label      ,					self.width      )
								if self.inner_table is not None:
									self.html_code += text_to_html_table(self.inner_table      )
								if self.child_bar is not None:
									self.html_code += self.child_bar.html_code
								if self.output is None:
									a      :		Dict	=				disp.display(disp.HTML(self.html_code      )      ,					display_id=lowerCAmelCase__      )
								else:
									self.output.update(disp.HTML(self.html_code      )      )






							def     __a   (   self      ,					lowerCAmelCase__      )	-> List[Any]:
								if self.inner_table is None:
									a      :		Any	=				[list(values.keys()      ), list(values.values()      )]
								else:
									a      :		str	=				self.inner_table[0]
									if len(self.inner_table      ) == 1:
										# We give a chance to update the column names at the first iteration
										for key in values.keys():
											if key not in columns:
												columns.append(lowerCAmelCase__      )
										a      :		Optional[int]	=				columns
									self.inner_table.append([values[c] for c in columns]      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=300      )	-> Optional[Any]:
								a      :		List[str]	=				NotebookProgressBar(lowerCAmelCase__      ,					prefix=lowerCAmelCase__      ,					parent=self      ,					width=lowerCAmelCase__      )
								return self.child_bar






							def     __a   (   self      )	-> List[str]:
								a      :		Any	=				None
								self.display()


class        __UpperCamelCase				(						a__						):






							def __init__(   self      )	-> Union[str, Any]:
								a      :		Optional[int]	=				None
								a      :		List[str]	=				None
								a      :		Tuple	=				False






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					**lowerCAmelCase__      )	-> Optional[Any]:
								a      :		Optional[Any]	=				"Epoch" if args.evaluation_strategy == IntervalStrategy.EPOCH else "Step"
								a      :		Tuple	=				0
								a      :		Any	=				0
								a      :		List[str]	=				[self.first_column] + ["Training Loss"]
								if args.evaluation_strategy != IntervalStrategy.NO:
									column_names.append("Validation Loss"      )
								a      :		Dict	=				NotebookTrainingTracker(state.max_steps      ,					lowerCAmelCase__      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					**lowerCAmelCase__      )	-> Optional[int]:
								a      :		Any	=				int(state.epoch      ) if int(state.epoch      ) == state.epoch else f"""{state.epoch:.2f}"""
								self.training_tracker.update(
								    state.global_step + 1      ,					comment=f"""Epoch {epoch}/{state.num_train_epochs}"""      ,					force_update=self._force_next_update      ,					)
								a      :		Union[str, Any]	=				False






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      ,					**lowerCAmelCase__      )	-> Optional[Any]:
								if not has_length(lowerCAmelCase__      ):
									return
								if self.prediction_bar is None:
									if self.training_tracker is not None:
										a      :		List[str]	=				self.training_tracker.add_child(len(lowerCAmelCase__      )      )
									else:
										a      :		Union[str, Any]	=				NotebookProgressBar(len(lowerCAmelCase__      )      )
									self.prediction_bar.update(1      )
								else:
									self.prediction_bar.update(self.prediction_bar.value + 1      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					**lowerCAmelCase__      )	-> Union[str, Any]:
								if self.prediction_bar is not None:
									self.prediction_bar.close()
								a      :		Optional[Any]	=				None






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      ,					**lowerCAmelCase__      )	-> Optional[int]:
								# Only for when there is no evaluation
								if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
									a      :		List[Any]	=				{"Training Loss": logs["loss"]}
									# First column is necessarily Step sine we're not in epoch eval strategy
									a      :		Optional[Any]	=				state.global_step
									self.training_tracker.write_line(lowerCAmelCase__      )






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__=None      ,					**lowerCAmelCase__      )	-> Optional[Any]:
								if self.training_tracker is not None:
									a      :		Dict	=				{"Training Loss": "No log", "Validation Loss": "No log"}
									for log in reversed(state.log_history      ):
										if "loss" in log:
											a      :		Union[str, Any]	=				log["loss"]
											break

									if self.first_column == "Epoch":
										a      :		str	=				int(state.epoch      )
									else:
										a      :		Dict	=				state.global_step
									a      :		Dict	=				"eval"
									for k in metrics:
										if k.endswith("_loss"      ):
											a      :		Any	=				re.sub(R"\_loss$"      ,					""      ,					lowerCAmelCase__      )
									a      :		Optional[Any]	=				metrics.pop("total_flos"      ,					lowerCAmelCase__      )
									a      :		Optional[int]	=				metrics.pop("epoch"      ,					lowerCAmelCase__      )
									a      :		int	=				metrics.pop(f"""{metric_key_prefix}_runtime"""      ,					lowerCAmelCase__      )
									a      :		Tuple	=				metrics.pop(f"""{metric_key_prefix}_samples_per_second"""      ,					lowerCAmelCase__      )
									a      :		str	=				metrics.pop(f"""{metric_key_prefix}_steps_per_second"""      ,					lowerCAmelCase__      )
									a      :		Any	=				metrics.pop(f"""{metric_key_prefix}_jit_compilation_time"""      ,					lowerCAmelCase__      )
									for k, v in metrics.items():
										if k == f"""{metric_key_prefix}_loss""":
											a      :		Union[str, Any]	=				v
										else:
											a      :		Tuple	=				k.split("_"      )
											a      :		str	=				" ".join([part.capitalize() for part in splits[1:]]      )
											a      :		Union[str, Any]	=				v
									self.training_tracker.write_line(lowerCAmelCase__      )
									self.training_tracker.remove_child()
									a      :		Any	=				None
									# Evaluation takes a long time so we should force the next update.
									a      :		Optional[Any]	=				True






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					lowerCAmelCase__      ,					**lowerCAmelCase__      )	-> Any:
								self.training_tracker.update(
								    state.global_step      ,					comment=f"""Epoch {int(state.epoch      )}/{state.num_train_epochs}"""      ,					force_update=lowerCAmelCase__      )
								a      :		Tuple	=				None








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
 from .tokenization_fnet import FNetTokenizer
else:
 lowerCamelCase				:		List[Any]     =   None

lowerCamelCase				:		int     =   logging.get_logger(__name__)
lowerCamelCase				:		Optional[int]     =   {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

lowerCamelCase				:		Optional[int]     =   {
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json",
    },
}

lowerCamelCase				:		List[Any]     =   {
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

lowerCamelCase				:		Any     =   "▁"







class   A(						UpperCamelCase   ):



 '''simple docstring'''


 UpperCamelCase						   =						VOCAB_FILES_NAMES
 UpperCamelCase						   =						PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase						   =						['''input_ids''', '''token_type_ids''']
 UpperCamelCase						   =						FNetTokenizer

 def __init__(				self					:  Optional[int]						,							A_					:  Any=None						,							A_					:  int=None						,							A_					:  int=False						,							A_					:  Optional[int]=True						,							A_					:  List[Any]=True						,							A_					:  Tuple="<unk>"						,							A_					:  Optional[int]="[SEP]"						,							A_					:  List[Any]="<pad>"						,							A_					:  Optional[int]="[CLS]"						,							A_					:  Optional[Any]="[MASK]"						,							**A_					:  Dict						,							)   ->	List[str]:

       """simple docstring"""






       lowerCamelCase_		     =    (
           AddedToken(A_						,							lstrip=A_						,							rstrip=A_						,							normalized=A_  )
           if isinstance(A_						,							A_  )
           else mask_token
       )

       super().__init__(
           A_						,							tokenizer_file=A_						,							do_lower_case=A_						,							remove_space=A_						,							keep_accents=A_						,							unk_token=A_						,							sep_token=A_						,							pad_token=A_						,							cls_token=A_						,							mask_token=A_						,							**A_						,							)

       lowerCamelCase_		     =    do_lower_case
       lowerCamelCase_		     =    remove_space
       lowerCamelCase_		     =    keep_accents
       lowerCamelCase_		     =    vocab_file
       lowerCamelCase_		     =    False if not self.vocab_file else True

 def 				a__ (				self					:  List[Any]						,							A_					:  List[int]						,							A_					:  Optional[List[int]] = None  )   ->	List[int]:

       """simple docstring"""






       lowerCamelCase_		     =    [self.sep_token_id]
       lowerCamelCase_		     =    [self.cls_token_id]
       if token_ids_a is None:
             return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep

 def 				a__ (				self					:  Tuple						,							A_					:  List[int]						,							A_					:  Optional[List[int]] = None  )   ->	List[int]:

       """simple docstring"""






       lowerCamelCase_		     =    [self.sep_token_id]
       lowerCamelCase_		     =    [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]





 def 				a__ (				self					:  Dict						,							A_					:  str						,							A_					:  Optional[str] = None  )   ->	Tuple[str]:

       """simple docstring"""






       if not os.path.isdir(A_  ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
             return
       lowerCamelCase_		     =    os.path.join(
           A_						,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

       if os.path.abspath(self.vocab_file  ) != os.path.abspath(A_  ):
             copyfile(self.vocab_file						,							A_  )

       return (out_vocab_file,)



