
'''simple docstring'''

from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

UpperCAmelCase__   : int					=		{
    name: (getattr(transformers, name), getattr(transformers, name + "Fast")) for name in SLOW_TO_FAST_CONVERTERS
}

UpperCAmelCase__   : Union[str, Any]					=		datasets.load_dataset("xnli", split="test+validation")

UpperCAmelCase__   : Union[str, Any]					=		0
UpperCAmelCase__   : List[Any]					=		0
UpperCAmelCase__   : Dict					=		0
UpperCAmelCase__   : Dict					=		0







def   A     (      UpperCamelCase_					:     Optional[Any]						,	UpperCamelCase_					:     Any						,	UpperCamelCase_					:     List[Any]						,	UpperCamelCase_					:     str      )					->       Tuple:







			'''simple docstring'''



			if spm_diff == list(reversed(UpperCamelCase_      )      ):
						# AAA -> AA+A vs A+AA case.
						return True
			elif len(UpperCamelCase_      ) == len(UpperCamelCase_      ) and fast.decode(UpperCamelCase_      ) == fast.decode(UpperCamelCase_      ):
						# Second order OK
						# Barrich -> Barr + ich vs Bar + rich
						return True
			lowerCAmelCase__								=							slow.encode(slow.decode(UpperCamelCase_      )      )
			lowerCAmelCase__								=							fast.encode(fast.decode(UpperCamelCase_      )      )
			if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
						# Type 3 error.
						# Snehagatha ->
						#       Sne, h, aga, th, a
						#       Sne, ha, gat, ha
						# Encoding the wrong with sp does not even recover what spm gave us
						# It fits tokenizer however...
						return True
			return False







def   A     (      UpperCamelCase_					:     str						,	UpperCamelCase_					:     str						,	UpperCamelCase_					:     str      )					->       Any:







			'''simple docstring'''



			lowerCAmelCase__								=							fast.encode_plus(UpperCamelCase_      )[0]
			lowerCAmelCase__								=							enc.offsets
			lowerCAmelCase__	,lowerCAmelCase__								=							offsets[idx], offsets[idx - 1]
			if curr is not None and line[curr[0] : curr[1]] == "\u200f":
						return True
			if prev is not None and line[prev[0] : prev[1]] == "\u200f":
						return True







def   A     (      UpperCamelCase_					:     Optional[Any]						,	UpperCamelCase_					:     Tuple						,	UpperCamelCase_					:     List[str]						,	UpperCamelCase_					:     List[Any]						,	UpperCamelCase_					:     str      )					->       Optional[Any]:







			'''simple docstring'''



			for i, (spm_id, tok_id) in enumerate(zip(UpperCamelCase_						,	UpperCamelCase_      )      ):
						if spm_id != tok_id:
									break
			lowerCAmelCase__								=							i
			for i, (spm_id, tok_id) in enumerate(zip(reversed(UpperCamelCase_      )						,	reversed(UpperCamelCase_      )      )      ):
						if spm_id != tok_id:
									break
			lowerCAmelCase__								=							len(UpperCamelCase_      ) - i

			lowerCAmelCase__								=							spm_ids[first:last]
			lowerCAmelCase__								=							tok_ids[first:last]

			if check_diff(UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_      ):
						return True

			if check_LTR_mark(UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_      ):
						return True

			if last - first > 5:
						# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
						lowerCAmelCase__								=							Counter(spm_ids[first:last]      )
						lowerCAmelCase__								=							Counter(tok_ids[first:last]      )

						lowerCAmelCase__								=							{spm_ for (spm_, si) in spms.items() if toks.get(spm_						,	0      ) == si}
						lowerCAmelCase__								=							3
						for i in range(last - first - min_width      ):
									if all(spm_ids[first + i + j] in removable_tokens for j in range(UpperCamelCase_      )      ):
												lowerCAmelCase__								=							[
												    k
												    for k in range(last - first - min_width      )
												    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
												]
												for j in possible_matches:
															if check_diff(spm_ids[first : first + i]						,	tok_ids[first : first + j]						,	UpperCamelCase_						,	UpperCamelCase_      ) and check_details(
															    UpperCamelCase_						,	spm_ids[first + i : last]						,	tok_ids[first + j : last]						,	UpperCamelCase_						,	UpperCamelCase_						,	):
																		return True

			print(F"""Spm: {[fast.decode([spm_ids[i]]      ) for i in range(UpperCamelCase_						,	UpperCamelCase_      )]}"""      )
			try:
						print(F"""Tok: {[fast.decode([tok_ids[i]]      ) for i in range(UpperCamelCase_						,	UpperCamelCase_      )]}"""      )
			except Exception:
						pass

			lowerCAmelCase__								=							fast.decode(spm_ids[:first]      )
			lowerCAmelCase__								=							fast.decode(spm_ids[last:]      )
			lowerCAmelCase__								=							fast.decode(spm_ids[first:last]      )
			print()
			print(UpperCamelCase_      )
			return False







def   A     (      UpperCamelCase_					:     Optional[int]						,	UpperCamelCase_					:     Union[str, Any]						,	UpperCamelCase_					:     Tuple      )					->       List[str]:







			'''simple docstring'''



			global perfect
			global imperfect
			global wrong
			global total

			lowerCAmelCase__								=							slow.encode(UpperCamelCase_      )
			lowerCAmelCase__								=							fast.encode(UpperCamelCase_      )

			lowerCAmelCase__								=							False
			total += 1

			if slow_ids != fast_ids:
						if check_details(UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_      ):
									lowerCAmelCase__								=							True
									imperfect += 1
						else:
									wrong += 1
			else:
						perfect += 1

			if total % 1_00_00 == 0:
						print(F"""({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"""      )

			if skip_assert:
						return

			assert (
			    slow_ids == fast_ids
			), F"""line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(UpperCamelCase_      )}\n{fast.tokenize(UpperCamelCase_      )}"""







def   A     (      UpperCamelCase_					:     Optional[Any]						,	UpperCamelCase_					:     str      )					->       Tuple:







			'''simple docstring'''



			global batch_total
			for i in range(len(UpperCamelCase_      )      ):
						# premise, all languages
						for text in dataset[i]["premise"].values():
									test_string(UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_      )

						# hypothesis, all languages
						for text in dataset[i]["hypothesis"]["translation"]:
									test_string(UpperCamelCase_						,	UpperCamelCase_						,	UpperCamelCase_      )


if __name__ == "__main__":
		for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
				UpperCAmelCase__   : Tuple					=		list(slow_class.max_model_input_sizes.keys())
				for checkpoint in checkpoint_names:
						UpperCAmelCase__   : Optional[int]					=		0
						UpperCAmelCase__   : Union[str, Any]					=		0
						UpperCAmelCase__   : Optional[int]					=		0
						UpperCAmelCase__   : Optional[int]					=		0

						print(F"========================== Checking {name}: {checkpoint} ==========================")
						UpperCAmelCase__   : List[str]					=		slow_class.from_pretrained(checkpoint, force_download=True)
						UpperCAmelCase__   : int					=		fast_class.from_pretrained(checkpoint, force_download=True)
						test_tokenizer(slow, fast)
						print(F"Accuracy {perfect * 1_00 / total:.2f}")







'''simple docstring'''

from __future__ import annotations







def   A     (      UpperCamelCase_					:     int      )					->       list[int]:







			'''simple docstring'''



			assert (
			    isinstance(UpperCamelCase_						,	UpperCamelCase_      ) and mask > 0
			), F"""mask needs to be positive integer, your input {mask}"""
			lowerCAmelCase__								=							[]
			lowerCAmelCase__								=							mask

			while submask:
						all_submasks.append(UpperCamelCase_      )
						lowerCAmelCase__								=							(submask - 1) & mask

			return all_submasks


if __name__ == "__main__":
		import doctest

		doctest.testmod()





