code: from collections import Counter
  import datasets
  import transformers
  from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
  
  from transformers.utils import logging
  
  logging.set_verbosity_info()
  
  _SCREAMING_SNAKE_CASE   						=    {
      name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
  }
  
  _SCREAMING_SNAKE_CASE   						=    datasets.load_dataset("""xnli""", split="""test+validation""")
  
  _SCREAMING_SNAKE_CASE   						=    0
  _SCREAMING_SNAKE_CASE   						=    0
  _SCREAMING_SNAKE_CASE   						=    0
  _SCREAMING_SNAKE_CASE   						=    0
  def 				__lowercase	(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->Union[str, Any]:
  
    '''simple docstring'''
  
  
  
  
    if spm_diff == list(reversed(lowerCamelCase__					)					):
      # AAA -> AA+A vs A+AA case.
      return True
    elif len(lowerCamelCase__					) == len(lowerCamelCase__					) and fast.decode(lowerCamelCase__					) == fast.decode(lowerCamelCase__					):
      # Second order OK
      # Barrich -> Barr + ich vs Bar + rich
      return True
    a__     		=     slow.encode(slow.decode(lowerCamelCase__					)					)
    a__     		=     fast.encode(fast.decode(lowerCamelCase__					)					)
    if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
      # Type 3 error.
      # Snehagatha ->
      #       Sne, h, aga, th, a
      #       Sne, ha, gat, ha
      # Encoding the wrong with sp does not even recover what spm gave us
      # It fits tokenizer however...
      return True
    return False
  def 				__lowercase	(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->Optional[Any]:
  
    '''simple docstring'''
  
  
  
  
    a__     		=     fast.encode_plus(lowerCamelCase__					)[0]
    a__     		=     enc.offsets
    a__		,a__     		=     offsets[idx], offsets[idx - 1]
    if curr is not None and line[curr[0] : curr[1]] == "\u200f":
      return True
    if prev is not None and line[prev[0] : prev[1]] == "\u200f":
      return True
  def 				__lowercase	(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->Union[str, Any]:
  
    '''simple docstring'''
  
  
  
  
    for i, (spm_id, tok_id) in enumerate(zip(lowerCamelCase__ ,				lowerCamelCase__					)					):
      if spm_id != tok_id:
        break
    a__     		=     i
    for i, (spm_id, tok_id) in enumerate(zip(reversed(lowerCamelCase__					) ,				reversed(lowerCamelCase__					)					)					):
      if spm_id != tok_id:
        break
    a__     		=     len(lowerCamelCase__					) - i
  
    a__     		=     spm_ids[first:last]
    a__     		=     tok_ids[first:last]
  
    if check_diff(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					):
      return True
  
    if check_LTR_mark(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					):
      return True
  
    if last - first > 5:
      # We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
      a__     		=     Counter(spm_ids[first:last]					)
      a__     		=     Counter(tok_ids[first:last]					)
  
      a__     		=     {spm_ for (spm_, si) in spms.items() if toks.get(spm_ ,				0					) == si}
      a__     		=     3
      for i in range(last - first - min_width					):
        if all(spm_ids[first + i + j] in removable_tokens for j in range(lowerCamelCase__					)					):
          a__     		=     [
              k
              for k in range(last - first - min_width					)
              if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
          ]
          for j in possible_matches:
            if check_diff(spm_ids[first : first + i] ,				tok_ids[first : first + j] ,				lowerCamelCase__ ,				lowerCamelCase__					) and check_details(
                lowerCamelCase__ ,				spm_ids[first + i : last] ,				tok_ids[first + j : last] ,				lowerCamelCase__ ,				lowerCamelCase__ ,				):
              return True
  
    print(f"Spm: {[fast.decode([spm_ids[i]]					) for i in range(lowerCamelCase__ ,				lowerCamelCase__					)]}"					)
    try:
      print(f"Tok: {[fast.decode([tok_ids[i]]					) for i in range(lowerCamelCase__ ,				lowerCamelCase__					)]}"					)
    except Exception:
      pass
  
    a__     		=     fast.decode(spm_ids[:first]					)
    a__     		=     fast.decode(spm_ids[last:]					)
    a__     		=     fast.decode(spm_ids[first:last]					)
    print()
    print(lowerCamelCase__					)
    return False
  def 				__lowercase	(						lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)				->int:
  
    '''simple docstring'''
  
  
  
  
    global perfect
    global imperfect
    global wrong
    global total
  
    a__     		=     slow.encode(lowerCamelCase__					)
    a__     		=     fast.encode(lowerCamelCase__					)
  
    a__     		=     False
    total += 1
  
    if slow_ids != fast_ids:
      if check_details(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					):
        a__     		=     True
        imperfect += 1
      else:
        wrong += 1
    else:
      perfect += 1
  
    if total % 1_00_00 == 0:
      print(f"({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})"					)
  
    if skip_assert:
      return
  
    assert (
        slow_ids == fast_ids
    ), f"line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(lowerCamelCase__					)}\n{fast.tokenize(lowerCamelCase__					)}"
  def 				__lowercase	(						lowerCamelCase__ ,				lowerCamelCase__					)				->Any:
  
    '''simple docstring'''
  
  
  
  
    global batch_total
    for i in range(len(lowerCamelCase__					)					):
      # premise, all languages
      for text in dataset[i]["premise"].values():
        test_string(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)
  
      # hypothesis, all languages
      for text in dataset[i]["hypothesis"]["translation"]:
        test_string(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__					)
  
  
  if __name__ == "__main__":
       for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
            _SCREAMING_SNAKE_CASE   						=    list(slow_class.max_model_input_sizes.keys())
            for checkpoint in checkpoint_names:
                 _SCREAMING_SNAKE_CASE   						=    0
                 _SCREAMING_SNAKE_CASE   						=    0
                 _SCREAMING_SNAKE_CASE   						=    0
                 _SCREAMING_SNAKE_CASE   						=    0
  
                 print(F"""========================== Checking {name}: {checkpoint} ==========================""")
                 _SCREAMING_SNAKE_CASE   						=    slow_class.from_pretrained(checkpoint, force_download=True)
                 _SCREAMING_SNAKE_CASE   						=    fast_class.from_pretrained(checkpoint, force_download=True)
                 test_tokenizer(slow, fast)
                 print(F"""Accuracy {perfect * 100 / total:.2f}""")
  
  
style_context: _SCREAMING_SNAKE_CASE   						=    [0.485, 0.456, 0.406]
  _SCREAMING_SNAKE_CASE   						=    [0.229, 0.224, 0.225]
  _SCREAMING_SNAKE_CASE   						=    [0.5, 0.5, 0.5]
  _SCREAMING_SNAKE_CASE   						=    [0.5, 0.5, 0.5]
  _SCREAMING_SNAKE_CASE   						=    [0.48145466, 0.4578275, 0.40821073]
  _SCREAMING_SNAKE_CASE   						=    [0.26862954, 0.26130258, 0.27577711]
  
  