code: 
  import argparse
  import logging
  import os
  import sys
  from collections import namedtuple
  
  import torch
  from modeling_bertabs import BertAbs, build_predictor
  from torch.utils.data import DataLoader, SequentialSampler
  from tqdm import tqdm
  
  from transformers import BertTokenizer
  
  from .utils_summarization import (
      CNNDMDataset,
      build_mask,
      compute_token_type_ids,
      encode_for_summarization,
      truncate_or_pad,
  )
  
  
  snake_case				:	int							   =						logging.getLogger(__name__)
  logging.basicConfig(stream=sys.stdout, level=logging.INFO)
  
  
  snake_case				:	Union[str, Any]							   =						namedtuple("Batch", ["document_names", "batch_size", "src", "segs", "mask_src", "tgt_str"])
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						List[str]     )       ->	Union[str, Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Optional[Any]			     =  BertTokenizer.from_pretrained("bert-base-uncased"   ,     do_lower_case=_snake_case     )
  	__magic_name__	:	List[str]			     =  BertAbs.from_pretrained("remi/bertabs-finetuned-extractive-abstractive-summarization"     )
  	model.to(args.device     )
  	model.eval()
  
  	__magic_name__	:	Dict			     =  {
  	    "BOS": tokenizer.vocab["[unused0]"],
  	    "EOS": tokenizer.vocab["[unused1]"],
  	    "PAD": tokenizer.vocab["[PAD]"],
  	}
  
  	if args.compute_rouge:
  		__magic_name__	:	Optional[Any]			     =  []
  		__magic_name__	:	List[str]			     =  []
  
  		import nltk
  		import rouge
  
  		nltk.download("punkt"     )
  		__magic_name__	:	Optional[int]			     =  rouge.Rouge(
  		    metrics=["rouge-n", "rouge-l"]   ,     max_n=2   ,     limit_length=_snake_case   ,     length_limit=args.beam_size   ,     length_limit_type="words"   ,     apply_avg=_snake_case   ,     apply_best=_snake_case   ,     alpha=0.5   ,     weight_factor=1.2   ,     stemming=_snake_case   ,     )
  
  	# these (unused) arguments are defined to keep the compatibility
  	# with the legacy code and will be deleted in a next iteration.
  	__magic_name__	:	Any			     =  ""
  	__magic_name__	:	Tuple			     =  ""
  
  	__magic_name__	:	Union[str, Any]			     =  build_data_iterator(_snake_case   ,     _snake_case     )
  	__magic_name__	:	List[str]			     =  build_predictor(_snake_case   ,     _snake_case   ,     _snake_case   ,     _snake_case     )
  
  	logger.info("***** Running evaluation *****"     )
  	logger.info("  Number examples = %d"   ,     len(data_iterator.dataset     )     )
  	logger.info("  Batch size = %d"   ,     args.batch_size     )
  	logger.info(""     )
  	logger.info("***** Beam Search parameters *****"     )
  	logger.info("  Beam size = %d"   ,     args.beam_size     )
  	logger.info("  Minimum length = %d"   ,     args.min_length     )
  	logger.info("  Maximum length = %d"   ,     args.max_length     )
  	logger.info("  Alpha (length penalty) = %.2f"   ,     args.alpha     )
  	logger.info("  Trigrams %s be blocked"   ,     ("will" if args.block_trigram else "will NOT")     )
  
  	for batch in tqdm(_snake_case     ):
  		__magic_name__	:	Union[str, Any]			     =  predictor.translate_batch(_snake_case     )
  		__magic_name__	:	Optional[int]			     =  predictor.from_batch(_snake_case     )
  		__magic_name__	:	Tuple			     =  [format_summary(_snake_case     ) for t in translations]
  		save_summaries(_snake_case   ,     args.summaries_output_dir   ,     batch.document_names     )
  
  		if args.compute_rouge:
  			reference_summaries += batch.tgt_str
  			generated_summaries += summaries
  
  	if args.compute_rouge:
  		__magic_name__	:	str			     =  rouge_evaluator.get_scores(_snake_case   ,     _snake_case     )
  		__magic_name__	:	Dict			     =  format_rouge_scores(_snake_case     )
  		save_rouge_scores(_snake_case     )
  		print(_snake_case     )
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Tuple   ,     _snake_case   :						Tuple   ,     _snake_case   :						List[Any]     )       ->	Optional[Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	for summary, document_name in zip(_snake_case   ,     _snake_case     ):
  		# Prepare the summary file's name
  		if "." in document_name:
  			__magic_name__	:	List[str]			     =  ".".join(document_name.split("."     )[:-1]     )
  			__magic_name__	:	str			     =  document_name.split("."     )[-1]
  			__magic_name__	:	Any			     =  bare_document_name + "_summary." + extension
  		else:
  			__magic_name__	:	Dict			     =  document_name + "_summary"
  
  		__magic_name__	:	Optional[Any]			     =  os.path.join(_snake_case   ,     _snake_case     )
  		with open(_snake_case   ,     "w"     ) as output:
  			output.write(_snake_case     )
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						str     )       ->	Union[str, Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Any			     =  translation
  	__magic_name__	:	Any			     =  (
  	    raw_summary.replace("[unused0]"   ,     ""     )
  	    .replace("[unused3]"   ,     ""     )
  	    .replace("[PAD]"   ,     ""     )
  	    .replace("[unused1]"   ,     ""     )
  	    .replace(R" +"   ,     " "     )
  	    .replace(" [unused2] "   ,     ". "     )
  	    .replace("[unused2]"   ,     ""     )
  	    .strip()
  	)
  
  	return summary
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Optional[int]     )       ->	int:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	return """\n
  ****** ROUGE SCORES ******
  
  ** ROUGE 1
  F1        >> {:.3f}
  Precision >> {:.3f}
  Recall    >> {:.3f}
  
  ** ROUGE 2
  F1        >> {:.3f}
  Precision >> {:.3f}
  Recall    >> {:.3f}
  
  ** ROUGE L
  F1        >> {:.3f}
  Precision >> {:.3f}
  Recall    >> {:.3f}""".format(
  	    scores["rouge-1"]["f"]   ,     scores["rouge-1"]["p"]   ,     scores["rouge-1"]["r"]   ,     scores["rouge-2"]["f"]   ,     scores["rouge-2"]["p"]   ,     scores["rouge-2"]["r"]   ,     scores["rouge-l"]["f"]   ,     scores["rouge-l"]["p"]   ,     scores["rouge-l"]["r"]   ,     )
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						int     )       ->	Any:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	with open("rouge_scores.txt"   ,     "w"     ) as output:
  		output.write(_snake_case     )
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Tuple   ,     _snake_case   :						Optional[int]     )       ->	int:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Optional[Any]			     =  load_and_cache_examples(_snake_case   ,     _snake_case     )
  	__magic_name__	:	Optional[int]			     =  SequentialSampler(_snake_case     )
  
  	def collate_fn(_snake_case   :						Dict     ):
  		return collate(_snake_case   ,     _snake_case   ,     block_size=512   ,     device=args.device     )
  
  	__magic_name__	:	Dict			     =  DataLoader(
  	    _snake_case   ,     sampler=_snake_case   ,     batch_size=args.batch_size   ,     collate_fn=_snake_case   ,     )
  
  	return iterator
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Dict   ,     _snake_case   :						str     )       ->	Union[str, Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	Tuple			     =  CNNDMDataset(args.documents_dir     )
  	return dataset
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Union[str, Any]   ,     _snake_case   :						Dict   ,     _snake_case   :						List[Any]   ,     _snake_case   :						Union[str, Any]     )       ->	Union[str, Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	int			     =  [x for x in data if not len(x[1]     ) == 0]  # remove empty_files
  	__magic_name__	:	List[str]			     =  [name for name, _, _ in data]
  	__magic_name__	:	Tuple			     =  [" ".join(_snake_case     ) for _, _, summary_list in data]
  
  	__magic_name__	:	Optional[Any]			     =  [encode_for_summarization(_snake_case   ,     _snake_case   ,     _snake_case     ) for _, story, summary in data]
  	__magic_name__	:	Optional[int]			     =  torch.tensor(
  	    [truncate_or_pad(_snake_case   ,     _snake_case   ,     tokenizer.pad_token_id     ) for story, _ in encoded_text]     )
  	__magic_name__	:	Union[str, Any]			     =  compute_token_type_ids(_snake_case   ,     tokenizer.cls_token_id     )
  	__magic_name__	:	List[str]			     =  build_mask(_snake_case   ,     tokenizer.pad_token_id     )
  
  	__magic_name__	:	List[Any]			     =  Batch(
  	    document_names=_snake_case   ,     batch_size=len(_snake_case     )   ,     src=encoded_stories.to(_snake_case     )   ,     segs=encoder_token_type_ids.to(_snake_case     )   ,     mask_src=encoder_mask.to(_snake_case     )   ,     tgt_str=_snake_case   ,     )
  
  	return batch
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						List[str]   ,     _snake_case   :						Optional[Any]     )       ->	List[Any]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	int			     =  summary_tokens.to("cpu"     ).numpy()
  	__magic_name__	:	Dict			     =  tokenizer.decode(_snake_case     )
  	__magic_name__	:	Union[str, Any]			     =  summary.split("."     )
  	__magic_name__	:	Union[str, Any]			     =  [s + "." for s in sentences]
  	return sentences
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      )       ->	Dict:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	__magic_name__	:	str			     =  argparse.ArgumentParser()
  	parser.add_argument(
  	    "--documents_dir"   ,     default=_snake_case   ,     type=_snake_case   ,     required=_snake_case   ,     help="The folder where the documents to summarize are located."   ,     )
  	parser.add_argument(
  	    "--summaries_output_dir"   ,     default=_snake_case   ,     type=_snake_case   ,     required=_snake_case   ,     help="The folder in wich the summaries should be written. Defaults to the folder where the documents are"   ,     )
  	parser.add_argument(
  	    "--compute_rouge"   ,     default=_snake_case   ,     type=_snake_case   ,     required=_snake_case   ,     help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset."   ,     )
  	# EVALUATION options
  	parser.add_argument(
  	    "--no_cuda"   ,     default=_snake_case   ,     type=_snake_case   ,     help="Whether to force the execution on CPU."   ,     )
  	parser.add_argument(
  	    "--batch_size"   ,     default=4   ,     type=_snake_case   ,     help="Batch size per GPU/CPU for training."   ,     )
  	# BEAM SEARCH arguments
  	parser.add_argument(
  	    "--min_length"   ,     default=50   ,     type=_snake_case   ,     help="Minimum number of tokens for the summaries."   ,     )
  	parser.add_argument(
  	    "--max_length"   ,     default=200   ,     type=_snake_case   ,     help="Maixmum number of tokens for the summaries."   ,     )
  	parser.add_argument(
  	    "--beam_size"   ,     default=5   ,     type=_snake_case   ,     help="The number of beams to start with for each example."   ,     )
  	parser.add_argument(
  	    "--alpha"   ,     default=0.95   ,     type=_snake_case   ,     help="The value of alpha for the length penalty in the beam search."   ,     )
  	parser.add_argument(
  	    "--block_trigram"   ,     default=_snake_case   ,     type=_snake_case   ,     help="Whether to block the existence of repeating trigrams in the text generated by beam search."   ,     )
  	__magic_name__	:	Union[str, Any]			     =  parser.parse_args()
  
  	# Select device (distibuted not available)
  	__magic_name__	:	List[Any]			     =  torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu"     )
  
  	# Check the existence of directories
  	if not args.summaries_output_dir:
  		__magic_name__	:	Tuple			     =  args.documents_dir
  
  	if not documents_dir_is_valid(args.documents_dir     ):
  		raise FileNotFoundError(
  		    "We could not find the directory you specified for the documents to summarize, or it was empty. Please"
  		    " specify a valid path."     )
  	os.makedirs(args.summaries_output_dir   ,     exist_ok=_snake_case     )
  
  	evaluate(_snake_case     )
  
  
  
  
  
  
  def 						lowerCAmelCase_		(      _snake_case   :						Optional[Any]     )       ->	List[str]:
  
  
  
  
  
  
  
  	'''simple docstring'''
  
  
  	if not os.path.exists(_snake_case     ):
  		return False
  
  	__magic_name__	:	Tuple			     =  os.listdir(_snake_case     )
  	if len(_snake_case     ) == 0:
  		return False
  
  	return True
  
  
  if __name__ == "__main__":
  				main()
  
style_context: 
  import os
  import unittest
  
  from transformers import ElectraTokenizerFast
  from transformers.models.electra.tokenization_electra import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      ElectraTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class    _snake_case      (  snake_case       ,		unittest.TestCase       ):
  			UpperCamelCase__					   =					ElectraTokenizer
  			UpperCamelCase__					   =					ElectraTokenizerFast
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					True
  			UpperCamelCase__					   =					filter_non_english
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				super().setUp()
  
  				__magic_name__	:	Optional[Any]			     =  [
  				    "[UNK]",
  				    "[CLS]",
  				    "[SEP]",
  				    "[PAD]",
  				    "[MASK]",
  				    "want",
  				    "##want",
  				    "##ed",
  				    "wa",
  				    "un",
  				    "runn",
  				    "##ing",
  				    ",",
  				    "low",
  				    "lowest",
  				]
  				__magic_name__	:	Dict			     =  os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]	)
  				with open(self.vocab_file		,     "w"		,     encoding="utf-8"	) as vocab_writer:
  					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self		,     _a	):
  				__magic_name__	:	Dict			     =  "UNwant\u00E9d,running"
  				__magic_name__	:	int			     =  "unwanted, running"
  				return input_text, output_text
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  self.tokenizer_class(self.vocab_file	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.tokenize("UNwant\u00E9d,running"	)
  				self.assertListEqual(_a		,     ["un", "##want", "##ed", ",", "runn", "##ing"]	)
  				self.assertListEqual(tokenizer.convert_tokens_to_ids(_a	)		,     [9, 6, 7, 12, 10, 11]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				if not self.test_rust_tokenizer:
  					return
  
  				__magic_name__	:	str			     =  self.get_tokenizer()
  				__magic_name__	:	Optional[int]			     =  self.get_rust_tokenizer()
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	List[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	str			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	int			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	Union[str, Any]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Tuple			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Union[str, Any]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				# With lower casing
  				__magic_name__	:	Optional[Any]			     =  self.get_tokenizer(do_lower_case=_a	)
  				__magic_name__	:	Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=_a	)
  
  				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"
  
  				__magic_name__	:	Optional[Any]			     =  tokenizer.tokenize(_a	)
  				__magic_name__	:	List[Any]			     =  rust_tokenizer.tokenize(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Any			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
  				__magic_name__	:	List[str]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
  				self.assertListEqual(_a		,     _a	)
  
  				__magic_name__	:	Dict			     =  self.get_rust_tokenizer()
  				__magic_name__	:	Optional[int]			     =  tokenizer.encode(_a	)
  				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
  				self.assertListEqual(_a		,     _a	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Tuple			     =  BasicTokenizer()
  
  				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)		,     ["ah", "\u535A", "\u63A8", "zz"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["hello", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hällo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["h\u00E9llo"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Optional[int]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
  				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     never_split=["[UNK]"]	)
  
  				self.assertListEqual(
  				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	Dict			     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]
  
  				__magic_name__	:	Optional[int]			     =  {}
  				for i, token in enumerate(_a	):
  					__magic_name__	:	Tuple			     =  i
  				__magic_name__	:	Tuple			     =  WordpieceTokenizer(vocab=_a		,     unk_token="[UNK]"	)
  
  				self.assertListEqual(tokenizer.tokenize(""	)		,     []	)
  
  				self.assertListEqual(tokenizer.tokenize("unwanted running"	)		,     ["un", "##want", "##ed", "runn", "##ing"]	)
  
  				self.assertListEqual(tokenizer.tokenize("unwantedX running"	)		,     ["[UNK]", "runn", "##ing"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_whitespace(" "	)	)
  				self.assertTrue(_is_whitespace("\t"	)	)
  				self.assertTrue(_is_whitespace("\r"	)	)
  				self.assertTrue(_is_whitespace("\n"	)	)
  				self.assertTrue(_is_whitespace("\u00A0"	)	)
  
  				self.assertFalse(_is_whitespace("A"	)	)
  				self.assertFalse(_is_whitespace("-"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_control("\u0005"	)	)
  
  				self.assertFalse(_is_control("A"	)	)
  				self.assertFalse(_is_control(" "	)	)
  				self.assertFalse(_is_control("\t"	)	)
  				self.assertFalse(_is_control("\r"	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				self.assertTrue(_is_punctuation("-"	)	)
  				self.assertTrue(_is_punctuation("$"	)	)
  				self.assertTrue(_is_punctuation("`"	)	)
  				self.assertTrue(_is_punctuation("."	)	)
  
  				self.assertFalse(_is_punctuation("A"	)	)
  				self.assertFalse(_is_punctuation(" "	)	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	List[Any]			     =  self.get_tokenizer()
  				__magic_name__	:	List[str]			     =  self.get_rust_tokenizer()
  
  				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  				self.assertListEqual([tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  				self.assertListEqual(
  				    [rust_tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)
  
  
  
  
  
  
  
  			@slow
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	int			     =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)
  
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("sequence builders"		,     add_special_tokens=_a	)
  				__magic_name__	:	List[Any]			     =  tokenizer.encode("multi-sequence build"		,     add_special_tokens=_a	)
  
  				__magic_name__	:	Optional[int]			     =  tokenizer.build_inputs_with_special_tokens(_a	)
  				__magic_name__	:	Tuple			     =  tokenizer.build_inputs_with_special_tokens(_a		,     _a	)
  
  				assert encoded_sentence == [101] + text + [102]
  				assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	str			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Optional[Any]			     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  						__magic_name__	:	Any			     =  tokenizer_r.encode_plus(
  						    _a		,     return_attention_mask=_a		,     return_token_type_ids=_a		,     return_offsets_mapping=_a		,     add_special_tokens=_a		,     )
  
  						__magic_name__	:	Any			     =  tokenizer_r.do_lower_case if hasattr(_a		,     "do_lower_case"	) else False
  						__magic_name__	:	int			     =  (
  						    [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "A"),
  						        ((1, 2), ","),
  						        ((3, 5), "na"),
  						        ((5, 6), "##ï"),
  						        ((6, 8), "##ve"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "Allen"),
  						        ((21, 23), "##NL"),
  						        ((23, 24), "##P"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						    if not do_lower_case
  						    else [
  						        ((0, 0), tokenizer_r.cls_token),
  						        ((0, 1), "a"),
  						        ((1, 2), ","),
  						        ((3, 8), "naive"),
  						        ((9, 15), tokenizer_r.mask_token),
  						        ((16, 21), "allen"),
  						        ((21, 23), "##nl"),
  						        ((23, 24), "##p"),
  						        ((25, 33), "sentence"),
  						        ((33, 34), "."),
  						        ((0, 0), tokenizer_r.sep_token),
  						    ]
  						)
  
  						self.assertEqual(
  						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
  						self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]	)
  
  
  
  
  
  
  
  			def     SCREAMING_SNAKE_CASE					(       self	):
  				__magic_name__	:	str			     =  ["的", "人", "有"]
  				__magic_name__	:	List[str]			     =  "".join(_a	)
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
  						__magic_name__	:	Optional[int]			     =  True
  						__magic_name__	:	Optional[int]			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	Union[str, Any]			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Tuple			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that each Chinese character is not preceded by "##"
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  
  						__magic_name__	:	Tuple			     =  False
  						__magic_name__	:	Tuple			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
  						__magic_name__	:	str			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
  
  						__magic_name__	:	Dict			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
  						__magic_name__	:	Dict			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
  
  						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
  						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)
  
  						# it is expected that only the first Chinese character is not preceded by "##".
  						__magic_name__	:	Tuple			     =  [
  						    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a	)
  						]
  						self.assertListEqual(_a		,     _a	)
  						self.assertListEqual(_a		,     _a	)
  