code: 
  '''simple docstring'''
  
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class 			A							(       __UpperCAmelCase		,						unittest.TestCase				):
  			lowerCamelCase							:							List[str]       		=  RealmTokenizer
  			lowerCamelCase							:							int       		=  RealmTokenizerFast
  			lowerCamelCase							:							Optional[Any]       		=  True
  			lowerCamelCase							:							List[Any]       		=  True
  			lowerCamelCase							:							Dict       		=  filter_non_english
  
  
  
  			def A__ (					self       )			->		Optional[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									super().setUp()
  
  									lowercase__							 =       [
  									    """[UNK]""",
  									    """[CLS]""",
  									    """[SEP]""",
  									    """[PAD]""",
  									    """[MASK]""",
  									    """want""",
  									    """##want""",
  									    """##ed""",
  									    """wa""",
  									    """un""",
  									    """runn""",
  									    """##ing""",
  									    """,""",
  									    """low""",
  									    """lowest""",
  									]
  									lowercase__							 =       os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES["""vocab_file"""]       )
  									with open(self.vocab_file       ,     """w"""       ,     encoding="""utf-8"""       ) as vocab_writer:
  															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
  
  
  
  			def A__ (					self       ,     lowerCamelCase__       )			->		str:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       """UNwant\u00E9d,running"""
  									lowercase__							 =       """unwanted, running"""
  									return input_text, output_text
  
  
  
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       self.tokenizer_class(self.vocab_file       )
  
  									lowercase__							 =       tokenizer.tokenize("""UNwant\u00E9d,running"""       )
  									self.assertListEqual(lowerCamelCase__       ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]       )
  									self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__       )       ,     [9, 6, 7, 12, 10, 11]       )
  
  
  
  			def A__ (					self       )			->		Union[str, Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									if not self.test_rust_tokenizer:
  															return
  
  									lowercase__							 =       self.get_tokenizer()
  									lowercase__							 =       self.get_rust_tokenizer()
  
  									lowercase__							 =       """UNwant\u00E9d,running"""
  
  									lowercase__							 =       tokenizer.tokenize(lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.tokenize(lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									lowercase__							 =       tokenizer.encode(lowerCamelCase__       ,     add_special_tokens=lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.encode(lowerCamelCase__       ,     add_special_tokens=lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									lowercase__							 =       self.get_rust_tokenizer()
  									lowercase__							 =       tokenizer.encode(lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.encode(lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									# With lower casing
  									lowercase__							 =       self.get_tokenizer(do_lower_case=lowerCamelCase__       )
  									lowercase__							 =       self.get_rust_tokenizer(do_lower_case=lowerCamelCase__       )
  
  									lowercase__							 =       """UNwant\u00E9d,running"""
  
  									lowercase__							 =       tokenizer.tokenize(lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.tokenize(lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									lowercase__							 =       tokenizer.encode(lowerCamelCase__       ,     add_special_tokens=lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.encode(lowerCamelCase__       ,     add_special_tokens=lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									lowercase__							 =       self.get_rust_tokenizer()
  									lowercase__							 =       tokenizer.encode(lowerCamelCase__       )
  									lowercase__							 =       rust_tokenizer.encode(lowerCamelCase__       )
  									self.assertListEqual(lowerCamelCase__       ,     lowerCamelCase__       )
  
  
  
  			def A__ (					self       )			->		List[str]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer()
  
  									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       )       ,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]       )
  
  
  
  			def A__ (					self       )			->		Tuple:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )       ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
  									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )       ,     ["""hello"""]       )
  
  
  
  			def A__ (					self       )			->		str:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       ,     strip_accents=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )       ,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
  									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )       ,     ["""h\u00E9llo"""]       )
  
  
  
  			def A__ (					self       )			->		Optional[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       ,     strip_accents=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )       ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
  									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )       ,     ["""hello"""]       )
  
  
  
  			def A__ (					self       )			->		int:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )       ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
  									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )       ,     ["""hello"""]       )
  
  
  
  			def A__ (					self       )			->		Tuple:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )       ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )
  
  
  
  			def A__ (					self       )			->		Union[str, Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       ,     strip_accents=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )       ,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )
  
  
  
  			def A__ (					self       )			->		Union[str, Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       ,     strip_accents=lowerCamelCase__       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )       ,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )
  
  
  
  			def A__ (					self       )			->		List[Any]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       BasicTokenizer(do_lower_case=lowerCamelCase__       ,     never_split=["""[UNK]"""]       )
  
  									self.assertListEqual(
  									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       )       ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )
  
  
  
  			def A__ (					self       )			->		str:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]
  
  									lowercase__							 =       {}
  									for i, token in enumerate(lowerCamelCase__       ):
  															lowercase__							 =       i
  									lowercase__							 =       WordpieceTokenizer(vocab=lowerCamelCase__       ,     unk_token="""[UNK]"""       )
  
  									self.assertListEqual(tokenizer.tokenize(""""""       )       ,     []       )
  
  									self.assertListEqual(tokenizer.tokenize("""unwanted running"""       )       ,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )
  
  									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       )       ,     ["""[UNK]""", """runn""", """##ing"""]       )
  
  
  
  			def A__ (					self       )			->		Optional[int]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									self.assertTrue(_is_whitespace(""" """       )       )
  									self.assertTrue(_is_whitespace("""\t"""       )       )
  									self.assertTrue(_is_whitespace("""\r"""       )       )
  									self.assertTrue(_is_whitespace("""\n"""       )       )
  									self.assertTrue(_is_whitespace("""\u00A0"""       )       )
  
  									self.assertFalse(_is_whitespace("""A"""       )       )
  									self.assertFalse(_is_whitespace("""-"""       )       )
  
  
  
  			def A__ (					self       )			->		Dict:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									self.assertTrue(_is_control("""\u0005"""       )       )
  
  									self.assertFalse(_is_control("""A"""       )       )
  									self.assertFalse(_is_control(""" """       )       )
  									self.assertFalse(_is_control("""\t"""       )       )
  									self.assertFalse(_is_control("""\r"""       )       )
  
  
  
  			def A__ (					self       )			->		int:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									self.assertTrue(_is_punctuation("""-"""       )       )
  									self.assertTrue(_is_punctuation("""$"""       )       )
  									self.assertTrue(_is_punctuation("""`"""       )       )
  									self.assertTrue(_is_punctuation("""."""       )       )
  
  									self.assertFalse(_is_punctuation("""A"""       )       )
  									self.assertFalse(_is_punctuation(""" """       )       )
  
  
  
  			def A__ (					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       self.get_tokenizer()
  
  									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  									self.assertListEqual([tokenizer.tokenize(lowerCamelCase__       ) for t in ["""Test""", """\xad""", """test"""]]       ,     [["""[UNK]"""], [], ["""[UNK]"""]]       )
  
  									if self.test_rust_tokenizer:
  															lowercase__							 =       self.get_rust_tokenizer()
  															self.assertListEqual(
  															    [rust_tokenizer.tokenize(lowerCamelCase__       ) for t in ["""Test""", """\xad""", """test"""]]       ,     [["""[UNK]"""], [], ["""[UNK]"""]]       )
  
  
  
  			@slow
  			def A__ (					self       )			->		Tuple:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       self.tokenizer_class.from_pretrained("""bert-base-uncased"""       )
  
  									lowercase__							 =       tokenizer.encode("""sequence builders"""       ,     add_special_tokens=lowerCamelCase__       )
  									lowercase__							 =       tokenizer.encode("""multi-sequence build"""       ,     add_special_tokens=lowerCamelCase__       )
  
  									lowercase__							 =       tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       )
  									lowercase__							 =       tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       ,     lowerCamelCase__       )
  
  									assert encoded_sentence == [101] + text + [102]
  									assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  			def A__ (					self       )			->		List[str]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
  																					lowercase__							 =       self.rust_tokenizer_class.from_pretrained(lowerCamelCase__       ,     **lowerCamelCase__       )
  
  																					lowercase__							 =       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  																					lowercase__							 =       tokenizer_r.encode_plus(
  																					    lowerCamelCase__       ,     return_attention_mask=lowerCamelCase__       ,     return_token_type_ids=lowerCamelCase__       ,     return_offsets_mapping=lowerCamelCase__       ,     add_special_tokens=lowerCamelCase__       ,     )
  
  																					lowercase__							 =       tokenizer_r.do_lower_case if hasattr(lowerCamelCase__       ,     """do_lower_case"""       ) else False
  																					lowercase__							 =       (
  																					    [
  																					        ((0, 0), tokenizer_r.cls_token),
  																					        ((0, 1), """A"""),
  																					        ((1, 2), ""","""),
  																					        ((3, 5), """na"""),
  																					        ((5, 6), """##ï"""),
  																					        ((6, 8), """##ve"""),
  																					        ((9, 15), tokenizer_r.mask_token),
  																					        ((16, 21), """Allen"""),
  																					        ((21, 23), """##NL"""),
  																					        ((23, 24), """##P"""),
  																					        ((25, 33), """sentence"""),
  																					        ((33, 34), """."""),
  																					        ((0, 0), tokenizer_r.sep_token),
  																					    ]
  																					    if not do_lower_case
  																					    else [
  																					        ((0, 0), tokenizer_r.cls_token),
  																					        ((0, 1), """a"""),
  																					        ((1, 2), ""","""),
  																					        ((3, 8), """naive"""),
  																					        ((9, 15), tokenizer_r.mask_token),
  																					        ((16, 21), """allen"""),
  																					        ((21, 23), """##nl"""),
  																					        ((23, 24), """##p"""),
  																					        ((25, 33), """sentence"""),
  																					        ((33, 34), """."""),
  																					        ((0, 0), tokenizer_r.sep_token),
  																					    ]
  																					)
  
  																					self.assertEqual(
  																					    [e[1] for e in expected_results]       ,     tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
  																					self.assertEqual([e[0] for e in expected_results]       ,     tokens["""offset_mapping"""]       )
  
  
  
  			@slow
  			def A__ (					self       )			->		int:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
  																					lowercase__							 =       self.rust_tokenizer_class.from_pretrained(lowerCamelCase__       ,     **lowerCamelCase__       )
  																					lowercase__							 =       self.tokenizer_class.from_pretrained(lowerCamelCase__       ,     **lowerCamelCase__       )
  																					lowercase__							 =       [["""Hello world!""", """Nice to meet you!"""], ["""The cute cat.""", """The adorable dog."""]]
  
  																					lowercase__							 =       tokenizer_r.batch_encode_candidates(lowerCamelCase__       ,     max_length=10       ,     return_tensors="""np"""       )
  																					lowercase__							 =       tokenizer_p.batch_encode_candidates(lowerCamelCase__       ,     max_length=10       ,     return_tensors="""np"""       )
  
  																					lowercase__							 =       (2, 2, 10)
  
  																					self.assertEqual(encoded_sentence_r["""input_ids"""].shape       ,     lowerCamelCase__       )
  																					self.assertEqual(encoded_sentence_r["""attention_mask"""].shape       ,     lowerCamelCase__       )
  																					self.assertEqual(encoded_sentence_r["""token_type_ids"""].shape       ,     lowerCamelCase__       )
  
  																					self.assertEqual(encoded_sentence_p["""input_ids"""].shape       ,     lowerCamelCase__       )
  																					self.assertEqual(encoded_sentence_p["""attention_mask"""].shape       ,     lowerCamelCase__       )
  																					self.assertEqual(encoded_sentence_p["""token_type_ids"""].shape       ,     lowerCamelCase__       )
  
  
  
  
  
  
  
style_context: 
  '''simple docstring'''
  
  
  
  import bisect
  import copy
  from collections import defaultdict
  
  import numpy as np
  from torch.utils.data import BatchSampler, Sampler
  
  from utils import logger
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__	):
  						lowercase__							 =       copy.deepcopy(lowercase__	)
  						lowercase__							 =       sorted(lowercase__	)
  						lowercase__							 =       [bisect.bisect_right(lowercase__		,		lowercase__	) for y in x]
  						return quantized
  
  
  
  
  
  def  _A		(							lowercase__		,		lowercase__=0	):
  						lowercase__							 =       np.arange(start=3		,		stop=lowercase__		,		step=4	).tolist() if k > 0 else [10]
  						lowercase__							 =       _quantize(lowercase__		,		lowercase__	)
  						# count number of elements per group
  						lowercase__							 =       np.unique(lowercase__		,		return_counts=lowercase__	)[1]
  						lowercase__							 =       [0] + bins + [np.inf]
  						logger.info("""Using {} as bins for aspect lengths quantization""".format(lowercase__	)	)
  						logger.info("""Count of instances per bin: {}""".format(lowercase__	)	)
  						return groups
  
  
  
  
  
  
  
  class 			A							(       __UpperCAmelCase				):
  
  
  
  			def __init__(					self       ,     lowerCamelCase__       ,     lowerCamelCase__       ,     lowerCamelCase__       )			->		Optional[int]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									if not isinstance(lowerCamelCase__       ,     lowerCamelCase__       ):
  															raise ValueError(
  															    """sampler should be an instance of torch.utils.data.Sampler, but got sampler={}""".format(lowerCamelCase__       )       )
  									lowercase__							 =       sampler
  									lowercase__							 =       group_ids
  									lowercase__							 =       batch_size
  
  
  
  			def __iter__(					self       )			->		List[str]:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									lowercase__							 =       defaultdict(lowerCamelCase__       )
  									lowercase__							 =       defaultdict(lowerCamelCase__       )
  
  									lowercase__							 =       0
  									for idx in self.sampler:
  															lowercase__							 =       self.group_ids[idx]
  															buffer_per_group[group_id].append(lowerCamelCase__       )
  															samples_per_group[group_id].append(lowerCamelCase__       )
  															if len(buffer_per_group[group_id]       ) == self.batch_size:
  																					yield buffer_per_group[group_id]  # TODO
  																					num_batches += 1
  																					del buffer_per_group[group_id]
  															assert len(buffer_per_group[group_id]       ) < self.batch_size
  
  									# now we have run out of elements that satisfy
  									# the group criteria, let's return the remaining
  									# elements so that the size of the sampler is
  									# deterministic
  									lowercase__							 =       len(self       )
  									lowercase__							 =       expected_num_batches - num_batches
  									if num_remaining > 0:
  															# for the remaining batches, group the batches by similar lengths
  															lowercase__							 =       []
  															for group_id, idxs in sorted(buffer_per_group.items()       ,     key=lambda lowerCamelCase__       : x[0]       ):
  																					batch_idx.extend(lowerCamelCase__       )
  																					if len(lowerCamelCase__       ) >= self.batch_size:
  																											yield batch_idx[: self.batch_size]
  																											lowercase__							 =       batch_idx[self.batch_size :]
  																											num_remaining -= 1
  															if len(lowerCamelCase__       ) > 0:
  																					yield batch_idx
  																					num_remaining -= 1
  									assert num_remaining == 0
  
  
  
  			def __len__(					self       )			->		Any:
  
  
  
  
  									'''simple docstring'''
  
  
  
  									return (len(self.sampler       ) + self.batch_size - 1) // self.batch_size
  
  
  
  
  
  
  