




"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 					lowercase(     __a       ,			unittest.TestCase ):







 '''simple docstring'''





 lowercase__ 					=					RoCBertTokenizer
 lowercase__ 					=					None
 lowercase__ 					=					False
 lowercase__ 					=					True
 lowercase__ 					=					filter_non_english


 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   super().setUp()

   _snake_case							:		List[str]								=  ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
   _snake_case							:		Tuple								=  {}
   _snake_case							:		Any								=  {}
   for i, value in enumerate(a_		):
     _snake_case							:		List[str]								=  i
     _snake_case							:		Optional[int]								=  i
   _snake_case							:		List[Any]								=  os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES["""vocab_file"""]		)
   _snake_case							:		Dict								=  os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES["""word_shape_file"""]		)
   _snake_case							:		Optional[Any]								=  os.path.join(self.tmpdirname,      VOCAB_FILES_NAMES["""word_pronunciation_file"""]		)
   with open(self.vocab_file,      """w""",      encoding="""utf-8"""		) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
   with open(self.word_shape_file,      """w""",      encoding="""utf-8"""		) as word_shape_writer:
     json.dump(a_,      a_,      ensure_ascii=a_		)
   with open(self.word_pronunciation_file,      """w""",      encoding="""utf-8"""		) as word_pronunciation_writer:
     json.dump(a_,      a_,      ensure_ascii=a_		)


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.tokenizer_class(self.vocab_file,      self.word_shape_file,      self.word_pronunciation_file		)

   _snake_case							:		str								=  tokenizer.tokenize("""你好[SEP]你是谁"""		)
   self.assertListEqual(a_,      ["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]		)
   self.assertListEqual(tokenizer.convert_tokens_to_ids(a_		),      [5, 6, 2, 5, 7, 8]		)
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(a_		),      [5, 6, 2, 5, 7, 8]		)
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(a_		),      [5, 6, 2, 5, 7, 8]		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		Any								=  RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""		),      ["""ah""", """\u535A""", """\u63A8""", """zz"""]		)


 def  UpperCamelCase_   (				self:		Any		):




   '''simple docstring'''



   _snake_case							:		Tuple								=  RoCBertBasicTokenizer(do_lower_case=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		),      ["""hello""", """!""", """how""", """are""", """you""", """?"""]		)
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		),      ["""hello"""]		)


 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  RoCBertBasicTokenizer(do_lower_case=a_,      strip_accents=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		),      ["""hällo""", """!""", """how""", """are""", """you""", """?"""]		)
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		),      ["""h\u00E9llo"""]		)


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Any								=  RoCBertBasicTokenizer(do_lower_case=a_,      strip_accents=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		),      ["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		),      ["""hello"""]		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  RoCBertBasicTokenizer(do_lower_case=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		),      ["""hallo""", """!""", """how""", """are""", """you""", """?"""]		)
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""		),      ["""hello"""]		)


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  RoCBertBasicTokenizer(do_lower_case=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """		),      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


 def  UpperCamelCase_   (				self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		str								=  RoCBertBasicTokenizer(do_lower_case=a_,      strip_accents=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		),      ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


 def  UpperCamelCase_   (				self:		Any		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  RoCBertBasicTokenizer(do_lower_case=a_,      strip_accents=a_		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """		),      ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]		)


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		str								=  RoCBertBasicTokenizer(do_lower_case=a_,      never_split=["""[UNK]"""]		)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""		),      ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]		)


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   _snake_case							:		Optional[Any]								=  {}
   for i, token in enumerate(a_		):
     _snake_case							:		Dict								=  i
   _snake_case							:		Optional[Any]								=  RoCBertWordpieceTokenizer(vocab=a_,      unk_token="""[UNK]"""		)

   self.assertListEqual(tokenizer.tokenize(""""""		),      []		)

   self.assertListEqual(tokenizer.tokenize("""unwanted running"""		),      ["""un""", """##want""", """##ed""", """runn""", """##ing"""]		)

   self.assertListEqual(tokenizer.tokenize("""unwantedX running"""		),      ["""[UNK]""", """runn""", """##ing"""]		)


 def  UpperCamelCase_   (				self:		List[str]		):




   '''simple docstring'''



   self.assertTrue(_is_whitespace(""" """		)		)
   self.assertTrue(_is_whitespace("""\t"""		)		)
   self.assertTrue(_is_whitespace("""\r"""		)		)
   self.assertTrue(_is_whitespace("""\n"""		)		)
   self.assertTrue(_is_whitespace("""\u00A0"""		)		)

   self.assertFalse(_is_whitespace("""A"""		)		)
   self.assertFalse(_is_whitespace("""-"""		)		)


 def  UpperCamelCase_   (				self:		Optional[int]		):




   '''simple docstring'''



   self.assertTrue(_is_control("""\u0005"""		)		)

   self.assertFalse(_is_control("""A"""		)		)
   self.assertFalse(_is_control(""" """		)		)
   self.assertFalse(_is_control("""\t"""		)		)
   self.assertFalse(_is_control("""\r"""		)		)


 def  UpperCamelCase_   (				self:		Optional[Any]		):




   '''simple docstring'''



   self.assertTrue(_is_punctuation("""-"""		)		)
   self.assertTrue(_is_punctuation("""$"""		)		)
   self.assertTrue(_is_punctuation("""`"""		)		)
   self.assertTrue(_is_punctuation("""."""		)		)

   self.assertFalse(_is_punctuation("""A"""		)		)
   self.assertFalse(_is_punctuation(""" """		)		)


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(a_		) for t in ["""Test""", """\xad""", """test"""]],      [["""[UNK]"""], [], ["""[UNK]"""]]		)

   if self.test_rust_tokenizer:
     _snake_case							:		Optional[int]								=  self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(a_		) for t in ["""Test""", """\xad""", """test"""]],      [["""[UNK]"""], [], ["""[UNK]"""]]		)


 def  UpperCamelCase_   (				self:		Any		):




   '''simple docstring'''



   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
       _snake_case							:		Optional[int]								=  self.rust_tokenizer_class.from_pretrained(a_,      **a_		)

       _snake_case							:		List[Any]								=  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
       _snake_case							:		List[Any]								=  tokenizer_r.encode_plus(
           a_,      return_attention_mask=a_,      return_token_type_ids=a_,      return_offsets_mapping=a_,      add_special_tokens=a_,      )

       _snake_case							:		Optional[Any]								=  tokenizer_r.do_lower_case if hasattr(a_,      """do_lower_case"""		) else False
       _snake_case							:		Optional[Any]								=  (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """A"""),
               ((1, 2), ""","""),
               ((3, 5), """na"""),
               ((5, 6), """##ï"""),
               ((6, 8), """##ve"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """Allen"""),
               ((21, 23), """##NL"""),
               ((23, 24), """##P"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """a"""),
               ((1, 2), ""","""),
               ((3, 8), """naive"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """allen"""),
               ((21, 23), """##nl"""),
               ((23, 24), """##p"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results],      tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]		)		)
       self.assertEqual([e[0] for e in expected_results],      tokens["""offset_mapping"""]		)


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  ["""的""", """人""", """有"""]
   _snake_case							:		Any								=  """""".join(a_		)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
       _snake_case							:		int								=  True
       _snake_case							:		Tuple								=  self.tokenizer_class.from_pretrained(a_,      **a_		)
       _snake_case							:		List[Any]								=  self.rust_tokenizer_class.from_pretrained(a_,      **a_		)

       _snake_case							:		Optional[Any]								=  tokenizer_p.encode(a_,      add_special_tokens=a_		)
       _snake_case							:		int								=  tokenizer_r.encode(a_,      add_special_tokens=a_		)

       _snake_case							:		Optional[Any]								=  tokenizer_r.convert_ids_to_tokens(a_		)
       _snake_case							:		Optional[int]								=  tokenizer_p.convert_ids_to_tokens(a_		)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(a_,      a_		)
       self.assertListEqual(a_,      a_		)

       _snake_case							:		List[str]								=  False
       _snake_case							:		Union[str, Any]								=  self.rust_tokenizer_class.from_pretrained(a_,      **a_		)
       _snake_case							:		str								=  self.tokenizer_class.from_pretrained(a_,      **a_		)

       _snake_case							:		Optional[Any]								=  tokenizer_r.encode(a_,      add_special_tokens=a_		)
       _snake_case							:		Any								=  tokenizer_p.encode(a_,      add_special_tokens=a_		)

       _snake_case							:		Optional[int]								=  tokenizer_r.convert_ids_to_tokens(a_		)
       _snake_case							:		Tuple								=  tokenizer_p.convert_ids_to_tokens(a_		)

       # it is expected that only the first Chinese character is not preceded by "##".
       _snake_case							:		List[Any]								=  [
           f"##{token}" if idx != 0 else token for idx, token in enumerate(a_		)
       ]
       self.assertListEqual(a_,      a_		)
       self.assertListEqual(a_,      a_		)


 @slow
 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.tokenizer_class(self.vocab_file,      self.word_shape_file,      self.word_pronunciation_file		)

   _snake_case							:		Tuple								=  tokenizer.encode("""你好""",      add_special_tokens=a_		)
   _snake_case							:		Optional[int]								=  tokenizer.encode("""你是谁""",      add_special_tokens=a_		)

   _snake_case							:		str								=  tokenizer.build_inputs_with_special_tokens(a_		)
   _snake_case							:		int								=  tokenizer.build_inputs_with_special_tokens(a_,      a_		)

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]


 def  UpperCamelCase_   (				self:		str		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  self.get_tokenizers(do_lower_case=a_		)
   for tokenizer in tokenizers:
     with self.subTest(f"{tokenizer.__class__.__name__}"		):
       _snake_case							:		int								=  """你好，你是谁"""
       _snake_case							:		List[str]								=  tokenizer.tokenize(a_		)
       _snake_case							:		List[str]								=  tokenizer.convert_tokens_to_ids(a_		)
       _snake_case							:		List[Any]								=  tokenizer.convert_tokens_to_shape_ids(a_		)
       _snake_case							:		Optional[Any]								=  tokenizer.convert_tokens_to_pronunciation_ids(a_		)
       _snake_case							:		Dict								=  tokenizer.prepare_for_model(
           a_,      a_,      a_,      add_special_tokens=a_		)

       _snake_case							:		Dict								=  tokenizer.encode_plus(a_,      add_special_tokens=a_		)

       self.assertEqual(a_,      a_		)







"""simple docstring"""







import random

def 				UpperCamelCase_    (    lowerCAmelCase__      :				int    ,   lowerCAmelCase__      :				float    ,   lowerCAmelCase__      :				bool = False						)						->   dict:

  """simple docstring"""




  lowerCAmelCase_   :				dict								= {i: [] for i in range(lowerCAmelCase__						)}

  # if probability is greater or equal than 1, then generate a complete graph
  if probability >= 1:
    return complete_graph(lowerCAmelCase__						)
  # if probability is lower or equal than 0, then return a graph without edges
  if probability <= 0:
    return graph

  # for each couple of nodes, add an edge from u to v
  # if the number randomly generated is greater than probability probability
  for i in range(lowerCAmelCase__						):
    for j in range(i + 1    ,   lowerCAmelCase__						):
      if random.random() < probability:
        graph[i].append(lowerCAmelCase__						)
        if not directed:
          # if the graph is undirected, add an edge in from j to i, either
          graph[j].append(lowerCAmelCase__						)
  return graph

def 				UpperCamelCase_    (    lowerCAmelCase__      :				int						)						->   dict:

  """simple docstring"""




  return {
      i: [j for j in range(lowerCAmelCase__						) if i != j] for i in range(lowerCAmelCase__						)
  }


if __name__ == "__main__":
     import doctest

     doctest.testmod()





