




"""simple docstring"""






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class lowerCAmelCase__	(     lowercase,	unittest.TestCase						):






			'''simple docstring'''



			lowerCamelCase__      =			BertJapaneseTokenizer
			lowerCamelCase__      =			False
			lowerCamelCase__      =			True
			def 					A_    (  self  ):
							super().setUp()

							_lowerCamelCase     :			Optional[Any]    =    [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    'こんにちは',
							    'こん',
							    'にちは',
							    'ばんは',
							    '##こん',
							    '##にちは',
							    '##ばんは',
							    '世界',
							    '##世界',
							    '、',
							    '##、',
							    '。',
							    '##。',
							]

							_lowerCamelCase     :			Union[str, Any]    =    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']  )
							with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'  ) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			int    =    'こんにちは、世界。 \nこんばんは、世界。'
							_lowerCamelCase     :			Union[str, Any]    =    'こんにちは 、 世界 。 こんばんは 、 世界 。'
							return input_text, output_text
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase, _lowerCamelCase     :			Any    =    self.get_input_output_texts(lowercase  )
							_lowerCamelCase     :			Tuple    =    tokenizer.encode(lowercase  ,      add_special_tokens=lowercase  )
							_lowerCamelCase     :			Dict    =    tokenizer.decode(lowercase  ,      clean_up_tokenization_spaces=lowercase  )
							return text, ids
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[int]    =    self.tokenizer_class(self.vocab_file  )

							_lowerCamelCase     :			Union[str, Any]    =    tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'  )
							self.assertListEqual(lowercase  ,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase  )  ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Tuple    =    self.tokenizer_class(self.vocab_file  ,      word_tokenizer_type='mecab'  )
							self.assertIsNotNone(lowercase  )

							_lowerCamelCase     :			int    =    'こんにちは、世界。\nこんばんは、世界。'
							_lowerCamelCase     :			Tuple    =    tokenizer.tokenize(lowercase  )
							self.assertListEqual(lowercase  ,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase  )  ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

							_lowerCamelCase     :			Union[str, Any]    =    os.path.join(self.tmpdirname  ,      'tokenizer.bin'  )
							with open(lowercase  ,      'wb'  ) as handle:
											pickle.dump(lowercase  ,      lowercase  )

							with open(lowercase  ,      'rb'  ) as handle:
											_lowerCamelCase     :			Dict    =    pickle.load(lowercase  )

							_lowerCamelCase     :			Dict    =    tokenizer_new.tokenize(lowercase  )

							self.assertListEqual(lowercase  ,      lowercase  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Union[str, Any]    =    MecabTokenizer(mecab_dic='ipadic'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']  ,      )
			def 					A_    (  self  ):
							try:
											_lowerCamelCase     :			Dict    =    MecabTokenizer(mecab_dic='unidic_lite'  )
							except ModuleNotFoundError:
											return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']  ,      )
			def 					A_    (  self  ):
							try:
											_lowerCamelCase     :			Any    =    MecabTokenizer(mecab_dic='unidic'  )
							except ModuleNotFoundError:
											return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']  ,      )
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[Any]    =    MecabTokenizer(do_lower_case=lowercase  ,      mecab_dic='ipadic'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']  ,      )
			def 					A_    (  self  ):
							try:
											_lowerCamelCase     :			Tuple    =    MecabTokenizer(
											    do_lower_case=lowercase  ,      normalize_text=lowercase  ,      mecab_option='-d /usr/local/lib/mecab/dic/jumandic'  )
							except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']  ,      )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Union[str, Any]    =    MecabTokenizer(normalize_text=lowercase  ,      mecab_dic='ipadic'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']  ,      )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[int]    =    self.tokenizer_class(self.vocab_file  ,      word_tokenizer_type='sudachi'  )
							self.assertIsNotNone(lowercase  )

							_lowerCamelCase     :			int    =    'こんにちは、世界。\nこんばんは、世界。'
							_lowerCamelCase     :			Tuple    =    tokenizer.tokenize(lowercase  )
							self.assertListEqual(lowercase  ,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase  )  ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

							_lowerCamelCase     :			Union[str, Any]    =    os.path.join(self.tmpdirname  ,      'tokenizer.bin'  )
							with open(lowercase  ,      'wb'  ) as handle:
											pickle.dump(lowercase  ,      lowercase  )

							with open(lowercase  ,      'rb'  ) as handle:
											_lowerCamelCase     :			int    =    pickle.load(lowercase  )

							_lowerCamelCase     :			List[Any]    =    tokenizer_new.tokenize(lowercase  )

							self.assertListEqual(lowercase  ,      lowercase  )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[Any]    =    SudachiTokenizer(sudachi_dict_type='core'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']  ,      )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			str    =    SudachiTokenizer(sudachi_dict_type='core'  ,      sudachi_split_mode='A'  )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'  )  ,      ['外国', '人', '参政', '権']  )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			Tuple    =    SudachiTokenizer(sudachi_dict_type='core'  ,      sudachi_split_mode='B'  )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'  )  ,      ['外国人', '参政権']  )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[Any]    =    SudachiTokenizer(sudachi_dict_type='core'  ,      sudachi_split_mode='C'  )

							self.assertListEqual(tokenizer.tokenize('外国人参政権'  )  ,      ['外国人参政権']  )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			Tuple    =    SudachiTokenizer(do_lower_case=lowercase  ,      sudachi_dict_type='core'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']  ,      )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			Tuple    =    SudachiTokenizer(normalize_text=lowercase  ,      sudachi_dict_type='core'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']  ,      )
			@require_sudachi
			def 					A_    (  self  ):
							_lowerCamelCase     :			str    =    SudachiTokenizer(trim_whitespace=lowercase  ,      sudachi_dict_type='core'  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']  ,      )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			Union[str, Any]    =    self.tokenizer_class(self.vocab_file  ,      word_tokenizer_type='jumanpp'  )
							self.assertIsNotNone(lowercase  )

							_lowerCamelCase     :			Optional[Any]    =    'こんにちは、世界。\nこんばんは、世界。'
							_lowerCamelCase     :			Optional[Any]    =    tokenizer.tokenize(lowercase  )
							self.assertListEqual(lowercase  ,      ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']  )
							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase  )  ,      [3, 12, 10, 14, 4, 9, 12, 10, 14]  )

							_lowerCamelCase     :			int    =    os.path.join(self.tmpdirname  ,      'tokenizer.bin'  )
							with open(lowercase  ,      'wb'  ) as handle:
											pickle.dump(lowercase  ,      lowercase  )

							with open(lowercase  ,      'rb'  ) as handle:
											_lowerCamelCase     :			Tuple    =    pickle.load(lowercase  )

							_lowerCamelCase     :			Dict    =    tokenizer_new.tokenize(lowercase  )

							self.assertListEqual(lowercase  ,      lowercase  )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[Any]    =    JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']  ,      )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			str    =    JumanppTokenizer(do_lower_case=lowercase  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']  ,      )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			List[Any]    =    JumanppTokenizer(normalize_text=lowercase  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']  ,      )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			Union[str, Any]    =    JumanppTokenizer(trim_whitespace=lowercase  )

							self.assertListEqual(
							    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '  )  ,      ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']  ,      )
			@require_jumanpp
			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[int]    =    JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'  )  ,      ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']  ,      )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    ['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

							_lowerCamelCase     :			int    =    {}
							for i, token in enumerate(lowercase  ):
											_lowerCamelCase     :			List[str]    =    i
							_lowerCamelCase     :			Tuple    =    WordpieceTokenizer(vocab=lowercase  ,      unk_token='[UNK]'  )

							self.assertListEqual(tokenizer.tokenize(''  )  ,      []  )

							self.assertListEqual(tokenizer.tokenize('こんにちは'  )  ,      ['こんにちは']  )

							self.assertListEqual(tokenizer.tokenize('こんばんは'  )  ,      ['こん', '##ばんは']  )

							self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'  )  ,      ['こん', '##ばんは', '[UNK]', 'こんにちは']  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[Any]    =    BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'  )
							_lowerCamelCase     :			Optional[int]    =    tokenizer.subword_tokenizer

							_lowerCamelCase     :			Union[str, Any]    =    subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'  )
							self.assertListEqual(lowercase  ,      ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']  )

							_lowerCamelCase     :			Optional[int]    =    subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'  )
							self.assertListEqual(lowercase  ,      ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']  )






			def 					A_    (  self  ):
							_lowerCamelCase     :			str    =    self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'  )

							_lowerCamelCase     :			int    =    tokenizer.encode('ありがとう。'  ,      add_special_tokens=lowercase  )
							_lowerCamelCase     :			Any    =    tokenizer.encode('どういたしまして。'  ,      add_special_tokens=lowercase  )

							_lowerCamelCase     :			Union[str, Any]    =    tokenizer.build_inputs_with_special_tokens(lowercase  )
							_lowerCamelCase     :			Union[str, Any]    =    tokenizer.build_inputs_with_special_tokens(lowercase  ,      lowercase  )

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class lowerCAmelCase__	(     lowercase,	unittest.TestCase						):






			'''simple docstring'''



			lowerCamelCase__      =			BertJapaneseTokenizer
			lowerCamelCase__      =			False
			def 					A_    (  self  ):
							super().setUp()

							_lowerCamelCase     :			int    =    ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							_lowerCamelCase     :			Optional[int]    =    os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES['vocab_file']  )
							with open(self.vocab_file  ,      'w'  ,      encoding='utf-8'  ) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
			def 					A_    (  self  ,      **lowercase  ):
							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname  ,      subword_tokenizer_type='character'  ,      **lowercase  )
			def 					A_    (  self  ,      lowercase  ):
							_lowerCamelCase     :			Tuple    =    'こんにちは、世界。 \nこんばんは、世界。'
							_lowerCamelCase     :			Dict    =    'こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
							return input_text, output_text
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							pass  # TODO add if relevant
			def 					A_    (  self  ):
							_lowerCamelCase     :			Dict    =    self.tokenizer_class(self.vocab_file  ,      subword_tokenizer_type='character'  )

							_lowerCamelCase     :			List[Any]    =    tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'  )
							self.assertListEqual(
							    lowercase  ,      ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']  )
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(lowercase  )  ,      [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]  )
			def 					A_    (  self  ):
							_lowerCamelCase     :			Union[str, Any]    =    ['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

							_lowerCamelCase     :			Optional[int]    =    {}
							for i, token in enumerate(lowercase  ):
											_lowerCamelCase     :			str    =    i
							_lowerCamelCase     :			List[Any]    =    CharacterTokenizer(vocab=lowercase  ,      unk_token='[UNK]'  )

							self.assertListEqual(tokenizer.tokenize(''  )  ,      []  )

							self.assertListEqual(tokenizer.tokenize('こんにちは'  )  ,      ['こ', 'ん', 'に', 'ち', 'は']  )

							self.assertListEqual(tokenizer.tokenize('こんにちほ'  )  ,      ['こ', 'ん', 'に', 'ち', '[UNK]']  )






			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[Any]    =    self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'  )

							_lowerCamelCase     :			str    =    tokenizer.encode('ありがとう。'  ,      add_special_tokens=lowercase  )
							_lowerCamelCase     :			Optional[Any]    =    tokenizer.encode('どういたしまして。'  ,      add_special_tokens=lowercase  )

							_lowerCamelCase     :			Dict    =    tokenizer.build_inputs_with_special_tokens(lowercase  )
							_lowerCamelCase     :			Optional[int]    =    tokenizer.build_inputs_with_special_tokens(lowercase  ,      lowercase  )

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			def 					A_    (  self  ):
							_lowerCamelCase     :			Optional[int]    =    'cl-tohoku/bert-base-japanese'
							_lowerCamelCase     :			str    =    AutoTokenizer.from_pretrained(lowercase  )
							self.assertIsInstance(lowercase  ,      lowercase  )






class lowerCAmelCase__	(     unittest.TestCase						):






			'''simple docstring'''



			def 					A_    (  self  ):
							_lowerCamelCase     :			Any    =    'cl-tohoku/bert-base-japanese'
							with self.assertLogs('transformers'  ,      level='WARNING'  ) as cm:
											BertTokenizer.from_pretrained(lowercase  )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
											        ' is called from.'  )  )
							_lowerCamelCase     :			Optional[Any]    =    'bert-base-cased'
							with self.assertLogs('transformers'  ,      level='WARNING'  ) as cm:
											BertJapaneseTokenizer.from_pretrained(lowercase  )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
											        ' is called from.'  )  )







from pathlib import PurePosixPath
from typing import Optional

import fsspec
from fsspec import AbstractFileSystem
from huggingface_hub.hf_api import DatasetInfo

from ..utils.file_utils import get_authentication_headers_for_url
from ..utils.hub import hf_hub_url
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       """"""
				lowercase__											=       """hf-legacy"""  # "hf://"" is reserved for hffs




				def __init__(     self						:    List[str],  lowerCamelCase						:    Optional[DatasetInfo] = None,  lowerCamelCase						:    Optional[str] = None,  **lowerCamelCase						:    str,  ):

								'''simple docstring'''
								super().__init__(self,  **lowerCamelCase  )
								lowercase__    =			repo_info
								lowercase__    =			token
								lowercase__    =			None




				def 				lowercase__   (     self						:    Optional[Any]  ):

								'''simple docstring'''
								if self.dir_cache is None:
												lowercase__    =			{}
												for hf_file in self.repo_info.siblings:
																# TODO(QL): add sizes
																lowercase__    =			{
																    '''name''': hf_file.rfilename,
																    '''size''': None,
																    '''type''': '''file''',
																}
																self.dir_cache.update(
																    {
																        str(lowerCamelCase  ): {'''name''': str(lowerCamelCase  ), '''size''': None, '''type''': '''directory'''}
																        for d in list(PurePosixPath(hf_file.rfilename  ).parents  )[:-1]
																    }  )




				def 				lowercase__   (     self						:    Any,  lowerCamelCase						:    str,  lowerCamelCase						:    str = "rb",  **lowerCamelCase						:    Any,  ):

								'''simple docstring'''
								if not isinstance(self.repo_info,  lowerCamelCase  ):
												raise NotImplementedError(F"""Open is only implemented for dataset repositories, but got {self.repo_info}"""  )
								lowercase__    =			hf_hub_url(self.repo_info.id,  lowerCamelCase,  revision=self.repo_info.sha  )
								return fsspec.open(
								    lowerCamelCase,  mode=lowerCamelCase,  headers=get_authentication_headers_for_url(lowerCamelCase,  use_auth_token=self.token  ),  client_kwargs={'''trust_env''': True},  ).open()




				def 				lowercase__   (     self						:    Dict,  lowerCamelCase						:    Any,  **lowerCamelCase						:    int  ):

								'''simple docstring'''
								self._get_dirs()
								lowercase__    =			self._strip_protocol(lowerCamelCase  )
								if path in self.dir_cache:
												return self.dir_cache[path]
								else:
												raise FileNotFoundError(lowerCamelCase  )




				def 				lowercase__   (     self						:    Optional[Any],  lowerCamelCase						:    List[str],  lowerCamelCase						:    Optional[int]=False,  **lowerCamelCase						:    str  ):

								'''simple docstring'''
								self._get_dirs()
								lowercase__    =			PurePosixPath(path.strip('''/'''  )  )
								lowercase__    =			{}
								for p, f in self.dir_cache.items():
												lowercase__    =			PurePosixPath(p.strip('''/'''  )  )
												lowercase__    =			p.parent
												if root == path:
																lowercase__    =			f
								lowercase__    =			list(paths.values()  )
								if detail:
												return out
								else:
												return sorted(f['''name'''] for f in out  )




