






# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


UpperCamelCase 				=	get_logger(__name__)



class       __UpperCAmelCase		:
			__snake_case				:      Tuple 						=							"dummy_data"
			__snake_case				:      Any 						=							"datasets"
			__snake_case				:      str 						=							False







			def __init__(	self:  Any      ,	UpperCAmelCase_:  str      ,	UpperCAmelCase_:  str      ,	UpperCAmelCase_:  Union[Version, str]      ,	UpperCAmelCase_:  Optional[str] = None      ,	UpperCAmelCase_:  bool = False      ,	UpperCAmelCase_:  bool = True      ,	UpperCAmelCase_:  Optional[List[Callable]] = None      ,	):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  0
									_SCREAMING_SNAKE_CASE	      =  dataset_name
									_SCREAMING_SNAKE_CASE	      =  cache_dir
									_SCREAMING_SNAKE_CASE	      =  use_local_dummy_data
									_SCREAMING_SNAKE_CASE	      =  config
									# download_callbacks take a single url as input
									_SCREAMING_SNAKE_CASE	      =  download_callbacks or []
									# if False, it doesn't load existing files and it returns the paths of the dummy files relative
									# to the dummy_data zip file root
									_SCREAMING_SNAKE_CASE	      =  load_existing_dummy_data

									# TODO(PVP, QL) might need to make this more general
									_SCREAMING_SNAKE_CASE	      =  str(_lowercase     )
									# to be downloaded
									_SCREAMING_SNAKE_CASE	      =  None
									_SCREAMING_SNAKE_CASE	      =  None







			@property
			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									if self._dummy_file is None:
															_SCREAMING_SNAKE_CASE	      =  self.download_dummy_data()
									return self._dummy_file







			@property
			def 							UpperCamelCase      (	self:  Union[str, Any]     ):







									'''simple docstring'''




									if self.config is not None:
															# structure is dummy / config_name / version_name
															return os.path.join("""dummy"""      ,	self.config.name      ,	self.version_name     )
									# structure is dummy / version_name
									return os.path.join("""dummy"""      ,	self.version_name     )







			@property
			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									return os.path.join(self.dummy_data_folder      ,	"""dummy_data.zip"""     )







			def 							UpperCamelCase      (	self:  Optional[Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  (
									    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
									)

									_SCREAMING_SNAKE_CASE	      =  cached_path(
									    _lowercase      ,	cache_dir=self.cache_dir      ,	extract_compressed_file=_lowercase      ,	force_extract=_lowercase     )

									return os.path.join(_lowercase      ,	self.dummy_file_name     )







			@property
			def 							UpperCamelCase      (	self:  List[Any]     ):







									'''simple docstring'''




									return os.path.join(self.datasets_scripts_dir      ,	self.dataset_name      ,	self.dummy_zip_file     )







			@property
			def 							UpperCamelCase      (	self:  int     ):







									'''simple docstring'''




									if self._bucket_url is None:
															_SCREAMING_SNAKE_CASE	      =  hf_github_url(self.dataset_name      ,	self.dummy_zip_file.replace(os.sep      ,	"""/"""     )     )
									return self._bucket_url







			@property
			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									if os.path.isdir(self.dummy_file     ):
															return self.dummy_file
									# else cut off path to file -> example `xsum`.
									return "/".join(self.dummy_file.replace(os.sep      ,	"""/"""     ).split("""/"""     )[:-1]     )







			def 							UpperCamelCase      (	self:  Union[str, Any]      ,	UpperCAmelCase_:  Optional[Any]      ,	*UpperCAmelCase_:  Dict     ):







									'''simple docstring'''




									if self.load_existing_dummy_data:
															# dummy data is downloaded and tested
															_SCREAMING_SNAKE_CASE	      =  self.dummy_file
									else:
															# dummy data cannot be downloaded and only the path to dummy file is returned
															_SCREAMING_SNAKE_CASE	      =  self.dummy_file_name

									# special case when data_url is a dict
									if isinstance(_lowercase      ,	_lowercase     ):
															return self.create_dummy_data_dict(_lowercase      ,	_lowercase     )
									elif isinstance(_lowercase      ,	(list, tuple)     ):
															return self.create_dummy_data_list(_lowercase      ,	_lowercase     )
									else:
															return self.create_dummy_data_single(_lowercase      ,	_lowercase     )







			def 							UpperCamelCase      (	self:  str      ,	UpperCAmelCase_:  List[str]      ,	*UpperCAmelCase_:  List[Any]     ):







									'''simple docstring'''




									return self.download_and_extract(_lowercase     )







			def 							UpperCamelCase      (	self:  int      ,	UpperCAmelCase_:  Union[str, Any]      ,	UpperCAmelCase_:  str     ):







									'''simple docstring'''




									return self.download_and_extract(_lowercase     )







			def 							UpperCamelCase      (	self:  Tuple      ,	UpperCAmelCase_:  List[Any]      ,	*UpperCAmelCase_:  Union[str, Any]      ,	**UpperCAmelCase_:  Optional[int]     ):







									'''simple docstring'''




									return path







			def 							UpperCamelCase      (	self:  Optional[Any]     ):







									'''simple docstring'''




									return {}







			def 							UpperCamelCase      (	self:  Any      ,	UpperCAmelCase_:  Tuple      ,	UpperCAmelCase_:  List[str]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  {}
									for key, single_urls in data_url.items():
															for download_callback in self.download_callbacks:
																					if isinstance(_lowercase      ,	_lowercase     ):
																											for single_url in single_urls:
																																	download_callback(_lowercase     )
																					else:
																											_SCREAMING_SNAKE_CASE	      =  single_urls
																											download_callback(_lowercase     )
																											 # we force the name of each key to be the last file / folder name of the url path
																											 # if the url has arguments, we need to encode them with urllib.parse.quote_plus
															if isinstance(_lowercase      ,	_lowercase     ):
																					_SCREAMING_SNAKE_CASE	      =  [os.path.join(_lowercase      ,	urllib.parse.quote_plus(Path(_lowercase     ).name     )     ) for x in single_urls]
															else:
																					_SCREAMING_SNAKE_CASE	      =  single_urls
																					_SCREAMING_SNAKE_CASE	      =  os.path.join(_lowercase      ,	urllib.parse.quote_plus(Path(_lowercase     ).name     )     )
															_SCREAMING_SNAKE_CASE	      =  value

									# make sure that values are unique
									if all(isinstance(_lowercase      ,	_lowercase     ) for i in dummy_data_dict.values()     ) and len(set(dummy_data_dict.values()     )     ) < len(
									    dummy_data_dict.values()     ):
															# append key to value to make its name unique
															_SCREAMING_SNAKE_CASE	      =  {key: value + key for key, value in dummy_data_dict.items()}

									return dummy_data_dict







			def 							UpperCamelCase      (	self:  List[str]      ,	UpperCAmelCase_:  int      ,	UpperCAmelCase_:  Any     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  []
									# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
									_SCREAMING_SNAKE_CASE	      =  all(bool(re.findall("""[0-9]{3,}-of-[0-9]{3,}"""      ,	_lowercase     )     ) for url in data_url     )
									_SCREAMING_SNAKE_CASE	      =  all(
									    url.startswith("""https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed"""     ) for url in data_url     )
									if data_url and (is_tf_records or is_pubmed_records):
															_SCREAMING_SNAKE_CASE	      =  [data_url[0]] * len(_lowercase     )
									for single_url in data_url:
															for download_callback in self.download_callbacks:
																					download_callback(_lowercase     )
															# we force the name of each key to be the last file / folder name of the url path
															# if the url has arguments, we need to encode them with urllib.parse.quote_plus
															_SCREAMING_SNAKE_CASE	      =  os.path.join(_lowercase      ,	urllib.parse.quote_plus(single_url.split("""/"""     )[-1]     )     )
															dummy_data_list.append(_lowercase     )
									return dummy_data_list







			def 							UpperCamelCase      (	self:  Tuple      ,	UpperCAmelCase_:  Tuple      ,	UpperCAmelCase_:  Optional[Any]     ):







									'''simple docstring'''




									for download_callback in self.download_callbacks:
															download_callback(_lowercase     )
									# we force the name of each key to be the last file / folder name of the url path
									# if the url has arguments, we need to encode them with urllib.parse.quote_plus
									_SCREAMING_SNAKE_CASE	      =  os.path.join(_lowercase      ,	urllib.parse.quote_plus(data_url.split("""/"""     )[-1]     )     )
									if os.path.exists(_lowercase     ) or not self.load_existing_dummy_data:
															return value
									else:
															# Backward compatibility, maybe deprecate at one point.
															# For many datasets with single url calls to dl_manager.download_and_extract,
															# the dummy_data.zip file is actually the zipped downloaded file
															# while now we expected the dummy_data.zip file to be a directory containing
															# the downloaded file.
															return path_to_dummy_data







			def 							UpperCamelCase      (	self:  Dict     ):







									'''simple docstring'''




									pass







			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									pass







			def 							UpperCamelCase      (	self:  Any      ,	UpperCAmelCase_:  List[Any]     ):







									'''simple docstring'''




									def _iter_archive_members(UpperCAmelCase_:  Dict     ):
															# this preserves the order of the members inside the ZIP archive
															_SCREAMING_SNAKE_CASE	      =  Path(self.dummy_file     ).parent
															_SCREAMING_SNAKE_CASE	      =  path.relative_to(_lowercase     )
															with ZipFile(self.local_path_to_dummy_data     ) as zip_file:
																					_SCREAMING_SNAKE_CASE	      =  zip_file.namelist()
															for member in members:
																					if member.startswith(relative_path.as_posix()     ):
																											yield dummy_parent_path.joinpath(_lowercase     )

									_SCREAMING_SNAKE_CASE	      =  Path(_lowercase     )
									_SCREAMING_SNAKE_CASE	      =  _iter_archive_members(_lowercase     ) if self.use_local_dummy_data else path.rglob("""*"""     )
									for file_path in file_paths:
															if file_path.is_file() and not file_path.name.startswith((""".""", """__""")     ):
																					yield file_path.relative_to(_lowercase     ).as_posix(), file_path.open("""rb"""     )







			def 							UpperCamelCase      (	self:  Tuple      ,	UpperCAmelCase_:  Optional[Any]     ):







									'''simple docstring'''




									if not isinstance(_lowercase      ,	_lowercase     ):
															_SCREAMING_SNAKE_CASE	      =  [paths]
									for path in paths:
															if os.path.isfile(_lowercase     ):
																					if os.path.basename(_lowercase     ).startswith((""".""", """__""")     ):
																											return
																					yield path
															else:
																					for dirpath, dirnames, filenames in os.walk(_lowercase     ):
																											if os.path.basename(_lowercase     ).startswith((""".""", """__""")     ):
																																	continue
																											dirnames.sort()
																											for filename in sorted(_lowercase     ):
																																	if filename.startswith((""".""", """__""")     ):
																																							continue
																																	yield os.path.join(_lowercase      ,	_lowercase     )


import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
__lowerCamelCase			: List[str]							   =		object()

# For specifying empty leaf dict `{}`
__lowerCamelCase			: Optional[int]							   =		object()



def 	__SCREAMING_SNAKE_CASE  (					__UpperCamelCase    :       List[Any] ,     __UpperCamelCase    :       List[str]	)							->					Optional[Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__										=      tuple((re.compile(x + """$"""	) for x in qs)	)
  for i in range(len(__UpperCamelCase	) - len(__UpperCamelCase	) + 1	):
    SCREAMING_SNAKE_CASE__										=      [x.match(__UpperCamelCase	) for x, y in zip(__UpperCamelCase ,     ks[i:]	)]
    if matches and all(__UpperCamelCase	):
      return True
  return False



def 	__SCREAMING_SNAKE_CASE  (					__UpperCamelCase    :       Optional[int]	)							->					List[str]:







  """simple docstring"""





  def replace(__UpperCamelCase    :       Tuple ,     __UpperCamelCase    :       Any	):
    for rule, replacement in rules:
      if _match(__UpperCamelCase ,     __UpperCamelCase	):
        return replacement
    return val

  return replace



def 	__SCREAMING_SNAKE_CASE  (					)							->					List[Any]:







  """simple docstring"""





  return [
      # embeddings
      (("transformer", "wpe", "embedding"), P("""mp""" ,     __UpperCamelCase	)),
      (("transformer", "wte", "embedding"), P("""mp""" ,     __UpperCamelCase	)),
      # atention
      (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(__UpperCamelCase ,     """mp"""	)),
      (("attention", "out_proj", "kernel"), P("""mp""" ,     __UpperCamelCase	)),
      (("attention", "out_proj", "bias"), None),
      # mlp
      (("mlp", "c_fc", "kernel"), P(__UpperCamelCase ,     """mp"""	)),
      (("mlp", "c_fc", "bias"), P("""mp"""	)),
      (("mlp", "c_proj", "kernel"), P("""mp""" ,     __UpperCamelCase	)),
      (("mlp", "c_proj", "bias"), None),
      # layer norms
      ((r"ln_\d+", "bias"), None),
      ((r"\d+", r"ln_\d+", "scale"), None),
      (("ln_f", "bias"), None),
      (("ln_f", "scale"), None),
  ]






def 	__SCREAMING_SNAKE_CASE  (					__UpperCamelCase    :       Union[str, Any]	)							->					Dict:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__										=      _get_partition_rules()
  SCREAMING_SNAKE_CASE__										=      _replacement_rules(__UpperCamelCase	)
  SCREAMING_SNAKE_CASE__										=      {k: _unmatched for k in flatten_dict(__UpperCamelCase	)}
  SCREAMING_SNAKE_CASE__										=      {k: replace(__UpperCamelCase ,     __UpperCamelCase	) for k, v in initd.items()}
  assert _unmatched not in result.values(), "Incomplete partition spec."
  return freeze(unflatten_dict(__UpperCamelCase	)	)
