'''simple docstring'''

import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


_A			:      List[Any]					      =				datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
 import pyspark







@dataclass
class 				_lowercase     (				datasets.BuilderConfig			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       Optional[datasets.Features]      	=					None







def      UpperCamelCase_						(					snake_case_       :       "pyspark.sql.DataFrame"						,		snake_case_       :       List[int]						,		)					->     List[Any]:






 '''simple docstring'''

 import pyspark

 def generate_fn():
  __lowerCAmelCase											=						df.select("""*"""						,		pyspark.sql.functions.spark_partition_id().alias("""part_id"""      )      )
  for partition_id in partition_order:
   __lowerCAmelCase											=						df_with_partition_id.select("""*"""      ).where(f"""part_id = {partition_id}"""      ).drop("""part_id"""      )
   __lowerCAmelCase											=						partition_df.collect()
   __lowerCAmelCase											=						0
   for row in rows:
    yield f"""{partition_id}_{row_id}""", row.asDict()
    row_id += 1

 return generate_fn







class 				_lowercase     (				_BaseExamplesIterable			):


 '''simple docstring'''






 def __init__(      self   :	Tuple      ,       SCREAMING_SNAKE_CASE__   :	"pyspark.sql.DataFrame"      ,       SCREAMING_SNAKE_CASE__   :	Union[str, Any]=None      ,       )  ->    List[Any]:
  __lowerCAmelCase											=						df
  __lowerCAmelCase											=						partition_order or range(self.df.rdd.getNumPartitions()							)
  __lowerCAmelCase											=						_generate_iterable_examples(self.df      ,       self.partition_order							)
 def __iter__(      self   :	str							)  ->    Union[str, Any]:
  yield from self.generate_examples_fn()
 def a					(      self   :	Any      ,       SCREAMING_SNAKE_CASE__   :	np.random.Generator							)  ->    "SparkExamplesIterable":
  __lowerCAmelCase											=						list(range(self.df.rdd.getNumPartitions()							)							)
  generator.shuffle(SCREAMING_SNAKE_CASE__							)
  return SparkExamplesIterable(self.df      ,       partition_order=SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	List[str]      ,       SCREAMING_SNAKE_CASE__   :	int      ,       SCREAMING_SNAKE_CASE__   :	int							)  ->    "SparkExamplesIterable":
  __lowerCAmelCase											=						self.split_shard_indices_by_worker(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
  return SparkExamplesIterable(self.df      ,       partition_order=SCREAMING_SNAKE_CASE__							)





 @property
 def a					(      self   :	Optional[int]							)  ->    int:
  return len(self.partition_order							)







class 				_lowercase     (				datasets.DatasetBuilder			):


 '''simple docstring'''






 _SCREAMING_SNAKE_CASE     :       List[Any]      	=					SparkConfig
 def __init__(      self   :	Tuple      ,       SCREAMING_SNAKE_CASE__   :	"pyspark.sql.DataFrame"      ,       SCREAMING_SNAKE_CASE__   :	str = None      ,       SCREAMING_SNAKE_CASE__   :	str = None      ,       **SCREAMING_SNAKE_CASE__   :	Any      ,       )  ->    str:
  import pyspark

  __lowerCAmelCase											=						pyspark.sql.SparkSession.builder.getOrCreate()
  __lowerCAmelCase											=						df
  __lowerCAmelCase											=						working_dir

  super().__init__(
      cache_dir=SCREAMING_SNAKE_CASE__      ,       config_name=str(self.df.semanticHash()							)      ,       **SCREAMING_SNAKE_CASE__      ,       )
 def a					(      self   :	Any							)  ->    Optional[Any]:
  # Returns the path of the created file.
  def create_cache_and_write_probe(SCREAMING_SNAKE_CASE__   :	str							):
   # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
   # already exist.
   os.makedirs(self._cache_dir      ,       exist_ok=SCREAMING_SNAKE_CASE__							)
   __lowerCAmelCase											=						os.path.join(self._cache_dir      ,       """fs_test""" + uuid.uuida().hex							)
   # Opening the file in append mode will create a new file unless it already exists, in which case it will not
   # change the file contents.
   open(SCREAMING_SNAKE_CASE__      ,       """a"""							)
   return [probe_file]

  if self._spark.conf.get("""spark.master"""      ,       """"""							).startswith("""local"""							):
   return

  # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
  # accessible to the driver.
  # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
  if self._cache_dir:
   __lowerCAmelCase											=						(
       self._spark.sparkContext.parallelize(range(1							)      ,       1							).mapPartitions(SCREAMING_SNAKE_CASE__							).collect()
   )
   if os.path.isfile(probe[0]							):
    return

  raise ValueError(
      """When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"""							)
 def a					(      self   :	str							)  ->    Optional[int]:
  return datasets.DatasetInfo(features=self.config.features							)
 def a					(      self   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	datasets.download.download_manager.DownloadManager							)  ->    List[Any]:
  return [datasets.SplitGenerator(name=datasets.Split.TRAIN							)]
 def a					(      self   :	str      ,       SCREAMING_SNAKE_CASE__   :	Optional[Any]							)  ->    Optional[int]:
  import pyspark

  def get_arrow_batch_size(SCREAMING_SNAKE_CASE__   :	Any							):
   for batch in it:
    yield pa.RecordBatch.from_pydict({"""batch_bytes""": [batch.nbytes]}							)

  __lowerCAmelCase											=						self.df.count()
  __lowerCAmelCase											=						df_num_rows if df_num_rows <= 1_00 else 1_00
  # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
  __lowerCAmelCase											=						(
      self.df.limit(SCREAMING_SNAKE_CASE__							)
      .repartition(1							)
      .mapInArrow(SCREAMING_SNAKE_CASE__      ,       """batch_bytes: long"""							)
      .agg(pyspark.sql.functions.sum("""batch_bytes"""							).alias("""sample_bytes"""							)							)
      .collect()[0]
      .sample_bytes
      / sample_num_rows
  )
  __lowerCAmelCase											=						approx_bytes_per_row * df_num_rows
  if approx_total_size > max_shard_size:
   # Make sure there is at least one row per partition.
   __lowerCAmelCase											=						min(SCREAMING_SNAKE_CASE__      ,       int(approx_total_size / max_shard_size							)							)
   __lowerCAmelCase											=						self.df.repartition(SCREAMING_SNAKE_CASE__							)
 def a					(      self   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	str      ,       SCREAMING_SNAKE_CASE__   :	str      ,       SCREAMING_SNAKE_CASE__   :	int      ,       )  ->    Iterable[Tuple[int, bool, Union[int, tuple]]]:
  import pyspark

  __lowerCAmelCase											=						ParquetWriter if file_format == """parquet""" else ArrowWriter
  __lowerCAmelCase											=						os.path.join(self._working_dir      ,       os.path.basename(SCREAMING_SNAKE_CASE__							)							) if self._working_dir else fpath
  __lowerCAmelCase											=						file_format == """parquet"""

  # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
  # pickling the SparkContext.
  __lowerCAmelCase											=						self.config.features
  __lowerCAmelCase											=						self._writer_batch_size
  __lowerCAmelCase											=						self._fs.storage_options

  def write_arrow(SCREAMING_SNAKE_CASE__   :	Any							):
   # Within the same SparkContext, no two task attempts will share the same attempt ID.
   __lowerCAmelCase											=						pyspark.TaskContext().taskAttemptId()
   __lowerCAmelCase											=						next(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)
   if first_batch is None:
    # Some partitions might not receive any data.
    return pa.RecordBatch.from_arrays(
        [[task_id], [0], [0]]      ,       names=["""task_id""", """num_examples""", """num_bytes"""]      ,       )
   __lowerCAmelCase											=						0
   __lowerCAmelCase											=						writer_class(
       features=SCREAMING_SNAKE_CASE__      ,       path=working_fpath.replace("""SSSSS"""      ,       f"""{shard_id:05d}"""							).replace("""TTTTT"""      ,       f"""{task_id:05d}"""							)      ,       writer_batch_size=SCREAMING_SNAKE_CASE__      ,       storage_options=SCREAMING_SNAKE_CASE__      ,       embed_local_files=SCREAMING_SNAKE_CASE__      ,       )
   __lowerCAmelCase											=						pa.Table.from_batches([first_batch]							)
   writer.write_table(SCREAMING_SNAKE_CASE__							)
   for batch in it:
    if max_shard_size is not None and writer._num_bytes >= max_shard_size:
     __lowerCAmelCase							,       __lowerCAmelCase											=						writer.finalize()
     writer.close()
     yield pa.RecordBatch.from_arrays(
         [[task_id], [num_examples], [num_bytes]]      ,       names=["""task_id""", """num_examples""", """num_bytes"""]      ,       )
     shard_id += 1
     __lowerCAmelCase											=						writer_class(
         features=writer._features      ,       path=working_fpath.replace("""SSSSS"""      ,       f"""{shard_id:05d}"""							).replace("""TTTTT"""      ,       f"""{task_id:05d}"""							)      ,       writer_batch_size=SCREAMING_SNAKE_CASE__      ,       storage_options=SCREAMING_SNAKE_CASE__      ,       embed_local_files=SCREAMING_SNAKE_CASE__      ,       )
    __lowerCAmelCase											=						pa.Table.from_batches([batch]							)
    writer.write_table(SCREAMING_SNAKE_CASE__							)

   if writer._num_bytes > 0:
    __lowerCAmelCase							,       __lowerCAmelCase											=						writer.finalize()
    writer.close()
    yield pa.RecordBatch.from_arrays(
        [[task_id], [num_examples], [num_bytes]]      ,       names=["""task_id""", """num_examples""", """num_bytes"""]      ,       )

   if working_fpath != fpath:
    for file in os.listdir(os.path.dirname(SCREAMING_SNAKE_CASE__							)							):
     __lowerCAmelCase											=						os.path.join(os.path.dirname(SCREAMING_SNAKE_CASE__							)      ,       os.path.basename(SCREAMING_SNAKE_CASE__							)							)
     shutil.move(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							)

  __lowerCAmelCase											=						(
      self.df.mapInArrow(SCREAMING_SNAKE_CASE__      ,       """task_id: long, num_examples: long, num_bytes: long"""							)
      .groupBy("""task_id"""							)
      .agg(
          pyspark.sql.functions.sum("""num_examples"""							).alias("""total_num_examples"""							)      ,       pyspark.sql.functions.sum("""num_bytes"""							).alias("""total_num_bytes"""							)      ,       pyspark.sql.functions.count("""num_bytes"""							).alias("""num_shards"""							)      ,       pyspark.sql.functions.collect_list("""num_examples"""							).alias("""shard_lengths"""							)      ,       )
      .collect()
  )
  for row in stats:
   yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)
 def a					(      self   :	List[str]      ,       SCREAMING_SNAKE_CASE__   :	"datasets.SplitGenerator"      ,       SCREAMING_SNAKE_CASE__   :	str = "arrow"      ,       SCREAMING_SNAKE_CASE__   :	Optional[Union[str, int]] = None      ,       SCREAMING_SNAKE_CASE__   :	Optional[int] = None      ,       **SCREAMING_SNAKE_CASE__   :	List[Any]      ,       )  ->    Union[str, Any]:
  self._validate_cache_dir()

  __lowerCAmelCase											=						convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE							)
  self._repartition_df_if_needed(SCREAMING_SNAKE_CASE__							)
  __lowerCAmelCase											=						not is_remote_filesystem(self._fs							)
  __lowerCAmelCase											=						os.path.join if is_local else posixpath.join

  __lowerCAmelCase											=						"""-TTTTT-SSSSS-of-NNNNN"""
  __lowerCAmelCase											=						f"""{self.name}-{split_generator.name}{SUFFIX}.{file_format}"""
  __lowerCAmelCase											=						path_join(self._output_dir      ,       SCREAMING_SNAKE_CASE__							)

  __lowerCAmelCase											=						0
  __lowerCAmelCase											=						0
  __lowerCAmelCase											=						0
  __lowerCAmelCase											=						[]
  __lowerCAmelCase											=						[]

  for task_id, content in self._prepare_split_single(SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__      ,       SCREAMING_SNAKE_CASE__							):
   (
       (
       __lowerCAmelCase
   )							,       (
       __lowerCAmelCase
   )							,       (
       __lowerCAmelCase
   )							,       (
       __lowerCAmelCase
   )							,       
   )											=						content
   if num_bytes > 0:
    total_num_examples += num_examples
    total_num_bytes += num_bytes
    total_shards += num_shards
    task_id_and_num_shards.append((task_id, num_shards)							)
    all_shard_lengths.extend(SCREAMING_SNAKE_CASE__							)

  __lowerCAmelCase											=						total_num_examples
  __lowerCAmelCase											=						total_num_bytes

  # should rename everything at the end
  logger.debug(f"""Renaming {total_shards} shards."""							)
  if total_shards > 1:
   __lowerCAmelCase											=						all_shard_lengths

   # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
   # pickling error due to pickling the SparkContext.
   __lowerCAmelCase											=						self._fs

   # use the -SSSSS-of-NNNNN pattern
   def _rename_shard(
       SCREAMING_SNAKE_CASE__   :	int      ,       SCREAMING_SNAKE_CASE__   :	int      ,       SCREAMING_SNAKE_CASE__   :	int      ,       ):
    rename(
        SCREAMING_SNAKE_CASE__      ,       fpath.replace("""SSSSS"""      ,       f"""{shard_id:05d}"""							).replace("""TTTTT"""      ,       f"""{task_id:05d}"""							)      ,       fpath.replace("""TTTTT-SSSSS"""      ,       f"""{global_shard_id:05d}"""							).replace("""NNNNN"""      ,       f"""{total_shards:05d}"""							)      ,       )

   __lowerCAmelCase											=						[]
   __lowerCAmelCase											=						0
   for i in range(len(SCREAMING_SNAKE_CASE__							)							):
    __lowerCAmelCase							,       __lowerCAmelCase											=						task_id_and_num_shards[i]
    for shard_id in range(SCREAMING_SNAKE_CASE__							):
     args.append([task_id, shard_id, global_shard_id]							)
     global_shard_id += 1
   self._spark.sparkContext.parallelize(SCREAMING_SNAKE_CASE__      ,       len(SCREAMING_SNAKE_CASE__							)							).map(lambda SCREAMING_SNAKE_CASE__							: _rename_shard(*SCREAMING_SNAKE_CASE__							)							).collect()
  else:
   # don't use any pattern
   __lowerCAmelCase											=						0
   __lowerCAmelCase											=						task_id_and_num_shards[0][0]
   self._rename(
       fpath.replace("""SSSSS"""      ,       f"""{shard_id:05d}"""							).replace("""TTTTT"""      ,       f"""{task_id:05d}"""							)      ,       fpath.replace(SCREAMING_SNAKE_CASE__      ,       """"""							)      ,       )





 def a					(      self   :	Optional[int]      ,       SCREAMING_SNAKE_CASE__   :	"datasets.SplitGenerator"      ,       )  ->    SparkExamplesIterable:
  return SparkExamplesIterable(self.df							)







'''simple docstring'''

import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


_A			:      List[Any]					      =				(
    '''https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py'''
)


_A			:      List[Any]					      =				logging.get_logger(__name__)  # pylint: disable=invalid-name







def      UpperCamelCase_						(					)					->     Dict:






 '''simple docstring'''

 __lowerCAmelCase											=						"""https://pypi.org/pypi/diffusers/json"""
 __lowerCAmelCase											=						json.loads(request.urlopen(snake_case_      ).read()      )["""releases"""].keys()
 return sorted(snake_case_						,		key=lambda snake_case_      : version.Version(snake_case_      )      )







def      UpperCamelCase_						(					)					->     int:






 '''simple docstring'''

 if HF_MODULES_CACHE in sys.path:
  return

 sys.path.append(snake_case_      )
 os.makedirs(snake_case_						,		exist_ok=snake_case_      )
 __lowerCAmelCase											=						Path(snake_case_      ) / """__init__.py"""
 if not init_path.exists():
  init_path.touch()







def      UpperCamelCase_						(					snake_case_       :       Union[str, os.PathLike]      )					->     Dict:






 '''simple docstring'''

 init_hf_modules()
 __lowerCAmelCase											=						Path(snake_case_      ) / name
 # If the parent module does not exist yet, recursively create it.
 if not dynamic_module_path.parent.exists():
  create_dynamic_module(dynamic_module_path.parent      )
 os.makedirs(snake_case_						,		exist_ok=snake_case_      )
 __lowerCAmelCase											=						dynamic_module_path / """__init__.py"""
 if not init_path.exists():
  init_path.touch()







def      UpperCamelCase_						(					snake_case_       :       int      )					->     str:






 '''simple docstring'''

 with open(snake_case_						,		"""r"""						,		encoding="""utf-8"""      ) as f:
  __lowerCAmelCase											=						f.read()

 # Imports of the form `import .xxx`
 __lowerCAmelCase											=						re.findall("""^\s*import\s+\.(\S+)\s*$"""						,		snake_case_						,		flags=re.MULTILINE      )
 # Imports of the form `from .xxx import yyy`
 relative_imports += re.findall("""^\s*from\s+\.(\S+)\s+import"""						,		snake_case_						,		flags=re.MULTILINE      )
 # Unique-ify
 return list(set(snake_case_      )      )







def      UpperCamelCase_						(					snake_case_       :       int      )					->     Dict:






 '''simple docstring'''

 __lowerCAmelCase											=						False
 __lowerCAmelCase											=						[module_file]
 __lowerCAmelCase											=						[]

 # Let's recurse through all relative imports
 while not no_change:
  __lowerCAmelCase											=						[]
  for f in files_to_check:
   new_imports.extend(get_relative_imports(snake_case_      )      )

  __lowerCAmelCase											=						Path(snake_case_      ).parent
  __lowerCAmelCase											=						[str(module_path / m      ) for m in new_imports]
  __lowerCAmelCase											=						[f for f in new_import_files if f not in all_relative_imports]
  __lowerCAmelCase											=						[f"""{f}.py""" for f in new_import_files]

  __lowerCAmelCase											=						len(snake_case_      ) == 0
  all_relative_imports.extend(snake_case_      )

 return all_relative_imports







def      UpperCamelCase_						(					snake_case_       :       List[Any]      )					->     Optional[Any]:






 '''simple docstring'''

 with open(snake_case_						,		"""r"""						,		encoding="""utf-8"""      ) as f:
  __lowerCAmelCase											=						f.read()

 # Imports of the form `import xxx`
 __lowerCAmelCase											=						re.findall("""^\s*import\s+(\S+)\s*$"""						,		snake_case_						,		flags=re.MULTILINE      )
 # Imports of the form `from xxx import yyy`
 imports += re.findall("""^\s*from\s+(\S+)\s+import"""						,		snake_case_						,		flags=re.MULTILINE      )
 # Only keep the top-level module
 __lowerCAmelCase											=						[imp.split("""."""      )[0] for imp in imports if not imp.startswith("""."""      )]

 # Unique-ify and test we got them all
 __lowerCAmelCase											=						list(set(snake_case_      )      )
 __lowerCAmelCase											=						[]
 for imp in imports:
  try:
   importlib.import_module(snake_case_      )
  except ImportError:
   missing_packages.append(snake_case_      )

 if len(snake_case_      ) > 0:
  raise ImportError(
      """This modeling file requires the following packages that were not found in your environment: """
      f"""{", ".join(snake_case_      )}. Run `pip install {" ".join(snake_case_      )}`"""      )

 return get_relative_imports(snake_case_      )







def      UpperCamelCase_						(					snake_case_       :       Dict						,		snake_case_       :       Optional[int]      )					->     List[str]:






 '''simple docstring'''

 __lowerCAmelCase											=						module_path.replace(os.path.sep						,		"""."""      )
 __lowerCAmelCase											=						importlib.import_module(snake_case_      )

 if class_name is None:
  return find_pipeline_class(snake_case_      )
 return getattr(snake_case_						,		snake_case_      )







def      UpperCamelCase_						(					snake_case_       :       str      )					->     Optional[Any]:






 '''simple docstring'''

 from ..pipelines import DiffusionPipeline

 __lowerCAmelCase											=						dict(inspect.getmembers(snake_case_						,		inspect.isclass      )      )

 __lowerCAmelCase											=						None
 for cls_name, cls in cls_members.items():
  if (
      cls_name != DiffusionPipeline.__name__
      and issubclass(cls						,		snake_case_      )
      and cls.__module__.split("""."""      )[0] != "diffusers"
  ):
   if pipeline_class is not None:
    raise ValueError(
        f"""Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:"""
        f""" {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in"""
        f""" {loaded_module}."""      )
   __lowerCAmelCase											=						cls

 return pipeline_class







def      UpperCamelCase_						(					snake_case_       :       Union[str, os.PathLike]						,		snake_case_       :       str						,		snake_case_       :       Optional[Union[str, os.PathLike]] = None						,		snake_case_       :       bool = False						,		snake_case_       :       bool = False						,		snake_case_       :       Optional[Dict[str, str]] = None						,		snake_case_       :       Optional[Union[bool, str]] = None						,		snake_case_       :       Optional[str] = None						,		snake_case_       :       bool = False						,		)					->     Any:






 '''simple docstring'''

 __lowerCAmelCase											=						str(snake_case_      )

 __lowerCAmelCase											=						os.path.join(snake_case_						,		snake_case_      )

 if os.path.isfile(snake_case_      ):
  __lowerCAmelCase											=						module_file_or_url
  __lowerCAmelCase											=						"""local"""
 elif pretrained_model_name_or_path.count("""/"""      ) == 0:
  __lowerCAmelCase											=						get_diffusers_versions()
  # cut ".dev0"
  __lowerCAmelCase											=						"""v""" + """.""".join(__version__.split("""."""      )[:3]      )

  # retrieve github version that matches
  if revision is None:
   __lowerCAmelCase											=						latest_version if latest_version[1:] in available_versions else """main"""
   logger.info(f"""Defaulting to latest_version: {revision}."""      )
  elif revision in available_versions:
   __lowerCAmelCase											=						f"""v{revision}"""
  elif revision == "main":
   __lowerCAmelCase											=						revision
  else:
   raise ValueError(
       f"""`custom_revision`: {revision} does not exist. Please make sure to choose one of"""
       f""" {", ".join(available_versions + ["main"]      )}."""      )

  # community pipeline on GitHub
  __lowerCAmelCase											=						COMMUNITY_PIPELINES_URL.format(revision=snake_case_						,		pipeline=snake_case_      )
  try:
   __lowerCAmelCase											=						cached_download(
       snake_case_						,		cache_dir=snake_case_						,		force_download=snake_case_						,		proxies=snake_case_						,		resume_download=snake_case_						,		local_files_only=snake_case_						,		use_auth_token=snake_case_						,		)
   __lowerCAmelCase											=						"""git"""
   __lowerCAmelCase											=						pretrained_model_name_or_path + """.py"""
  except EnvironmentError:
   logger.error(f"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""      )
   raise
 else:
  try:
   # Load from URL or cache if already cached
   __lowerCAmelCase											=						hf_hub_download(
       snake_case_						,		snake_case_						,		cache_dir=snake_case_						,		force_download=snake_case_						,		proxies=snake_case_						,		resume_download=snake_case_						,		local_files_only=snake_case_						,		use_auth_token=snake_case_						,		)
   __lowerCAmelCase											=						os.path.join("""local"""						,		"""--""".join(pretrained_model_name_or_path.split("""/"""      )      )      )
  except EnvironmentError:
   logger.error(f"""Could not locate the {module_file} inside {pretrained_model_name_or_path}."""      )
   raise

    # Check we have all the requirements in our environment
 __lowerCAmelCase											=						check_imports(snake_case_      )

 # Now we move the module inside our cached dynamic modules.
 __lowerCAmelCase											=						DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
 create_dynamic_module(snake_case_      )
 __lowerCAmelCase											=						Path(snake_case_      ) / full_submodule
 if submodule == "local" or submodule == "git":
  # We always copy local files (we could hash the file to see if there was a change, and give them the name of
  # that hash, to only copy when there is a modification but it seems overkill for now).
  # The only reason we do the copy is to avoid putting too many folders in sys.path.
  shutil.copy(snake_case_						,		submodule_path / module_file      )
  for module_needed in modules_needed:
   __lowerCAmelCase											=						f"""{module_needed}.py"""
   shutil.copy(os.path.join(snake_case_						,		snake_case_      )						,		submodule_path / module_needed      )
 else:
  # Get the commit hash
  # TODO: we will get this info in the etag soon, so retrieve it from there and not here.
  if isinstance(snake_case_						,		snake_case_      ):
   __lowerCAmelCase											=						use_auth_token
  elif use_auth_token is True:
   __lowerCAmelCase											=						HfFolder.get_token()
  else:
   __lowerCAmelCase											=						None

  __lowerCAmelCase											=						model_info(snake_case_						,		revision=snake_case_						,		token=snake_case_      ).sha

  # The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
  # benefit of versioning.
  __lowerCAmelCase											=						submodule_path / commit_hash
  __lowerCAmelCase											=						full_submodule + os.path.sep + commit_hash
  create_dynamic_module(snake_case_      )

  if not (submodule_path / module_file).exists():
   shutil.copy(snake_case_						,		submodule_path / module_file      )
  # Make sure we also have every file with relative
  for module_needed in modules_needed:
   if not (submodule_path / module_needed).exists():
    get_cached_module_file(
        snake_case_						,		f"""{module_needed}.py"""						,		cache_dir=snake_case_						,		force_download=snake_case_						,		resume_download=snake_case_						,		proxies=snake_case_						,		use_auth_token=snake_case_						,		revision=snake_case_						,		local_files_only=snake_case_						,		)
 return os.path.join(snake_case_						,		snake_case_      )







def      UpperCamelCase_						(					snake_case_       :       Union[str, os.PathLike]						,		snake_case_       :       str						,		snake_case_       :       Optional[str] = None						,		snake_case_       :       Optional[Union[str, os.PathLike]] = None						,		snake_case_       :       bool = False						,		snake_case_       :       bool = False						,		snake_case_       :       Optional[Dict[str, str]] = None						,		snake_case_       :       Optional[Union[bool, str]] = None						,		snake_case_       :       Optional[str] = None						,		snake_case_       :       bool = False						,		**snake_case_       :       Dict						,		)					->     Tuple:






 '''simple docstring'''

 __lowerCAmelCase											=						get_cached_module_file(
     snake_case_						,		snake_case_						,		cache_dir=snake_case_						,		force_download=snake_case_						,		resume_download=snake_case_						,		proxies=snake_case_						,		use_auth_token=snake_case_						,		revision=snake_case_						,		local_files_only=snake_case_						,		)
 return get_class_in_module(snake_case_						,		final_module.replace(""".py"""						,		""""""      )      )






