





"""simple docstring"""




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class 					_UpperCAmelCase    (  UpperCAmelCase__						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_ = None   , snake_case_ = None   , snake_case_ = None   , snake_case_ = False   , snake_case_ = False   , snake_case_ = None   , **snake_case_   , ):

        """simple docstring"""

        super().__init__(
            snake_case_   , split=snake_case_   , features=snake_case_   , cache_dir=snake_case_   , keep_in_memory=snake_case_   , streaming=snake_case_   , num_proc=snake_case_   , **snake_case_   , )
        A_    :			int									=    path_or_paths if isinstance(snake_case_   , snake_case_    ) else {self.split: path_or_paths}
        A_    :			List[str]									=    Csv(
            cache_dir=snake_case_   , data_files=snake_case_   , features=snake_case_   , **snake_case_   , )

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        if self.streaming:
          A_    :			List[str]									=    self.builder.as_streaming_dataset(split=self.split    )
        # Build regular (map-style) dataset
        else:
          A_    :			List[Any]									=    None
          A_    :			Optional[int]									=    None
          A_    :			Optional[int]									=    None
          A_    :			List[str]									=    None

          self.builder.download_and_prepare(
              download_config=snake_case_   , download_mode=snake_case_   , verification_mode=snake_case_   , base_path=snake_case_   , num_proc=self.num_proc   , )
          A_    :			List[Any]									=    self.builder.as_dataset(
              split=self.split   , verification_mode=snake_case_   , in_memory=self.keep_in_memory    )
        return dataset


class 					_UpperCAmelCase    :



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_ = None   , snake_case_ = None   , **snake_case_   , ):

        """simple docstring"""

        if num_proc is not None and num_proc <= 0:
          raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""    )

        A_    :			Optional[int]									=    dataset
        A_    :			List[str]									=    path_or_buf
        A_    :			Optional[int]									=    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        A_    :			Optional[int]									=    num_proc
        A_    :			Optional[int]									=    'utf-8'
        A_    :			int									=    to_csv_kwargs

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Union[str, Any]									=    self.to_csv_kwargs.pop('path_or_buf'   , snake_case_    )
        A_    :			str									=    self.to_csv_kwargs.pop('header'   , snake_case_    )
        A_    :			Optional[Any]									=    self.to_csv_kwargs.pop('index'   , snake_case_    )

        if isinstance(self.path_or_buf   , (str, bytes, os.PathLike)    ):
          with open(self.path_or_buf   , 'wb+'    ) as buffer:
            A_    :			Optional[Any]									=    self._write(file_obj=snake_case_   , header=snake_case_   , index=snake_case_   , **self.to_csv_kwargs    )
        else:
          A_    :			int									=    self._write(file_obj=self.path_or_buf   , header=snake_case_   , index=snake_case_   , **self.to_csv_kwargs    )
        return written

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_		,			A_    :			Union[str, Any]									=    args

        A_    :			Dict									=    query_table(
            table=self.dataset.data   , key=slice(snake_case_   , offset + self.batch_size    )   , indices=self.dataset._indices   , )
        A_    :			List[str]									=    batch.to_pandas().to_csv(
            path_or_buf=snake_case_   , header=header if (offset == 0) else False   , index=snake_case_   , **snake_case_    )
        return csv_str.encode(self.encoding    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_   , snake_case_   , **snake_case_    ):

        """simple docstring"""

        A_    :			Any									=    0

        if self.num_proc is None or self.num_proc == 1:
          for offset in logging.tqdm(
              range(0   , len(self.dataset    )   , self.batch_size    )   , unit='ba'   , disable=not logging.is_progress_bar_enabled()   , desc='Creating CSV from Arrow format'   , ):
            A_    :			Optional[int]									=    self._batch_csv((offset, header, index, to_csv_kwargs)    )
            written += file_obj.write(snake_case_    )

        else:
          A_		,			A_    :			Optional[int]									=    len(self.dataset    ), self.batch_size
          with multiprocessing.Pool(self.num_proc    ) as pool:
            for csv_str in logging.tqdm(
                pool.imap(
                    self._batch_csv   , [(offset, header, index, to_csv_kwargs) for offset in range(0   , snake_case_   , snake_case_    )]   , )   , total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   , unit='ba'   , disable=not logging.is_progress_bar_enabled()   , desc='Creating CSV from Arrow format'   , ):
              written += file_obj.write(snake_case_    )

        return written






"""simple docstring"""




import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			List[Any]									=    key.shape[-3:]
  A_    :			Dict									=    value.shape[-1]
  A_    :			Union[str, Any]									=    min(_UpperCAmelCase					,   _UpperCAmelCase   )
  A_    :			str									=    query / jnp.sqrt(_UpperCAmelCase   )

  @functools.partial(jax.checkpoint					,   prevent_cse=_UpperCAmelCase   )
  def summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   ):
    A_    :			Any									=    jnp.einsum('...qhd,...khd->...qhk'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )

    A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=-1					,   keepdims=_UpperCAmelCase   )
    A_    :			List[str]									=    jax.lax.stop_gradient(_UpperCAmelCase   )
    A_    :			Dict									=    jnp.exp(attn_weights - max_score   )

    A_    :			int									=    jnp.einsum('...vhf,...qhv->...qhf'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )
    A_    :			Optional[Any]									=    jnp.einsum('...qhk->...qh'					,   _UpperCAmelCase   )

    return (exp_values, exp_weights.sum(axis=-1   ), max_score)

  def chunk_scanner(_UpperCAmelCase   ):
    # julienne key array
    A_    :			Dict									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(key.shape[:-3]   ) + [key_chunk_size, num_heads, k_features]					,   )

    # julienne value array
    A_    :			Optional[Any]									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(value.shape[:-3]   ) + [key_chunk_size, num_heads, v_features]					,   )

    return summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   )

  A_		,			A_		,			A_    :			Dict									=    jax.lax.map(f=_UpperCAmelCase					,   xs=jnp.arange(0					,   _UpperCAmelCase					,   _UpperCAmelCase   )   )

  A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=0					,   keepdims=_UpperCAmelCase   )
  A_    :			Tuple									=    jnp.exp(chunk_max - global_max   )

  chunk_values *= jnp.expand_dims(_UpperCAmelCase					,   axis=-1   )
  chunk_weights *= max_diffs

  A_    :			List[Any]									=    chunk_values.sum(axis=0   )
  A_    :			Dict									=    jnp.expand_dims(_UpperCAmelCase					,   -1   ).sum(axis=0   )

  return all_values / all_weights





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase=jax.lax.Precision.HIGHEST					,   _UpperCAmelCase = 1024					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			Any									=    query.shape[-3:]

  def chunk_scanner(_UpperCAmelCase					,   _UpperCAmelCase   ):
    # julienne query array
    A_    :			Tuple									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,   slice_sizes=list(query.shape[:-3]   ) + [min(_UpperCAmelCase					,   _UpperCAmelCase   ), num_heads, q_features]					,   )

    return (
        chunk_idx + query_chunk_size,  # unused ignore it
        _query_chunk_attention(
            query=_UpperCAmelCase					,   key=_UpperCAmelCase					,   value=_UpperCAmelCase					,   precision=_UpperCAmelCase					,   key_chunk_size=_UpperCAmelCase   ),
    )

  A_		,			A_    :			Union[str, Any]									=    jax.lax.scan(
      f=_UpperCAmelCase					,   init=0					,   xs=_UpperCAmelCase					,   length=math.ceil(num_q / query_chunk_size   )   )

  return jnp.concatenate(_UpperCAmelCase					,   axis=-3   )  # fuse the chunked result back

class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int             =  8
      lowercase_ :					int             =  64
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    self.dim_head * self.heads
        A_    :			Tuple									=    self.dim_head**-0.5

        # Weights were exported with old names {to_q, to_k, to_v, to_out}
        A_    :			Any									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_q'    )
        A_    :			int									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_k'    )
        A_    :			Optional[int]									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_v'    )

        A_    :			Union[str, Any]									=    nn.Dense(self.query_dim   , dtype=self.dtype   , name='to_out_0'    )
        A_    :			str									=    nn.Dropout(rate=self.dropout    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			Union[str, Any]									=    tensor.shape
        A_    :			List[str]									=    self.heads
        A_    :			str									=    tensor.reshape(snake_case_   , snake_case_   , snake_case_   , dim // head_size    )
        A_    :			Optional[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size * head_size   , snake_case_   , dim // head_size    )
        return tensor

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			str									=    tensor.shape
        A_    :			Optional[Any]									=    self.heads
        A_    :			List[Any]									=    tensor.reshape(batch_size // head_size   , snake_case_   , snake_case_   , snake_case_    )
        A_    :			List[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size // head_size   , snake_case_   , dim * head_size    )
        return tensor

      def __call__(	self   , snake_case_   , snake_case_=None   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    hidden_states if context is None else context

        A_    :			str									=    self.query(snake_case_    )
        A_    :			Any									=    self.key(snake_case_    )
        A_    :			Union[str, Any]									=    self.value(snake_case_    )

        A_    :			int									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			List[str]									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			str									=    self.reshape_heads_to_batch_dim(snake_case_    )

        if self.use_memory_efficient_attention:
          A_    :			str									=    query_states.transpose(1   , 0   , 2    )
          A_    :			Optional[int]									=    key_states.transpose(1   , 0   , 2    )
          A_    :			Optional[Any]									=    value_states.transpose(1   , 0   , 2    )

          # this if statement create a chunk size for each layer of the unet
          # the chunk size is equal to the query_length dimension of the deepest layer of the unet

          A_    :			Union[str, Any]									=    query_states.shape[-3]
          if flatten_latent_dim % 6_4 == 0:
            A_    :			Any									=    int(flatten_latent_dim / 6_4    )
          elif flatten_latent_dim % 1_6 == 0:
            A_    :			Optional[int]									=    int(flatten_latent_dim / 1_6    )
          elif flatten_latent_dim % 4 == 0:
            A_    :			Dict									=    int(flatten_latent_dim / 4    )
          else:
            A_    :			Optional[int]									=    int(snake_case_    )

          A_    :			Any									=    jax_memory_efficient_attention(
              snake_case_   , snake_case_   , snake_case_   , query_chunk_size=snake_case_   , key_chunk_size=4_0_9_6 * 4    )

          A_    :			Any									=    hidden_states.transpose(1   , 0   , 2    )
        else:
          # compute attentions
          A_    :			str									=    jnp.einsum('b i d, b j d->b i j'   , snake_case_   , snake_case_    )
          A_    :			Dict									=    attention_scores * self.scale
          A_    :			List[str]									=    nn.softmax(snake_case_   , axis=2    )

          # attend to values
          A_    :			Optional[Any]									=    jnp.einsum('b i j, b j d -> b i d'   , snake_case_   , snake_case_    )

        A_    :			List[str]									=    self.reshape_batch_dim_to_heads(snake_case_    )
        A_    :			Tuple									=    self.proj_attn(snake_case_    )
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        # cross attention
        A_    :			Optional[Any]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        A_    :			Optional[Any]									=    FlaxFeedForward(dim=self.dim   , dropout=self.dropout   , dtype=self.dtype    )
        A_    :			Any									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Dict									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Optional[int]									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[int]									=    hidden_states
        if self.only_cross_attention:
          A_    :			Union[str, Any]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        else:
          A_    :			Tuple									=    self.attna(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[Any]									=    hidden_states + residual

        # cross attention
        A_    :			List[str]									=    hidden_states
        A_    :			List[str]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    hidden_states + residual

        # feed forward
        A_    :			Dict									=    hidden_states
        A_    :			str									=    self.ff(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[int]									=    hidden_states + residual

        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int             =  1
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    nn.GroupNorm(num_groups=3_2   , epsilon=1E-5    )

        A_    :			Dict									=    self.n_heads * self.d_head
        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Any									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    [
            FlaxBasicTransformerBlock(
                snake_case_   , self.n_heads   , self.d_head   , dropout=self.dropout   , only_cross_attention=self.only_cross_attention   , dtype=self.dtype   , use_memory_efficient_attention=self.use_memory_efficient_attention   , )
            for _ in range(self.depth    )
        ]

        if self.use_linear_projection:
          A_    :			List[Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Dict									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_		,			A_		,			A_		,			A_    :			Any									=    hidden_states.shape
        A_    :			Any									=    hidden_states
        A_    :			List[Any]									=    self.norm(snake_case_    )
        if self.use_linear_projection:
          A_    :			str									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )
          A_    :			Optional[Any]									=    self.proj_in(snake_case_    )
        else:
          A_    :			List[str]									=    self.proj_in(snake_case_    )
          A_    :			List[Any]									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )

        for transformer_block in self.transformer_blocks:
          A_    :			int									=    transformer_block(snake_case_   , snake_case_   , deterministic=snake_case_    )

        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )
          A_    :			Any									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
        else:
          A_    :			int									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )

        A_    :			Union[str, Any]									=    hidden_states + residual
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxGEGLU(self.dim   , self.dropout   , self.dtype    )
        A_    :			int									=    nn.Dense(self.dim   , dtype=self.dtype    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Dict									=    self.net_a(snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    self.net_a(snake_case_    )
        return hidden_states


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[int]									=    self.dim * 4
        A_    :			List[str]									=    nn.Dense(inner_dim * 2   , dtype=self.dtype    )
        A_    :			int									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    self.proj(snake_case_    )
        A_		,			A_    :			Any									=    jnp.split(snake_case_   , 2   , axis=2    )
        return self.dropout_layer(hidden_linear * nn.gelu(snake_case_    )   , deterministic=snake_case_    )