




import tensorflow as tf

from ...tf_utils import shape_list







class 						lowercase							(   tf.keras.layers.Layer		):

     def __init__(  self		,     _a		,     _a		,     _a		,     _a		,     _a=1		,     _a=False		,     **_a				)						->							Optional[int]:
           super().__init__(**_a				)

           _A							:     List[Any]			=						vocab_size
           _A							:     Union[str, Any]			=						d_embed
           _A							:     str			=						d_proj

           _A							:     List[str]			=						cutoffs + [vocab_size]
           _A							:     Optional[int]			=						[0] + self.cutoffs
           _A							:     Optional[Any]			=						div_val

           _A							:     List[str]			=						self.cutoffs[0]
           _A							:     List[str]			=						len(self.cutoffs				) - 1
           _A							:     Tuple			=						self.shortlist_size + self.n_clusters
           _A							:     List[str]			=						keep_order

           _A							:     Tuple			=						[]
           _A							:     int			=						[]

     def 					a__	(  self		,     _a				)						->							int:
           if self.n_clusters > 0:
                 _A							:     int			=						self.add_weight(
                     shape=(self.n_clusters, self.d_embed)		,     initializer="""zeros"""		,     trainable=_a		,     name="""cluster_weight"""				)
                 _A							:     List[str]			=						self.add_weight(
                     shape=(self.n_clusters,)		,     initializer="""zeros"""		,     trainable=_a		,     name="""cluster_bias"""				)

           if self.div_val == 1:
                 for i in range(len(self.cutoffs				)				):
                       if self.d_proj != self.d_embed:
                             _A							:     int			=						self.add_weight(
                                 shape=(self.d_embed, self.d_proj)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_projs_._{i}'''		,     )
                             self.out_projs.append(_a				)
                       else:
                             self.out_projs.append(_a				)
                       _A							:     str			=						self.add_weight(
                           shape=(self.vocab_size, self.d_embed)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_layers_._{i}_._weight'''		,     )
                       _A							:     Union[str, Any]			=						self.add_weight(
                           shape=(self.vocab_size,)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_layers_._{i}_._bias'''		,     )
                       self.out_layers.append((weight, bias)				)
           else:
                 for i in range(len(self.cutoffs				)				):
                       _A			,   _A							:     List[Any]			=						self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       _A							:     Optional[int]			=						self.d_embed // (self.div_val**i)

                       _A							:     Dict			=						self.add_weight(
                           shape=(d_emb_i, self.d_proj)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_projs_._{i}'''				)
                       self.out_projs.append(_a				)
                       _A							:     Optional[int]			=						self.add_weight(
                           shape=(r_idx - l_idx, d_emb_i)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_layers_._{i}_._weight'''		,     )
                       _A							:     int			=						self.add_weight(
                           shape=(r_idx - l_idx,)		,     initializer="""zeros"""		,     trainable=_a		,     name=F'''out_layers_._{i}_._bias'''		,     )
                       self.out_layers.append((weight, bias)				)
           super().build(_a				)

     @staticmethod
     def 					a__	(  _a		,     _a		,     _a		,     _a=None				)						->							Dict:
           _A							:     Tuple			=						x
           if proj is not None:
                 _A							:     Any			=						tf.einsum("""ibd,ed->ibe"""		,     _a		,     _a				)
           return tf.einsum("""ibd,nd->ibn"""		,     _a		,     _a				) + b

     @staticmethod
     def 					a__	(  _a		,     _a				)						->							Optional[Any]:
           _A							:     Any			=						shape_list(_a				)
           _A							:     List[str]			=						tf.range(lp_size[0]		,     dtype=target.dtype				)
           _A							:     Optional[Any]			=						tf.stack([r, target]		,     1				)
           return tf.gather_nd(_a		,     _a				)



     def 					a__	(  self		,     _a		,     _a		,     _a=True		,     _a=False				)						->							Optional[Any]:
           _A							:     Union[str, Any]			=						0
           if self.n_clusters == 0:
                 _A							:     List[Any]			=						self._logit(_a		,     self.out_layers[0][0]		,     self.out_layers[0][1]		,     self.out_projs[0]				)
                 if target is not None:
                       _A							:     int			=						tf.nn.sparse_softmax_cross_entropy_with_logits(labels=_a		,     logits=_a				)
                 _A							:     Union[str, Any]			=						tf.nn.log_softmax(_a		,     axis=-1				)
           else:
                 _A							:     Optional[Any]			=						shape_list(_a				)
                 _A							:     Optional[Any]			=						[]
                 _A							:     Optional[Any]			=						tf.zeros(hidden_sizes[:2]				)
                 for i in range(len(self.cutoffs				)				):
                       _A			,   _A							:     List[str]			=						self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       if target is not None:
                             _A							:     int			=						(target >= l_idx) & (target < r_idx)
                             _A							:     str			=						tf.where(_a				)
                             _A							:     Dict			=						tf.boolean_mask(_a		,     _a				) - l_idx

                       if self.div_val == 1:
                             _A							:     str			=						self.out_layers[0][0][l_idx:r_idx]
                             _A							:     List[str]			=						self.out_layers[0][1][l_idx:r_idx]
                       else:
                             _A							:     List[Any]			=						self.out_layers[i][0]
                             _A							:     int			=						self.out_layers[i][1]

                       if i == 0:
                             _A							:     int			=						tf.concat([cur_W, self.cluster_weight]		,     0				)
                             _A							:     int			=						tf.concat([cur_b, self.cluster_bias]		,     0				)

                             _A							:     Dict			=						self._logit(_a		,     _a		,     _a		,     self.out_projs[0]				)
                             _A							:     Optional[Any]			=						tf.nn.log_softmax(_a				)
                             out.append(head_logprob[..., : self.cutoffs[0]]				)
                             if target is not None:
                                   _A							:     Tuple			=						tf.boolean_mask(_a		,     _a				)
                                   _A							:     Union[str, Any]			=						self._gather_logprob(_a		,     _a				)
                       else:
                             _A							:     Dict			=						self._logit(_a		,     _a		,     _a		,     self.out_projs[i]				)
                             _A							:     Union[str, Any]			=						tf.nn.log_softmax(_a				)
                             _A							:     Union[str, Any]			=						self.cutoffs[0] + i - 1  # No probability for the head cluster
                             _A							:     Dict			=						head_logprob[..., cluster_prob_idx, None] + tail_logprob
                             out.append(_a				)
                             if target is not None:
                                   _A							:     Optional[int]			=						tf.boolean_mask(_a		,     _a				)
                                   _A							:     Optional[Any]			=						tf.boolean_mask(_a		,     _a				)
                                   _A							:     Tuple			=						self._gather_logprob(_a		,     _a				)
                                   cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                       if target is not None:
                             loss += tf.scatter_nd(_a		,     -cur_logprob		,     shape_list(_a				)				)
                 _A							:     Optional[Any]			=						tf.concat(_a		,     axis=-1				)

           if target is not None:
                 if return_mean:
                       _A							:     Tuple			=						tf.reduce_mean(_a				)
                 # Add the training-time loss value to the layer using `self.add_loss()`.
                 self.add_loss(_a				)

                 # Log the loss as a metric (we could log arbitrary metrics,
                 # including different metrics for training and inference.
                 self.add_metric(_a		,     name=self.name		,     aggregation="""mean""" if return_mean else """"""				)

           return out






'''simple docstring'''




import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def __init__(self       ,      lowercase       ,      lowercase = None       ,      lowercase = None       ,      lowercase = None       ,      lowercase = False       ,      lowercase = False       ,      lowercase = None       ,      lowercase = None       ,      **lowercase       ,      ):
				super().__init__(
				    lowercase       ,      split=lowercase       ,      features=lowercase       ,      cache_dir=lowercase       ,      keep_in_memory=lowercase       ,      streaming=lowercase       ,      num_proc=lowercase       ,      **lowercase       ,      )
				A_	:						Any	=     field
				A_	:						Optional[int]	=     path_or_paths if isinstance(lowercase       ,      lowercase							) else {self.split: path_or_paths}
				A_	:						str	=     Json(
				    cache_dir=lowercase       ,      data_files=lowercase       ,      features=lowercase       ,      field=lowercase       ,      **lowercase       ,      )


	def     _a	(self							):
				# Build iterable dataset
				if self.streaming:
							A_	:						Optional[int]	=     self.builder.as_streaming_dataset(split=self.split							)
				# Build regular (map-style) dataset
				else:
							A_	:						Optional[Any]	=     None
							A_	:						Optional[Any]	=     None
							A_	:						Tuple	=     None
							A_	:						Optional[int]	=     None

							self.builder.download_and_prepare(
							    download_config=lowercase       ,      download_mode=lowercase       ,      verification_mode=lowercase       ,      base_path=lowercase       ,      num_proc=self.num_proc       ,      )
							A_	:						Union[str, Any]	=     self.builder.as_dataset(
							    split=self.split       ,      verification_mode=lowercase       ,      in_memory=self.keep_in_memory							)
				return dataset







class 	_lowerCAmelCase							:


	def __init__(self       ,      lowercase       ,      lowercase       ,      lowercase = None       ,      lowercase = None       ,      **lowercase       ,      ):
				if num_proc is not None and num_proc <= 0:
							raise ValueError(F'num_proc {num_proc} must be an integer > 0.'							)

				A_	:						Union[str, Any]	=     dataset
				A_	:						Optional[int]	=     path_or_buf
				A_	:						str	=     batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
				A_	:						List[str]	=     num_proc
				A_	:						Union[str, Any]	=     """utf-8"""
				A_	:						Dict	=     to_json_kwargs


	def     _a	(self							):
				A_	:						Optional[Any]	=     self.to_json_kwargs.pop("""path_or_buf"""       ,      lowercase							)
				A_	:						Tuple	=     self.to_json_kwargs.pop("""orient"""       ,      """records"""							)
				A_	:						Union[str, Any]	=     self.to_json_kwargs.pop("""lines"""       ,      True if orient == """records""" else False							)
				A_	:						str	=     self.to_json_kwargs.pop("""index"""       ,      False if orient in ["""split""", """table"""] else True							)
				A_	:						Dict	=     self.to_json_kwargs.pop("""compression"""       ,      lowercase							)

				if compression not in [None, "infer", "gzip", "bz2", "xz"]:
							raise NotImplementedError(F'`datasets` currently does not support {compression} compression'							)

				if isinstance(self.path_or_buf       ,      (str, bytes, os.PathLike)							):
							with fsspec.open(self.path_or_buf       ,      """wb"""       ,      compression=lowercase							) as buffer:
										A_	:						List[str]	=     self._write(file_obj=lowercase       ,      orient=lowercase       ,      lines=lowercase       ,      index=lowercase       ,      **self.to_json_kwargs							)
				else:
							if compression:
										raise NotImplementedError(
										    F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
										    """ was passed. Please provide a local path instead."""							)
							A_	:						List[Any]	=     self._write(
							    file_obj=self.path_or_buf       ,      orient=lowercase       ,      lines=lowercase       ,      index=lowercase       ,      **self.to_json_kwargs							)
				return written


	def     _a	(self       ,      lowercase							):
				A_,    A_,    A_,    A_,    A_	:						Dict	=     args

				A_	:						Any	=     query_table(
				    table=self.dataset.data       ,      key=slice(lowercase       ,      offset + self.batch_size							)       ,      indices=self.dataset._indices       ,      )
				A_	:						Union[str, Any]	=     batch.to_pandas().to_json(
				    path_or_buf=lowercase       ,      orient=lowercase       ,      lines=lowercase       ,      index=lowercase       ,      **lowercase							)
				if not json_str.endswith("""\n"""							):
							json_str += "\n"
				return json_str.encode(self.encoding							)


	def     _a	(self       ,      lowercase       ,      lowercase       ,      lowercase       ,      lowercase       ,      **lowercase       ,      ):
				A_	:						Optional[Any]	=     0

				if self.num_proc is None or self.num_proc == 1:
							for offset in logging.tqdm(
							    range(0       ,      len(self.dataset							)       ,      self.batch_size							)       ,      unit="""ba"""       ,      disable=not logging.is_progress_bar_enabled()       ,      desc="""Creating json from Arrow format"""       ,      ):
										A_	:						Any	=     self._batch_json((offset, orient, lines, index, to_json_kwargs)							)
										written += file_obj.write(lowercase							)
				else:
							A_,    A_	:						List[Any]	=     len(self.dataset							), self.batch_size
							with multiprocessing.Pool(self.num_proc							) as pool:
										for json_str in logging.tqdm(
										    pool.imap(
										        self._batch_json       ,      [(offset, orient, lines, index, to_json_kwargs) for offset in range(0       ,      lowercase       ,      lowercase							)]       ,      )       ,      total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size       ,      unit="""ba"""       ,      disable=not logging.is_progress_bar_enabled()       ,      desc="""Creating json from Arrow format"""       ,      ):
													written += file_obj.write(lowercase							)

				return written