



'''simple docstring'''





import torch
from torch import nn






class 					lowerCamelCase_    (				nn.Module     ):
  """simple docstring"""




  def __init__( self      :	Any			,	_a      :	Dict			,	_a      :	List[Any]			,	_a      :	str			,	_a      :	List[Any]			,	_a      :	Optional[Any]=1			,	_a      :	str=False   )					->					Any:
    super().__init__()

    __lowerCamelCase       :       Dict           = n_token
    __lowerCamelCase       :       Union[str, Any]           = d_embed
    __lowerCamelCase       :       Dict           = d_proj

    __lowerCamelCase       :       Any           = cutoffs + [n_token]
    __lowerCamelCase       :       Dict           = [0] + self.cutoffs
    __lowerCamelCase       :       str           = div_val

    __lowerCamelCase       :       Dict           = self.cutoffs[0]
    __lowerCamelCase       :       List[Any]           = len(self.cutoffs   ) - 1
    __lowerCamelCase       :       Union[str, Any]           = self.shortlist_size + self.n_clusters

    if self.n_clusters > 0:
      __lowerCamelCase       :       Any           = nn.Parameter(torch.zeros(self.n_clusters			,	self.d_embed   )   )
      __lowerCamelCase       :       Union[str, Any]           = nn.Parameter(torch.zeros(self.n_clusters   )   )

    __lowerCamelCase       :       Optional[int]           = nn.ModuleList()
    __lowerCamelCase       :       Optional[Any]           = nn.ParameterList()

    if div_val == 1:
      for i in range(len(self.cutoffs   )   ):
        if d_proj != d_embed:
          self.out_projs.append(nn.Parameter(torch.FloatTensor(_a			,	_a   )   )   )
        else:
          self.out_projs.append(_a   )

      self.out_layers.append(nn.Linear(_a			,	_a   )   )
    else:
      for i in range(len(self.cutoffs   )   ):
        __lowerCamelCase       ,__lowerCamelCase       :       Tuple           = self.cutoff_ends[i], self.cutoff_ends[i + 1]
        __lowerCamelCase       :       Union[str, Any]           = d_embed // (div_val**i)

        self.out_projs.append(nn.Parameter(torch.FloatTensor(_a			,	_a   )   )   )

        self.out_layers.append(nn.Linear(_a			,	r_idx - l_idx   )   )

    __lowerCamelCase       :       str           = keep_order




  def 						_lowercase     ( self      :	Optional[int]			,	_a      :	str			,	_a      :	Tuple			,	_a      :	List[str]			,	_a      :	Union[str, Any]   )					->					Any:
    if proj is None:
      __lowerCamelCase       :       Optional[Any]           = nn.functional.linear(_a			,	_a			,	bias=_a   )
    else:
      # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
      __lowerCamelCase       :       int           = nn.functional.linear(_a			,	proj.t().contiguous()   )
      __lowerCamelCase       :       int           = nn.functional.linear(_a			,	_a			,	bias=_a   )
      # else:
      #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
      #     if bias is not None:
      #         logit = logit + bias

    return logit




  def 						_lowercase     ( self      :	Any			,	_a      :	List[str]			,	_a      :	Optional[int]=None			,	_a      :	Optional[Any]=False   )					->					Tuple:

    if labels is not None:
      # Shift so that tokens < n predict n
      __lowerCamelCase       :       str           = hidden[..., :-1, :].contiguous()
      __lowerCamelCase       :       str           = labels[..., 1:].contiguous()
      __lowerCamelCase       :       List[Any]           = hidden.view(-1			,	hidden.size(-1   )   )
      __lowerCamelCase       :       Optional[Any]           = labels.view(-1   )
      if hidden.size(0   ) != labels.size(0   ):
        raise RuntimeError('Input and labels should have the same size in the batch dimension.'   )
    else:
      __lowerCamelCase       :       int           = hidden.view(-1			,	hidden.size(-1   )   )

    if self.n_clusters == 0:
      __lowerCamelCase       :       int           = self._compute_logit(_a			,	self.out_layers[0].weight			,	self.out_layers[0].bias			,	self.out_projs[0]   )
      if labels is not None:
        __lowerCamelCase       :       Any           = labels != -100
        __lowerCamelCase       :       Dict           = torch.zeros_like(_a			,	dtype=hidden.dtype			,	device=hidden.device   )
        __lowerCamelCase       :       str           = (
            -nn.functional.log_softmax(_a			,	dim=-1   )[mask].gather(1			,	labels[mask].unsqueeze(1   )   ).squeeze(1   )
        )
      else:
        __lowerCamelCase       :       Dict           = nn.functional.log_softmax(_a			,	dim=-1   )
    else:
      # construct weights and biases
      __lowerCamelCase       ,__lowerCamelCase       :       int           = [], []
      for i in range(len(self.cutoffs   )   ):
        if self.div_val == 1:
          __lowerCamelCase       ,__lowerCamelCase       :       List[Any]           = self.cutoff_ends[i], self.cutoff_ends[i + 1]
          __lowerCamelCase       :       Optional[Any]           = self.out_layers[0].weight[l_idx:r_idx]
          __lowerCamelCase       :       Tuple           = self.out_layers[0].bias[l_idx:r_idx]
        else:
          __lowerCamelCase       :       Dict           = self.out_layers[i].weight
          __lowerCamelCase       :       int           = self.out_layers[i].bias

        if i == 0:
          __lowerCamelCase       :       List[Any]           = torch.cat([weight_i, self.cluster_weight]			,	dim=0   )
          __lowerCamelCase       :       Union[str, Any]           = torch.cat([bias_i, self.cluster_bias]			,	dim=0   )

        weights.append(_a   )
        biases.append(_a   )

      __lowerCamelCase       ,__lowerCamelCase       ,__lowerCamelCase       :       Tuple           = weights[0], biases[0], self.out_projs[0]

      __lowerCamelCase       :       Optional[Any]           = self._compute_logit(_a			,	_a			,	_a			,	_a   )
      __lowerCamelCase       :       Optional[int]           = nn.functional.log_softmax(_a			,	dim=1   )

      if labels is None:
        __lowerCamelCase       :       Tuple           = hidden.new_empty((head_logit.size(0   ), self.n_token)   )
      else:
        __lowerCamelCase       :       List[str]           = torch.zeros_like(_a			,	dtype=hidden.dtype			,	device=hidden.device   )

      __lowerCamelCase       :       str           = 0
      __lowerCamelCase       :       Optional[Any]           = [0] + self.cutoffs
      for i in range(len(_a   ) - 1   ):
        __lowerCamelCase       ,__lowerCamelCase       :       int           = cutoff_values[i], cutoff_values[i + 1]

        if labels is not None:
          __lowerCamelCase       :       List[Any]           = (labels >= l_idx) & (labels < r_idx)
          __lowerCamelCase       :       str           = mask_i.nonzero().squeeze()

          if indices_i.numel() == 0:
            continue

          __lowerCamelCase       :       str           = labels.index_select(0			,	_a   ) - l_idx
          __lowerCamelCase       :       Tuple           = head_logprob.index_select(0			,	_a   )
          __lowerCamelCase       :       Dict           = hidden.index_select(0			,	_a   )
        else:
          __lowerCamelCase       :       Optional[int]           = hidden

        if i == 0:
          if labels is not None:
            __lowerCamelCase       :       Dict           = head_logprob_i.gather(1			,	target_i[:, None]   ).squeeze(1   )
          else:
            __lowerCamelCase       :       int           = head_logprob[:, : self.cutoffs[0]]
        else:
          __lowerCamelCase       ,__lowerCamelCase       ,__lowerCamelCase       :       Optional[int]           = weights[i], biases[i], self.out_projs[i]

          __lowerCamelCase       :       int           = self._compute_logit(_a			,	_a			,	_a			,	_a   )
          __lowerCamelCase       :       int           = nn.functional.log_softmax(_a			,	dim=1   )
          __lowerCamelCase       :       Any           = self.cutoffs[0] + i - 1  # No probability for the head cluster
          if labels is not None:
            __lowerCamelCase       :       List[Any]           = head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                1			,	target_i[:, None]   ).squeeze(1   )
          else:
            __lowerCamelCase       :       Optional[Any]           = head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
            __lowerCamelCase       :       Optional[Any]           = logprob_i

        if labels is not None:
          if (hasattr(self			,	'keep_order'   ) and self.keep_order) or keep_order:
            out.index_copy_(0			,	_a			,	-logprob_i   )
          else:
            out[offset : offset + logprob_i.size(0   )].copy_(-logprob_i   )
          offset += logprob_i.size(0   )

    return out




  def 						_lowercase     ( self      :	List[str]			,	_a      :	Union[str, Any]   )					->					Optional[int]:
    if self.n_clusters == 0:
      __lowerCamelCase       :       str           = self._compute_logit(_a			,	self.out_layers[0].weight			,	self.out_layers[0].bias			,	self.out_projs[0]   )
      return nn.functional.log_softmax(_a			,	dim=-1   )
    else:
      # construct weights and biases
      __lowerCamelCase       ,__lowerCamelCase       :       Dict           = [], []
      for i in range(len(self.cutoffs   )   ):
        if self.div_val == 1:
          __lowerCamelCase       ,__lowerCamelCase       :       Optional[Any]           = self.cutoff_ends[i], self.cutoff_ends[i + 1]
          __lowerCamelCase       :       Optional[Any]           = self.out_layers[0].weight[l_idx:r_idx]
          __lowerCamelCase       :       List[str]           = self.out_layers[0].bias[l_idx:r_idx]
        else:
          __lowerCamelCase       :       Tuple           = self.out_layers[i].weight
          __lowerCamelCase       :       Any           = self.out_layers[i].bias

        if i == 0:
          __lowerCamelCase       :       Dict           = torch.cat([weight_i, self.cluster_weight]			,	dim=0   )
          __lowerCamelCase       :       Tuple           = torch.cat([bias_i, self.cluster_bias]			,	dim=0   )

        weights.append(_a   )
        biases.append(_a   )

      __lowerCamelCase       ,__lowerCamelCase       ,__lowerCamelCase       :       Optional[Any]           = weights[0], biases[0], self.out_projs[0]
      __lowerCamelCase       :       List[str]           = self._compute_logit(_a			,	_a			,	_a			,	_a   )

      __lowerCamelCase       :       Union[str, Any]           = hidden.new_empty((head_logit.size(0   ), self.n_token)   )
      __lowerCamelCase       :       Union[str, Any]           = nn.functional.log_softmax(_a			,	dim=1   )

      __lowerCamelCase       :       str           = [0] + self.cutoffs
      for i in range(len(_a   ) - 1   ):
        __lowerCamelCase       ,__lowerCamelCase       :       Union[str, Any]           = cutoff_values[i], cutoff_values[i + 1]

        if i == 0:
          __lowerCamelCase       :       Union[str, Any]           = head_logprob[:, : self.cutoffs[0]]
        else:
          __lowerCamelCase       ,__lowerCamelCase       ,__lowerCamelCase       :       str           = weights[i], biases[i], self.out_projs[i]

          __lowerCamelCase       :       List[Any]           = self._compute_logit(_a			,	_a			,	_a			,	_a   )
          __lowerCamelCase       :       Optional[Any]           = nn.functional.log_softmax(_a			,	dim=1   )

          __lowerCamelCase       :       Union[str, Any]           = head_logprob[:, -i] + tail_logprob_i
          __lowerCamelCase       :       List[Any]           = logprob_i

      return out











'''simple docstring'''





from __future__ import annotations

import math
from collections.abc import Callable

def        a_     (							_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase       ,_lowerCAmelCase = 100       ,)      ->      float:
  __lowerCamelCase       :       Dict           = x_start
  __lowerCamelCase       :       int           = fnc(_lowerCAmelCase       )
  __lowerCamelCase       :       Dict           = 0.0

  for _ in range(_lowerCAmelCase       ):
    # Approximates curve as a sequence of linear lines and sums their length
    __lowerCamelCase       :       List[str]           = (x_end - x_start) / steps + xa
    __lowerCamelCase       :       List[Any]           = fnc(_lowerCAmelCase       )
    length += math.hypot(xa - xa       ,fxa - fxa       )

    # Increment step
    __lowerCamelCase       :       Any           = xa
    __lowerCamelCase       :       Tuple           = fxa

  return length


if __name__ == "__main__":

   def        a_     (							_lowerCAmelCase       )      ->      Dict:
     return math.sin(10 * x       )

   print('f(x) = sin(10 * x)')
   print('The length of the curve from x = -10 to x = 10 is:')
   _UpperCamelCase											=						10
   while i <= 100000:
      print(f'''With {i} steps: {line_length(f, -10, 10, i)}''')
      i *= 10






