




'''simple docstring'''





import inspect
from typing import Callable, List, Optional, Set, Tuple, Union

import torch
from packaging import version
from safetensors.torch import storage_ptr, storage_size
from torch import nn

from .utils import logging


a		:	List[str]       =      [nn.LayerNorm]

a		:	int       =      logging.get_logger(__name__)

a		:	str       =      version.parse(version.parse(torch.__version__).base_version)

a		:	Tuple       =      parsed_torch_version_base >= version.parse('''2.0''')
a		:	Optional[int]       =      parsed_torch_version_base >= version.parse('''1.12''')
a		:	Dict       =      parsed_torch_version_base >= version.parse('''1.11''')
a		:	Optional[int]       =      parsed_torch_version_base < version.parse('''1.11''')





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	self						):
    from torch import _softmax_backward_data

    if is_torch_less_than_1_11:
        return _softmax_backward_data(UpperCAmelCase_   ,	UpperCAmelCase_   ,	parent.dim   ,	self						)
    else:
        return _softmax_backward_data(UpperCAmelCase_   ,	UpperCAmelCase_   ,	parent.dim   ,	self.dtype						)





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_ = 0						):
    _a				:							Any							       =      index.to(layer.weight.device						)
    _a				:							Optional[int]							       =      layer.weight.index_select(UpperCAmelCase_   ,	UpperCAmelCase_						).clone().detach()
    if layer.bias is not None:
        if dim == 1:
            _a				:							int							       =      layer.bias.clone().detach()
        else:
            _a				:							List[str]							       =      layer.bias[index].clone().detach()
    _a				:							List[str]							       =      list(layer.weight.size()						)
    _a				:							str							       =      len(UpperCAmelCase_						)
    _a				:							Union[str, Any]							       =      nn.Linear(new_size[1]   ,	new_size[0]   ,	bias=layer.bias is not None						).to(layer.weight.device						)
    _a				:							int							       =      False
    new_layer.weight.copy_(W.contiguous()						)
    _a				:							Optional[int]							       =      True
    if layer.bias is not None:
        _a				:							Any							       =      False
        new_layer.bias.copy_(b.contiguous()						)
        _a				:							int							       =      True
    return new_layer




class      UpperCamelCase       (	nn.Module       ):



  def __init__(			self		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      super().__init__()
      _a				:							Tuple							       =      nf
      _a				:							Union[str, Any]							       =      nn.Parameter(torch.empty(__lowerCAmelCase		,     __lowerCAmelCase ) )
      _a				:							Optional[int]							       =      nn.Parameter(torch.zeros(__lowerCAmelCase ) )
      nn.init.normal_(self.weight		,     std=0.02 )



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Union[str, Any]							       =      x.size()[:-1] + (self.nf,)
      _a				:							List[str]							       =      torch.addmm(self.bias		,     x.view(-1		,     x.size(-1 ) )		,     self.weight )
      _a				:							Any							       =      x.view(__lowerCAmelCase )
      return x






def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_ = 1						):
    _a				:							int							       =      index.to(layer.weight.device						)
    _a				:							int							       =      layer.weight.index_select(UpperCAmelCase_   ,	UpperCAmelCase_						).clone().detach()
    if dim == 0:
        _a				:							List[Any]							       =      layer.bias.clone().detach()
    else:
        _a				:							Optional[Any]							       =      layer.bias[index].clone().detach()
    _a				:							List[str]							       =      list(layer.weight.size()						)
    _a				:							List[Any]							       =      len(UpperCAmelCase_						)
    _a				:							str							       =      ConvaD(new_size[1]   ,	new_size[0]						).to(layer.weight.device						)
    _a				:							Tuple							       =      False
    new_layer.weight.copy_(W.contiguous()						)
    _a				:							List[Any]							       =      True
    _a				:							str							       =      False
    new_layer.bias.copy_(b.contiguous()						)
    _a				:							Union[str, Any]							       =      True
    return new_layer





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_ = None						):
    if isinstance(UpperCAmelCase_   ,	nn.Linear						):
        return prune_linear_layer(UpperCAmelCase_   ,	UpperCAmelCase_   ,	dim=0 if dim is None else dim						)
    elif isinstance(UpperCAmelCase_   ,	UpperCAmelCase_						):
        return prune_convad_layer(UpperCAmelCase_   ,	UpperCAmelCase_   ,	dim=1 if dim is None else dim						)
    else:
        raise ValueError(F'''Can\'t prune layer of class {layer.__class__}'''						)





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	*UpperCAmelCase_						):
    assert len(UpperCAmelCase_						) > 0, F'''{input_tensors} has to be a tuple/list of tensors'''

    # inspect.signature exist since python 3.5 and is a python method -> no problem with backward compatibility
    _a				:							Tuple							       =      len(inspect.signature(UpperCAmelCase_						).parameters						)
    if num_args_in_forward_chunk_fn != len(UpperCAmelCase_						):
        raise ValueError(
            F'''forward_chunk_fn expects {num_args_in_forward_chunk_fn} arguments, but only {len(UpperCAmelCase_						)} input '''
            """tensors are given"""						)

    if chunk_size > 0:
        _a				:							List[str]							       =      input_tensors[0].shape[chunk_dim]
        for input_tensor in input_tensors:
            if input_tensor.shape[chunk_dim] != tensor_shape:
                raise ValueError(
                    F'''All input tenors have to be of the same shape: {tensor_shape}, '''
                    F'''found shape {input_tensor.shape[chunk_dim]}'''						)

        if input_tensors[0].shape[chunk_dim] % chunk_size != 0:
            raise ValueError(
                F'''The dimension to be chunked {input_tensors[0].shape[chunk_dim]} has to be a multiple of the chunk '''
                F'''size {chunk_size}'''						)

        _a				:							str							       =      input_tensors[0].shape[chunk_dim] // chunk_size

        # chunk input tensor into tuples
        _a				:							List[Any]							       =      tuple(input_tensor.chunk(UpperCAmelCase_   ,	dim=UpperCAmelCase_						) for input_tensor in input_tensors						)
        # apply forward fn to every tuple
        _a				:							str							       =      tuple(forward_fn(*UpperCAmelCase_						) for input_tensors_chunk in zip(*UpperCAmelCase_						)						)
        # concatenate output at same dimension
        return torch.cat(UpperCAmelCase_   ,	dim=UpperCAmelCase_						)

    return forward_fn(*UpperCAmelCase_						)





def UpperCAmelCase    (       UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_   ,	UpperCAmelCase_						):
    _a				:							int							       =      torch.ones(UpperCAmelCase_   ,	UpperCAmelCase_						)
    _a				:							List[str]							       =      set(UpperCAmelCase_						) - already_pruned_heads  # Convert to set and remove already pruned heads
    for head in heads:
        # Compute how many pruned heads are before the head and move the index accordingly
        _a				:							Tuple							       =      head - sum(1 if h < head else 0 for h in already_pruned_heads						)
        _a				:							List[Any]							       =      0
    _a				:							int							       =      mask.view(-1						).contiguous().eq(1						)
    _a				:							torch.LongTensor							       =      torch.arange(len(UpperCAmelCase_						)						)[mask].long()
    return heads, index





def UpperCAmelCase    (       *UpperCAmelCase_   ,	UpperCAmelCase_ = None						):
    return torch.meshgrid(*UpperCAmelCase_   ,	indexing=UpperCAmelCase_						)





def UpperCAmelCase    (       UpperCAmelCase_						):
    return tensor.device, storage_ptr(UpperCAmelCase_						), storage_size(UpperCAmelCase_						)







"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase				:     Dict								=  {
    """configuration_camembert""": ["""CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CamembertConfig""", """CamembertOnnxConfig"""],
}

try:
    if not is_sentencepiece_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase				:     Optional[Any]								=  ["""CamembertTokenizer"""]

try:
    if not is_tokenizers_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase				:     Optional[Any]								=  ["""CamembertTokenizerFast"""]

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase				:     int								=  [
        """CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """CamembertForCausalLM""",
        """CamembertForMaskedLM""",
        """CamembertForMultipleChoice""",
        """CamembertForQuestionAnswering""",
        """CamembertForSequenceClassification""",
        """CamembertForTokenClassification""",
        """CamembertModel""",
        """CamembertPreTrainedModel""",
    ]

try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCamelCase				:     str								=  [
        """TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """TFCamembertForCausalLM""",
        """TFCamembertForMaskedLM""",
        """TFCamembertForMultipleChoice""",
        """TFCamembertForQuestionAnswering""",
        """TFCamembertForSequenceClassification""",
        """TFCamembertForTokenClassification""",
        """TFCamembertModel""",
        """TFCamembertPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig, CamembertOnnxConfig

    try:
        if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_camembert import CamembertTokenizer

    try:
        if not is_tokenizers_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .tokenization_camembert_fast import CamembertTokenizerFast

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_camembert import (
            CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            CamembertForCausalLM,
            CamembertForMaskedLM,
            CamembertForMultipleChoice,
            CamembertForQuestionAnswering,
            CamembertForSequenceClassification,
            CamembertForTokenClassification,
            CamembertModel,
            CamembertPreTrainedModel,
        )

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_camembert import (
            TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
            TFCamembertForCausalLM,
            TFCamembertForMaskedLM,
            TFCamembertForMultipleChoice,
            TFCamembertForQuestionAnswering,
            TFCamembertForSequenceClassification,
            TFCamembertForTokenClassification,
            TFCamembertModel,
            TFCamembertPreTrainedModel,
        )

else:
    import sys

    lowerCamelCase				:     List[Any]								=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
