


"""simple docstring"""





import numpy as np

import datasets


snake_case__						:   Dict			 = '''
Compute the Mahalanobis Distance

Mahalonobis distance is the distance between a point and a distribution.
And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance.
It was introduced by Prof. P. C. Mahalanobis in 1936
and has been used in various statistical applications ever since
[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/]
'''

snake_case__						:   Union[str, Any]			 = '''\
@article{de2000mahalanobis,
  title={The mahalanobis distance},
  author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L},
  journal={Chemometrics and intelligent laboratory systems},
  volume={50},
  number={1},
  pages={1--18},
  year={2000},
  publisher={Elsevier}
}
'''

snake_case__						:   Any			 = '''
Args:
    X: List of datapoints to be compared with the `reference_distribution`.
    reference_distribution: List of datapoints from the reference distribution we want to compare to.
Returns:
    mahalanobis: The Mahalonobis distance for each datapoint in `X`.
Examples:

    >>> mahalanobis_metric = datasets.load_metric("mahalanobis")
    >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]])
    >>> print(results)
    {\'mahalanobis\': array([0.5])}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,    _KWARGS_DESCRIPTION	)
class 						snake_case_(				datasets.Metric	):






     def    lowerCamelCase__   (		self     :    List[str] ):
           return datasets.MetricInfo(
               description=_DESCRIPTION    , citation=_CITATION    , inputs_description=_KWARGS_DESCRIPTION    , features=datasets.Features(
                   {
                       '''X''': datasets.Sequence(datasets.Value('''float'''    , id='''sequence''' )    , id='''X''' ),
                   } )    , )






     def    lowerCamelCase__   (		self     :    Optional[int]    , UpperCamelCase_     :    Optional[int]    , UpperCamelCase_     :    int ):
           # convert to numpy arrays
           lowerCAmelCase    :		List[str]				=      np.array(UpperCamelCase_ )
           lowerCAmelCase    :		Optional[Any]				=      np.array(UpperCamelCase_ )

           # Assert that arrays are 2D
           if len(X.shape ) != 2:
                 raise ValueError('''Expected `X` to be a 2D vector''' )
           if len(reference_distribution.shape ) != 2:
                 raise ValueError('''Expected `reference_distribution` to be a 2D vector''' )
           if reference_distribution.shape[0] < 2:
                 raise ValueError(
                     '''Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension''' )

           # Get mahalanobis distance for each prediction
           lowerCAmelCase    :		List[Any]				=      X - np.mean(UpperCamelCase_ )
           lowerCAmelCase    :		Union[str, Any]				=      np.cov(reference_distribution.T )
           try:
                 lowerCAmelCase    :		Dict				=      np.linalg.inv(UpperCamelCase_ )
           except np.linalg.LinAlgError:
                 lowerCAmelCase    :		List[str]				=      np.linalg.pinv(UpperCamelCase_ )
           lowerCAmelCase    :		Optional[int]				=      np.dot(UpperCamelCase_    , UpperCamelCase_ )
           lowerCAmelCase    :		Optional[Any]				=      np.dot(UpperCamelCase_    , X_minus_mu.T ).diagonal()

           return {"mahalanobis": mahal_dist}









"""simple docstring"""
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
lowerCamelCase_								=		logging.getLogger(__name__)



def      __lowerCamelCase   (							)			->      int:
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  argparse.ArgumentParser(
          description='''Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'''							)
      parser.add_argument('''--file_path''' ,   type=a_ ,   default='''data/dump.txt''' ,   help='''The path to the data.'''							)
      parser.add_argument('''--tokenizer_type''' ,   type=a_ ,   default='''bert''' ,   choices=['''bert''', '''roberta''', '''gpt2''']							)
      parser.add_argument('''--tokenizer_name''' ,   type=a_ ,   default='''bert-base-uncased''' ,   help='''The tokenizer to use.'''							)
      parser.add_argument('''--dump_file''' ,   type=a_ ,   default='''data/dump''' ,   help='''The dump file prefix.'''							)
      __SCREAMING_SNAKE_CASE   :Any						=  parser.parse_args()

      logger.info(f'''Loading Tokenizer ({args.tokenizer_name})'''							)
      if args.tokenizer_type == "bert":
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  BertTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.special_tokens_map['''cls_token''']  # `[CLS]`
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tokenizer.special_tokens_map['''sep_token''']  # `[SEP]`
      elif args.tokenizer_type == "roberta":
            __SCREAMING_SNAKE_CASE   :str						=  RobertaTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''cls_token''']  # `<s>`
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''sep_token''']  # `</s>`
      elif args.tokenizer_type == "gpt2":
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  GPTaTokenizer.from_pretrained(args.tokenizer_name							)
            __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.special_tokens_map['''bos_token''']  # `<|endoftext|>`
            __SCREAMING_SNAKE_CASE   :str						=  tokenizer.special_tokens_map['''eos_token''']  # `<|endoftext|>`

      logger.info(f'''Loading text from {args.file_path}'''							)
      with open(args.file_path ,   '''r''' ,   encoding='''utf8'''							) as fp:
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  fp.readlines()

      logger.info('''Start encoding'''							)
      logger.info(f'''{len(a_							)} examples to process.'''							)

      __SCREAMING_SNAKE_CASE   :Optional[int]						=  []
      __SCREAMING_SNAKE_CASE   :List[str]						=  0
      __SCREAMING_SNAKE_CASE   :Optional[Any]						=  1_00_00
      __SCREAMING_SNAKE_CASE   :List[Any]						=  time.time()
      for text in data:
            __SCREAMING_SNAKE_CASE   :Any						=  f'''{bos} {text.strip()} {sep}'''
            __SCREAMING_SNAKE_CASE   :int						=  tokenizer.encode(a_ ,   add_special_tokens=a_							)
            rslt.append(a_							)

            iter += 1
            if iter % interval == 0:
                  __SCREAMING_SNAKE_CASE   :Any						=  time.time()
                  logger.info(f'''{iter} examples processed. - {(end-start):.2f}s/{interval}expl'''							)
                  __SCREAMING_SNAKE_CASE   :Any						=  time.time()
      logger.info('''Finished binarization'''							)
      logger.info(f'''{len(a_							)} examples processed.'''							)

      __SCREAMING_SNAKE_CASE   :Optional[int]						=  f'''{args.dump_file}.{args.tokenizer_name}.pickle'''
      __SCREAMING_SNAKE_CASE   :str						=  tokenizer.vocab_size
      if vocab_size < (1 << 16):
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  [np.uintaa(a_							) for d in rslt]
      else:
            __SCREAMING_SNAKE_CASE   :List[Any]						=  [np.intaa(a_							) for d in rslt]
      random.shuffle(rslt_							)
      logger.info(f'''Dump to {dp_file}'''							)
      with open(a_ ,   '''wb'''							) as handle:
            pickle.dump(rslt_ ,   a_ ,   protocol=pickle.HIGHEST_PROTOCOL							)


if __name__ == "__main__":
  main()