

import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase_									=   logging.get_logger(__name__)

lowercase_									=   '▁'

lowercase_									=   {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

lowercase_									=   {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

lowercase_									=   {
    'facebook/m2m100_418M': 1_0_2_4,
}

# fmt: off
lowercase_									=   {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}





class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE							):
  _a									=  VOCAB_FILES_NAMES
  _a									=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _a									=  PRETRAINED_VOCAB_FILES_MAP
  _a									=  ['input_ids', 'attention_mask']

  _a									=  []
  _a									=  []




  def __init__(				self							,   lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase=None							,   lowerCAmelCase=None							,   lowerCAmelCase="<s>"							,   lowerCAmelCase="</s>"							,   lowerCAmelCase="</s>"							,   lowerCAmelCase="<pad>"							,   lowerCAmelCase="<unk>"							,   lowerCAmelCase="m2m100"							,   lowerCAmelCase = None							,   lowerCAmelCase=8							,   **lowerCAmelCase							,   )  -> None:
        '''simple docstring'''






        _lowercase	       ={} if sp_model_kwargs is None else sp_model_kwargs

        _lowercase	       =language_codes
        _lowercase	       =FAIRSEQ_LANGUAGE_CODES[language_codes]
        _lowercase	       ={lang_code: F'''__{lang_code}__''' for lang_code in fairseq_language_code}

        _lowercase	       =kwargs.get('additional_special_tokens'							,   []					)
        kwargs["additional_special_tokens"] += [
            self.get_lang_token(lowerCAmelCase					)
            for lang_code in fairseq_language_code
            if self.get_lang_token(lowerCAmelCase					) not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=lowerCAmelCase							,   tgt_lang=lowerCAmelCase							,   bos_token=lowerCAmelCase							,   eos_token=lowerCAmelCase							,   sep_token=lowerCAmelCase							,   unk_token=lowerCAmelCase							,   pad_token=lowerCAmelCase							,   language_codes=lowerCAmelCase							,   sp_model_kwargs=self.sp_model_kwargs							,   num_madeup_words=lowerCAmelCase							,   **lowerCAmelCase							,   )

        _lowercase	       =vocab_file
        _lowercase	       =load_json(lowerCAmelCase					)
        _lowercase	       ={v: k for k, v in self.encoder.items()}
        _lowercase	       =spm_file
        _lowercase	       =load_spm(lowerCAmelCase							,   self.sp_model_kwargs					)

        _lowercase	       =len(self.encoder					)

        _lowercase	       ={
            self.get_lang_token(lowerCAmelCase					): self.encoder_size + i for i, lang_code in enumerate(lowerCAmelCase					)
        }
        _lowercase	       ={lang_code: self.encoder_size + i for i, lang_code in enumerate(lowerCAmelCase					)}
        _lowercase	       ={v: k for k, v in self.lang_token_to_id.items()}

        _lowercase	       =src_lang if src_lang is not None else 'en'
        _lowercase	       =tgt_lang
        _lowercase	       =self.get_lang_id(self._src_lang					)
        self.set_src_lang_special_tokens(self._src_lang					)

        _lowercase	       =num_madeup_words




  @property
  def   A__     (				self					)  -> int:
        '''simple docstring'''






        return len(self.encoder					) + len(self.lang_token_to_id					)




  @property
  def   A__     (				self					)  -> str:
        '''simple docstring'''






        return self._src_lang




  @src_lang.setter
  def   A__     (				self							,   lowerCAmelCase					)  -> None:
        '''simple docstring'''






        _lowercase	       =new_src_lang
        self.set_src_lang_special_tokens(self._src_lang					)




  def   A__     (				self							,   lowerCAmelCase					)  -> List[str]:
        '''simple docstring'''






        return self.sp_model.encode(lowerCAmelCase							,   out_type=lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase					)  -> Union[str, Any]:
        '''simple docstring'''






        if token in self.lang_token_to_id:
              return self.lang_token_to_id[token]
        return self.encoder.get(lowerCAmelCase							,   self.encoder[self.unk_token]					)




  def   A__     (				self							,   lowerCAmelCase					)  -> str:
        '''simple docstring'''






        if index in self.id_to_lang_token:
              return self.id_to_lang_token[index]
        return self.decoder.get(lowerCAmelCase							,   self.unk_token					)




  def   A__     (				self							,   lowerCAmelCase					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =[]
        _lowercase	       =''
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    out_string += self.sp_model.decode(lowerCAmelCase					) + token
                    _lowercase	       =[]
              else:
                    current_sub_tokens.append(lowerCAmelCase					)
        out_string += self.sp_model.decode(lowerCAmelCase					)
        return out_string.strip()




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase = None							,   lowerCAmelCase = False					)  -> List[int]:
        '''simple docstring'''






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowerCAmelCase							,   token_ids_a=lowerCAmelCase							,   already_has_special_tokens=lowerCAmelCase					)

        _lowercase	       =[1] * len(self.prefix_tokens					)
        _lowercase	       =[1] * len(self.suffix_tokens					)
        if token_ids_a is None:
              return prefix_ones + ([0] * len(lowerCAmelCase					)) + suffix_ones
        return prefix_ones + ([0] * len(lowerCAmelCase					)) + ([0] * len(lowerCAmelCase					)) + suffix_ones




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase = None					)  -> List[int]:
        '''simple docstring'''






        if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       ={self.convert_ids_to_tokens(lowerCAmelCase					): i for i in range(self.vocab_size					)}
        vocab.update(self.added_tokens_encoder					)
        return vocab




  def __getstate__(				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =self.__dict__.copy()
        _lowercase	       =None
        return state




  def __setstate__(				self							,   lowerCAmelCase					)  -> None:
        '''simple docstring'''






        _lowercase	       =d

        # for backward compatibility
        if not hasattr(self							,   'sp_model_kwargs'					):
              _lowercase	       ={}

        _lowercase	       =load_spm(self.spm_file							,   self.sp_model_kwargs					)




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase = None					)  -> Tuple[str]:
        '''simple docstring'''






        _lowercase	       =Path(lowerCAmelCase					)
        if not save_dir.is_dir():
              raise OSError(F'''{save_directory} should be a directory'''					)
        _lowercase	       =save_dir / (
            (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
        )
        _lowercase	       =save_dir / (
            (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
        )

        save_json(self.encoder							,   lowerCAmelCase					)

        if os.path.abspath(self.spm_file					) != os.path.abspath(lowerCAmelCase					) and os.path.isfile(self.spm_file					):
              copyfile(self.spm_file							,   lowerCAmelCase					)
        elif not os.path.isfile(self.spm_file					):
              with open(lowerCAmelCase							,   'wb'					) as fi:
                    _lowercase	       =self.sp_model.serialized_model_proto()
                    fi.write(lowerCAmelCase					)

        return (str(lowerCAmelCase					), str(lowerCAmelCase					))




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase = "en"							,   lowerCAmelCase = None							,   lowerCAmelCase = "ro"							,   **lowerCAmelCase							,   )  -> BatchEncoding:
        '''simple docstring'''






        _lowercase	       =src_lang
        _lowercase	       =tgt_lang
        self.set_src_lang_special_tokens(self.src_lang					)
        return super().prepare_seqaseq_batch(lowerCAmelCase							,   lowerCAmelCase							,   **lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase							,   lowerCAmelCase							,   **lowerCAmelCase					)  -> Union[str, Any]:
        '''simple docstring'''






        if src_lang is None or tgt_lang is None:
              raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'					)
        _lowercase	       =src_lang
        _lowercase	       =self(lowerCAmelCase							,   add_special_tokens=lowerCAmelCase							,   **lowerCAmelCase					)
        _lowercase	       =self.get_lang_id(lowerCAmelCase					)
        _lowercase	       =tgt_lang_id
        return inputs




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        self.set_src_lang_special_tokens(self.src_lang					)




  def   A__     (				self					)  -> Tuple:
        '''simple docstring'''






        self.set_tgt_lang_special_tokens(self.tgt_lang					)




  def   A__     (				self							,   lowerCAmelCase					)  -> None:
        '''simple docstring'''






        _lowercase	       =self.get_lang_token(lowerCAmelCase					)
        _lowercase	       =self.lang_token_to_id[lang_token]
        _lowercase	       =[self.cur_lang_id]
        _lowercase	       =[self.eos_token_id]




  def   A__     (				self							,   lowerCAmelCase					)  -> None:
        '''simple docstring'''






        _lowercase	       =self.get_lang_token(lowerCAmelCase					)
        _lowercase	       =self.lang_token_to_id[lang_token]
        _lowercase	       =[self.cur_lang_id]
        _lowercase	       =[self.eos_token_id]




  def   A__     (				self							,   lowerCAmelCase					)  -> str:
        '''simple docstring'''






        return self.lang_code_to_token[lang]




  def   A__     (				self							,   lowerCAmelCase					)  -> int:
        '''simple docstring'''






        _lowercase	       =self.get_lang_token(lowerCAmelCase					)
        return self.lang_token_to_id[lang_token]
def      a      (			A__	:   str      ,   A__	:   Optional[int]     )    ->      Tuple:





      """simple docstring"""



      _lowercase	       =sentencepiece.SentencePieceProcessor(**__A     )
      spm.Load(str(__A     )     )
      return spm
def      a      (			A__	:   Union[str, Any]     )    ->      Dict:





      """simple docstring"""



      with open(__A      ,   'r'     ) as f:
            return json.load(__A     )







def      a      (			A__	:   List[str]      ,   A__	:   Optional[Any]     )    ->      Dict:





      """simple docstring"""



      with open(__A      ,   'w'     ) as f:
            json.dump(__A      ,   __A      ,   indent=2     )







'''simple docstring'''





from typing import List

import datasets
from datasets.tasks import AudioClassification

from ..folder_based_builder import folder_based_builder


lowercase_		       =  datasets.utils.logging.get_logger(__name__)






class __A			(  folder_based_builder.FolderBasedBuilderConfig						):



 '''simple docstring'''







 __lowerCamelCase    :       bool          =       None
 __lowerCamelCase    :       bool          =       None






class __A			(  folder_based_builder.FolderBasedBuilder						):



 '''simple docstring'''







 __lowerCamelCase    :       int          =       datasets.Audio()
 __lowerCamelCase    :       str          =       'audio'
 __lowerCamelCase    :       Optional[Any]          =       AudioFolderConfig
 __lowerCamelCase    :       List[str]  # definition at the bottom of the script
 __lowerCamelCase    :       Union[str, Any]          =       AudioClassification(audio_column='audio'   ,   label_column='label'						)





lowercase_		       =  [
    ".aiff",
    ".au",
    ".avr",
    ".caf",
    ".flac",
    ".htk",
    ".svx",
    ".mat4",
    ".mat5",
    ".mpc2k",
    ".ogg",
    ".paf",
    ".pvf",
    ".raw",
    ".rf64",
    ".sd2",
    ".sds",
    ".ircam",
    ".voc",
    ".w64",
    ".wav",
    ".nist",
    ".wavex",
    ".wve",
    ".xi",
    ".mp3",
    ".opus",
]
lowercase_		       =  AUDIO_EXTENSIONS



