
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__          =		logging.get_logger(__name__)

lowerCamelCase__          =		"""▁"""

lowerCamelCase__          =		{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

lowerCamelCase__          =		{
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

lowerCamelCase__          =		{
    """facebook/m2m100_418M""": 1024,
}

# fmt: off
lowerCamelCase__          =		{
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}







class      A__							(     __magic_name__	):
				lowercase			    =  VOCAB_FILES_NAMES
				lowercase			    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase			    =  PRETRAINED_VOCAB_FILES_MAP
				lowercase			    =  ['input_ids', 'attention_mask']

				lowercase			    =  []
				lowercase			    =  []






				def __init__(						self					:      Optional[int]    ,     a					:      Union[str, Any]    ,     a					:      Dict    ,     a					:      List[Any]=None    ,     a					:      int=None    ,     a					:      Optional[Any]="<s>"    ,     a					:      Optional[Any]="</s>"    ,     a					:      List[Any]="</s>"    ,     a					:      Optional[Any]="<pad>"    ,     a					:      Union[str, Any]="<unk>"    ,     a					:      Tuple="m2m100"    ,     a					:      Optional[Dict[str, Any]] = None    ,     a					:      Tuple=8    ,     **a					:      Union[str, Any]    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		{} if sp_model_kwargs is None else sp_model_kwargs

							lowerCAmelCase__      :      Optional[Any]         =		language_codes
							lowerCAmelCase__      :      int         =		FAIRSEQ_LANGUAGE_CODES[language_codes]
							lowerCAmelCase__      :      List[str]         =		{lang_code: f'''__{lang_code}__''' for lang_code in fairseq_language_code}

							lowerCAmelCase__      :      Optional[Any]         =		kwargs.get('additional_special_tokens'    ,     []   )
							kwargs["additional_special_tokens"] += [
							    self.get_lang_token(a   )
							    for lang_code in fairseq_language_code
							    if self.get_lang_token(a   ) not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=a    ,     tgt_lang=a    ,     bos_token=a    ,     eos_token=a    ,     sep_token=a    ,     unk_token=a    ,     pad_token=a    ,     language_codes=a    ,     sp_model_kwargs=self.sp_model_kwargs    ,     num_madeup_words=a    ,     **a    ,     )

							lowerCAmelCase__      :      str         =		vocab_file
							lowerCAmelCase__      :      List[Any]         =		load_json(a   )
							lowerCAmelCase__      :      Optional[int]         =		{v: k for k, v in self.encoder.items()}
							lowerCAmelCase__      :      Optional[Any]         =		spm_file
							lowerCAmelCase__      :      List[str]         =		load_spm(a    ,     self.sp_model_kwargs   )

							lowerCAmelCase__      :      List[str]         =		len(self.encoder   )

							lowerCAmelCase__      :      int         =		{
							    self.get_lang_token(a   ): self.encoder_size + i for i, lang_code in enumerate(a   )
							}
							lowerCAmelCase__      :      Tuple         =		{lang_code: self.encoder_size + i for i, lang_code in enumerate(a   )}
							lowerCAmelCase__      :      Optional[Any]         =		{v: k for k, v in self.lang_token_to_id.items()}

							lowerCAmelCase__      :      str         =		src_lang if src_lang is not None else 'en'
							lowerCAmelCase__      :      Any         =		tgt_lang
							lowerCAmelCase__      :      int         =		self.get_lang_id(self._src_lang   )
							self.set_src_lang_special_tokens(self._src_lang   )

							lowerCAmelCase__      :      Union[str, Any]         =		num_madeup_words






				@property
				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							return len(self.encoder   ) + len(self.lang_token_to_id   )






				@property
				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							return self._src_lang






				@src_lang.setter
				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		new_src_lang
							self.set_src_lang_special_tokens(self._src_lang   )






				def 							_lowerCamelCase		(						self					:      str    ,     a					:      str   ):






							'''simple docstring'''







							return self.sp_model.encode(a    ,     out_type=a   )






				def 							_lowerCamelCase		(						self					:      int    ,     a					:      Dict   ):






							'''simple docstring'''







							if token in self.lang_token_to_id:
										return self.lang_token_to_id[token]
							return self.encoder.get(a    ,     self.encoder[self.unk_token]   )






				def 							_lowerCamelCase		(						self					:      int    ,     a					:      int   ):






							'''simple docstring'''







							if index in self.id_to_lang_token:
										return self.id_to_lang_token[index]
							return self.decoder.get(a    ,     self.unk_token   )






				def 							_lowerCamelCase		(						self					:      List[Any]    ,     a					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		[]
							lowerCAmelCase__      :      List[str]         =		''
							for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
													out_string += self.sp_model.decode(a   ) + token
													lowerCAmelCase__      :      str         =		[]
										else:
													current_sub_tokens.append(a   )
							out_string += self.sp_model.decode(a   )
							return out_string.strip()






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      List[int]    ,     a					:      Optional[List[int]] = None    ,     a					:      bool = False   ):






							'''simple docstring'''







							if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=a    ,     token_ids_a=a    ,     already_has_special_tokens=a   )

							lowerCAmelCase__      :      Tuple         =		[1] * len(self.prefix_tokens   )
							lowerCAmelCase__      :      Dict         =		[1] * len(self.suffix_tokens   )
							if token_ids_a is None:
										return prefix_ones + ([0] * len(a   )) + suffix_ones
							return prefix_ones + ([0] * len(a   )) + ([0] * len(a   )) + suffix_ones






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      List[int]    ,     a					:      Optional[List[int]] = None   ):






							'''simple docstring'''







							if token_ids_a is None:
										return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		{self.convert_ids_to_tokens(a   ): i for i in range(self.vocab_size   )}
							vocab.update(self.added_tokens_encoder   )
							return vocab






				def __getstate__(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		self.__dict__.copy()
							lowerCAmelCase__      :      List[Any]         =		None
							return state






				def __setstate__(						self					:      List[Any]    ,     a					:      Dict   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		d

							# for backward compatibility
							if not hasattr(self    ,     'sp_model_kwargs'   ):
										lowerCAmelCase__      :      Optional[Any]         =		{}

							lowerCAmelCase__      :      List[Any]         =		load_spm(self.spm_file    ,     self.sp_model_kwargs   )






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      str    ,     a					:      Optional[str] = None   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		Path(a   )
							if not save_dir.is_dir():
										raise OSError(f'''{save_directory} should be a directory'''   )
							lowerCAmelCase__      :      Optional[int]         =		save_dir / (
							    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
							)
							lowerCAmelCase__      :      Optional[Any]         =		save_dir / (
							    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
							)

							save_json(self.encoder    ,     a   )

							if os.path.abspath(self.spm_file   ) != os.path.abspath(a   ) and os.path.isfile(self.spm_file   ):
										copyfile(self.spm_file    ,     a   )
							elif not os.path.isfile(self.spm_file   ):
										with open(a    ,     'wb'   ) as fi:
													lowerCAmelCase__      :      Optional[Any]         =		self.sp_model.serialized_model_proto()
													fi.write(a   )

							return (str(a   ), str(a   ))






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      List[str]    ,     a					:      str = "en"    ,     a					:      Optional[List[str]] = None    ,     a					:      str = "ro"    ,     **a					:      Union[str, Any]    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		src_lang
							lowerCAmelCase__      :      str         =		tgt_lang
							self.set_src_lang_special_tokens(self.src_lang   )
							return super().prepare_seqaseq_batch(a    ,     a    ,     **a   )






				def 							_lowerCamelCase		(						self					:      Dict    ,     a					:      List[str]    ,     a					:      Optional[str]    ,     a					:      Optional[str]    ,     **a					:      str   ):






							'''simple docstring'''







							if src_lang is None or tgt_lang is None:
										raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'   )
							lowerCAmelCase__      :      Dict         =		src_lang
							lowerCAmelCase__      :      Any         =		self(a    ,     add_special_tokens=a    ,     **a   )
							lowerCAmelCase__      :      Union[str, Any]         =		self.get_lang_id(a   )
							lowerCAmelCase__      :      List[Any]         =		tgt_lang_id
							return inputs






				def 							_lowerCamelCase		(						self					:      str   ):






							'''simple docstring'''







							self.set_src_lang_special_tokens(self.src_lang   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							self.set_tgt_lang_special_tokens(self.tgt_lang   )






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		self.get_lang_token(a   )
							lowerCAmelCase__      :      Tuple         =		self.lang_token_to_id[lang_token]
							lowerCAmelCase__      :      Optional[int]         =		[self.cur_lang_id]
							lowerCAmelCase__      :      Any         =		[self.eos_token_id]






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		self.get_lang_token(a   )
							lowerCAmelCase__      :      Optional[Any]         =		self.lang_token_to_id[lang_token]
							lowerCAmelCase__      :      List[str]         =		[self.cur_lang_id]
							lowerCAmelCase__      :      str         =		[self.eos_token_id]






				def 							_lowerCamelCase		(						self					:      List[Any]    ,     a					:      str   ):






							'''simple docstring'''







							return self.lang_code_to_token[lang]






				def 							_lowerCamelCase		(						self					:      Union[str, Any]    ,     a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		self.get_lang_token(a   )
							return self.lang_token_to_id[lang_token]






def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		sentencepiece.SentencePieceProcessor:
			lowerCAmelCase__      :      Dict         =		sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE_     )
			spm.Load(str(SCREAMING_SNAKE_CASE_     )     )
			return spm
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Union[Dict, List]:
			with open(SCREAMING_SNAKE_CASE_       ,							'r'     ) as f:
						return json.load(SCREAMING_SNAKE_CASE_     )







def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		None:
			with open(SCREAMING_SNAKE_CASE_       ,							'w'     ) as f:
						json.dump(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							indent=2     )

import datetime
import platform
import subprocess
from typing import Optional, Tuple, Union

import numpy as np
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		np.array:
			lowerCAmelCase__      :      Dict         =		F'''{sampling_rate}'''
			lowerCAmelCase__      :      Any         =		'1'
			lowerCAmelCase__      :      Optional[Any]         =		'f32le'
			lowerCAmelCase__      :      Any         =		[
			    'ffmpeg',
			    '-i',
			    'pipe:0',
			    '-ac',
			    ac,
			    '-ar',
			    ar,
			    '-f',
			    format_for_conversion,
			    '-hide_banner',
			    '-loglevel',
			    'quiet',
			    'pipe:1',
			]

			try:
						with subprocess.Popen(SCREAMING_SNAKE_CASE_       ,							stdin=subprocess.PIPE       ,							stdout=subprocess.PIPE     ) as ffmpeg_process:
									lowerCAmelCase__      :      List[Any]         =		ffmpeg_process.communicate(SCREAMING_SNAKE_CASE_     )
			except FileNotFoundError as error:
						raise ValueError('ffmpeg was not found but is required to load audio files from filename'     ) from error
			lowerCAmelCase__      :      List[str]         =		output_stream[0]
			lowerCAmelCase__      :      str         =		np.frombuffer(SCREAMING_SNAKE_CASE_       ,							np.floataa     )
			if audio.shape[0] == 0:
						raise ValueError('Malformed soundfile'     )
			return audio
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_ = "f32le"       ,							)				->		Dict:
			lowerCAmelCase__      :      Optional[Any]         =		F'''{sampling_rate}'''
			lowerCAmelCase__      :      Any         =		'1'
			if format_for_conversion == "s16le":
						lowerCAmelCase__      :      Dict         =		2
			elif format_for_conversion == "f32le":
						lowerCAmelCase__      :      List[str]         =		4
			else:
						raise ValueError(F'''Unhandled format `{format_for_conversion}`. Please use `s16le` or `f32le`'''     )

			lowerCAmelCase__      :      Tuple         =		platform.system()
			if system == "Linux":
						lowerCAmelCase__      :      str         =		'alsa'
						lowerCAmelCase__      :      str         =		'default'
			elif system == "Darwin":
						lowerCAmelCase__      :      Any         =		'avfoundation'
						lowerCAmelCase__      :      Tuple         =		':0'
			elif system == "Windows":
						lowerCAmelCase__      :      Any         =		'dshow'
						lowerCAmelCase__      :      int         =		'default'

			lowerCAmelCase__      :      Any         =		[
			    'ffmpeg',
			    '-f',
			    format_,
			    '-i',
			    input_,
			    '-ac',
			    ac,
			    '-ar',
			    ar,
			    '-f',
			    format_for_conversion,
			    '-fflags',
			    'nobuffer',
			    '-hide_banner',
			    '-loglevel',
			    'quiet',
			    'pipe:1',
			]
			lowerCAmelCase__      :      List[str]         =		int(round(sampling_rate * chunk_length_s     )     ) * size_of_sample
			lowerCAmelCase__      :      str         =		_ffmpeg_stream(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
			for item in iterator:
						yield item
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_ = None       ,							SCREAMING_SNAKE_CASE_ = None       ,							SCREAMING_SNAKE_CASE_ = "f32le"       ,							)				->		str:
			if stream_chunk_s is not None:
						lowerCAmelCase__      :      Union[str, Any]         =		stream_chunk_s
			else:
						lowerCAmelCase__      :      Tuple         =		chunk_length_s

			lowerCAmelCase__      :      Any         =		ffmpeg_microphone(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							format_for_conversion=SCREAMING_SNAKE_CASE_     )
			if format_for_conversion == "s16le":
						lowerCAmelCase__      :      Optional[Any]         =		np.intaa
						lowerCAmelCase__      :      Optional[Any]         =		2
			elif format_for_conversion == "f32le":
						lowerCAmelCase__      :      Optional[Any]         =		np.floataa
						lowerCAmelCase__      :      Optional[Any]         =		4
			else:
						raise ValueError(F'''Unhandled format `{format_for_conversion}`. Please use `s16le` or `f32le`'''     )

			if stride_length_s is None:
						lowerCAmelCase__      :      Dict         =		chunk_length_s / 6
			lowerCAmelCase__      :      int         =		int(round(sampling_rate * chunk_length_s     )     ) * size_of_sample
			if isinstance(SCREAMING_SNAKE_CASE_       ,							(int, float)     ):
						lowerCAmelCase__      :      Dict         =		[stride_length_s, stride_length_s]

			lowerCAmelCase__      :      Union[str, Any]         =		int(round(sampling_rate * stride_length_s[0]     )     ) * size_of_sample
			lowerCAmelCase__      :      List[Any]         =		int(round(sampling_rate * stride_length_s[1]     )     ) * size_of_sample
			lowerCAmelCase__      :      Any         =		datetime.datetime.now()
			lowerCAmelCase__      :      Any         =		datetime.timedelta(seconds=SCREAMING_SNAKE_CASE_     )
			for item in chunk_bytes_iter(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							stride=(stride_left, stride_right)       ,							stream=SCREAMING_SNAKE_CASE_     ):
						# Put everything back in numpy scale
						lowerCAmelCase__      :      Any         =		np.frombuffer(item['raw']       ,							dtype=SCREAMING_SNAKE_CASE_     )
						lowerCAmelCase__      :      Optional[Any]         =		(
						    item['stride'][0] // size_of_sample,
						    item['stride'][1] // size_of_sample,
						)
						lowerCAmelCase__      :      Optional[int]         =		sampling_rate
						audio_time += delta
						if datetime.datetime.now() > audio_time + 10 * delta:
									# We're late !! SKIP
									continue
						yield item
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_ = False     )				->		Optional[int]:
			lowerCAmelCase__      :      Union[str, Any]         =		b''
			lowerCAmelCase__      ,				lowerCAmelCase__      :      Union[str, Any]         =		stride
			if stride_left + stride_right >= chunk_len:
						raise ValueError(
						    F'''Stride needs to be strictly smaller than chunk_len: ({stride_left}, {stride_right}) vs {chunk_len}'''     )
			lowerCAmelCase__      :      List[str]         =		0
			for raw in iterator:
						acc += raw
						if stream and len(SCREAMING_SNAKE_CASE_     ) < chunk_len:
									lowerCAmelCase__      :      Tuple         =		(_stride_left, 0)
									yield {"raw": acc[:chunk_len], "stride": stride, "partial": True}
						else:
									while len(SCREAMING_SNAKE_CASE_     ) >= chunk_len:
												# We are flushing the accumulator
												lowerCAmelCase__      :      Dict         =		(_stride_left, stride_right)
												lowerCAmelCase__      :      Any         =		{'raw': acc[:chunk_len], 'stride': stride}
												if stream:
															lowerCAmelCase__      :      Optional[int]         =		False
												yield item
												lowerCAmelCase__      :      Optional[int]         =		stride_left
												lowerCAmelCase__      :      Optional[int]         =		acc[chunk_len - stride_left - stride_right :]
    # Last chunk
			if len(SCREAMING_SNAKE_CASE_     ) > stride_left:
						lowerCAmelCase__      :      Tuple         =		{'raw': acc, 'stride': (_stride_left, 0)}
						if stream:
									lowerCAmelCase__      :      Any         =		False
						yield item







def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		str:
			lowerCAmelCase__      :      int         =		2**24  # 16Mo
			try:
						with subprocess.Popen(SCREAMING_SNAKE_CASE_       ,							stdout=subprocess.PIPE       ,							bufsize=SCREAMING_SNAKE_CASE_     ) as ffmpeg_process:
									while True:
												lowerCAmelCase__      :      List[str]         =		ffmpeg_process.stdout.read(SCREAMING_SNAKE_CASE_     )
												if raw == b"":
															break
												yield raw
			except FileNotFoundError as error:
						raise ValueError('ffmpeg was not found but is required to stream audio files from filename'     ) from error