
'''simple docstring'''







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase    :							Any      =					logging.get_logger(__name__)

lowercase    :							Optional[int]      =					'''▁'''

lowercase    :							Tuple      =					{
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
}

lowercase    :							Any      =					{
    '''vocab_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'''
        ),
    },
    '''spm_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'''
        )
    },
}

lowercase    :							Optional[int]      =					{
    '''facebook/s2t-small-librispeech-asr''': 1024,
}

lowercase    :							str      =					['''pt''', '''fr''', '''ru''', '''nl''', '''ro''', '''it''', '''es''', '''de''']

lowercase    :							Tuple      =					{'''mustc''': MUSTC_LANGS}


class 						__UpperCAmelCase      (					lowerCamelCase_						):
							__lowercase      		=       VOCAB_FILES_NAMES
							__lowercase      		=       PRETRAINED_VOCAB_FILES_MAP
							__lowercase      		=       MAX_MODEL_INPUT_SIZES
							__lowercase      		=       ["""input_ids""", """attention_mask"""]

							__lowercase      		=       []


							def __init__(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_    ,   lowerCAmelCase_="<s>"    ,   lowerCAmelCase_="</s>"    ,   lowerCAmelCase_="<pad>"    ,   lowerCAmelCase_="<unk>"    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=False    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_=None    ,   lowerCAmelCase_ = None    ,   **lowerCAmelCase_    ,   ):





								"""simple docstring"""






								_snake_case   						=		{} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=_lowercase    ,   eos_token=_lowercase    ,   unk_token=_lowercase    ,   pad_token=_lowercase    ,   do_upper_case=_lowercase    ,   do_lower_case=_lowercase    ,   tgt_lang=_lowercase    ,   lang_codes=_lowercase    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **_lowercase    ,   )
								_snake_case   						=		do_upper_case
								_snake_case   						=		do_lower_case

								_snake_case   						=		load_json(_lowercase  )
								_snake_case   						=		{v: k for k, v in self.encoder.items()}
								_snake_case   						=		spm_file
								_snake_case   						=		load_spm(_lowercase    ,   self.sp_model_kwargs  )

								if lang_codes is not None:
									_snake_case   						=		lang_codes
									_snake_case   						=		LANGUAGES[lang_codes]
									_snake_case   						=		[F'<lang:{lang}>' for lang in self.langs]
									_snake_case   						=		{lang: self.sp_model.PieceToId(F'<lang:{lang}>'  ) for lang in self.langs}

									_snake_case   						=		self.lang_tokens
									_snake_case   						=		tgt_lang if tgt_lang is not None else self.langs[0]

									self.set_tgt_lang_special_tokens(self._tgt_lang  )
								else:
									_snake_case   						=		{}


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return len(self.encoder  )


							@property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return self._tgt_lang


							@tgt_lang.setter
							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		new_tgt_lang
								self.set_tgt_lang_special_tokens(_lowercase  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		self.lang_code_to_id[tgt_lang]
								_snake_case   						=		[lang_code_id]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.sp_model.encode(_lowercase    ,   out_type=_lowercase  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.encoder.get(_lowercase    ,   self.encoder[self.unk_token]  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								return self.decoder.get(_lowercase    ,   self.unk_token  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		[]
								_snake_case   						=		''
								for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
										_snake_case   						=		self.sp_model.decode(_lowercase  )
										out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
										_snake_case   						=		[]
									else:
										current_sub_tokens.append(_lowercase  )
								_snake_case   						=		self.sp_model.decode(_lowercase  )
								out_string += decoded.upper() if self.do_upper_case else decoded
								return out_string.strip()


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_=None  ):





								"""simple docstring"""






								if token_ids_a is None:
									return self.prefix_tokens + token_ids_a + [self.eos_token_id]
								# We don't expect to process pairs, but leave the pair logic for API consistency
								return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None    ,   lowerCAmelCase_ = False  ):





								"""simple docstring"""






								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=_lowercase    ,   token_ids_a=_lowercase    ,   already_has_special_tokens=_lowercase  )

								_snake_case   						=		[1] * len(self.prefix_tokens  )
								_snake_case   						=		[1]
								if token_ids_a is None:
									return prefix_ones + ([0] * len(_lowercase  )) + suffix_ones
								return prefix_ones + ([0] * len(_lowercase  )) + ([0] * len(_lowercase  )) + suffix_ones


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.encoder.copy()
								vocab.update(self.added_tokens_encoder  )
								return vocab


							def __getstate__(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.__dict__.copy()
								_snake_case   						=		None
								return state


							def __setstate__(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		d

								# for backward compatibility
								if not hasattr(self    ,   'sp_model_kwargs'  ):
									_snake_case   						=		{}

								_snake_case   						=		load_spm(self.spm_file    ,   self.sp_model_kwargs  )




							def  lowerCamelCase				(       self    ,   lowerCAmelCase_    ,   lowerCAmelCase_ = None  ):





								"""simple docstring"""






								_snake_case   						=		Path(_lowercase  )
								assert save_dir.is_dir(), F'{save_directory} should be a directory'
								_snake_case   						=		save_dir / (
								    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
								)
								_snake_case   						=		save_dir / (
								    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
								)

								save_json(self.encoder    ,   _lowercase  )

								if os.path.abspath(self.spm_file  ) != os.path.abspath(_lowercase  ) and os.path.isfile(self.spm_file  ):
									copyfile(self.spm_file    ,   _lowercase  )
								elif not os.path.isfile(self.spm_file  ):
									with open(_lowercase    ,   'wb'  ) as fi:
										_snake_case   						=		self.sp_model.serialized_model_proto()
										fi.write(_lowercase  )

								return (str(_lowercase  ), str(_lowercase  ))







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A	)       ->	sentencepiece.SentencePieceProcessor:
	_snake_case   						=		sentencepiece.SentencePieceProcessor(**__UpperCamelCase	)
	spm.Load(str(__UpperCamelCase	)	)
	return spm







def 				SCREAMING_SNAKE_CASE__	(   __A	)       ->	Union[Dict, List]:





	with open(__UpperCamelCase			,				'r'	) as f:
		return json.load(__UpperCamelCase	)







def 				SCREAMING_SNAKE_CASE__	(   __A			,				__A	)       ->	None:





	with open(__UpperCamelCase			,				'w'	) as f:
		json.dump(__UpperCamelCase			,				__UpperCamelCase			,				indent=2	)


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


__lowerCamelCase			: Union[str, Any]							   =		'''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
   import torch

   print('''Torch version:''', torch.__version__)
   print('''Cuda available:''', torch.cuda.is_available())
   print('''Cuda version:''', torch.version.cuda)
   print('''CuDNN version:''', torch.backends.cudnn.version())
   print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
   print('''Torch version:''', None)

try:
   import transformers

   print('''transformers version:''', transformers.__version__)
except ImportError:
   print('''transformers version:''', None)
