code: 
  
  
  import json
  import os
  import re
  import warnings
  from pathlib import Path
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple, Union
  
  import sentencepiece
  
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
  
  
  A_														=      logging.get_logger(__name__)
  
  A_														=      {
      "source_spm": "source.spm",
      "target_spm": "target.spm",
      "vocab": "vocab.json",
      "target_vocab_file": "target_vocab.json",
      "tokenizer_config_file": "tokenizer_config.json",
  }
  
  A_														=      {
      "source_spm": {
          "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm"
      },
      "target_spm": {
          "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm"
      },
      "vocab": {
          "Helsinki-NLP/opus-mt-en-de": "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json"
      },
      "tokenizer_config_file": {
          "Helsinki-NLP/opus-mt-en-de": (
              "https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json"
          )
      },
  }
  
  A_														=      {"Helsinki-NLP/opus-mt-en-de": 5_1_2}
  A_														=      {}
  
  class 						snake_case			(       lowerCAmelCase__	):
  			'''simple docstring'''
  
  			UpperCAmelCase       :       int       = VOCAB_FILES_NAMES
  			UpperCAmelCase       :       Any       = PRETRAINED_VOCAB_FILES_MAP
  			UpperCAmelCase       :       int       = PRETRAINED_INIT_CONFIGURATION
  			UpperCAmelCase       :       int       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  			UpperCAmelCase       :       Optional[Any]       = ["""input_ids""", """attention_mask"""]
  			UpperCAmelCase       :       Any       = re.compile(""">>.+<<"""	)  # type: re.Pattern
  
  			def __init__(	self     :     int							,							lowerCAmelCase_     :     Any							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     List[str]							,							lowerCAmelCase_     :     List[str]=None							,							lowerCAmelCase_     :     Optional[int]=None							,							lowerCAmelCase_     :     Dict=None							,							lowerCAmelCase_     :     int="<unk>"							,							lowerCAmelCase_     :     Any="</s>"							,							lowerCAmelCase_     :     Any="<pad>"							,							lowerCAmelCase_     :     int=512							,							lowerCAmelCase_     :     Optional[Dict[str, Any]] = None							,							lowerCAmelCase_     :     List[str]=False							,							**lowerCAmelCase_     :     List[Any]							,							)					->					None:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       {} if sp_model_kwargs is None else sp_model_kwargs
  
  								super().__init__(
  								    # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
  								    source_lang=lowerCAmelCase_							,							target_lang=lowerCAmelCase_							,							unk_token=lowerCAmelCase_							,							eos_token=lowerCAmelCase_							,							pad_token=lowerCAmelCase_							,							model_max_length=lowerCAmelCase_							,							sp_model_kwargs=self.sp_model_kwargs							,							target_vocab_file=lowerCAmelCase_							,							separate_vocabs=lowerCAmelCase_							,							**lowerCAmelCase_							,							)
  								assert Path(lowerCAmelCase_					).exists(), F'''cannot find spm source {source_spm}'''
  
  								SCREAMING_SNAKE_CASE_	      =       separate_vocabs
  								SCREAMING_SNAKE_CASE_	      =       load_json(lowerCAmelCase_					)
  								if self.unk_token not in self.encoder:
  													raise KeyError('''<unk> token must be in vocab'''					)
  								assert self.pad_token in self.encoder
  
  								if separate_vocabs:
  													SCREAMING_SNAKE_CASE_	      =       load_json(lowerCAmelCase_					)
  													SCREAMING_SNAKE_CASE_	      =       {v: k for k, v in self.target_encoder.items()}
  													SCREAMING_SNAKE_CASE_	      =       []
  								else:
  													SCREAMING_SNAKE_CASE_	      =       {v: k for k, v in self.encoder.items()}
  													SCREAMING_SNAKE_CASE_	      =       [k for k in self.encoder if k.startswith('''>>'''					) and k.endswith('''<<'''					)]
  
  								SCREAMING_SNAKE_CASE_	      =       source_lang
  								SCREAMING_SNAKE_CASE_	      =       target_lang
  								SCREAMING_SNAKE_CASE_	      =       [source_spm, target_spm]
  
  								# load SentencePiece model for pre-processing
  								SCREAMING_SNAKE_CASE_	      =       load_spm(lowerCAmelCase_							,							self.sp_model_kwargs					)
  								SCREAMING_SNAKE_CASE_	      =       load_spm(lowerCAmelCase_							,							self.sp_model_kwargs					)
  								SCREAMING_SNAKE_CASE_	      =       self.spm_source
  								SCREAMING_SNAKE_CASE_	      =       self.encoder
  
  								# Multilingual target side: default to using first supported language code.
  
  								self._setup_normalizer()
  
  			def _lowercase (	self     :     Optional[int]					)					->					Union[str, Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								try:
  													from sacremoses import MosesPunctNormalizer
  
  													SCREAMING_SNAKE_CASE_	      =       MosesPunctNormalizer(self.source_lang					).normalize
  								except (ImportError, FileNotFoundError):
  													warnings.warn('''Recommended: pip install sacremoses.'''					)
  													SCREAMING_SNAKE_CASE_	      =       lambda lowerCAmelCase_					: x
  
  			def _lowercase (	self     :     Optional[Any]							,							lowerCAmelCase_     :     str					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.punc_normalizer(lowerCAmelCase_					) if x else ""
  
  			def _lowercase (	self     :     List[str]							,							lowerCAmelCase_     :     List[Any]					)					->					int:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.current_encoder.get(lowerCAmelCase_							,							self.current_encoder[self.unk_token]					)
  
  			def _lowercase (	self     :     Union[str, Any]							,							lowerCAmelCase_     :     str					)					->					Any:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.language_code_re.match(lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       [match.group(0					)] if match else []
  								return code, self.language_code_re.sub(''''''							,							lowerCAmelCase_					)
  
  			def _lowercase (	self     :     Any							,							lowerCAmelCase_     :     str					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_	      =       self.remove_language_code(lowerCAmelCase_					)
  								SCREAMING_SNAKE_CASE_	      =       self.current_spm.encode(lowerCAmelCase_							,							out_type=lowerCAmelCase_					)
  								return code + pieces
  
  			def _lowercase (	self     :     Dict							,							lowerCAmelCase_     :     int					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.decoder.get(lowerCAmelCase_							,							self.unk_token					)
  
  			def _lowercase (	self     :     Optional[int]							,							lowerCAmelCase_     :     List[Any]							,							**lowerCAmelCase_     :     Any					)					->					List[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return super().batch_decode(lowerCAmelCase_							,							**lowerCAmelCase_					)
  
  			def _lowercase (	self     :     Optional[int]							,							lowerCAmelCase_     :     str							,							**lowerCAmelCase_     :     Union[str, Any]					)					->					Tuple:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return super().decode(lowerCAmelCase_							,							**lowerCAmelCase_					)
  
  			def _lowercase (	self     :     Optional[Any]							,							lowerCAmelCase_     :     List[str]					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.spm_source if self._decode_use_source_tokenizer else self.spm_target
  								SCREAMING_SNAKE_CASE_	      =       []
  								SCREAMING_SNAKE_CASE_	      =       ''''''
  								for token in tokens:
  													# make sure that special tokens are not decoded using sentencepiece model
  													if token in self.all_special_tokens:
  																		out_string += sp_model.decode_pieces(lowerCAmelCase_					) + token + " "
  																		SCREAMING_SNAKE_CASE_	      =       []
  													else:
  																		current_sub_tokens.append(lowerCAmelCase_					)
  								out_string += sp_model.decode_pieces(lowerCAmelCase_					)
  								return out_string.strip()
  
  			def _lowercase (	self     :     str							,							lowerCAmelCase_     :     Optional[Any]							,							lowerCAmelCase_     :     Tuple=None					)					->					List[int]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if token_ids_a is None:
  													return token_ids_a + [self.eos_token_id]
  								# We don't expect to process pairs, but leave the pair logic for API consistency
  								return token_ids_a + token_ids_a + [self.eos_token_id]
  
  			def _lowercase (	self     :     List[Any]					)					->					List[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.spm_source
  								SCREAMING_SNAKE_CASE_	      =       self.encoder
  
  			def _lowercase (	self     :     List[Any]					)					->					List[Any]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.spm_target
  								if self.separate_vocabs:
  													SCREAMING_SNAKE_CASE_	      =       self.target_encoder
  
  			@property
  			def _lowercase (	self     :     Any					)					->					int:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return len(self.encoder					)
  
  			def _lowercase (	self     :     Optional[Any]							,							lowerCAmelCase_     :     str							,							lowerCAmelCase_     :     Optional[str] = None					)					->					Tuple[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if not os.path.isdir(lowerCAmelCase_					):
  													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
  													return
  								SCREAMING_SNAKE_CASE_	      =       []
  
  								if self.separate_vocabs:
  													SCREAMING_SNAKE_CASE_	      =       os.path.join(
  													    lowerCAmelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']							,							)
  													SCREAMING_SNAKE_CASE_	      =       os.path.join(
  													    lowerCAmelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''target_vocab_file''']							,							)
  													save_json(self.encoder							,							lowerCAmelCase_					)
  													save_json(self.target_encoder							,							lowerCAmelCase_					)
  													saved_files.append(lowerCAmelCase_					)
  													saved_files.append(lowerCAmelCase_					)
  								else:
  													SCREAMING_SNAKE_CASE_	      =       os.path.join(
  													    lowerCAmelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']					)
  													save_json(self.encoder							,							lowerCAmelCase_					)
  													saved_files.append(lowerCAmelCase_					)
  
  								for spm_save_filename, spm_orig_path, spm_model in zip(
  								    [VOCAB_FILES_NAMES['''source_spm'''], VOCAB_FILES_NAMES['''target_spm''']]							,							self.spm_files							,							[self.spm_source, self.spm_target]							,							):
  													SCREAMING_SNAKE_CASE_	      =       os.path.join(
  													    lowerCAmelCase_							,							(filename_prefix + '''-''' if filename_prefix else '''''') + spm_save_filename					)
  													if os.path.abspath(lowerCAmelCase_					) != os.path.abspath(lowerCAmelCase_					) and os.path.isfile(lowerCAmelCase_					):
  																		copyfile(lowerCAmelCase_							,							lowerCAmelCase_					)
  																		saved_files.append(lowerCAmelCase_					)
  													elif not os.path.isfile(lowerCAmelCase_					):
  																		with open(lowerCAmelCase_							,							'''wb'''					) as fi:
  																							SCREAMING_SNAKE_CASE_	      =       spm_model.serialized_model_proto()
  																							fi.write(lowerCAmelCase_					)
  																		saved_files.append(lowerCAmelCase_					)
  
  								return tuple(lowerCAmelCase_					)
  
  			def _lowercase (	self     :     Tuple					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return self.get_src_vocab()
  
  			def _lowercase (	self     :     List[Any]					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return dict(self.encoder							,							**self.added_tokens_encoder					)
  
  			def _lowercase (	self     :     int					)					->					int:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return dict(self.target_encoder							,							**self.added_tokens_decoder					)
  
  			def __getstate__(	self     :     Tuple					)					->					Dict:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       self.__dict__.copy()
  								state.update(
  								    {k: None for k in ['''spm_source''', '''spm_target''', '''current_spm''', '''punc_normalizer''', '''target_vocab_file''']}					)
  								return state
  
  			def __setstate__(	self     :     Optional[int]							,							lowerCAmelCase_     :     Dict					)					->					None:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       d
  
  								# for backward compatibility
  								if not hasattr(self							,							'''sp_model_kwargs'''					):
  													SCREAMING_SNAKE_CASE_	      =       {}
  
  								SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_	      =       (load_spm(lowerCAmelCase_							,							self.sp_model_kwargs					) for f in self.spm_files)
  								SCREAMING_SNAKE_CASE_	      =       self.spm_source
  								self._setup_normalizer()
  
  			def _lowercase (	self     :     int							,							*lowerCAmelCase_     :     Optional[Any]							,							**lowerCAmelCase_     :     Any					)					->					List[str]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								return 1
  
  			def _lowercase (	self     :     List[Any]							,							lowerCAmelCase_     :     Tuple					)					->					str:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								SCREAMING_SNAKE_CASE_	      =       set(self.all_special_ids					)  # call it once instead of inside list comp
  								all_special_ids.remove(self.unk_token_id					)  # <unk> is only sometimes special
  								return [1 if x in all_special_ids else 0 for x in seq]
  
  
  
  
  			def _lowercase (	self     :     Tuple							,							lowerCAmelCase_     :     List							,							lowerCAmelCase_     :     Optional[List] = None							,							lowerCAmelCase_     :     bool = False					)					->					List[int]:
  
  
  
  
  								"""simple docstring"""
  
  
  
  
  
  
  								if already_has_special_tokens:
  													return self._special_token_mask(lowerCAmelCase_					)
  								elif token_ids_a is None:
  													return self._special_token_mask(lowerCAmelCase_					) + [1]
  								else:
  													return self._special_token_mask(token_ids_a + token_ids_a					) + [1]
  
  
  
  
  
  
  
  def 							UpperCAmelCase				(     UpperCAmelCase				,UpperCAmelCase      )->		sentencepiece.SentencePieceProcessor:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					SCREAMING_SNAKE_CASE_	      =       sentencepiece.SentencePieceProcessor(**UpperCAmelCase      )
  					spm.Load(UpperCAmelCase      )
  					return spm
  
  
  
  
  
  
  
  def 							UpperCAmelCase				(     UpperCAmelCase				,UpperCAmelCase      )->		None:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					with open(UpperCAmelCase				,'''w'''      ) as f:
  										json.dump(UpperCAmelCase				,UpperCAmelCase				,indent=2      )
  
  
  
  
  
  
  
  def 							UpperCAmelCase				(     UpperCAmelCase      )->		Union[Dict, List]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					with open(UpperCAmelCase				,'''r'''      ) as f:
  										return json.load(UpperCAmelCase      )
  
  
  
style_context: 
  
  
  def 							UpperCAmelCase				(     UpperCAmelCase      )->		bool:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					if not isinstance(UpperCAmelCase				,UpperCAmelCase      ):
  										SCREAMING_SNAKE_CASE_	      =       f'''Input value of [number={number}] must be an integer'''
  										raise TypeError(UpperCAmelCase      )
  					if number < 0 or number % 2 == 1:
  										return False
  					SCREAMING_SNAKE_CASE_	      =       int(number**0.5      )
  					return number == number_sqrt * (number_sqrt + 1)
  
  
  if __name__ == "__main__":
  		import doctest
  
  		doctest.testmod()
  
  
  