code: 
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
  		from .tokenization_xlm_roberta import XLMRobertaTokenizer
  else:
  		lowercase_								=    None
  
  
  lowercase_								=    logging.get_logger(__name__)
  
  lowercase_								=    {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}
  
  lowercase_								=    {
      'vocab_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
          ),
      },
      'tokenizer_file': {
          'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
          'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
          'xlm-roberta-large-finetuned-conll02-dutch': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll02-spanish': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-english': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
          ),
          'xlm-roberta-large-finetuned-conll03-german': (
              'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
          ),
      },
  }
  
  lowercase_								=    {
      'xlm-roberta-base': 5_12,
      'xlm-roberta-large': 5_12,
      'xlm-roberta-large-finetuned-conll02-dutch': 5_12,
      'xlm-roberta-large-finetuned-conll02-spanish': 5_12,
      'xlm-roberta-large-finetuned-conll03-english': 5_12,
      'xlm-roberta-large-finetuned-conll03-german': 5_12,
  }
  
  
  
  
  class 					_UpperCamelCase (      lowerCamelCase__	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  							_A  				=  VOCAB_FILES_NAMES
  							_A  				=  PRETRAINED_VOCAB_FILES_MAP
  							_A  				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							_A  				=  ["input_ids", "attention_mask"]
  							_A  				=  XLMRobertaTokenizer
  
  
  
  
  
  
  
  							def __init__(		self  :				List[Any]		,				SCREAMING_SNAKE_CASE_  :				Tuple=None		,				SCREAMING_SNAKE_CASE_  :				Optional[int]=None		,				SCREAMING_SNAKE_CASE_  :				Union[str, Any]="<s>"		,				SCREAMING_SNAKE_CASE_  :				Optional[int]="</s>"		,				SCREAMING_SNAKE_CASE_  :				str="</s>"		,				SCREAMING_SNAKE_CASE_  :				Any="<s>"		,				SCREAMING_SNAKE_CASE_  :				List[str]="<unk>"		,				SCREAMING_SNAKE_CASE_  :				int="<pad>"		,				SCREAMING_SNAKE_CASE_  :				int="<mask>"		,				**SCREAMING_SNAKE_CASE_  :				str		,				):
  													# Mask token behave like a normal word, i.e. include the space before it
  													_a     =			AddedToken(SCREAMING_SNAKE_CASE_		,				lstrip=SCREAMING_SNAKE_CASE_		,				rstrip=SCREAMING_SNAKE_CASE_		) if isinstance(SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		) else mask_token
  
  													super().__init__(
  													    SCREAMING_SNAKE_CASE_		,				tokenizer_file=SCREAMING_SNAKE_CASE_		,				bos_token=SCREAMING_SNAKE_CASE_		,				eos_token=SCREAMING_SNAKE_CASE_		,				sep_token=SCREAMING_SNAKE_CASE_		,				cls_token=SCREAMING_SNAKE_CASE_		,				unk_token=SCREAMING_SNAKE_CASE_		,				pad_token=SCREAMING_SNAKE_CASE_		,				mask_token=SCREAMING_SNAKE_CASE_		,				**SCREAMING_SNAKE_CASE_		,				)
  
  													_a     =			vocab_file
  													_a     =			False if not self.vocab_file else True
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Optional[Any]		,				SCREAMING_SNAKE_CASE_  :				List[int]		,				SCREAMING_SNAKE_CASE_  :				Optional[List[int]] = None		):
  
  													if token_ids_a is None:
  																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  													_a     =			[self.cls_token_id]
  													_a     =			[self.sep_token_id]
  													return cls + token_ids_a + sep + sep + token_ids_a + sep
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Dict		,				SCREAMING_SNAKE_CASE_  :				List[int]		,				SCREAMING_SNAKE_CASE_  :				Optional[List[int]] = None		):
  													_a     =			[self.sep_token_id]
  													_a     =			[self.cls_token_id]
  
  													if token_ids_a is None:
  																			return len(cls + token_ids_a + sep		) * [0]
  													return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Union[str, Any]		,				SCREAMING_SNAKE_CASE_  :				str		,				SCREAMING_SNAKE_CASE_  :				Optional[str] = None		):
  													if not self.can_save_slow_tokenizer:
  																			raise ValueError(
  																			    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
  																			    'tokenizer.'		)
  
  													if not os.path.isdir(SCREAMING_SNAKE_CASE_		):
  																			logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""		)
  																			return
  													_a     =			os.path.join(
  													    SCREAMING_SNAKE_CASE_		,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
  
  													if os.path.abspath(self.vocab_file		) != os.path.abspath(SCREAMING_SNAKE_CASE_		):
  																			copyfile(self.vocab_file		,				SCREAMING_SNAKE_CASE_		)
  
  													return (out_vocab_file,)
  
  
  
style_context: 
  
  
  from .chunk_utils import chunk_layer
  from .data_transforms import make_atomaa_masks
  from .feats import atomaa_to_atomaa, frames_and_literature_positions_to_atomaa_pos, torsion_angles_to_frames
  from .loss import compute_predicted_aligned_error, compute_tm
  from .protein import Protein as OFProtein
  from .protein import to_pdb
  from .rigid_utils import Rigid, Rotation
  from .tensor_utils import dict_multimap, flatten_final_dims, permute_final_dims
  
  
  