

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase					:			Optional[Any]  				=			logging.get_logger(__name__)


_lowerCAmelCase					:			List[Any]  				=			{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
_lowerCAmelCase					:			Tuple  				=			{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

_lowerCAmelCase					:			Tuple  				=			{
    "facebook/bart-base": 1_024,
    "facebook/bart-large": 1_024,
    "facebook/bart-large-mnli": 1_024,
    "facebook/bart-large-cnn": 1_024,
    "facebook/bart-large-xsum": 1_024,
    "yjernite/bart_eli5": 1_024,
}






@lru_cache()
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =(
			    list(range(ord('!'	)       ,     ord('~'	) + 1	)	) + list(range(ord('¡'	)       ,     ord('¬'	) + 1	)	) + list(range(ord('®'	)       ,     ord('ÿ'	) + 1	)	)
			)
			__a        =bs[:]
			__a        =0
			for b in range(2**8	):
						if b not in bs:
									bs.append(_snake_case	)
									cs.append(2**8 + n	)
									n += 1
			__a        =[chr(_snake_case	) for n in cs]
			return dict(zip(_snake_case       ,     _snake_case	)	)






def        UpperCamelCase_(							_snake_case   :  Optional[Any]	):




			"""simple docstring"""






			__a        =set()
			__a        =word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)	)
						__a        =char
			return pairs






class 	__magic_name__    (			lowerCAmelCase_      ):
			SCREAMING_SNAKE_CASE							   =						VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE							   =						PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE							   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE							   =						['input_ids', 'attention_mask']





			def __init__(		self					,  __snake_case					,  __snake_case					,  __snake_case="replace"					,  __snake_case="<s>"					,  __snake_case="</s>"					,  __snake_case="</s>"					,  __snake_case="<s>"					,  __snake_case="<unk>"					,  __snake_case="<pad>"					,  __snake_case="<mask>"					,  __snake_case=False					,  **__snake_case					,  )   -> Union[str, Any]:




						'''simple docstring'''
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else bos_token
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else eos_token
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else sep_token
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else cls_token
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else unk_token
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						__a        =AddedToken(__snake_case					,  lstrip=__snake_case					,  rstrip=__snake_case					) if isinstance(__snake_case					,  __snake_case					) else mask_token

						super().__init__(
						    errors=__snake_case					,  bos_token=__snake_case					,  eos_token=__snake_case					,  unk_token=__snake_case					,  sep_token=__snake_case					,  cls_token=__snake_case					,  pad_token=__snake_case					,  mask_token=__snake_case					,  add_prefix_space=__snake_case					,  **__snake_case					,  )

						with open(__snake_case					,  encoding='utf-8'					) as vocab_handle:
									__a        =json.load(__snake_case					)
						__a        ={v: k for k, v in self.encoder.items()}
						__a        =errors  # how to handle errors in decoding
						__a        =bytes_to_unicode()
						__a        ={v: k for k, v in self.byte_encoder.items()}
						with open(__snake_case					,  encoding='utf-8'					) as merges_handle:
									__a        =merges_handle.read().split('\n'					)[1:-1]
						__a        =[tuple(merge.split()					) for merge in bpe_merges]
						__a        =dict(zip(__snake_case					,  range(len(__snake_case					)					)					)					)
						__a        ={}
						__a        =add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						__a        =re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'					)





			@property
			def 		__magic_name__    (		self					)   -> List[Any]:




						'''simple docstring'''
						return len(self.encoder					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						return dict(self.encoder					,  **self.added_tokens_encoder					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> Tuple:




						'''simple docstring'''
						if token in self.cache:
									return self.cache[token]
						__a        =tuple(__snake_case					)
						__a        =get_pairs(__snake_case					)

						if not pairs:
									return token

						while True:
									__a        =min(__snake_case					,  key=lambda __snake_case					: self.bpe_ranks.get(__snake_case					,  float('inf'					)					)					)
									if bigram not in self.bpe_ranks:
												break
									__a      ,    __a        =bigram
									__a        =[]
									__a        =0
									while i < len(__snake_case					):
												try:
															__a        =word.index(__snake_case					,  __snake_case					)
												except ValueError:
															new_word.extend(word[i:]					)
															break
												else:
															new_word.extend(word[i:j]					)
															__a        =j

												if word[i] == first and i < len(__snake_case					) - 1 and word[i + 1] == second:
															new_word.append(first + second					)
															i += 2
												else:
															new_word.append(word[i]					)
															i += 1
									__a        =tuple(__snake_case					)
									__a        =new_word
									if len(__snake_case					) == 1:
												break
									else:
												__a        =get_pairs(__snake_case					)
						__a        =' '.join(__snake_case					)
						__a        =word
						return word





			def 		__magic_name__    (		self					,  __snake_case					)   -> int:




						'''simple docstring'''
						__a        =[]
						for token in re.findall(self.pat					,  __snake_case					):
									__a        =''.join(
									    self.byte_encoder[b] for b in token.encode('utf-8'					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
									bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case					).split(' '					)					)
						return bpe_tokens





			def 		__magic_name__    (		self					,  __snake_case					)   -> Any:




						'''simple docstring'''
						return self.encoder.get(__snake_case					,  self.encoder.get(self.unk_token					)					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> int:




						'''simple docstring'''
						return self.decoder.get(__snake_case					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> int:




						'''simple docstring'''
						__a        =''.join(__snake_case					)
						__a        =bytearray([self.byte_decoder[c] for c in text]					).decode('utf-8'					,  errors=self.errors					)
						return text





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					)   -> Tuple[str]:




						'''simple docstring'''
						if not os.path.isdir(__snake_case					):
									logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
									return
						__a        =os.path.join(
						    __snake_case					,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
						__a        =os.path.join(
						    __snake_case					,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

						with open(__snake_case					,  'w'					,  encoding='utf-8'					) as f:
									f.write(json.dumps(self.encoder					,  indent=2					,  sort_keys=__snake_case					,  ensure_ascii=__snake_case					) + '\n'					)

						__a        =0
						with open(__snake_case					,  'w'					,  encoding='utf-8'					) as writer:
									writer.write('#version: 0.2\n'					)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,  key=lambda __snake_case					: kv[1]					):
												if index != token_index:
															logger.warning(
															    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
															    ' Please check that the tokenizer is not corrupted!'					)
															__a        =token_index
												writer.write(' '.join(__snake_case					) + '\n'					)
												index += 1

						return vocab_file, merge_file





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					)   -> List[int]:




						'''simple docstring'''
						if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__a        =[self.cls_token_id]
						__a        =[self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					,  __snake_case = False					)   -> List[int]:




						'''simple docstring'''
						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__snake_case					,  token_ids_a=__snake_case					,  already_has_special_tokens=__snake_case					)

						if token_ids_a is None:
									return [1] + ([0] * len(__snake_case					)) + [1]
						return [1] + ([0] * len(__snake_case					)) + [1, 1] + ([0] * len(__snake_case					)) + [1]





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case = None					)   -> List[int]:




						'''simple docstring'''
						__a        =[self.sep_token_id]
						__a        =[self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep					) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]





			def 		__magic_name__    (		self					,  __snake_case					,  __snake_case=False					,  **__snake_case					)   -> Union[str, Any]:




						'''simple docstring'''
						__a        =kwargs.pop('add_prefix_space'					,  self.add_prefix_space					)
						if (is_split_into_words or add_prefix_space) and (len(__snake_case					) > 0 and not text[0].isspace()):
									__a        =' ' + text
						return (text, kwargs)









import argparse

import torch

from transformers import OpenAIGPTConfig, OpenAIGPTModel, load_tf_weights_in_openai_gpt
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()






def        UpperCamelCase_(							_snake_case   :  Dict       ,     _snake_case   :  Optional[int]       ,     _snake_case   :  str	):




			"""simple docstring"""






			if openai_config_file == "":
						__a        =OpenAIGPTConfig()
			else:
						__a        =OpenAIGPTConfig.from_json_file(_snake_case	)
			__a        =OpenAIGPTModel(_snake_case	)

			# Load weights from numpy
			load_tf_weights_in_openai_gpt(_snake_case       ,     _snake_case       ,     _snake_case	)

			# Save pytorch-model
			__a        =pytorch_dump_folder_path + '/' + WEIGHTS_NAME
			__a        =pytorch_dump_folder_path + '/' + CONFIG_NAME
			print(F'Save PyTorch model to {pytorch_weights_dump_path}'	)
			torch.save(model.state_dict()       ,     _snake_case	)
			print(F'Save configuration file to {pytorch_config_dump_path}'	)
			with open(_snake_case       ,     'w'       ,     encoding='utf-8'	) as f:
						f.write(config.to_json_string()	)


if __name__ == "__main__":
						_lowerCAmelCase					:			Tuple  				=			argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    "--openai_checkpoint_folder_path",
						    default=None,
						    type=str,
						    required=True,
						    help="Path to the TensorFlow checkpoint path.",
						)
						parser.add_argument(
						    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
						)
						parser.add_argument(
						    "--openai_config_file",
						    default="",
						    type=str,
						    help=(
						        "An optional config json file corresponding to the pre-trained OpenAI model. \n"
						        "This specifies the model architecture."
						    ),
						)
						_lowerCAmelCase					:			int  				=			parser.parse_args()
						convert_openai_checkpoint_to_pytorch(
						    args.openai_checkpoint_folder_path, args.openai_config_file, args.pytorch_dump_folder_path
						)






