
'''simple docstring'''




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
      import torch


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


lowercase			:				List[Any]    		=			logging.get_logger(__name__)
lowercase			:				Any    		=			{'vocab_file': 'spiece.model'}

lowercase			:				int    		=			{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

lowercase			:				List[str]    		=			{
    'AI-Sweden/gpt-sw3-126m': 20_48,
    'AI-Sweden/gpt-sw3-350m': 20_48,
    'AI-Sweden/gpt-sw3-1.6b': 20_48,
    'AI-Sweden/gpt-sw3-6.7b': 20_48,
    'AI-Sweden/gpt-sw3-20b': 20_48,
}





class  A				(							__snake_case						):
     __magic_name__						=		VOCAB_FILES_NAMES
     __magic_name__						=		PRETRAINED_VOCAB_FILES_MAP
     __magic_name__						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __magic_name__						=		['''input_ids''', '''attention_mask''']





     def __init__(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE=False						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE=None						,			SCREAMING_SNAKE_CASE = None						,			**SCREAMING_SNAKE_CASE						,			) ->				None:






         """simple docstring"""






         A       :   List[str]					=						{} if sp_model_kwargs is None else sp_model_kwargs

         A       :   Optional[int]					=						kwargs.get('''name_or_path'''	)
         if name_or_path is None:
             logger.warning(
                 '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
                 ''' you are testing the model, this can safely be ignored'''	)
             A       :   List[Any]					=						'''None'''

         # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
         A       :   int					=						'''<|endoftext|>''' if eos_token is None else eos_token
         A       :   Dict					=						'''<unk>''' if unk_token is None else unk_token
         if "gpt-sw3-7b" in name_or_path:
             A       :   List[Any]					=						unk_token if pad_token is None else pad_token
             A       :   List[str]					=						eos_token if bos_token is None else bos_token
         else:
             A       :   Optional[Any]					=						'''<pad>''' if pad_token is None else pad_token
             A       :   Optional[int]					=						'''<s>''' if bos_token is None else bos_token

         super().__init__(
             do_lower_case=SCREAMING_SNAKE_CASE						,			remove_space=SCREAMING_SNAKE_CASE						,			keep_accents=SCREAMING_SNAKE_CASE						,			bos_token=SCREAMING_SNAKE_CASE						,			eos_token=SCREAMING_SNAKE_CASE						,			unk_token=SCREAMING_SNAKE_CASE						,			pad_token=SCREAMING_SNAKE_CASE						,			sp_model_kwargs=self.sp_model_kwargs						,			**SCREAMING_SNAKE_CASE						,			)

         A       :   List[str]					=						do_lower_case
         A       :   List[str]					=						remove_space
         A       :   Union[str, Any]					=						keep_accents
         A       :   Union[str, Any]					=						vocab_file

         A       :   Tuple					=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.Load(SCREAMING_SNAKE_CASE	)

         # Used for whitespace normalization in input texts
         # fmt : off
         A       :   Any					=						{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
         # fmt : on

         # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
         A       :   List[str]					=						re.compile(
             F'[{"".join(map(SCREAMING_SNAKE_CASE						,			list(range(0						,			9	)	) + list(range(11						,			32	)	) + list(range(127						,			160	)	) + [160, 173, 8203]	)	)}]'	)





     def __getstate__(							self	) ->				Dict:






         """simple docstring"""






         A       :   Tuple					=						self.__dict__.copy()
         A       :   List[Any]					=						None
         return state





     def __setstate__(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         A       :   Tuple					=						d

         # for backward compatibility
         if not hasattr(self						,			'''sp_model_kwargs'''	):
             A       :   Any					=						{}

         A       :   Dict					=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.Load(self.vocab_file	)





     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def 				__lowerCAmelCase							(							self	) ->				int:






         """simple docstring"""






         return len(self.sp_model	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         A       :   List[str]					=						self.non_printing_characters_re.sub(''''''						,			SCREAMING_SNAKE_CASE	)

         # Normalize whitespaces
         A       :   Union[str, Any]					=						''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]	)

         # NFC Unicode normalization
         A       :   Optional[int]					=						unicodedata.normalize('''NFC'''						,			SCREAMING_SNAKE_CASE	)
         return text





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			**SCREAMING_SNAKE_CASE	) ->				List[str]:






         """simple docstring"""






         A       :   Optional[int]					=						self.preprocess_text(SCREAMING_SNAKE_CASE	)
         return self.sp_model.encode(SCREAMING_SNAKE_CASE						,			out_type=SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				int:






         """simple docstring"""






         return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE	)





     @staticmethod
     def 				__lowerCAmelCase							(							SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         return out_string





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         A       :   Optional[int]					=						[]
         A       :   Optional[Any]					=						''''''
         A       :   List[str]					=						False
         for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                 # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                 if not prev_is_special:
                     out_string += " "

                 out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	) + token
                 A       :   Dict					=						True
                 A       :   int					=						[]
             else:
                 current_sub_tokens.append(SCREAMING_SNAKE_CASE	)
                 A       :   List[Any]					=						False
         out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	)

         return out_string





     def 				__lowerCAmelCase							(							self	) ->				Dict[str, int]:






         """simple docstring"""






         A       :   Tuple					=						{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE	): i for i in range(self.vocab_size	)}
         vocab.update(self.added_tokens_encoder	)
         return vocab





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = None	) ->				Tuple[str]:






         """simple docstring"""






         if not os.path.isdir(SCREAMING_SNAKE_CASE	):
             logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
             return
         A       :   Tuple					=						os.path.join(
             SCREAMING_SNAKE_CASE						,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	) and os.path.isfile(self.vocab_file	):
             copyfile(self.vocab_file						,			SCREAMING_SNAKE_CASE	)
         elif not os.path.isfile(self.vocab_file	):
             with open(SCREAMING_SNAKE_CASE						,			'''wb'''	) as fi:
                 A       :   List[str]					=						self.sp_model.serialized_model_proto()
                 fi.write(SCREAMING_SNAKE_CASE	)

         return (out_vocab_file,)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE = False	) ->				Union[List[int], List[List[int]], "torch.Tensor"]:






         """simple docstring"""






         if isinstance(SCREAMING_SNAKE_CASE						,			SCREAMING_SNAKE_CASE	):
             A       :   List[str]					=						self.preprocess_text(SCREAMING_SNAKE_CASE	)
             A       :   Any					=						self.sp_model.encode(SCREAMING_SNAKE_CASE	)
         else:
             A       :   Any					=						[self.preprocess_text(SCREAMING_SNAKE_CASE	) for t in text]
             A       :   Optional[int]					=						self.sp_model.encode(SCREAMING_SNAKE_CASE	)

         if return_tensors is True or return_tensors == "pt":
             A       :   Union[str, Any]					=						torch.tensor(SCREAMING_SNAKE_CASE	)

         return token_ids





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				str:






         """simple docstring"""






         return self.sp_model.decode(SCREAMING_SNAKE_CASE	)





     def 				__lowerCAmelCase							(							self						,			SCREAMING_SNAKE_CASE	) ->				List[int]:






         """simple docstring"""






         A       :   int					=						[F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
         A       :   Tuple					=						(
             F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(SCREAMING_SNAKE_CASE	) + F'{self.bos_token}Bot:'
         )
         return self.encode(text=SCREAMING_SNAKE_CASE	)





"""simple docstring"""






import argparse

import torch

from transformers import FunnelBaseModel, FunnelConfig, FunnelModel, load_tf_weights_in_funnel
from transformers.utils import logging


logging.set_verbosity_info()



def SCREAMING_SNAKE_CASE__			(   __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase )  ->  List[Any]:
   # Initialise PyTorch model
   lowercase__:   Optional[Any]					=      FunnelConfig.from_json_file(__UpperCAmelCase )
   print(F"""Building PyTorch model from configuration: {config}""" )
   lowercase__:   List[Any]					=      FunnelBaseModel(__UpperCAmelCase ) if base_model else FunnelModel(__UpperCAmelCase )

   # Load weights from tf checkpoint
   load_tf_weights_in_funnel(__UpperCAmelCase       ,    __UpperCAmelCase       ,    __UpperCAmelCase )

   # Save pytorch-model
   print(F"""Save PyTorch model to {pytorch_dump_path}""" )
   torch.save(model.state_dict()       ,    __UpperCAmelCase )


if __name__ == "__main__":
       __A			   = argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
       )
       parser.add_argument(
           "--config_file",
           default=None,
           type=str,
           required=True,
           help="The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.",
       )
       parser.add_argument(
           "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
       )
       parser.add_argument(
           "--base_model", action="store_true", help="Whether you want just the base model (no decoder) or not."
       )
       __A			   = parser.parse_args()
       convert_tf_checkpoint_to_pytorch(
           args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.base_model
       )
