






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A__	:			List[Any]			   = logging.get_logger(__name__)

A__	:			str			   = {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__	:			int			   = {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__	:			Optional[Any]			   = {
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


A__	:			List[str]			   = {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}
class       _UpperCAmelCase					(       A__	):

				"""simple docstring"""




				lowercase__											=       VOCAB_FILES_NAMES
				lowercase__											=       PRETRAINED_VOCAB_FILES_MAP
				lowercase__											=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__											=       PRETRAINED_INIT_CONFIGURATION
				lowercase__											=       ["""input_ids""", """attention_mask"""]
				lowercase__											=       DistilBertTokenizer




				def __init__(     self						:    List[Any],  lowerCamelCase						:    List[Any]=None,  lowerCamelCase						:    Dict=None,  lowerCamelCase						:    str=True,  lowerCamelCase						:    Optional[int]="[UNK]",  lowerCamelCase						:    Optional[Any]="[SEP]",  lowerCamelCase						:    List[Any]="[PAD]",  lowerCamelCase						:    Any="[CLS]",  lowerCamelCase						:    Union[str, Any]="[MASK]",  lowerCamelCase						:    str=True,  lowerCamelCase						:    int=None,  **lowerCamelCase						:    Union[str, Any],  ):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,  tokenizer_file=lowerCamelCase,  do_lower_case=lowerCamelCase,  unk_token=lowerCamelCase,  sep_token=lowerCamelCase,  pad_token=lowerCamelCase,  cls_token=lowerCamelCase,  mask_token=lowerCamelCase,  tokenize_chinese_chars=lowerCamelCase,  strip_accents=lowerCamelCase,  **lowerCamelCase,  )

								lowercase__    =			json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
								if (
								    normalizer_state.get('''lowercase''',  lowerCamelCase  ) != do_lower_case
								    or normalizer_state.get('''strip_accents''',  lowerCamelCase  ) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',  lowerCamelCase  ) != tokenize_chinese_chars
								):
												lowercase__    =			getattr(lowerCamelCase,  normalizer_state.pop('''type'''  )  )
												lowercase__    =			do_lower_case
												lowercase__    =			strip_accents
												lowercase__    =			tokenize_chinese_chars
												lowercase__    =			normalizer_class(**lowerCamelCase  )

								lowercase__    =			do_lower_case




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    Optional[Any],  lowerCamelCase						:    List[Any]=None  ):

								'''simple docstring'''
								lowercase__    =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def 				lowercase__   (     self						:    Union[str, Any],  lowerCamelCase						:    List[int],  lowerCamelCase						:    Optional[List[int]] = None  ):

								'''simple docstring'''
								lowercase__    =			[self.sep_token_id]
								lowercase__    =			[self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]




				def 				lowercase__   (     self						:    str,  lowerCamelCase						:    str,  lowerCamelCase						:    Optional[str] = None  ):

								'''simple docstring'''
								lowercase__    =			self._tokenizer.model.save(lowerCamelCase,  name=lowerCamelCase  )
								return tuple(lowerCamelCase  )












import argparse
import os

import numpy as np
import tensorflow as tf
import torch

from transformers import BertModel






def   a			(						lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_				):






				'''simple docstring'''
				lowercase__    =			('''dense.weight''', '''attention.self.query''', '''attention.self.key''', '''attention.self.value''')

				lowercase__    =			(
				    ('''layer.''', '''layer_'''),
				    ('''word_embeddings.weight''', '''word_embeddings'''),
				    ('''position_embeddings.weight''', '''position_embeddings'''),
				    ('''token_type_embeddings.weight''', '''token_type_embeddings'''),
				    ('''.''', '''/'''),
				    ('''LayerNorm/weight''', '''LayerNorm/gamma'''),
				    ('''LayerNorm/bias''', '''LayerNorm/beta'''),
				    ('''weight''', '''kernel'''),
				)

				if not os.path.isdir(lowerCamelCase_				):
								os.makedirs(lowerCamelCase_				)

				lowercase__    =			model.state_dict()

				def to_tf_var_name(lowerCamelCase_				):
								for patt, repl in iter(lowerCamelCase_				):
												lowercase__    =			name.replace(lowerCamelCase_		,   lowerCamelCase_				)
								return F"""bert/{name}"""

				def create_tf_var(lowerCamelCase_		,   lowerCamelCase_		,   lowerCamelCase_				):
								lowercase__    =			tf.dtypes.as_dtype(tensor.dtype				)
								lowercase__    =			tf.get_variable(dtype=lowerCamelCase_		,   shape=tensor.shape		,   name=lowerCamelCase_		,   initializer=tf.zeros_initializer()				)
								session.run(tf.variables_initializer([tf_var]				)				)
								session.run(lowerCamelCase_				)
								return tf_var

				tf.reset_default_graph()
				with tf.Session() as session:
								for var_name in state_dict:
												lowercase__    =			to_tf_var_name(lowerCamelCase_				)
												lowercase__    =			state_dict[var_name].numpy()
												if any(x in var_name for x in tensors_to_transpose				):
																lowercase__    =			torch_tensor.T
												lowercase__    =			create_tf_var(tensor=lowerCamelCase_		,   name=lowerCamelCase_		,   session=lowerCamelCase_				)
												tf.keras.backend.set_value(lowerCamelCase_		,   lowerCamelCase_				)
												lowercase__    =			session.run(lowerCamelCase_				)
												print(F"""Successfully created {tf_name}: {np.allclose(lowerCamelCase_		,   lowerCamelCase_				)}"""				)

								lowercase__    =			tf.train.Saver(tf.trainable_variables()				)
								saver.save(lowerCamelCase_		,   os.path.join(lowerCamelCase_		,   model_name.replace('''-'''		,   '''_'''				) + '''.ckpt'''				)				)






def   a			(						lowerCamelCase_=None				):






				'''simple docstring'''
				lowercase__    =			argparse.ArgumentParser()
				parser.add_argument('''--model_name'''		,   type=lowerCamelCase_		,   required=lowerCamelCase_		,   help='''model name e.g. bert-base-uncased'''				)
				parser.add_argument(
				    '''--cache_dir'''		,   type=lowerCamelCase_		,   default=lowerCamelCase_		,   required=lowerCamelCase_		,   help='''Directory containing pytorch model'''				)
				parser.add_argument('''--pytorch_model_path'''		,   type=lowerCamelCase_		,   required=lowerCamelCase_		,   help='''/path/to/<pytorch-model-name>.bin'''				)
				parser.add_argument('''--tf_cache_dir'''		,   type=lowerCamelCase_		,   required=lowerCamelCase_		,   help='''Directory in which to save tensorflow model'''				)
				lowercase__    =			parser.parse_args(lowerCamelCase_				)

				lowercase__    =			BertModel.from_pretrained(
				    pretrained_model_name_or_path=args.model_name		,   state_dict=torch.load(args.pytorch_model_path				)		,   cache_dir=args.cache_dir		,   )

				convert_pytorch_checkpoint_to_tf(model=lowerCamelCase_		,   ckpt_dir=args.tf_cache_dir		,   model_name=args.model_name				)


if __name__ == "__main__":
							main()




