






import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowerCamelCase       :       Any        =				logging.getLogger(__name__)

lowerCamelCase       :       List[str]        =				tf.data.AUTOTUNE
def 					snake_case_						(   ):
							__lowercase					:							Dict			=   argparse.ArgumentParser(description="""Train a masked language model on TPU."""					)
							parser.add_argument(
							    """--pretrained_model_config"""				,				type=lowerCAmelCase_				,				default="""roberta-base"""				,				help="""The model config to use. Note that we don't copy the model's weights, only the config!"""				,				)
							parser.add_argument(
							    """--tokenizer"""				,				type=lowerCAmelCase_				,				default="""unigram-tokenizer-wikitext"""				,				help="""The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model's vocab size."""				,				)

							parser.add_argument(
							    """--per_replica_batch_size"""				,				type=lowerCAmelCase_				,				default=8				,				help="""Batch size per TPU core."""				,				)

							parser.add_argument(
							    """--no_tpu"""				,				action="""store_true"""				,				help="""If set, run on CPU and don't try to initialize a TPU. Useful for debugging on non-TPU instances."""				,				)

							parser.add_argument(
							    """--tpu_name"""				,				type=lowerCAmelCase_				,				help="""Name of TPU resource to initialize. Should be blank on Colab, and 'local' on TPU VMs."""				,				default="""local"""				,				)

							parser.add_argument(
							    """--tpu_zone"""				,				type=lowerCAmelCase_				,				help="""Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes."""				,				)

							parser.add_argument(
							    """--gcp_project"""				,				type=lowerCAmelCase_				,				help="""Google cloud project name. Only used for non-Colab TPU nodes."""					)

							parser.add_argument(
							    """--bfloat16"""				,				action="""store_true"""				,				help="""Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU."""				,				)

							parser.add_argument(
							    """--train_dataset"""				,				type=lowerCAmelCase_				,				help="""Path to training dataset to load. If the path begins with `gs://`"""
							    """ then the dataset will be loaded from a Google Cloud Storage bucket."""				,				)

							parser.add_argument(
							    """--shuffle_buffer_size"""				,				type=lowerCAmelCase_				,				default=2**18				,				help="""Size of the shuffle buffer (in samples)"""				,				)

							parser.add_argument(
							    """--eval_dataset"""				,				type=lowerCAmelCase_				,				help="""Path to evaluation dataset to load. If the path begins with `gs://`"""
							    """ then the dataset will be loaded from a Google Cloud Storage bucket."""				,				)

							parser.add_argument(
							    """--num_epochs"""				,				type=lowerCAmelCase_				,				default=1				,				help="""Number of epochs to train for."""				,				)

							parser.add_argument(
							    """--learning_rate"""				,				type=lowerCAmelCase_				,				default=1e-4				,				help="""Learning rate to use for training."""				,				)

							parser.add_argument(
							    """--weight_decay_rate"""				,				type=lowerCAmelCase_				,				default=1e-3				,				help="""Weight decay rate to use for training."""				,				)

							parser.add_argument(
							    """--max_length"""				,				type=lowerCAmelCase_				,				default=512				,				help="""Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py"""				,				)

							parser.add_argument(
							    """--mlm_probability"""				,				type=lowerCAmelCase_				,				default=0.15				,				help="""Fraction of tokens to mask during training."""				,				)

							parser.add_argument("""--output_dir"""				,				type=lowerCAmelCase_				,				required=lowerCAmelCase_				,				help="""Path to save model checkpoints to."""					)
							parser.add_argument("""--hub_model_id"""				,				type=lowerCAmelCase_				,				help="""Model ID to upload to on the Hugging Face Hub."""					)

							__lowercase					:							Any			=   parser.parse_args()
							return args
def 					snake_case_						(   lowerCAmelCase_				:  Any					):
							try:
														if args.tpu_name:
																					__lowercase					:							Tuple			=   tf.distribute.cluster_resolver.TPUClusterResolver(
																					    args.tpu_name				,				zone=args.tpu_zone				,				project=args.gcp_project					)
														else:
																					__lowercase					:							Dict			=   tf.distribute.cluster_resolver.TPUClusterResolver()
							except ValueError:
														raise RuntimeError(
														    """Couldn't connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or """
														    """--gcp_project. When running on a TPU VM, use --tpu_name local."""					)

							tf.config.experimental_connect_to_cluster(lowerCAmelCase_					)
							tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase_					)

							return tpu
def 					snake_case_						(   lowerCAmelCase_				:  List[str]					):
							__lowercase					:							str			=   0
							for file in file_list:
														__lowercase					:							List[Any]			=   file.split("""/"""					)[-1]
														__lowercase					:							Union[str, Any]			=   re.search(r"""-\d+-(\d+)\.tfrecord"""				,				lowerCAmelCase_					).group(1					)
														__lowercase					:							Any			=   int(lowerCAmelCase_					)
														num_samples += sample_count

							return num_samples
def 					snake_case_						(   lowerCAmelCase_				:  str				,				lowerCAmelCase_				:  Any				,				lowerCAmelCase_				:  str				,				lowerCAmelCase_				:  Union[str, Any]				,				lowerCAmelCase_				:  Any				,				lowerCAmelCase_				:  List[str]=None					):
							__lowercase					:							str			=   count_samples(lowerCAmelCase_					)
							__lowercase					:							List[str]			=   tf.data.Dataset.from_tensor_slices(lowerCAmelCase_					)
							if shuffle:
														__lowercase					:							Optional[int]			=   dataset.shuffle(len(lowerCAmelCase_					)					)
							__lowercase					:							List[str]			=   tf.data.TFRecordDataset(lowerCAmelCase_				,				num_parallel_reads=lowerCAmelCase_					)
							# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
							__lowercase					:							Tuple			=   dataset.apply(tf.data.experimental.assert_cardinality(lowerCAmelCase_					)					)
							__lowercase					:							Dict			=   dataset.map(lowerCAmelCase_				,				num_parallel_calls=lowerCAmelCase_					)
							if shuffle:
														assert shuffle_buffer_size is not None
														__lowercase					:							Tuple			=   dataset.shuffle(args.shuffle_buffer_size					)
							__lowercase					:							str			=   dataset.batch(lowerCAmelCase_				,				drop_remainder=lowerCAmelCase_					)
							__lowercase					:							Any			=   dataset.map(lowerCAmelCase_				,				num_parallel_calls=lowerCAmelCase_					)
							__lowercase					:							Tuple			=   dataset.prefetch(lowerCAmelCase_					)
							return dataset
def 					snake_case_						(   lowerCAmelCase_				:  Any					):
							if not args.no_tpu:
														__lowercase					:							List[str]			=   initialize_tpu(lowerCAmelCase_					)
														__lowercase					:							str			=   tf.distribute.TPUStrategy(lowerCAmelCase_					)
							else:
														__lowercase					:							Optional[int]			=   tf.distribute.OneDeviceStrategy(device="""/gpu:0"""					)

							if args.bfloataa:
														tf.keras.mixed_precision.set_global_policy("""mixed_bfloat16"""					)

							__lowercase					:							Any			=   AutoTokenizer.from_pretrained(args.tokenizer					)
							__lowercase					:							Union[str, Any]			=   AutoConfig.from_pretrained(args.pretrained_model_config					)
							__lowercase					:							Optional[int]			=   tokenizer.vocab_size

							__lowercase					:							Optional[Any]			=   tf.io.gfile.glob(os.path.join(args.train_dataset				,				"""*.tfrecord"""					)					)
							if not training_records:
														raise ValueError(F"No .tfrecord files found in {args.train_dataset}."					)
							__lowercase					:							Optional[int]			=   tf.io.gfile.glob(os.path.join(args.eval_dataset				,				"""*.tfrecord"""					)					)
							if not eval_records:
														raise ValueError(F"No .tfrecord files found in {args.eval_dataset}."					)

							__lowercase					:							Tuple			=   count_samples(lowerCAmelCase_					)

							__lowercase					:							Any			=   num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
							__lowercase					:							str			=   steps_per_epoch * args.num_epochs

							with strategy.scope():
														__lowercase					:							str			=   TFAutoModelForMaskedLM.from_config(lowerCAmelCase_					)
														model(model.dummy_inputs					)  # Pass some dummy inputs through the model to ensure all the weights are built
														__lowercase	,	__lowercase					:							Optional[int]			=   create_optimizer(
														    num_train_steps=lowerCAmelCase_				,				num_warmup_steps=total_train_steps // 20				,				init_lr=args.learning_rate				,				weight_decay_rate=args.weight_decay_rate				,				)

														# Transformers models compute the right loss for their task by default when labels are passed, and will
														# use this for training unless you specify your own loss function in compile().
														model.compile(optimizer=lowerCAmelCase_				,				metrics=["""accuracy"""]					)

							def decode_fn(lowerCAmelCase_				:  int					):
														__lowercase					:							Optional[Any]			=   {
														    """input_ids""": tf.io.FixedLenFeature(dtype=tf.intaa				,				shape=(args.max_length,)					),
														    """attention_mask""": tf.io.FixedLenFeature(dtype=tf.intaa				,				shape=(args.max_length,)					),
														}
														return tf.io.parse_single_example(lowerCAmelCase_				,				lowerCAmelCase_					)

							# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
							# use their methods in our data pipeline.
							__lowercase					:							Union[str, Any]			=   DataCollatorForLanguageModeling(
							    tokenizer=lowerCAmelCase_				,				mlm_probability=args.mlm_probability				,				mlm=lowerCAmelCase_				,				return_tensors="""tf"""					)

							def mask_with_collator(lowerCAmelCase_				:  Optional[int]					):
														# TF really needs an isin() function
														__lowercase					:							Any			=   (
														    ~tf.cast(batch["""attention_mask"""]				,				tf.bool					)
														    | (batch["""input_ids"""] == tokenizer.cls_token_id)
														    | (batch["""input_ids"""] == tokenizer.sep_token_id)
														)
														__lowercase	,	__lowercase					:							Optional[int]			=   data_collator.tf_mask_tokens(
														    batch["""input_ids"""]				,				vocab_size=len(lowerCAmelCase_					)				,				mask_token_id=tokenizer.mask_token_id				,				special_tokens_mask=lowerCAmelCase_				,				)
														return batch

							__lowercase					:							Optional[Any]			=   args.per_replica_batch_size * strategy.num_replicas_in_sync

							__lowercase					:							Tuple			=   prepare_dataset(
							    lowerCAmelCase_				,				decode_fn=lowerCAmelCase_				,				mask_fn=lowerCAmelCase_				,				batch_size=lowerCAmelCase_				,				shuffle=lowerCAmelCase_				,				shuffle_buffer_size=args.shuffle_buffer_size				,				)

							__lowercase					:							List[Any]			=   prepare_dataset(
							    lowerCAmelCase_				,				decode_fn=lowerCAmelCase_				,				mask_fn=lowerCAmelCase_				,				batch_size=lowerCAmelCase_				,				shuffle=lowerCAmelCase_				,				)

							__lowercase					:							Any			=   []
							if args.hub_model_id:
														callbacks.append(
														    PushToHubCallback(output_dir=args.output_dir				,				hub_model_id=args.hub_model_id				,				tokenizer=lowerCAmelCase_					)					)

							model.fit(
							    lowerCAmelCase_				,				validation_data=lowerCAmelCase_				,				epochs=args.num_epochs				,				callbacks=lowerCAmelCase_				,				)

							model.save_pretrained(args.output_dir					)


if __name__ == "__main__":
	lowerCamelCase       :       List[str]        =				parse_args()
	main(args)







import argparse

from transformers import (
    TapasConfig,
    TapasForMaskedLM,
    TapasForQuestionAnswering,
    TapasForSequenceClassification,
    TapasModel,
    TapasTokenizer,
    load_tf_weights_in_tapas,
)
from transformers.utils import logging


logging.set_verbosity_info()
def 					snake_case_						(   lowerCAmelCase_				:  List[Any]				,				lowerCAmelCase_				:  Dict				,				lowerCAmelCase_				:  Any				,				lowerCAmelCase_				:  Tuple				,				lowerCAmelCase_				:  int					):
							# Initialise PyTorch model.
							# If you want to convert a checkpoint that uses absolute position embeddings, make sure to set reset_position_index_per_cell of
							# TapasConfig to False.

							# initialize configuration from json file
							__lowercase					:							Optional[int]			=   TapasConfig.from_json_file(lowerCAmelCase_					)
							# set absolute/relative position embeddings parameter
							__lowercase					:							Optional[Any]			=   reset_position_index_per_cell

							# set remaining parameters of TapasConfig as well as the model based on the task
							if task == "SQA":
														__lowercase					:							Union[str, Any]			=   TapasForQuestionAnswering(config=lowerCAmelCase_					)
							elif task == "WTQ":
														# run_task_main.py hparams
														__lowercase					:							List[Any]			=   4
														__lowercase					:							Union[str, Any]			=   True
														# hparam_utils.py hparams
														__lowercase					:							Any			=   0.664_694
														__lowercase					:							Tuple			=   0.207_951
														__lowercase					:							Dict			=   0.121_194
														__lowercase					:							List[str]			=   True
														__lowercase					:							str			=   True
														__lowercase					:							Dict			=   False
														__lowercase					:							Tuple			=   0.0_352_513

														__lowercase					:							List[Any]			=   TapasForQuestionAnswering(config=lowerCAmelCase_					)
							elif task == "WIKISQL_SUPERVISED":
														# run_task_main.py hparams
														__lowercase					:							Optional[int]			=   4
														__lowercase					:							int			=   False
														# hparam_utils.py hparams
														__lowercase					:							Tuple			=   36.4_519
														__lowercase					:							str			=   0.903_421
														__lowercase					:							List[Any]			=   222.088
														__lowercase					:							Union[str, Any]			=   True
														__lowercase					:							Tuple			=   True
														__lowercase					:							Union[str, Any]			=   True
														__lowercase					:							Optional[Any]			=   0.763_141

														__lowercase					:							str			=   TapasForQuestionAnswering(config=lowerCAmelCase_					)
							elif task == "TABFACT":
														__lowercase					:							List[Any]			=   TapasForSequenceClassification(config=lowerCAmelCase_					)
							elif task == "MLM":
														__lowercase					:							Optional[int]			=   TapasForMaskedLM(config=lowerCAmelCase_					)
							elif task == "INTERMEDIATE_PRETRAINING":
														__lowercase					:							Dict			=   TapasModel(config=lowerCAmelCase_					)
							else:
														raise ValueError(F"Task {task} not supported."					)

							print(F"Building PyTorch model from configuration: {config}"					)
							# Load weights from tf checkpoint
							load_tf_weights_in_tapas(lowerCAmelCase_				,				lowerCAmelCase_				,				lowerCAmelCase_					)

							# Save pytorch-model (weights and configuration)
							print(F"Save PyTorch model to {pytorch_dump_path}"					)
							model.save_pretrained(lowerCAmelCase_					)

							# Save tokenizer files
							print(F"Save tokenizer files to {pytorch_dump_path}"					)
							__lowercase					:							Any			=   TapasTokenizer(vocab_file=tf_checkpoint_path[:-10] + """vocab.txt"""				,				model_max_length=512					)
							tokenizer.save_pretrained(lowerCAmelCase_					)

							print("""Used relative position embeddings:"""				,				model.config.reset_position_index_per_cell					)


if __name__ == "__main__":
	lowerCamelCase       :       List[Any]        =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--task''', default='''SQA''', type=str, help='''Model task for which to convert a checkpoint. Defaults to SQA.'''
	)
	parser.add_argument(
	    '''--reset_position_index_per_cell''',
	    default=False,
	    action='''store_true''',
	    help='''Whether to use relative position embeddings or not. Defaults to True.''',
	)
	parser.add_argument(
	    '''--tf_checkpoint_path''', default=None, type=str, required=True, help='''Path to the TensorFlow checkpoint path.'''
	)
	parser.add_argument(
	    '''--tapas_config_file''',
	    default=None,
	    type=str,
	    required=True,
	    help=(
	        '''The config json file corresponding to the pre-trained TAPAS model. \n'''
	        '''This specifies the model architecture.'''
	    ),
	)
	parser.add_argument(
	    '''--pytorch_dump_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
	)
	lowerCamelCase       :       int        =				parser.parse_args()
	convert_tf_checkpoint_to_pytorch(
	    args.task,
	    args.reset_position_index_per_cell,
	    args.tf_checkpoint_path,
	    args.tapas_config_file,
	    args.pytorch_dump_path,
	)