


"""simple docstring"""





import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


snake_case__						:   Optional[Any]			 = logging.getLogger(__name__)


def    _snake_case  (  ):
      lowerCAmelCase    :		Tuple				=      argparse.ArgumentParser(
          description='''Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset.'''	)
      parser.add_argument(
          '''--dataset_name'''       ,      type=_snake_case       ,      default='''wikitext'''       ,      help='''Name of the training. Explore datasets at: hf.co/datasets.'''       ,      )
      parser.add_argument(
          '''--dataset_config'''       ,      type=_snake_case       ,      default='''wikitext-103-raw-v1'''       ,      help='''Configuration name of the dataset.'''	)
      parser.add_argument(
          '''--tokenizer_name_or_path'''       ,      type=_snake_case       ,      default='''sayakpaul/unigram-tokenizer-wikitext'''       ,      help='''Tokenizer identifier. Can be a local filepath or a Hub identifier.'''       ,      )
      parser.add_argument(
          '''--shard_size'''       ,      type=_snake_case       ,      default=1000       ,      help='''Number of entries to go in a single shard.'''       ,      )
      parser.add_argument('''--split'''       ,      type=_snake_case       ,      default='''train'''       ,      choices=['''train''', '''test''', '''validation''']	)
      parser.add_argument(
          '''--limit'''       ,      default=_snake_case       ,      type=_snake_case       ,      help='''Limit the number of shards (used for debugging).'''       ,      )
      parser.add_argument(
          '''--max_length'''       ,      type=_snake_case       ,      default=512       ,      help='''Maximum sequence length. For training on TPUs, it helps to have a maximum'''
          ''' sequence length that is a multiple of 8.'''       ,      )
      parser.add_argument(
          '''--output_dir'''       ,      default='''tf-tpu'''       ,      type=_snake_case       ,      help='''Output directory where the TFRecord shards will be saved. If the'''
          ''' path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord'''
          ''' shards will be directly saved to a Google Cloud Storage bucket.'''       ,      )

      lowerCAmelCase    :		Optional[Any]				=      parser.parse_args()
      return args


def    _snake_case  (  _snake_case :	Optional[Any]	):
      def fn(_snake_case :	Optional[int]	):
            return tokenizer(examples['''text''']	)

      return fn


def    _snake_case  (  _snake_case :	List[Any]	):
      lowerCAmelCase    :		Optional[Any]				=      []
      for i in range(len(tokenized_data['''input_ids''']	)	):
            lowerCAmelCase    :		Optional[int]				=      {
                '''input_ids''': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data['''input_ids'''][i]	)	),
                '''attention_mask''': tf.train.Feature(
                    intaa_list=tf.train.IntaaList(value=tokenized_data['''attention_mask'''][i]	)	),
            }
            lowerCAmelCase    :		Optional[Any]				=      tf.train.Features(feature=_snake_case	)
            lowerCAmelCase    :		int				=      tf.train.Example(features=_snake_case	)
            lowerCAmelCase    :		Dict				=      example.SerializeToString()
            records.append(_snake_case	)
      return records


def    _snake_case  (  _snake_case :	List[Any]	):
      lowerCAmelCase    :		str				=      datasets.load_dataset(args.dataset_name       ,      args.dataset_config       ,      split=args.split	)

      if args.limit is not None:
            lowerCAmelCase    :		Dict				=      min(len(_snake_case	)       ,      args.limit	)
            lowerCAmelCase    :		List[Any]				=      dataset.select(range(_snake_case	)	)
            print(f'''Limiting the dataset to {args.limit} entries.'''	)

      lowerCAmelCase    :		Any				=      AutoTokenizer.from_pretrained(args.tokenizer_name_or_path	)

      # Handle output directory creation.
      # For serializing into a Google Cloud Storage Bucket, one needs to first
      # create a bucket.
      if "gs" not in args.output_dir:
            if not os.path.exists(args.output_dir	):
                  os.makedirs(args.output_dir	)
            lowerCAmelCase    :		Union[str, Any]				=      os.path.join(args.output_dir       ,      args.split	)
            if not os.path.exists(_snake_case	):
                  os.makedirs(_snake_case	)
      else:
            lowerCAmelCase    :		int				=      os.path.join(args.output_dir       ,      args.split	)

      # Tokenize the whole dataset at once.
      lowerCAmelCase    :		Optional[int]				=      tokenize_function(_snake_case	)
      lowerCAmelCase    :		Tuple				=      dataset.map(_snake_case       ,      batched=_snake_case       ,      num_proc=4       ,      remove_columns=['''text''']	)

      # We need to concatenate all our texts together, and then split the result
      # into chunks of a fixed size, which we will call block_size. To do this, we
      # will use the map method again, with the option batched=True. When we use batched=True,
      # the function we pass to map() will be passed multiple inputs at once, allowing us
      # to group them into more or fewer examples than we had in the input.
      # This allows us to create our new fixed-length samples. The advantage of this
      # method is that we don't lose a whole lot of content from the dataset compared to the
      # case where we simply tokenize with a pre-defined max_length.

      def group_texts(_snake_case :	Tuple	):
            # Concatenate all texts.
            lowerCAmelCase    :		int				=      {k: sum(examples[k]       ,      []	) for k in examples.keys()}
            lowerCAmelCase    :		int				=      len(concatenated_examples[list(examples.keys()	)[0]]	)
            # We drop the small remainder, though you could add padding instead if the model supports it
            # In this, as in all things, we advise you to follow your heart 🫀
            lowerCAmelCase    :		Optional[Any]				=      (total_length // args.max_length) * args.max_length
            # Split by chunks of max_len.
            lowerCAmelCase    :		Any				=      {
                k: [t[i : i + args.max_length] for i in range(0       ,      _snake_case       ,      args.max_length	)]
                for k, t in concatenated_examples.items()
            }
            return result

      lowerCAmelCase    :		Tuple				=      dataset_tokenized.map(_snake_case       ,      batched=_snake_case       ,      batch_size=1000       ,      num_proc=4	)

      lowerCAmelCase    :		str				=      0
      lowerCAmelCase    :		Any				=      0
      for shard in range(0       ,      len(_snake_case	)       ,      args.shard_size	):
            lowerCAmelCase    :		Dict				=      grouped_dataset[shard : shard + args.shard_size]
            lowerCAmelCase    :		List[str]				=      len(dataset_snapshot['''input_ids''']	)
            lowerCAmelCase    :		List[str]				=      os.path.join(_snake_case       ,      f'''dataset-{shard_count}-{records_containing}.tfrecord'''	)
            lowerCAmelCase    :		Union[str, Any]				=      get_serialized_examples(_snake_case	)

            with tf.io.TFRecordWriter(_snake_case	) as out_file:
                  for i in range(len(_snake_case	)	):
                        lowerCAmelCase    :		Dict				=      serialized_examples[i]
                        out_file.write(_snake_case	)
                  print('''Wrote file {} containing {} records'''.format(_snake_case       ,      _snake_case	)	)

            shard_count += 1
            total_records += records_containing

      with open(f'''split-{args.split}-records-count.txt'''       ,      '''w'''	) as f:
            print(f'''Total {args.split} records: {total_records}'''       ,      file=_snake_case	)


if __name__ == "__main__":
 snake_case__						:   str			 = parse_args()
 main(args)








import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Any			,			__UpperCAmelCase						:	str=False							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			[]
      for i in range(config.num_hidden_layers							):
            # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
            rename_keys.append((f"module.blocks.{i}.norm1.weight", f"vit.encoder.layer.{i}.layernorm_before.weight")							)
            rename_keys.append((f"module.blocks.{i}.norm1.bias", f"vit.encoder.layer.{i}.layernorm_before.bias")							)
            rename_keys.append(
                (f"module.blocks.{i}.attn.proj.weight", f"vit.encoder.layer.{i}.attention.output.dense.weight")							)
            rename_keys.append((f"module.blocks.{i}.attn.proj.bias", f"vit.encoder.layer.{i}.attention.output.dense.bias")							)
            rename_keys.append((f"module.blocks.{i}.norm2.weight", f"vit.encoder.layer.{i}.layernorm_after.weight")							)
            rename_keys.append((f"module.blocks.{i}.norm2.bias", f"vit.encoder.layer.{i}.layernorm_after.bias")							)
            rename_keys.append((f"module.blocks.{i}.mlp.fc1.weight", f"vit.encoder.layer.{i}.intermediate.dense.weight")							)
            rename_keys.append((f"module.blocks.{i}.mlp.fc1.bias", f"vit.encoder.layer.{i}.intermediate.dense.bias")							)
            rename_keys.append((f"module.blocks.{i}.mlp.fc2.weight", f"vit.encoder.layer.{i}.output.dense.weight")							)
            rename_keys.append((f"module.blocks.{i}.mlp.fc2.bias", f"vit.encoder.layer.{i}.output.dense.bias")							)

      # projection layer + position embeddings
      rename_keys.extend(
          [
              ('module.cls_token', 'vit.embeddings.cls_token'),
              ('module.patch_embed.proj.weight', 'vit.embeddings.patch_embeddings.projection.weight'),
              ('module.patch_embed.proj.bias', 'vit.embeddings.patch_embeddings.projection.bias'),
              ('module.pos_embed', 'vit.embeddings.position_embeddings'),
          ]							)

      if base_model:
            # layernorm + pooler
            rename_keys.extend(
                [
                    ('module.norm.weight', 'layernorm.weight'),
                    ('module.norm.bias', 'layernorm.bias'),
                ]							)

            # if just the base model, we should remove "vit" from all keys that start with "vit"
            SCREAMING_SNAKE_CASE_							=			[(pair[0], pair[1][4:]) if pair[1].startswith('vit'							) else pair for pair in rename_keys]
      else:
            # layernorm + classification head
            rename_keys.extend(
                [
                    ('norm.weight', 'vit.layernorm.weight'),
                    ('norm.bias', 'vit.layernorm.bias'),
                    ('head.weight', 'classifier.weight'),
                    ('head.bias', 'classifier.bias'),
                ]							)

      return rename_keys

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Union[str, Any]			,			__UpperCAmelCase						:	Union[str, Any]			,			__UpperCAmelCase						:	Optional[Any]=False							)					-> Any:
      for i in range(config.num_hidden_layers							):
            if base_model:
                  SCREAMING_SNAKE_CASE_							=			''
            else:
                  SCREAMING_SNAKE_CASE_							=			'vit.'
            # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
            SCREAMING_SNAKE_CASE_							=			state_dict.pop(f"module.blocks.{i}.attn.qkv.weight"							)
            SCREAMING_SNAKE_CASE_							=			state_dict.pop(f"module.blocks.{i}.attn.qkv.bias"							)
            # next, add query, keys and values (in that order) to the state dict
            SCREAMING_SNAKE_CASE_							=			in_proj_weight[
                : config.hidden_size, :
            ]
            SCREAMING_SNAKE_CASE_							=			in_proj_bias[: config.hidden_size]
            SCREAMING_SNAKE_CASE_							=			in_proj_weight[
                config.hidden_size : config.hidden_size * 2, :
            ]
            SCREAMING_SNAKE_CASE_							=			in_proj_bias[
                config.hidden_size : config.hidden_size * 2
            ]
            SCREAMING_SNAKE_CASE_							=			in_proj_weight[
                -config.hidden_size :, :
            ]
            SCREAMING_SNAKE_CASE_							=			in_proj_bias[-config.hidden_size :]

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	List[Any]							)					-> Union[str, Any]:
      SCREAMING_SNAKE_CASE_							=			['head.weight', 'head.bias']
      for k in ignore_keys:
            state_dict.pop(__UpperCAmelCase			,			__UpperCAmelCase							)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[int]							)					-> str:
      # projection head is used in the self-supervised pre-training in MSN,
      # for downstream task it's not needed.
      SCREAMING_SNAKE_CASE_							=			[
          'module.fc.fc1.weight',
          'module.fc.fc1.bias',
          'module.fc.bn1.weight',
          'module.fc.bn1.bias',
          'module.fc.bn1.running_mean',
          'module.fc.bn1.running_var',
          'module.fc.bn1.num_batches_tracked',
          'module.fc.fc2.weight',
          'module.fc.fc2.bias',
          'module.fc.bn2.weight',
          'module.fc.bn2.bias',
          'module.fc.bn2.running_mean',
          'module.fc.bn2.running_var',
          'module.fc.bn2.num_batches_tracked',
          'module.fc.fc3.weight',
          'module.fc.fc3.bias',
      ]
      for k in ignore_keys:
            state_dict.pop(__UpperCAmelCase			,			__UpperCAmelCase							)

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Dict			,			__UpperCAmelCase						:	Any			,			__UpperCAmelCase						:	Any							)					-> Dict:
      SCREAMING_SNAKE_CASE_							=			dct.pop(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_							=			val

def 					UpperCAmelCase_							(      __UpperCAmelCase						:	Optional[int]			,			__UpperCAmelCase						:	Tuple							)					-> List[str]:
      SCREAMING_SNAKE_CASE_							=			ViTMSNConfig()
      SCREAMING_SNAKE_CASE_							=			10_00

      SCREAMING_SNAKE_CASE_							=			'datasets/huggingface/label-files'
      SCREAMING_SNAKE_CASE_							=			'imagenet-1k-id2label.json'
      SCREAMING_SNAKE_CASE_							=			json.load(open(hf_hub_download(__UpperCAmelCase			,			__UpperCAmelCase							)			,			'r'							)							)
      SCREAMING_SNAKE_CASE_							=			{int(__UpperCAmelCase							): v for k, v in idalabel.items()}
      SCREAMING_SNAKE_CASE_							=			idalabel
      SCREAMING_SNAKE_CASE_							=			{v: k for k, v in idalabel.items()}

      if "s16" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			3_84
            SCREAMING_SNAKE_CASE_							=			15_36
            SCREAMING_SNAKE_CASE_							=			6
      elif "l16" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			10_24
            SCREAMING_SNAKE_CASE_							=			40_96
            SCREAMING_SNAKE_CASE_							=			24
            SCREAMING_SNAKE_CASE_							=			16
            SCREAMING_SNAKE_CASE_							=			0.1
      elif "b4" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			4
      elif "l7" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			7
            SCREAMING_SNAKE_CASE_							=			10_24
            SCREAMING_SNAKE_CASE_							=			40_96
            SCREAMING_SNAKE_CASE_							=			24
            SCREAMING_SNAKE_CASE_							=			16
            SCREAMING_SNAKE_CASE_							=			0.1

      SCREAMING_SNAKE_CASE_							=			ViTMSNModel(__UpperCAmelCase							)

      SCREAMING_SNAKE_CASE_							=			torch.hub.load_state_dict_from_url(__UpperCAmelCase			,			map_location='cpu'							)['target_encoder']

      SCREAMING_SNAKE_CASE_							=			ViTImageProcessor(size=config.image_size							)

      remove_projection_head(__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_							=			create_rename_keys(__UpperCAmelCase			,			base_model=__UpperCAmelCase							)

      for src, dest in rename_keys:
            rename_key(__UpperCAmelCase			,			__UpperCAmelCase			,			__UpperCAmelCase							)
      read_in_q_k_v(__UpperCAmelCase			,			__UpperCAmelCase			,			base_model=__UpperCAmelCase							)

      model.load_state_dict(__UpperCAmelCase							)
      model.eval()

      SCREAMING_SNAKE_CASE_							=			'http://images.cocodataset.org/val2017/000000039769.jpg'

      SCREAMING_SNAKE_CASE_							=			Image.open(requests.get(__UpperCAmelCase			,			stream=__UpperCAmelCase							).raw							)
      SCREAMING_SNAKE_CASE_							=			ViTImageProcessor(
          size=config.image_size			,			image_mean=__UpperCAmelCase			,			image_std=__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_							=			image_processor(images=__UpperCAmelCase			,			return_tensors='pt'							)

      # forward pass
      torch.manual_seed(2							)
      SCREAMING_SNAKE_CASE_							=			model(**__UpperCAmelCase							)
      SCREAMING_SNAKE_CASE_							=			outputs.last_hidden_state

      # The following Colab Notebook was used to generate these outputs:
      # https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
      if "s16" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			torch.tensor([[-1.0_9_1_5, -1.4_8_7_6, -1.1_8_0_9]]							)
      elif "b16" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			torch.tensor([[1_4.2_8_8_9, -1_8.9_0_4_5, 1_1.7_2_8_1]]							)
      elif "l16" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			torch.tensor([[4_1.5_0_2_8, -2_2.8_6_8_1, 4_5.6_4_7_5]]							)
      elif "b4" in checkpoint_url:
            SCREAMING_SNAKE_CASE_							=			torch.tensor([[-4.3_8_6_8, 5.2_9_3_2, -0.4_1_3_7]]							)
      else:
            SCREAMING_SNAKE_CASE_							=			torch.tensor([[-0.1_7_9_2, -0.6_4_6_5, 2.4_2_6_3]]							)

      # verify logits
      assert torch.allclose(last_hidden_state[:, 0, :3]			,			__UpperCAmelCase			,			atol=1E-4							)

      print(f"Saving model to {pytorch_dump_folder_path}"							)
      model.save_pretrained(__UpperCAmelCase							)

      print(f"Saving image processor to {pytorch_dump_folder_path}"							)
      image_processor.save_pretrained(__UpperCAmelCase							)


if __name__ == "__main__":
   lowerCamelCase__ :   int					=							argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--checkpoint_url',
       default='https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar',
       type=str,
       help='URL of the checkpoint you\'d like to convert.',
   )
   parser.add_argument(
       '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
   )

   lowerCamelCase__ :   Union[str, Any]					=							parser.parse_args()
   convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)