



'''simple docstring'''




import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


a   :			List[Any]					  =    logging.getLogger(__name__)



def       __lowerCamelCase  (					)      ->			Dict:
							UpperCAmelCase     :		Tuple							     =		argparse.ArgumentParser(
							    description="""Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."""		)
							parser.add_argument(
							    """--dataset_name"""	,				type=lowerCAmelCase__	,				default="""wikitext"""	,				help="""Name of the training. Explore datasets at: hf.co/datasets."""	,				)
							parser.add_argument(
							    """--dataset_config"""	,				type=lowerCAmelCase__	,				default="""wikitext-103-raw-v1"""	,				help="""Configuration name of the dataset."""		)
							parser.add_argument(
							    """--tokenizer_name_or_path"""	,				type=lowerCAmelCase__	,				default="""sayakpaul/unigram-tokenizer-wikitext"""	,				help="""Tokenizer identifier. Can be a local filepath or a Hub identifier."""	,				)
							parser.add_argument(
							    """--shard_size"""	,				type=lowerCAmelCase__	,				default=1_0_0_0	,				help="""Number of entries to go in a single shard."""	,				)
							parser.add_argument("""--split"""	,				type=lowerCAmelCase__	,				default="""train"""	,				choices=["""train""", """test""", """validation"""]		)
							parser.add_argument(
							    """--limit"""	,				default=lowerCAmelCase__	,				type=lowerCAmelCase__	,				help="""Limit the number of shards (used for debugging)."""	,				)
							parser.add_argument(
							    """--max_length"""	,				type=lowerCAmelCase__	,				default=5_1_2	,				help="""Maximum sequence length. For training on TPUs, it helps to have a maximum"""
							    """ sequence length that is a multiple of 8."""	,				)
							parser.add_argument(
							    """--output_dir"""	,				default="""tf-tpu"""	,				type=lowerCAmelCase__	,				help="""Output directory where the TFRecord shards will be saved. If the"""
							    """ path is appended with `gs://` (\'gs://tf-tpu\', for example) then the TFRecord"""
							    """ shards will be directly saved to a Google Cloud Storage bucket."""	,				)

							UpperCAmelCase     :		List[Any]							     =		parser.parse_args()
							return args



def       __lowerCamelCase  (					_lowercase		)      ->			Union[str, Any]:




							def fn(_lowercase		):
														return tokenizer(examples["""text"""]		)

							return fn



def       __lowerCamelCase  (					_lowercase		)      ->			Dict:
							UpperCAmelCase     :		int							     =		[]
							for i in range(len(tokenized_data["""input_ids"""]		)		):
														UpperCAmelCase     :		Tuple							     =		{
														    'input_ids': tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["""input_ids"""][i]		)		),
														    'attention_mask': tf.train.Feature(
														        intaa_list=tf.train.IntaaList(value=tokenized_data["""attention_mask"""][i]		)		),
														}
														UpperCAmelCase     :		Union[str, Any]							     =		tf.train.Features(feature=lowerCAmelCase__		)
														UpperCAmelCase     :		Dict							     =		tf.train.Example(features=lowerCAmelCase__		)
														UpperCAmelCase     :		Optional[int]							     =		example.SerializeToString()
														records.append(lowerCAmelCase__		)
							return records



def       __lowerCamelCase  (					_lowercase		)      ->			Optional[int]:
							UpperCAmelCase     :		str							     =		datasets.load_dataset(args.dataset_name	,				args.dataset_config	,				split=args.split		)

							if args.limit is not None:
														UpperCAmelCase     :		Tuple							     =		min(len(lowerCAmelCase__		)	,				args.limit		)
														UpperCAmelCase     :		Any							     =		dataset.select(range(lowerCAmelCase__		)		)
														print(F'''Limiting the dataset to {args.limit} entries.'''		)

							UpperCAmelCase     :		List[str]							     =		AutoTokenizer.from_pretrained(args.tokenizer_name_or_path		)

							# Handle output directory creation.
							# For serializing into a Google Cloud Storage Bucket, one needs to first
							# create a bucket.
							if "gs" not in args.output_dir:
														if not os.path.exists(args.output_dir		):
																					os.makedirs(args.output_dir		)
														UpperCAmelCase     :		int							     =		os.path.join(args.output_dir	,				args.split		)
														if not os.path.exists(lowerCAmelCase__		):
																					os.makedirs(lowerCAmelCase__		)
							else:
														UpperCAmelCase     :		Dict							     =		os.path.join(args.output_dir	,				args.split		)

							# Tokenize the whole dataset at once.
							UpperCAmelCase     :		Dict							     =		tokenize_function(lowerCAmelCase__		)
							UpperCAmelCase     :		List[Any]							     =		dataset.map(lowerCAmelCase__	,				batched=lowerCAmelCase__	,				num_proc=4	,				remove_columns=["""text"""]		)

							# We need to concatenate all our texts together, and then split the result
							# into chunks of a fixed size, which we will call block_size. To do this, we
							# will use the map method again, with the option batched=True. When we use batched=True,
							# the function we pass to map() will be passed multiple inputs at once, allowing us
							# to group them into more or fewer examples than we had in the input.
							# This allows us to create our new fixed-length samples. The advantage of this
							# method is that we don't lose a whole lot of content from the dataset compared to the
							# case where we simply tokenize with a pre-defined max_length.

							def group_texts(_lowercase		):
														# Concatenate all texts.
														UpperCAmelCase     :		int							     =		{k: sum(examples[k]	,				[]		) for k in examples.keys()}
														UpperCAmelCase     :		Any							     =		len(concatenated_examples[list(examples.keys()		)[0]]		)
														# We drop the small remainder, though you could add padding instead if the model supports it
														# In this, as in all things, we advise you to follow your heart 🫀
														UpperCAmelCase     :		Union[str, Any]							     =		(total_length // args.max_length) * args.max_length
														# Split by chunks of max_len.
														UpperCAmelCase     :		Optional[Any]							     =		{
														    k: [t[i : i + args.max_length] for i in range(0	,				lowerCAmelCase__	,				args.max_length		)]
														    for k, t in concatenated_examples.items()
														}
														return result

							UpperCAmelCase     :		Optional[int]							     =		dataset_tokenized.map(lowerCAmelCase__	,				batched=lowerCAmelCase__	,				batch_size=1_0_0_0	,				num_proc=4		)

							UpperCAmelCase     :		Optional[int]							     =		0
							UpperCAmelCase     :		Optional[Any]							     =		0
							for shard in range(0	,				len(lowerCAmelCase__		)	,				args.shard_size		):
														UpperCAmelCase     :		Dict							     =		grouped_dataset[shard : shard + args.shard_size]
														UpperCAmelCase     :		Tuple							     =		len(dataset_snapshot["""input_ids"""]		)
														UpperCAmelCase     :		Optional[Any]							     =		os.path.join(lowerCAmelCase__	,				F'''dataset-{shard_count}-{records_containing}.tfrecord'''		)
														UpperCAmelCase     :		Tuple							     =		get_serialized_examples(lowerCAmelCase__		)

														with tf.io.TFRecordWriter(lowerCAmelCase__		) as out_file:
																					for i in range(len(lowerCAmelCase__		)		):
																												UpperCAmelCase     :		Dict							     =		serialized_examples[i]
																												out_file.write(lowerCAmelCase__		)
																					print("""Wrote file {} containing {} records""".format(lowerCAmelCase__	,				lowerCAmelCase__		)		)

														shard_count += 1
														total_records += records_containing

							with open(F'''split-{args.split}-records-count.txt'''	,				"""w"""		) as f:
														print(F'''Total {args.split} records: {total_records}'''	,				file=lowerCAmelCase__		)


if __name__ == "__main__":
							a   :			int					  =    parse_args()
							main(args)






"""simple docstring"""







from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
     from PIL import Image

     from transformers import AutoImageProcessor







class       UpperCamelCase__   :




      """simple docstring"""




      def __init__(      self				:      Dict    ,			SCREAMING_SNAKE_CASE_				:      str    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=3    ,			SCREAMING_SNAKE_CASE_				:      Any=3_2    ,			SCREAMING_SNAKE_CASE_				:      int=3    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=1_0    ,			SCREAMING_SNAKE_CASE_				:      Dict=[1_0, 2_0, 3_0, 4_0]    ,			SCREAMING_SNAKE_CASE_				:      int=[1, 1, 2, 1]    ,			SCREAMING_SNAKE_CASE_				:      int=True    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]=True    ,			SCREAMING_SNAKE_CASE_				:      int="relu"    ,			SCREAMING_SNAKE_CASE_				:      Union[str, Any]=3    ,			SCREAMING_SNAKE_CASE_				:      List[str]=None    ,			):
        lowerCAmelCase_   :				Optional[int]								= parent
        lowerCAmelCase_   :				Union[str, Any]								= batch_size
        lowerCAmelCase_   :				Optional[int]								= image_size
        lowerCAmelCase_   :				Optional[int]								= num_channels
        lowerCAmelCase_   :				Any								= embeddings_size
        lowerCAmelCase_   :				Dict								= hidden_sizes
        lowerCAmelCase_   :				Any								= depths
        lowerCAmelCase_   :				Optional[int]								= is_training
        lowerCAmelCase_   :				int								= use_labels
        lowerCAmelCase_   :				List[Any]								= hidden_act
        lowerCAmelCase_   :				Dict								= num_labels
        lowerCAmelCase_   :				Optional[int]								= scope
        lowerCAmelCase_   :				Tuple								= len(SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any       ):
        lowerCAmelCase_   :				List[str]								= floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]       )

        lowerCAmelCase_   :				Union[str, Any]								= None
        if self.use_labels:
          lowerCAmelCase_   :				int								= ids_tensor([self.batch_size]    ,			self.num_labels       )

        lowerCAmelCase_   :				List[str]								= self.get_config()
        return config, pixel_values, labels



      def      SCREAMING_SNAKE_CASE__ (      self				:      str       ):
        return RegNetConfig(
            num_channels=self.num_channels    ,			embeddings_size=self.embeddings_size    ,			hidden_sizes=self.hidden_sizes    ,			depths=self.depths    ,			hidden_act=self.hidden_act    ,			num_labels=self.num_labels    ,			)



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any    ,			SCREAMING_SNAKE_CASE_				:      List[str]    ,			SCREAMING_SNAKE_CASE_				:      List[Any]    ,			SCREAMING_SNAKE_CASE_				:      int       ):
        lowerCAmelCase_   :				List[Any]								= TFRegNetModel(config=SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				List[Any]								= model(SCREAMING_SNAKE_CASE_    ,			training=SCREAMING_SNAKE_CASE_       )
        # expected last hidden states: B, C, H // 32, W // 32
        self.parent.assertEqual(
            result.last_hidden_state.shape    ,			(self.batch_size, self.hidden_sizes[-1], self.image_size // 3_2, self.image_size // 3_2)    ,			)



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]    ,			SCREAMING_SNAKE_CASE_				:      Tuple    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			SCREAMING_SNAKE_CASE_				:      Optional[Any]       ):
        lowerCAmelCase_   :				int								= self.num_labels
        lowerCAmelCase_   :				int								= TFRegNetForImageClassification(SCREAMING_SNAKE_CASE_       )
        lowerCAmelCase_   :				Dict								= model(SCREAMING_SNAKE_CASE_    ,			labels=SCREAMING_SNAKE_CASE_    ,			training=SCREAMING_SNAKE_CASE_       )
        self.parent.assertEqual(result.logits.shape    ,			(self.batch_size, self.num_labels)       )





      def      SCREAMING_SNAKE_CASE__ (      self				:      List[str]       ):
        lowerCAmelCase_   :				str								= self.prepare_config_and_inputs()
        lowerCAmelCase_						,lowerCAmelCase_						,lowerCAmelCase_   :				Tuple								= config_and_inputs
        lowerCAmelCase_   :				Optional[Any]								= {'pixel_values': pixel_values}
        return config, inputs_dict







@require_tf
class       UpperCamelCase__   (	lowercase_,					lowercase_,					unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE  				=     (TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
      _SCREAMING_SNAKE_CASE  				=     (
          {"""feature-extraction""": TFRegNetModel, """image-classification""": TFRegNetForImageClassification}
          if is_tf_available()
          else {}
      )

      _SCREAMING_SNAKE_CASE  				=     False
      _SCREAMING_SNAKE_CASE  				=     False
      _SCREAMING_SNAKE_CASE  				=     False
      _SCREAMING_SNAKE_CASE  				=     False
      _SCREAMING_SNAKE_CASE  				=     False



      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				List[Any]								= TFRegNetModelTester(self       )
        lowerCAmelCase_   :				Tuple								= ConfigTester(self    ,			config_class=SCREAMING_SNAKE_CASE_    ,			has_text_modality=SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        return



      @unittest.skip(reason='RegNet does not use inputs_embeds'       )
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        pass



      @unittest.skipIf(
          not is_tf_available() or len(tf.config.list_physical_devices('GPU'       )       ) == 0    ,			reason='TF does not support backprop for grouped convolutions on CPU.'    ,			)
      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Tuple       ):
        super().test_keras_fit()



      @unittest.skip(reason='RegNet does not support input and output embeddings'       )
      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        pass



      def      SCREAMING_SNAKE_CASE__ (      self				:      int       ):
        lowerCAmelCase_						,lowerCAmelCase_   :				Optional[int]								= self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          lowerCAmelCase_   :				int								= model_class(SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Dict								= inspect.signature(model.call       )
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          lowerCAmelCase_   :				List[Any]								= [*signature.parameters.keys()]

          lowerCAmelCase_   :				Any								= ['pixel_values']
          self.assertListEqual(arg_names[:1]    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Any       ):
        lowerCAmelCase_   :				Dict								= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[int]       ):
        def check_hidden_states_output(SCREAMING_SNAKE_CASE_				:      Tuple    ,			SCREAMING_SNAKE_CASE_				:      int    ,			SCREAMING_SNAKE_CASE_				:      Tuple       ):
          lowerCAmelCase_   :				List[Any]								= model_class(SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Union[str, Any]								= model(**self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )    ,			training=SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				List[str]								= outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

          lowerCAmelCase_   :				Union[str, Any]								= self.model_tester.num_stages
          self.assertEqual(len(SCREAMING_SNAKE_CASE_       )    ,			expected_num_stages + 1       )

          # RegNet's feature maps are of shape (batch_size, num_channels, height, width)
          self.assertListEqual(
              list(hidden_states[0].shape[-2:]       )    ,			[self.model_tester.image_size // 2, self.model_tester.image_size // 2]    ,			)

        lowerCAmelCase_						,lowerCAmelCase_   :				List[Any]								= self.model_tester.prepare_config_and_inputs_for_common()
        lowerCAmelCase_   :				Optional[int]								= ['basic', 'bottleneck']
        for model_class in self.all_model_classes:
          for layer_type in layers_type:
            lowerCAmelCase_   :				str								= layer_type
            lowerCAmelCase_   :				List[str]								= True
            check_hidden_states_output(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

            # check that output_hidden_states also work using config
            del inputs_dict["output_hidden_states"]
            lowerCAmelCase_   :				List[str]								= True

            check_hidden_states_output(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Optional[Any]       ):
        lowerCAmelCase_						,lowerCAmelCase_   :				int								= self.model_tester.prepare_config_and_inputs_for_common()

        def check_equivalence(SCREAMING_SNAKE_CASE_				:      Dict    ,			SCREAMING_SNAKE_CASE_				:      str    ,			SCREAMING_SNAKE_CASE_				:      Optional[int]    ,			SCREAMING_SNAKE_CASE_				:      str={}       ):
          lowerCAmelCase_   :				Optional[Any]								= model(SCREAMING_SNAKE_CASE_    ,			return_dict=SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				List[Any]								= model(SCREAMING_SNAKE_CASE_    ,			return_dict=SCREAMING_SNAKE_CASE_    ,			**SCREAMING_SNAKE_CASE_       ).to_tuple()

          def recursive_check(SCREAMING_SNAKE_CASE_				:      int    ,			SCREAMING_SNAKE_CASE_				:      int       ):
            if isinstance(SCREAMING_SNAKE_CASE_    ,			(List, Tuple)       ):
              for tuple_iterable_value, dict_iterable_value in zip(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       ):
                recursive_check(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
            elif tuple_object is None:
              return
            else:
              self.assertTrue(
                  all(tf.equal(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )       )    ,			msg=(
                      'Tuple and dict output are not equal. Difference:'
                      F" {tf.math.reduce_max(tf.abs(tuple_object - dict_object       )       )}"
                  )    ,			)

            recursive_check(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

        for model_class in self.all_model_classes:
          lowerCAmelCase_   :				Union[str, Any]								= model_class(SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				List[Any]								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Union[str, Any]								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          check_equivalence(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				Optional[int]								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			return_labels=SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				str								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			return_labels=SCREAMING_SNAKE_CASE_       )
          check_equivalence(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )

          lowerCAmelCase_   :				Tuple								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				int								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_       )
          check_equivalence(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			{'output_hidden_states': True}       )

          lowerCAmelCase_   :				List[Any]								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			return_labels=SCREAMING_SNAKE_CASE_       )
          lowerCAmelCase_   :				Any								= self._prepare_for_class(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			return_labels=SCREAMING_SNAKE_CASE_       )
          check_equivalence(SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			SCREAMING_SNAKE_CASE_    ,			{'output_hidden_states': True}       )



      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        lowerCAmelCase_   :				List[str]								= self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*SCREAMING_SNAKE_CASE_       )





      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase_   :				int								= TFRegNetModel.from_pretrained(SCREAMING_SNAKE_CASE_       )
          self.assertIsNotNone(SCREAMING_SNAKE_CASE_       )




def 				UpperCamelCase_    (    )						->   List[str]:

  """simple docstring"""




  lowerCAmelCase_   :				Optional[Any]								= Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'						)
  return image







@require_tf
@require_vision
class       UpperCamelCase__   (	unittest.TestCase  ):




      """simple docstring"""




      @cached_property
      def      SCREAMING_SNAKE_CASE__ (      self				:      Union[str, Any]       ):
        return (
            AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]       )
            if is_vision_available()
            else None
        )





      @slow
      def      SCREAMING_SNAKE_CASE__ (      self				:      List[Any]       ):
        lowerCAmelCase_   :				int								= TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]       )

        lowerCAmelCase_   :				Tuple								= self.default_image_processor
        lowerCAmelCase_   :				Dict								= prepare_img()
        lowerCAmelCase_   :				Union[str, Any]								= image_processor(images=SCREAMING_SNAKE_CASE_    ,			return_tensors='tf'       )

        # forward pass
        lowerCAmelCase_   :				List[str]								= model(**SCREAMING_SNAKE_CASE_    ,			training=SCREAMING_SNAKE_CASE_       )

        # verify the logits
        lowerCAmelCase_   :				Optional[Any]								= tf.TensorShape((1, 1_0_0_0)       )
        self.assertEqual(outputs.logits.shape    ,			SCREAMING_SNAKE_CASE_       )

        lowerCAmelCase_   :				List[Any]								= tf.constant([-0.41_80, -1.50_51, -3.48_36]       )

        tf.debugging.assert_near(outputs.logits[0, :3]    ,			SCREAMING_SNAKE_CASE_    ,			atol=1E-4       )





