code: 
  
  
  
  
  
  import logging
  import os
  from dataclasses import dataclass, field
  from importlib import import_module
  from typing import Dict, List, Optional, Tuple
  
  import numpy as np
  from seqeval.metrics import classification_report, fa_score, precision_score, recall_score
  from utils_ner import Split, TFTokenClassificationDataset, TokenClassificationTask
  
  from transformers import (
      AutoConfig,
      AutoTokenizer,
      EvalPrediction,
      HfArgumentParser,
      TFAutoModelForTokenClassification,
      TFTrainer,
      TFTrainingArguments,
  )
  from transformers.utils import logging as hf_logging
  
  
  hf_logging.set_verbosity_info()
  hf_logging.enable_default_handler()
  hf_logging.enable_explicit_format()
  
  
  __lowerCAmelCase     =			logging.getLogger(__name__)
  
  @dataclass
  class   _UpperCAmelCase	:
  					__A          =   field(
  					    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}					)
  					__A          =   field(
  					    default=_lowercase ,       metadata={"help": "Pretrained config name or path if not the same as model_name"}					)
  					__A          =   field(
  					    default="NER" ,       metadata={"help": "Task type to fine tune in training (e.g. NER, POS, etc)"}					)
  					__A          =   field(
  					    default=_lowercase ,       metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}					)
  					__A          =   field(default=_lowercase ,       metadata={"help": "Set this flag to use fast tokenization."}					)
  					# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
  					# or just modify its tokenizer_config.json.
  					__A          =   field(
  					    default=_lowercase ,       metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"} ,       )
  
  
  
  @dataclass
  class   _UpperCAmelCase	:
  					__A          =   field(
  					    metadata={"help": "The input data dir. Should contain the .txt files for a CoNLL-2003-formatted task."}					)
  					__A          =   field(
  					    metadata={"help": "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."}					)
  					__A          =   field(
  					    default=1_28 ,       metadata={
  					        "help": (
  					            "The maximum total input sequence length after tokenization. Sequences longer "
  					            "than this will be truncated, sequences shorter will be padded."
  					        )
  					    } ,       )
  					__A          =   field(
  					    default=_lowercase ,       metadata={"help": "Overwrite the cached training and evaluation sets"}					)
  
  
  
  def     a						(      )     ->Optional[int]:
  		UpperCamelCase__         =	HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)       )
  		UpperCamelCase__         =	parser.parse_args_into_dataclasses()
  
  		if (
  		    os.path.exists(training_args.output_dir       )
  		    and os.listdir(training_args.output_dir       )
  		    and training_args.do_train
  		    and not training_args.overwrite_output_dir
  		):
  				raise ValueError(
  				    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
  				    """ --overwrite_output_dir to overcome."""       )
  
  		UpperCamelCase__         =	import_module("""tasks"""       )
  
  		try:
  				UpperCamelCase__         =	getattr(_UpperCAmelCase ,						model_args.task_type       )
  				UpperCamelCase__         =	token_classification_task_clazz()
  		except AttributeError:
  				raise ValueError(
  				    F"""Task {model_args.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. """
  				    F"""Available tasks classes are: {TokenClassificationTask.__subclasses__()}"""       )
  
  		# Setup logging
  		logging.basicConfig(
  		    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""" ,						datefmt="""%m/%d/%Y %H:%M:%S""" ,						level=logging.INFO ,						)
  		logger.info(
  		    """n_replicas: %s, distributed training: %s, 16-bits training: %s""" ,						training_args.n_replicas ,						bool(training_args.n_replicas > 1       ) ,						training_args.fpaa ,						)
  		logger.info("""Training/evaluation parameters %s""" ,						_UpperCAmelCase       )
  
  		# Prepare Token Classification task
  		UpperCamelCase__         =	token_classification_task.get_labels(data_args.labels       )
  		UpperCamelCase__         =	dict(enumerate(_UpperCAmelCase       )       )
  		UpperCamelCase__         =	len(_UpperCAmelCase       )
  
  		# Load pretrained model and tokenizer
  		#
  		# Distributed training:
  		# The .from_pretrained methods guarantee that only one local process can concurrently
  		# download model & vocab.
  
  		UpperCamelCase__         =	AutoConfig.from_pretrained(
  		    model_args.config_name if model_args.config_name else model_args.model_name_or_path ,						num_labels=_UpperCAmelCase ,						idalabel=_UpperCAmelCase ,						labelaid={label: i for i, label in enumerate(_UpperCAmelCase       )} ,						cache_dir=model_args.cache_dir ,						)
  		UpperCamelCase__         =	AutoTokenizer.from_pretrained(
  		    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path ,						cache_dir=model_args.cache_dir ,						use_fast=model_args.use_fast ,						)
  
  		with training_args.strategy.scope():
  				UpperCamelCase__         =	TFAutoModelForTokenClassification.from_pretrained(
  				    model_args.model_name_or_path ,						from_pt=bool(""".bin""" in model_args.model_name_or_path       ) ,						config=_UpperCAmelCase ,						cache_dir=model_args.cache_dir ,						)
  
  		# Get datasets
  		UpperCamelCase__         =	(
  		    TFTokenClassificationDataset(
  		        token_classification_task=_UpperCAmelCase ,						data_dir=data_args.data_dir ,						tokenizer=_UpperCAmelCase ,						labels=_UpperCAmelCase ,						model_type=config.model_type ,						max_seq_length=data_args.max_seq_length ,						overwrite_cache=data_args.overwrite_cache ,						mode=Split.train ,						)
  		    if training_args.do_train
  		    else None
  		)
  		UpperCamelCase__         =	(
  		    TFTokenClassificationDataset(
  		        token_classification_task=_UpperCAmelCase ,						data_dir=data_args.data_dir ,						tokenizer=_UpperCAmelCase ,						labels=_UpperCAmelCase ,						model_type=config.model_type ,						max_seq_length=data_args.max_seq_length ,						overwrite_cache=data_args.overwrite_cache ,						mode=Split.dev ,						)
  		    if training_args.do_eval
  		    else None
  		)
  
  		def align_predictions(_lowercase   :     List[str] ,						_lowercase   :     Union[str, Any]       ) -> Tuple[List[int], List[int]]:
  				UpperCamelCase__         =	np.argmax(_UpperCAmelCase ,						axis=2       )
  				UpperCamelCase__         =	preds.shape
  				UpperCamelCase__         =	[[] for _ in range(_UpperCAmelCase       )]
  				UpperCamelCase__         =	[[] for _ in range(_UpperCAmelCase       )]
  
  				for i in range(_UpperCAmelCase       ):
  						for j in range(_UpperCAmelCase       ):
  								if label_ids[i, j] != -1_0_0:
  										out_label_list[i].append(label_map[label_ids[i][j]]       )
  										preds_list[i].append(label_map[preds[i][j]]       )
  
  				return preds_list, out_label_list
  
  		def compute_metrics(_lowercase   :     Union[str, Any]       ) -> Dict:
  				UpperCamelCase__         =	align_predictions(p.predictions ,						p.label_ids       )
  
  				return {
  				    "precision": precision_score(_UpperCAmelCase ,						_UpperCAmelCase       ),
  				    "recall": recall_score(_UpperCAmelCase ,						_UpperCAmelCase       ),
  				    "f1": fa_score(_UpperCAmelCase ,						_UpperCAmelCase       ),
  				}
  
  		# Initialize our Trainer
  		UpperCamelCase__         =	TFTrainer(
  		    model=_UpperCAmelCase ,						args=_UpperCAmelCase ,						train_dataset=train_dataset.get_dataset() if train_dataset else None ,						eval_dataset=eval_dataset.get_dataset() if eval_dataset else None ,						compute_metrics=_UpperCAmelCase ,						)
  
  		# Training
  		if training_args.do_train:
  				trainer.train()
  				trainer.save_model()
  				tokenizer.save_pretrained(training_args.output_dir       )
  
  		# Evaluation
  		UpperCamelCase__         =	{}
  		if training_args.do_eval:
  				logger.info("""*** Evaluate ***"""       )
  
  				UpperCamelCase__         =	trainer.evaluate()
  				UpperCamelCase__         =	os.path.join(training_args.output_dir ,						"""eval_results.txt"""       )
  
  				with open(_UpperCAmelCase ,						"""w"""       ) as writer:
  						logger.info("""***** Eval results *****"""       )
  
  						for key, value in result.items():
  								logger.info("""  %s = %s""" ,						_UpperCAmelCase ,						_UpperCAmelCase       )
  								writer.write("""%s = %s\n""" % (key, value)       )
  
  						results.update(_UpperCAmelCase       )
  
      # Predict
  		if training_args.do_predict:
  				UpperCamelCase__         =	TFTokenClassificationDataset(
  				    token_classification_task=_UpperCAmelCase ,						data_dir=data_args.data_dir ,						tokenizer=_UpperCAmelCase ,						labels=_UpperCAmelCase ,						model_type=config.model_type ,						max_seq_length=data_args.max_seq_length ,						overwrite_cache=data_args.overwrite_cache ,						mode=Split.test ,						)
  
  				UpperCamelCase__         =	trainer.predict(test_dataset.get_dataset()       )
  				UpperCamelCase__         =	align_predictions(_UpperCAmelCase ,						_UpperCAmelCase       )
  				UpperCamelCase__         =	classification_report(_UpperCAmelCase ,						_UpperCAmelCase       )
  
  				logger.info("""\n%s""" ,						_UpperCAmelCase       )
  
  				UpperCamelCase__         =	os.path.join(training_args.output_dir ,						"""test_results.txt"""       )
  
  				with open(_UpperCAmelCase ,						"""w"""       ) as writer:
  						writer.write("""%s\n""" % report       )
  
  				# Save predictions
  				UpperCamelCase__         =	os.path.join(training_args.output_dir ,						"""test_predictions.txt"""       )
  
  				with open(_UpperCAmelCase ,						"""w"""       ) as writer:
  						with open(os.path.join(data_args.data_dir ,						"""test.txt"""       ) ,						"""r"""       ) as f:
  								UpperCamelCase__         =	0
  
  								for line in f:
  										if line.startswith("""-DOCSTART-"""       ) or line == "" or line == "\n":
  												writer.write(_UpperCAmelCase       )
  
  												if not preds_list[example_id]:
  														example_id += 1
  										elif preds_list[example_id]:
  												UpperCamelCase__         =	line.split()[0] + ' ' + preds_list[example_id].pop(0       ) + '\n'
  
  												writer.write(_UpperCAmelCase       )
  										else:
  												logger.warning("""Maximum sequence length exceeded: No prediction for \'%s\'.""" ,						line.split()[0]       )
  
  		return results
  
  
  if __name__ == "__main__":
  				main()
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  from typing import Dict, List, Optional, Union
  
  import numpy as np
  
  from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
  from ...image_transforms import (
      get_resize_output_image_size,
      normalize,
      pad,
      rescale,
      resize,
      to_channel_dimension_format,
  )
  from ...image_utils import (
      IMAGENET_STANDARD_MEAN,
      IMAGENET_STANDARD_STD,
      ChannelDimension,
      ImageInput,
      PILImageResampling,
      get_image_size,
      make_list_of_images,
      to_numpy_array,
      valid_images,
  )
  from ...utils import TensorType, logging
  from ...utils.import_utils import is_vision_available
  
  
  __lowerCAmelCase     =			logging.get_logger(__name__)
  
  
  if is_vision_available():
  				import PIL
  
  
  
  
  class   _UpperCAmelCase	(   _lowercase					):
  					__A          =   ["pixel_values"]
  
  
  
  
  					def __init__(						self    ,      __lowerCAmelCase = True    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = PILImageResampling.BILINEAR    ,      __lowerCAmelCase = True    ,      __lowerCAmelCase = False    ,      __lowerCAmelCase = True    ,      __lowerCAmelCase = True    ,      __lowerCAmelCase = 1 / 255    ,      __lowerCAmelCase = True    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      **__lowerCAmelCase    ,      )						->  None:
  							super().__init__(**__lowerCAmelCase       )
  
  							UpperCamelCase__         =	size if size is not None else {"""height""": 2_560, """width""": 1_920}
  							if isinstance(__lowerCAmelCase    ,      (tuple, list)       ):
  									# The previous feature extractor size parameter was in (width, height) format
  									UpperCamelCase__         =	size[::-1]
  							UpperCamelCase__         =	get_size_dict(__lowerCAmelCase       )
  
  							UpperCamelCase__         =	do_resize
  							UpperCamelCase__         =	size
  							UpperCamelCase__         =	resample
  							UpperCamelCase__         =	do_thumbnail
  							UpperCamelCase__         =	do_align_long_axis
  							UpperCamelCase__         =	do_pad
  							UpperCamelCase__         =	do_rescale
  							UpperCamelCase__         =	rescale_factor
  							UpperCamelCase__         =	do_normalize
  							UpperCamelCase__         =	image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
  							UpperCamelCase__         =	image_std if image_std is not None else IMAGENET_STANDARD_STD
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = None       )						->  np.ndarray:
  							UpperCamelCase__  ,			UpperCamelCase__         =	get_image_size(__lowerCAmelCase       )
  							UpperCamelCase__  ,			UpperCamelCase__         =	size["""height"""], size["""width"""]
  
  							if (output_width < output_height and input_width > input_height) or (
  							    output_width > output_height and input_width < input_height
  							):
  									UpperCamelCase__         =	np.rotaa(__lowerCAmelCase    ,      3       )
  
  							if data_format is not None:
  									UpperCamelCase__         =	to_channel_dimension_format(__lowerCAmelCase    ,      __lowerCAmelCase       )
  
  							return image
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = False    ,      __lowerCAmelCase = None    ,      )						->  np.ndarray:
  							UpperCamelCase__  ,			UpperCamelCase__         =	size["""height"""], size["""width"""]
  							UpperCamelCase__  ,			UpperCamelCase__         =	get_image_size(__lowerCAmelCase       )
  
  							UpperCamelCase__         =	output_width - input_width
  							UpperCamelCase__         =	output_height - input_height
  
  							if random_padding:
  									UpperCamelCase__         =	np.random.randint(low=0    ,      high=delta_height + 1       )
  									UpperCamelCase__         =	np.random.randint(low=0    ,      high=delta_width + 1       )
  							else:
  									UpperCamelCase__         =	delta_height // 2
  									UpperCamelCase__         =	delta_width // 2
  
  							UpperCamelCase__         =	delta_height - pad_top
  							UpperCamelCase__         =	delta_width - pad_left
  
  							UpperCamelCase__         =	((pad_top, pad_bottom), (pad_left, pad_right))
  							return pad(__lowerCAmelCase    ,      __lowerCAmelCase    ,      data_format=__lowerCAmelCase       )
  
  
  
  
  					def 	snake_case_		(						self    ,      *__lowerCAmelCase    ,      **__lowerCAmelCase       )						->  List[Any]:
  							logger.info("""pad is deprecated and will be removed in version 4.27. Please use pad_image instead."""       )
  							return self.pad_image(*__lowerCAmelCase    ,      **__lowerCAmelCase       )
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = PILImageResampling.BICUBIC    ,      __lowerCAmelCase = None    ,      **__lowerCAmelCase    ,      )						->  np.ndarray:
  							UpperCamelCase__  ,			UpperCamelCase__         =	get_image_size(__lowerCAmelCase       )
  							UpperCamelCase__  ,			UpperCamelCase__         =	size["""height"""], size["""width"""]
  
  							# We always resize to the smallest of either the input or output size.
  							UpperCamelCase__         =	min(__lowerCAmelCase    ,      __lowerCAmelCase       )
  							UpperCamelCase__         =	min(__lowerCAmelCase    ,      __lowerCAmelCase       )
  
  							if height == input_height and width == input_width:
  									return image
  
  							if input_height > input_width:
  									UpperCamelCase__         =	int(input_width * height / input_height       )
  							elif input_width > input_height:
  									UpperCamelCase__         =	int(input_height * width / input_width       )
  
  							return resize(
  							    __lowerCAmelCase    ,      size=(height, width)    ,      resample=__lowerCAmelCase    ,      reducing_gap=2.0    ,      data_format=__lowerCAmelCase    ,      **__lowerCAmelCase       )
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = PILImageResampling.BICUBIC    ,      __lowerCAmelCase = None    ,      **__lowerCAmelCase    ,      )						->  np.ndarray:
  							UpperCamelCase__         =	get_size_dict(__lowerCAmelCase       )
  							UpperCamelCase__         =	min(size["""height"""]    ,      size["""width"""]       )
  							UpperCamelCase__         =	get_resize_output_image_size(__lowerCAmelCase    ,      size=__lowerCAmelCase    ,      default_to_square=__lowerCAmelCase       )
  							UpperCamelCase__         =	resize(__lowerCAmelCase    ,      size=__lowerCAmelCase    ,      resample=__lowerCAmelCase    ,      data_format=__lowerCAmelCase    ,      **__lowerCAmelCase       )
  							return resized_image
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = None    ,      **__lowerCAmelCase    ,      )						->  List[str]:
  							return rescale(__lowerCAmelCase    ,      scale=__lowerCAmelCase    ,      data_format=__lowerCAmelCase    ,      **__lowerCAmelCase       )
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase    ,      __lowerCAmelCase = None    ,      **__lowerCAmelCase    ,      )						->  np.ndarray:
  							return normalize(__lowerCAmelCase    ,      mean=__lowerCAmelCase    ,      std=__lowerCAmelCase    ,      data_format=__lowerCAmelCase    ,      **__lowerCAmelCase       )
  
  
  
  
  					def 	snake_case_		(						self    ,      __lowerCAmelCase    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = False    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = None    ,      __lowerCAmelCase = ChannelDimension.FIRST    ,      **__lowerCAmelCase    ,      )						->  PIL.Image.Image:
  							UpperCamelCase__         =	do_resize if do_resize is not None else self.do_resize
  							UpperCamelCase__         =	size if size is not None else self.size
  							if isinstance(__lowerCAmelCase    ,      (tuple, list)       ):
  									# Previous feature extractor had size in (width, height) format
  									UpperCamelCase__         =	size[::-1]
  							UpperCamelCase__         =	get_size_dict(__lowerCAmelCase       )
  							UpperCamelCase__         =	resample if resample is not None else self.resample
  							UpperCamelCase__         =	do_thumbnail if do_thumbnail is not None else self.do_thumbnail
  							UpperCamelCase__         =	do_align_long_axis if do_align_long_axis is not None else self.do_align_long_axis
  							UpperCamelCase__         =	do_pad if do_pad is not None else self.do_pad
  							UpperCamelCase__         =	do_rescale if do_rescale is not None else self.do_rescale
  							UpperCamelCase__         =	rescale_factor if rescale_factor is not None else self.rescale_factor
  							UpperCamelCase__         =	do_normalize if do_normalize is not None else self.do_normalize
  							UpperCamelCase__         =	image_mean if image_mean is not None else self.image_mean
  							UpperCamelCase__         =	image_std if image_std is not None else self.image_std
  
  							UpperCamelCase__         =	make_list_of_images(__lowerCAmelCase       )
  
  							if not valid_images(__lowerCAmelCase       ):
  									raise ValueError(
  									    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
  									    """torch.Tensor, tf.Tensor or jax.ndarray."""       )
  
  							if do_resize and size is None:
  									raise ValueError("""Size must be specified if do_resize is True."""       )
  
  							if do_rescale and rescale_factor is None:
  									raise ValueError("""Rescale factor must be specified if do_rescale is True."""       )
  
  							if do_pad and size is None:
  									raise ValueError("""Size must be specified if do_pad is True."""       )
  
  							if do_normalize and (image_mean is None or image_std is None):
  									raise ValueError("""Image mean and std must be specified if do_normalize is True."""       )
  
  							# All transformations expect numpy arrays.
  							UpperCamelCase__         =	[to_numpy_array(__lowerCAmelCase       ) for image in images]
  
  							if do_align_long_axis:
  									UpperCamelCase__         =	[self.align_long_axis(__lowerCAmelCase    ,      size=__lowerCAmelCase       ) for image in images]
  
  							if do_resize:
  									UpperCamelCase__         =	[self.resize(image=__lowerCAmelCase    ,      size=__lowerCAmelCase    ,      resample=__lowerCAmelCase       ) for image in images]
  
  							if do_thumbnail:
  									UpperCamelCase__         =	[self.thumbnail(image=__lowerCAmelCase    ,      size=__lowerCAmelCase       ) for image in images]
  
  							if do_pad:
  									UpperCamelCase__         =	[self.pad_image(image=__lowerCAmelCase    ,      size=__lowerCAmelCase    ,      random_padding=__lowerCAmelCase       ) for image in images]
  
  							if do_rescale:
  									UpperCamelCase__         =	[self.rescale(image=__lowerCAmelCase    ,      scale=__lowerCAmelCase       ) for image in images]
  
  							if do_normalize:
  									UpperCamelCase__         =	[self.normalize(image=__lowerCAmelCase    ,      mean=__lowerCAmelCase    ,      std=__lowerCAmelCase       ) for image in images]
  
  							UpperCamelCase__         =	[to_channel_dimension_format(__lowerCAmelCase    ,      __lowerCAmelCase       ) for image in images]
  
  							UpperCamelCase__         =	{"""pixel_values""": images}
  							return BatchFeature(data=__lowerCAmelCase    ,      tensor_type=__lowerCAmelCase       )
  
  
  
  
  
  