code: import json
  import logging
  import os
  import random
  import sys
  from dataclasses import dataclass, field
  from typing import Optional
  
  import evaluate
  import numpy as np
  import torch
  from datasets import load_dataset
  from huggingface_hub import hf_hub_download
  from PIL import Image
  from torch import nn
  from torchvision import transforms
  from torchvision.transforms import functional
  
  import transformers
  from transformers import (
      AutoConfig,
      AutoImageProcessor,
      AutoModelForSemanticSegmentation,
      HfArgumentParser,
      Trainer,
      TrainingArguments,
      default_data_collator,
  )
  from transformers.trainer_utils import get_last_checkpoint
  from transformers.utils import check_min_version, send_example_telemetry
  from transformers.utils.versions import require_version
  UpperCAmelCase					=				logging.getLogger(__name__)
  
  # Will error if the minimal version of Transformers is not installed. Remove at your own risks.
  check_min_version('''4.31.0''')
  
  require_version('''datasets>=2.0.0''', '''To fix: pip install -r examples/pytorch/semantic-segmentation/requirements.txt''')
  
  
  def SCREAMING_SNAKE_CASE				(     lowercase_:	int		,						lowercase_:	str		,						lowercase_:	Tuple=0				):
         _UpperCamelCase     :       List[Any]  						=  (size, size) if isinstance(lowercase_		,						lowercase_				) else size
         _UpperCamelCase						,      _UpperCamelCase     :       Optional[Any]  						=  img.size
         _UpperCamelCase     :       List[str]  						=  size[1] - original_height if original_height < size[1] else 0
         _UpperCamelCase     :       str  						=  size[0] - original_width if original_width < size[0] else 0
         _UpperCamelCase     :       int  						=  functional.pad(lowercase_		,						(0, 0, pad_width, pad_height)		,						fill=lowercase_				)
         return img
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Any       ,			lowerCamelCase       :		List[Any]  ) ->							str:
          _UpperCamelCase     :       Dict  						=  transforms
  
   def __call__(			self       :		Union[str, Any]       ,			lowerCamelCase       :		List[str]       ,			lowerCamelCase       :		Any  ) ->							Dict:
          for t in self.transforms:
                 _UpperCamelCase						,      _UpperCamelCase     :       str  						=  t(lowerCamelCase       ,			lowerCamelCase  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Optional[int]  ) ->							List[str]:
          pass
  
   def __call__(			self       :		Union[str, Any]       ,			lowerCamelCase       :		Union[str, Any]       ,			lowerCamelCase       :		Dict  ) ->							Tuple:
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Optional[int]       ,			lowerCamelCase       :		Union[str, Any]  ) ->							List[Any]:
          _UpperCamelCase     :       Optional[int]  						=  size
  
   def __call__(			self       :		Tuple       ,			lowerCamelCase       :		Union[str, Any]       ,			lowerCamelCase       :		int  ) ->							Union[str, Any]:
          _UpperCamelCase     :       int  						=  functional.resize(lowerCamelCase       ,			self.size  )
          _UpperCamelCase     :       Union[str, Any]  						=  functional.resize(lowerCamelCase       ,			self.size       ,			interpolation=transforms.InterpolationMode.NEAREST  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Optional[Any]       ,			lowerCamelCase       :		Optional[int]       ,			lowerCamelCase       :		Tuple=None  ) ->							Optional[int]:
          _UpperCamelCase     :       int  						=  min_size
          if max_size is None:
                 _UpperCamelCase     :       str  						=  min_size
          _UpperCamelCase     :       Optional[Any]  						=  max_size
  
   def __call__(			self       :		Union[str, Any]       ,			lowerCamelCase       :		str       ,			lowerCamelCase       :		Dict  ) ->							Optional[Any]:
          _UpperCamelCase     :       Optional[int]  						=  random.randint(self.min_size       ,			self.max_size  )
          _UpperCamelCase     :       Optional[int]  						=  functional.resize(lowerCamelCase       ,			lowerCamelCase  )
          _UpperCamelCase     :       List[Any]  						=  functional.resize(lowerCamelCase       ,			lowerCamelCase       ,			interpolation=transforms.InterpolationMode.NEAREST  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		int       ,			lowerCamelCase       :		Union[str, Any]  ) ->							Optional[Any]:
          _UpperCamelCase     :       str  						=  size if isinstance(lowerCamelCase       ,			lowerCamelCase  ) else (size, size)
  
   def __call__(			self       :		Optional[int]       ,			lowerCamelCase       :		Union[str, Any]       ,			lowerCamelCase       :		Tuple  ) ->							Any:
          _UpperCamelCase     :       Optional[int]  						=  pad_if_smaller(lowerCamelCase       ,			self.size  )
          _UpperCamelCase     :       Optional[int]  						=  pad_if_smaller(lowerCamelCase       ,			self.size       ,			fill=2_5_5  )
          _UpperCamelCase     :       Dict  						=  transforms.RandomCrop.get_params(lowerCamelCase       ,			self.size  )
          _UpperCamelCase     :       List[str]  						=  functional.crop(lowerCamelCase       ,			*lowerCamelCase  )
          _UpperCamelCase     :       List[str]  						=  functional.crop(lowerCamelCase       ,			*lowerCamelCase  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Optional[int]       ,			lowerCamelCase       :		Tuple  ) ->							str:
          _UpperCamelCase     :       List[Any]  						=  flip_prob
  
   def __call__(			self       :		Dict       ,			lowerCamelCase       :		Tuple       ,			lowerCamelCase       :		Any  ) ->							Union[str, Any]:
          if random.random() < self.flip_prob:
                 _UpperCamelCase     :       str  						=  functional.hflip(lowerCamelCase  )
                 _UpperCamelCase     :       Tuple  						=  functional.hflip(lowerCamelCase  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __call__(			self       :		List[Any]       ,			lowerCamelCase       :		Tuple       ,			lowerCamelCase       :		str  ) ->							str:
          _UpperCamelCase     :       Any  						=  functional.pil_to_tensor(lowerCamelCase  )
          _UpperCamelCase     :       Optional[int]  						=  torch.as_tensor(np.array(lowerCamelCase  )       ,			dtype=torch.intaa  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Union[str, Any]       ,			lowerCamelCase       :		int  ) ->							Dict:
          _UpperCamelCase     :       List[str]  						=  dtype
  
   def __call__(			self       :		Dict       ,			lowerCamelCase       :		Tuple       ,			lowerCamelCase       :		Dict  ) ->							Tuple:
          _UpperCamelCase     :       int  						=  functional.convert_image_dtype(lowerCamelCase       ,			self.dtype  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		Tuple       ,			lowerCamelCase       :		Union[str, Any]       ,			lowerCamelCase       :		Any  ) ->							str:
          _UpperCamelCase     :       Optional[Any]  						=  mean
          _UpperCamelCase     :       Optional[Any]  						=  std
  
   def __call__(			self       :		Optional[int]       ,			lowerCamelCase       :		Optional[int]       ,			lowerCamelCase       :		Tuple  ) ->							int:
          _UpperCamelCase     :       str  						=  functional.normalize(lowerCamelCase       ,			mean=self.mean       ,			std=self.std  )
          return image, target
  
  
  
  
  
  
  
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
  
   def __call__(			self       :		str       ,			lowerCamelCase       :		List[str]       ,			lowerCamelCase       :		Any  ) ->							List[str]:
          if not isinstance(lowerCamelCase       ,			np.ndarray  ):
                 _UpperCamelCase     :       Any  						=  np.array(lowerCamelCase  ).astype(np.uinta  )
          # avoid using underflow conversion
          _UpperCamelCase     :       List[str]  						=  2_5_5
          _UpperCamelCase     :       List[Any]  						=  target - 1
          _UpperCamelCase     :       List[Any]  						=  2_5_5
  
          _UpperCamelCase     :       int  						=  Image.fromarray(lowerCamelCase  )
          return image, target
  
  
  
  
  
  
  
  @dataclass
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
   _lowerCamelCase						:	Optional[str]				    = field(
       default='segments/sidewalk-semantic',	metadata={
           'help': 'Name of a dataset from the hub (could be your own, possibly private dataset hosted on the hub).'
       },	)
   _lowerCamelCase						:	Optional[str]				    = field(
       default=a_,	metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'}						)
   _lowerCamelCase						:	Optional[float]				    = field(
       default=0.15,	metadata={'help': 'Percent to split off of train for validation.'}						)
   _lowerCamelCase						:	Optional[int]				    = field(
       default=a_,	metadata={
           'help': (
               'For debugging purposes or quicker training, truncate the number of training examples to this '
               'value if set.'
           )
       },	)
   _lowerCamelCase						:	Optional[int]				    = field(
       default=a_,	metadata={
           'help': (
               'For debugging purposes or quicker training, truncate the number of evaluation examples to this '
               'value if set.'
           )
       },	)
   _lowerCamelCase						:	Optional[bool]				    = field(
       default=a_,	metadata={'help': 'Whether or not to reduce all labels by 1 and replace background by 255.'},	)
  
   def 					__snake_case					(			self       :		Union[str, Any]  ) ->							str:
          if self.dataset_name is None and (self.train_dir is None and self.validation_dir is None):
                 raise ValueError(
                     'You must specify either a dataset name from the hub or a train and/or validation directory.'  )
  
  
  
  
  
  
  
  @dataclass
  class    __lowerCAmelCase  :
  
  
  
   '''simple docstring'''
   _lowerCamelCase						:	str				    = field(
       default='nvidia/mit-b0',	metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'},	)
   _lowerCamelCase						:	Optional[str]				    = field(
       default=a_,	metadata={'help': 'Pretrained config name or path if not the same as model_name'}						)
   _lowerCamelCase						:	Optional[str]				    = field(
       default=a_,	metadata={'help': 'Where do you want to store the pretrained models downloaded from s3'}						)
   _lowerCamelCase						:	str				    = field(
       default='main',	metadata={'help': 'The specific model version to use (can be a branch name, tag name or commit id).'},	)
   _lowerCamelCase						:	str				    = field(default=a_,	metadata={'help': 'Name or path of preprocessor config.'}						)
   _lowerCamelCase						:	bool				    = field(
       default=a_,	metadata={
           'help': (
               'Will use the token generated when running `huggingface-cli login` (necessary to use this script '
               'with private models).'
           )
       },	)
  
  
  
  
  
  
  
  def SCREAMING_SNAKE_CASE				(     ):
         # See all possible arguments in src/transformers/training_args.py
         # or by passing the --help flag to this script.
         # We now keep distinct sets of args, for a cleaner separation of concerns.
  
         _UpperCamelCase     :       Tuple  						=  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)
         if len(sys.argv				) == 2 and sys.argv[1].endswith('.json'				):
                # If we pass only one argument to the script and it's the path to a json file,
                # let's parse it to get our arguments.
                _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase     :       str  						=  parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]				)				)
         else:
                _UpperCamelCase						,      _UpperCamelCase						,      _UpperCamelCase     :       Any  						=  parser.parse_args_into_dataclasses()
  
         # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
         # information sent is the one passed as arguments along with your Python/PyTorch versions.
         send_example_telemetry('run_semantic_segmentation'		,						lowercase_		,						lowercase_				)
  
         # Setup logging
         logging.basicConfig(
             format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'		,						datefmt='%m/%d/%Y %H:%M:%S'		,						handlers=[logging.StreamHandler(sys.stdout				)]		,						)
  
         if training_args.should_log:
                # The default of training_args.log_level is passive, so we set log level at info here to have that default.
                transformers.utils.logging.set_verbosity_info()
  
         _UpperCamelCase     :       List[str]  						=  training_args.get_process_log_level()
         logger.setLevel(lowercase_				)
         transformers.utils.logging.set_verbosity(lowercase_				)
         transformers.utils.logging.enable_default_handler()
         transformers.utils.logging.enable_explicit_format()
  
         # Log on each process the small summary:
         logger.warning(
             F"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
             + F"distributed training: {bool(training_args.local_rank != -1				)}, 16-bits training: {training_args.fpaa}"				)
         logger.info(F"Training/evaluation parameters {training_args}"				)
  
         # Detecting last checkpoint.
         _UpperCamelCase     :       Tuple  						=  None
         if os.path.isdir(training_args.output_dir				) and training_args.do_train and not training_args.overwrite_output_dir:
                _UpperCamelCase     :       str  						=  get_last_checkpoint(training_args.output_dir				)
                if last_checkpoint is None and len(os.listdir(training_args.output_dir				)				) > 0:
                       raise ValueError(
                           F"Output directory ({training_args.output_dir}) already exists and is not empty. "
                           'Use --overwrite_output_dir to overcome.'				)
                elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
                       logger.info(
                           F"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                           'the `--output_dir` or add `--overwrite_output_dir` to train from scratch.'				)
  
      # Load dataset
      # In distributed training, the load_dataset function guarantees that only one local process can concurrently
      # download the dataset.
      # TODO support datasets from local folders
         _UpperCamelCase     :       Dict  						=  load_dataset(data_args.dataset_name		,						cache_dir=model_args.cache_dir				)
  
         # Rename column names to standardized names (only "image" and "label" need to be present)
         if "pixel_values" in dataset["train"].column_names:
                _UpperCamelCase     :       List[str]  						=  dataset.rename_columns({'pixel_values': 'image'}				)
         if "annotation" in dataset["train"].column_names:
                _UpperCamelCase     :       int  						=  dataset.rename_columns({'annotation': 'label'}				)
  
         # If we don't have a validation split, split off a percentage of train as validation.
         _UpperCamelCase     :       Any  						=  None if 'validation' in dataset.keys() else data_args.train_val_split
         if isinstance(data_args.train_val_split		,						lowercase_				) and data_args.train_val_split > 0.0:
                _UpperCamelCase     :       List[str]  						=  dataset['train'].train_test_split(data_args.train_val_split				)
                _UpperCamelCase     :       Optional[int]  						=  split['train']
                _UpperCamelCase     :       int  						=  split['test']
  
         # Prepare label mappings.
         # We'll include these in the model's config to get human readable labels in the Inference API.
         if data_args.dataset_name == "scene_parse_150":
                _UpperCamelCase     :       Any  						=  'huggingface/label-files'
                _UpperCamelCase     :       Optional[int]  						=  'ade20k-id2label.json'
         else:
                _UpperCamelCase     :       List[Any]  						=  data_args.dataset_name
                _UpperCamelCase     :       List[str]  						=  'id2label.json'
         _UpperCamelCase     :       int  						=  json.load(open(hf_hub_download(lowercase_		,						lowercase_		,						repo_type='dataset'				)		,						'r'				)				)
         _UpperCamelCase     :       List[str]  						=  {int(lowercase_				): v for k, v in idalabel.items()}
         _UpperCamelCase     :       Tuple  						=  {v: str(lowercase_				) for k, v in idalabel.items()}
  
         # Load the mean IoU metric from the datasets package
         _UpperCamelCase     :       Dict  						=  evaluate.load('mean_iou'				)
  
         # Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
         # predictions and label_ids field) and has to return a dictionary string to float.
         @torch.no_grad()
         def compute_metrics(lowercase_:	Any				):
                _UpperCamelCase						,      _UpperCamelCase     :       int  						=  eval_pred
                _UpperCamelCase     :       Optional[int]  						=  torch.from_numpy(lowercase_				)
                # scale the logits to the size of the label
                _UpperCamelCase     :       Optional[Any]  						=  nn.functional.interpolate(
                    lowercase_		,						size=labels.shape[-2:]		,						mode='bilinear'		,						align_corners=lowercase_		,						).argmax(dim=1				)
  
                _UpperCamelCase     :       List[str]  						=  logits_tensor.detach().cpu().numpy()
                _UpperCamelCase     :       List[Any]  						=  metric.compute(
                    predictions=lowercase_		,						references=lowercase_		,						num_labels=len(lowercase_				)		,						ignore_index=0		,						reduce_labels=image_processor.do_reduce_labels		,						)
                # add per category metrics as individual key-value pairs
                _UpperCamelCase     :       Union[str, Any]  						=  metrics.pop('per_category_accuracy'				).tolist()
                _UpperCamelCase     :       List[Any]  						=  metrics.pop('per_category_iou'				).tolist()
  
                metrics.update({F"accuracy_{idalabel[i]}": v for i, v in enumerate(lowercase_				)}				)
                metrics.update({F"iou_{idalabel[i]}": v for i, v in enumerate(lowercase_				)}				)
  
                return metrics
  
         _UpperCamelCase     :       Optional[Any]  						=  AutoConfig.from_pretrained(
             model_args.config_name or model_args.model_name_or_path		,						labelaid=lowercase_		,						idalabel=lowercase_		,						cache_dir=model_args.cache_dir		,						revision=model_args.model_revision		,						use_auth_token=True if model_args.use_auth_token else None		,						)
         _UpperCamelCase     :       Optional[Any]  						=  AutoModelForSemanticSegmentation.from_pretrained(
             model_args.model_name_or_path		,						from_tf=bool('.ckpt' in model_args.model_name_or_path				)		,						config=lowercase_		,						cache_dir=model_args.cache_dir		,						revision=model_args.model_revision		,						use_auth_token=True if model_args.use_auth_token else None		,						)
         _UpperCamelCase     :       Tuple  						=  AutoImageProcessor.from_pretrained(
             model_args.image_processor_name or model_args.model_name_or_path		,						cache_dir=model_args.cache_dir		,						revision=model_args.model_revision		,						use_auth_token=True if model_args.use_auth_token else None		,						)
  
         # Define torchvision transforms to be applied to each image + target.
         # Not that straightforward in torchvision: https://github.com/pytorch/vision/issues/9
         # Currently based on official torchvision references: https://github.com/pytorch/vision/blob/main/references/segmentation/transforms.py
         if "shortest_edge" in image_processor.size:
                # We instead set the target size as (shortest_edge, shortest_edge) to here to ensure all images are batchable.
                _UpperCamelCase     :       Dict  						=  (image_processor.size['shortest_edge'], image_processor.size['shortest_edge'])
         else:
                _UpperCamelCase     :       Optional[Any]  						=  (image_processor.size['height'], image_processor.size['width'])
         _UpperCamelCase     :       List[Any]  						=  Compose(
             [
                 ReduceLabels() if data_args.reduce_labels else Identity(),
                 RandomCrop(size=lowercase_				),
                 RandomHorizontalFlip(flip_prob=0.5				),
                 PILToTensor(),
                 ConvertImageDtype(torch.float				),
                 Normalize(mean=image_processor.image_mean		,						std=image_processor.image_std				),
             ]				)
         # Define torchvision transform to be applied to each image.
         # jitter = ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)
         _UpperCamelCase     :       Any  						=  Compose(
             [
                 ReduceLabels() if data_args.reduce_labels else Identity(),
                 Resize(size=lowercase_				),
                 PILToTensor(),
                 ConvertImageDtype(torch.float				),
                 Normalize(mean=image_processor.image_mean		,						std=image_processor.image_std				),
             ]				)
  
         def preprocess_train(lowercase_:	str				):
                _UpperCamelCase     :       List[Any]  						=  []
                _UpperCamelCase     :       List[str]  						=  []
                for image, target in zip(example_batch['image']		,						example_batch['label']				):
                       _UpperCamelCase						,      _UpperCamelCase     :       int  						=  train_transforms(image.convert('RGB'				)		,						lowercase_				)
                       pixel_values.append(lowercase_				)
                       labels.append(lowercase_				)
  
                _UpperCamelCase     :       Optional[Any]  						=  {}
                _UpperCamelCase     :       Tuple  						=  torch.stack(lowercase_				)
                _UpperCamelCase     :       Union[str, Any]  						=  torch.stack(lowercase_				)
  
                return encoding
  
         def preprocess_val(lowercase_:	str				):
                _UpperCamelCase     :       int  						=  []
                _UpperCamelCase     :       List[Any]  						=  []
                for image, target in zip(example_batch['image']		,						example_batch['label']				):
                       _UpperCamelCase						,      _UpperCamelCase     :       Union[str, Any]  						=  val_transforms(image.convert('RGB'				)		,						lowercase_				)
                       pixel_values.append(lowercase_				)
                       labels.append(lowercase_				)
  
                _UpperCamelCase     :       Any  						=  {}
                _UpperCamelCase     :       Tuple  						=  torch.stack(lowercase_				)
                _UpperCamelCase     :       List[str]  						=  torch.stack(lowercase_				)
  
                return encoding
  
         if training_args.do_train:
                if "train" not in dataset:
                       raise ValueError('--do_train requires a train dataset'				)
                if data_args.max_train_samples is not None:
                       _UpperCamelCase     :       List[str]  						=  (
                           dataset['train'].shuffle(seed=training_args.seed				).select(range(data_args.max_train_samples				)				)
                       )
                # Set the training transforms
                dataset["train"].set_transform(lowercase_				)
  
         if training_args.do_eval:
                if "validation" not in dataset:
                       raise ValueError('--do_eval requires a validation dataset'				)
                if data_args.max_eval_samples is not None:
                       _UpperCamelCase     :       Optional[Any]  						=  (
                           dataset['validation'].shuffle(seed=training_args.seed				).select(range(data_args.max_eval_samples				)				)
                       )
                # Set the validation transforms
                dataset["validation"].set_transform(lowercase_				)
  
         # Initalize our trainer
         _UpperCamelCase     :       Union[str, Any]  						=  Trainer(
             model=lowercase_		,						args=lowercase_		,						train_dataset=dataset['train'] if training_args.do_train else None		,						eval_dataset=dataset['validation'] if training_args.do_eval else None		,						compute_metrics=lowercase_		,						tokenizer=lowercase_		,						data_collator=lowercase_		,						)
  
         # Training
         if training_args.do_train:
                _UpperCamelCase     :       Any  						=  None
                if training_args.resume_from_checkpoint is not None:
                       _UpperCamelCase     :       str  						=  training_args.resume_from_checkpoint
                elif last_checkpoint is not None:
                       _UpperCamelCase     :       Optional[int]  						=  last_checkpoint
                _UpperCamelCase     :       Dict  						=  trainer.train(resume_from_checkpoint=lowercase_				)
                trainer.save_model()
                trainer.log_metrics('train'		,						train_result.metrics				)
                trainer.save_metrics('train'		,						train_result.metrics				)
                trainer.save_state()
  
         # Evaluation
         if training_args.do_eval:
                _UpperCamelCase     :       Optional[Any]  						=  trainer.evaluate()
                trainer.log_metrics('eval'		,						lowercase_				)
                trainer.save_metrics('eval'		,						lowercase_				)
  
         # Write model card and (optionally) push to hub
         _UpperCamelCase     :       Optional[Any]  						=  {
             'finetuned_from': model_args.model_name_or_path,
             'dataset': data_args.dataset_name,
             'tags': ['image-segmentation', 'vision'],
         }
         if training_args.push_to_hub:
                trainer.push_to_hub(**lowercase_				)
         else:
                trainer.create_model_card(**lowercase_				)
  
  
  if __name__ == "__main__":
   main()
  
  
  
  
style_context: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  UpperCAmelCase					=				logging.get_logger(__name__)
  
  UpperCAmelCase					=				{
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  UpperCAmelCase					=				{
      '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
      '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
  }
  
  UpperCAmelCase					=				{
      '''openai-gpt''': 5_12,
  }
  
  
  def SCREAMING_SNAKE_CASE				(     lowercase_:	int				):
         _UpperCamelCase     :       Optional[Any]  						=  text.strip()
         if not text:
                return []
         _UpperCamelCase     :       Optional[Any]  						=  text.split()
         return tokens
  
  
  
  
  class    __lowerCAmelCase  (       a_						):
  
  
  
   '''simple docstring'''
  
   def __init__(			self       :		str       ,			lowerCamelCase       :		Union[str, Any]=True       ,			lowerCamelCase       :		Optional[Any]=None       ,			lowerCamelCase       :		Dict=True       ,			lowerCamelCase       :		List[Any]=None       ,			lowerCamelCase       :		Optional[Any]=True       ,			) ->							Optional[Any]:
          if never_split is None:
                 _UpperCamelCase     :       Union[str, Any]  						=  []
          _UpperCamelCase     :       List[Any]  						=  do_lower_case
          _UpperCamelCase     :       List[Any]  						=  set(lowerCamelCase  )
          _UpperCamelCase     :       int  						=  tokenize_chinese_chars
          _UpperCamelCase     :       Dict  						=  strip_accents
          _UpperCamelCase     :       List[Any]  						=  do_split_on_punc
  
   def 					__snake_case					(			self       :		int       ,			lowerCamelCase       :		List[str]       ,			lowerCamelCase       :		List[Any]=None  ) ->							List[str]:
          _UpperCamelCase     :       Any  						=  self.never_split.union(set(lowerCamelCase  )  ) if never_split else self.never_split
          _UpperCamelCase     :       int  						=  self._clean_text(lowerCamelCase  )
  
          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
                 _UpperCamelCase     :       Optional[int]  						=  self._tokenize_chinese_chars(lowerCamelCase  )
          # prevents treating the same character with different unicode codepoints as different characters
          _UpperCamelCase     :       int  						=  unicodedata.normalize('NFC'       ,			lowerCamelCase  )
          _UpperCamelCase     :       Tuple  						=  whitespace_tokenize(lowerCamelCase  )
          _UpperCamelCase     :       Dict  						=  []
          for token in orig_tokens:
                 if token not in never_split:
                        if self.do_lower_case:
                               _UpperCamelCase     :       Tuple  						=  token.lower()
                               if self.strip_accents is not False:
                                      _UpperCamelCase     :       List[Any]  						=  self._run_strip_accents(lowerCamelCase  )
                        elif self.strip_accents:
                               _UpperCamelCase     :       List[str]  						=  self._run_strip_accents(lowerCamelCase  )
                 split_tokens.extend(self._run_split_on_punc(lowerCamelCase       ,			lowerCamelCase  )  )
  
          _UpperCamelCase     :       str  						=  whitespace_tokenize(' '.join(lowerCamelCase  )  )
          return output_tokens
  
   def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		Union[str, Any]  ) ->							str:
          _UpperCamelCase     :       Optional[Any]  						=  unicodedata.normalize('NFD'       ,			lowerCamelCase  )
          _UpperCamelCase     :       Dict  						=  []
          for char in text:
                 _UpperCamelCase     :       Tuple  						=  unicodedata.category(lowerCamelCase  )
                 if cat == "Mn":
                        continue
                 output.append(lowerCamelCase  )
          return "".join(lowerCamelCase  )
  
   def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		List[Any]       ,			lowerCamelCase       :		int=None  ) ->							List[Any]:
          if not self.do_split_on_punc or (never_split is not None and text in never_split):
                 return [text]
          _UpperCamelCase     :       Optional[int]  						=  list(lowerCamelCase  )
          _UpperCamelCase     :       Dict  						=  0
          _UpperCamelCase     :       Union[str, Any]  						=  True
          _UpperCamelCase     :       int  						=  []
          while i < len(lowerCamelCase  ):
                 _UpperCamelCase     :       List[str]  						=  chars[i]
                 if _is_punctuation(lowerCamelCase  ):
                        output.append([char]  )
                        _UpperCamelCase     :       str  						=  True
                 else:
                        if start_new_word:
                               output.append([]  )
                        _UpperCamelCase     :       List[str]  						=  False
                        output[-1].append(lowerCamelCase  )
                 i += 1
  
          return ["".join(lowerCamelCase  ) for x in output]
  
   def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		int  ) ->							Optional[Any]:
          _UpperCamelCase     :       Dict  						=  []
          for char in text:
                 _UpperCamelCase     :       Optional[int]  						=  ord(lowerCamelCase  )
                 if self._is_chinese_char(lowerCamelCase  ):
                        output.append(' '  )
                        output.append(lowerCamelCase  )
                        output.append(' '  )
                 else:
                        output.append(lowerCamelCase  )
          return "".join(lowerCamelCase  )
  
   def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		int  ) ->							Optional[Any]:
          # This defines a "chinese character" as anything in the CJK Unicode block:
          #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
          #
          # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
          # despite its name. The modern Korean Hangul alphabet is a different block,
          # as is Japanese Hiragana and Katakana. Those alphabets are used to write
          # space-separated words, so they are not treated specially and handled
          # like the all of the other languages.
          if (
              (cp >= 0x4_E00 and cp <= 0x9_FFF)
              or (cp >= 0x3_400 and cp <= 0x4_DBF)  #
              or (cp >= 0x20_000 and cp <= 0x2A_6DF)  #
              or (cp >= 0x2A_700 and cp <= 0x2B_73F)  #
              or (cp >= 0x2B_740 and cp <= 0x2B_81F)  #
              or (cp >= 0x2B_820 and cp <= 0x2C_EAF)  #
              or (cp >= 0xF_900 and cp <= 0xF_AFF)
              or (cp >= 0x2F_800 and cp <= 0x2F_A1F)  #
          ):  #
                 return True
  
          return False
  
   def 					__snake_case					(			self       :		Any       ,			lowerCamelCase       :		int  ) ->							Dict:
          _UpperCamelCase     :       int  						=  []
          for char in text:
                 _UpperCamelCase     :       str  						=  ord(lowerCamelCase  )
                 if cp == 0 or cp == 0xF_FFD or _is_control(lowerCamelCase  ):
                        continue
                 if _is_whitespace(lowerCamelCase  ):
                        output.append(' '  )
                 else:
                        output.append(lowerCamelCase  )
          return "".join(lowerCamelCase  )
  
  
  
  
  
  
  
  def SCREAMING_SNAKE_CASE				(     lowercase_:	Optional[Any]				):
         _UpperCamelCase     :       List[Any]  						=  set()
         _UpperCamelCase     :       Tuple  						=  word[0]
         for char in word[1:]:
                pairs.add((prev_char, char)				)
                _UpperCamelCase     :       Tuple  						=  char
         return pairs
  
  
  def SCREAMING_SNAKE_CASE				(     lowercase_:	Dict				):
         _UpperCamelCase     :       Union[str, Any]  						=  text.replace('—'		,						'-'				)
         _UpperCamelCase     :       int  						=  text.replace('–'		,						'-'				)
         _UpperCamelCase     :       Dict  						=  text.replace('―'		,						'-'				)
         _UpperCamelCase     :       Any  						=  text.replace('…'		,						'...'				)
         _UpperCamelCase     :       Dict  						=  text.replace('´'		,						'\''				)
         _UpperCamelCase     :       Union[str, Any]  						=  re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'		,						R' \1 '		,						lowercase_				)
         _UpperCamelCase     :       Dict  						=  re.sub(R'\s*\n\s*'		,						' \n '		,						lowercase_				)
         _UpperCamelCase     :       Union[str, Any]  						=  re.sub(R'[^\S\n]+'		,						' '		,						lowercase_				)
         return text.strip()
  
  
  
  
  
  class    __lowerCAmelCase  (       a_						):
  
  
  
   '''simple docstring'''
   _lowerCamelCase						:	int				    = VOCAB_FILES_NAMES
   _lowerCamelCase						:	Any				    = PRETRAINED_VOCAB_FILES_MAP
   _lowerCamelCase						:	Optional[int]				    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowerCamelCase						:	List[str]				    = ['input_ids', 'attention_mask']
  
   def __init__(			self       :		str       ,			lowerCamelCase       :		Optional[Any]       ,			lowerCamelCase       :		Tuple       ,			lowerCamelCase       :		Union[str, Any]="<unk>"       ,			**lowerCamelCase       :		Any  ) ->							Dict:
          super().__init__(unk_token=lowerCamelCase       ,			**lowerCamelCase  )
  
          try:
                 import ftfy
                 from spacy.lang.en import English
  
                 _UpperCamelCase     :       Any  						=  English()
                 _UpperCamelCase     :       Any  						=  _nlp.tokenizer
                 _UpperCamelCase     :       Dict  						=  ftfy.fix_text
          except ImportError:
                 logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'  )
                 _UpperCamelCase     :       int  						=  BasicTokenizer(do_lower_case=lowerCamelCase  )
                 _UpperCamelCase     :       List[Any]  						=  None
  
          with open(lowerCamelCase       ,			encoding='utf-8'  ) as vocab_handle:
                 _UpperCamelCase     :       Tuple  						=  json.load(lowerCamelCase  )
          _UpperCamelCase     :       List[Any]  						=  {v: k for k, v in self.encoder.items()}
          with open(lowerCamelCase       ,			encoding='utf-8'  ) as merges_handle:
                 _UpperCamelCase     :       List[str]  						=  merges_handle.read().split('\n'  )[1:-1]
          _UpperCamelCase     :       Optional[int]  						=  [tuple(merge.split()  ) for merge in merges]
          _UpperCamelCase     :       Dict  						=  dict(zip(lowerCamelCase       ,			range(len(lowerCamelCase  )  )  )  )
          _UpperCamelCase     :       Optional[Any]  						=  {}
  
   @property
   def 					__snake_case					(			self       :		str  ) ->							int:
          return True
  
   @property
   def 					__snake_case					(			self       :		int  ) ->							Dict:
          return len(self.encoder  )
  
   def 					__snake_case					(			self       :		Optional[Any]  ) ->							Union[str, Any]:
          return dict(self.encoder       ,			**self.added_tokens_encoder  )
  
   def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		Optional[int]  ) ->							Union[str, Any]:
          _UpperCamelCase     :       int  						=  tuple(token[:-1]  ) + (token[-1] + '</w>',)
          if token in self.cache:
                 return self.cache[token]
          _UpperCamelCase     :       Dict  						=  get_pairs(lowerCamelCase  )
  
          if not pairs:
                 return token + "</w>"
  
          while True:
                 _UpperCamelCase     :       List[str]  						=  min(lowerCamelCase       ,			key=lambda lowerCamelCase  : self.bpe_ranks.get(lowerCamelCase       ,			float('inf'  )  )  )
                 if bigram not in self.bpe_ranks:
                        break
                 _UpperCamelCase						,      _UpperCamelCase     :       Any  						=  bigram
                 _UpperCamelCase     :       Optional[Any]  						=  []
                 _UpperCamelCase     :       Optional[int]  						=  0
                 while i < len(lowerCamelCase  ):
                        try:
                               _UpperCamelCase     :       int  						=  word.index(lowerCamelCase       ,			lowerCamelCase  )
                        except ValueError:
                               new_word.extend(word[i:]  )
                               break
                        else:
                               new_word.extend(word[i:j]  )
                               _UpperCamelCase     :       Union[str, Any]  						=  j
  
                        if word[i] == first and i < len(lowerCamelCase  ) - 1 and word[i + 1] == second:
                               new_word.append(first + second  )
                               i += 2
                        else:
                               new_word.append(word[i]  )
                               i += 1
                 _UpperCamelCase     :       str  						=  tuple(lowerCamelCase  )
                 _UpperCamelCase     :       Optional[int]  						=  new_word
                 if len(lowerCamelCase  ) == 1:
                        break
                 else:
                        _UpperCamelCase     :       List[Any]  						=  get_pairs(lowerCamelCase  )
          _UpperCamelCase     :       Any  						=  ' '.join(lowerCamelCase  )
          if word == "\n  </w>":
                 _UpperCamelCase     :       Union[str, Any]  						=  '\n</w>'
          _UpperCamelCase     :       Tuple  						=  word
          return word
  
   def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		Optional[Any]  ) ->							List[Any]:
          _UpperCamelCase     :       Optional[int]  						=  []
          if self.fix_text is None:
                 # Using BERT's BasicTokenizer
                 _UpperCamelCase     :       int  						=  self.nlp.tokenize(lowerCamelCase  )
                 for token in text:
                        split_tokens.extend(list(self.bpe(lowerCamelCase  ).split(' '  )  )  )
          else:
                 # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                 _UpperCamelCase     :       Optional[Any]  						=  self.nlp(text_standardize(self.fix_text(lowerCamelCase  )  )  )
                 for token in text:
                        split_tokens.extend(list(self.bpe(token.text.lower()  ).split(' '  )  )  )
          return split_tokens
  
   def 					__snake_case					(			self       :		int       ,			lowerCamelCase       :		Optional[int]  ) ->							int:
          return self.encoder.get(lowerCamelCase       ,			self.encoder.get(self.unk_token  )  )
  
   def 					__snake_case					(			self       :		Dict       ,			lowerCamelCase       :		List[str]  ) ->							str:
          return self.decoder.get(lowerCamelCase       ,			self.unk_token  )
  
   def 					__snake_case					(			self       :		Union[str, Any]       ,			lowerCamelCase       :		Dict  ) ->							Tuple:
          _UpperCamelCase     :       str  						=  ''.join(lowerCamelCase  ).replace('</w>'       ,			' '  ).strip()
          return out_string
  
   def 					__snake_case					(			self       :		List[str]       ,			lowerCamelCase       :		str       ,			lowerCamelCase       :		Optional[str] = None  ) ->							Tuple[str]:
          if not os.path.isdir(lowerCamelCase  ):
                 logger.error(f"Vocabulary path ({save_directory}) should be a directory"  )
                 return
          _UpperCamelCase     :       str  						=  os.path.join(
              lowerCamelCase       ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
          _UpperCamelCase     :       List[str]  						=  os.path.join(
              lowerCamelCase       ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']  )
  
          with open(lowerCamelCase       ,			'w'       ,			encoding='utf-8'  ) as f:
                 f.write(json.dumps(self.encoder       ,			indent=2       ,			sort_keys=lowerCamelCase       ,			ensure_ascii=lowerCamelCase  ) + '\n'  )
  
          _UpperCamelCase     :       List[Any]  						=  0
          with open(lowerCamelCase       ,			'w'       ,			encoding='utf-8'  ) as writer:
                 writer.write('#version: 0.2\n'  )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,			key=lambda lowerCamelCase  : kv[1]  ):
                        if index != token_index:
                               logger.warning(
                                   f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                   ' Please check that the tokenizer is not corrupted!'  )
                               _UpperCamelCase     :       str  						=  token_index
                        writer.write(' '.join(lowerCamelCase  ) + '\n'  )
                        index += 1
  
          return vocab_file, merge_file
  
  
  
  