

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
	import botoa  # noqa: F401

def 						UpperCamelCase					(lowercase_: Tuple					)     ->      Optional[Any]:
	A__							:    Optional[Any]        =		botoa.client("""iam"""					)

	A__							:    Dict        =		{
	    """Version""": """2012-10-17""",
	    """Statement""": [
	        {"""Effect""": """Allow""", """Principal""": {"""Service""": """sagemaker.amazonaws.com"""}, """Action""": """sts:AssumeRole"""}
	    ],
	}
	try:
		# create the role, associated with the chosen trust policy
		iam_client.create_role(
		    RoleName=lowercase_    ,      AssumeRolePolicyDocument=json.dumps(lowercase_    ,      indent=2					)					)
		A__							:    Dict        =		{
		    """Version""": """2012-10-17""",
		    """Statement""": [
		        {
		            """Effect""": """Allow""",
		            """Action""": [
		                """sagemaker:*""",
		                """ecr:GetDownloadUrlForLayer""",
		                """ecr:BatchGetImage""",
		                """ecr:BatchCheckLayerAvailability""",
		                """ecr:GetAuthorizationToken""",
		                """cloudwatch:PutMetricData""",
		                """cloudwatch:GetMetricData""",
		                """cloudwatch:GetMetricStatistics""",
		                """cloudwatch:ListMetrics""",
		                """logs:CreateLogGroup""",
		                """logs:CreateLogStream""",
		                """logs:DescribeLogStreams""",
		                """logs:PutLogEvents""",
		                """logs:GetLogEvents""",
		                """s3:CreateBucket""",
		                """s3:ListBucket""",
		                """s3:GetBucketLocation""",
		                """s3:GetObject""",
		                """s3:PutObject""",
		            ],
		            """Resource""": """*""",
		        }
		    ],
		}
		# attach policy to role
		iam_client.put_role_policy(
		    RoleName=lowercase_    ,      PolicyName=f"""{role_name}_policy_permission"""    ,      PolicyDocument=json.dumps(lowercase_    ,      indent=2					)    ,      )
	except iam_client.exceptions.EntityAlreadyExistsException:
		print(f"""role {role_name} already exists. Using existing one"""					)

def 						UpperCamelCase					(lowercase_: str					)     ->      Dict:
	A__							:    Optional[Any]        =		botoa.client("""iam"""					)
	return iam_client.get_role(RoleName=lowercase_					)["Role"]["Arn"]


def 						UpperCamelCase					()     ->      Tuple:
	A__							:    Union[str, Any]        =		_ask_options(
	    """How do you want to authorize?"""    ,      ["""AWS Profile""", """Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) """]    ,      lowercase_    ,      )
	A__							:    str        =		None
	if credentials_configuration == 0:
		A__							:    Any        =		_ask_field("""Enter your AWS Profile name: [default] """    ,      default="""default"""					)
		A__							:    Any        =		aws_profile
	else:
		print(
		    """Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"""
		    """`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`"""					)
		A__							:    Optional[Any]        =		_ask_field("""AWS Access Key ID: """					)
		A__							:    Any        =		aws_access_key_id

		A__							:    Optional[Any]        =		_ask_field("""AWS Secret Access Key: """					)
		A__							:    List[Any]        =		aws_secret_access_key

	A__							:    Dict        =		_ask_field("""Enter your AWS Region: [us-east-1]"""    ,      default="""us-east-1"""					)
	A__							:    Union[str, Any]        =		aws_region

	A__							:    Tuple        =		_ask_options(
	    """Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"""    ,      ["""Provide IAM Role name""", """Create new IAM role using credentials"""]    ,      lowercase_    ,      )
	if role_management == 0:
		A__							:    str        =		_ask_field("""Enter your IAM role name: """					)
	else:
		A__							:    Tuple        =		"""accelerate_sagemaker_execution_role"""
		print(f"""Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials"""					)
		_create_iam_role_for_sagemaker(lowercase_					)

	A__							:    Any        =		_ask_field(
	    """Do you want to use custom Docker image? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
	A__							:    Dict        =		None
	if is_custom_docker_image:
		A__							:    Any        =		_ask_field("""Enter your Docker image: """    ,      lambda lowercase_					: str(lowercase_					).lower()					)

	A__							:    List[str]        =		_ask_field(
	    """Do you want to provide SageMaker input channels with data locations? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
	A__							:    Dict        =		None
	if is_sagemaker_inputs_enabled:
		A__							:    int        =		_ask_field(
		    """Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): """    ,      lambda lowercase_					: str(lowercase_					).lower()    ,      )

	A__							:    Optional[Any]        =		_ask_field(
	    """Do you want to enable SageMaker metrics? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
	A__							:    Tuple        =		None
	if is_sagemaker_metrics_enabled:
		A__							:    Optional[Any]        =		_ask_field(
		    """Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): """    ,      lambda lowercase_					: str(lowercase_					).lower()    ,      )

	A__							:    List[str]        =		_ask_options(
	    """What is the distributed mode?"""    ,      ["""No distributed training""", """Data parallelism"""]    ,      _convert_sagemaker_distributed_mode    ,      )
	A__							:    Any        =		{}
	A__							:    Union[str, Any]        =		_ask_field(
	    """Do you wish to optimize your script with torch dynamo?[yes/NO]:"""    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
	if use_dynamo:
		A__							:    Optional[int]        =		"""dynamo_"""
		A__							:    Optional[Any]        =		_ask_options(
		    """Which dynamo backend would you like to use?"""    ,      [x.lower() for x in DYNAMO_BACKENDS]    ,      _convert_dynamo_backend    ,      default=2    ,      )
		A__							:    Any        =		_ask_field(
		    """Do you want to customize the defaults sent to torch.compile? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )

		if use_custom_options:
			A__							:    Union[str, Any]        =		_ask_options(
			    """Which mode do you want to use?"""    ,      lowercase_    ,      lambda lowercase_					: TORCH_DYNAMO_MODES[int(lowercase_					)]    ,      default="""default"""    ,      )
			A__							:    Optional[Any]        =		_ask_field(
			    """Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
			A__							:    Dict        =		_ask_field(
			    """Do you want to enable dynamic shape tracing? [yes/NO]: """    ,      _convert_yes_no_to_bool    ,      default=lowercase_    ,      error_message="""Please enter yes or no."""    ,      )
	A__							:    Union[str, Any]        =		"""Which EC2 instance type you want to use for your training?"""
	if distributed_type != SageMakerDistributedType.NO:
		A__							:    Tuple        =		_ask_options(
		    lowercase_    ,      lowercase_    ,      lambda lowercase_					: SAGEMAKER_PARALLEL_EC2_INSTANCES[int(lowercase_					)]					)
	else:
		eca_instance_query += "? [ml.p3.2xlarge]:"
		A__							:    List[str]        =		_ask_field(lowercase_    ,      lambda lowercase_					: str(lowercase_					).lower()    ,      default="""ml.p3.2xlarge"""					)

	A__							:    Any        =		1
	if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
		A__							:    List[Any]        =		_ask_field(
		    """How many machines do you want use? [1]: """    ,      lowercase_    ,      default=1    ,      )

	A__							:    List[str]        =		_ask_options(
	    """Do you wish to use FP16 or BF16 (mixed precision)?"""    ,      ["""no""", """fp16""", """bf16""", """fp8"""]    ,      _convert_mixed_precision    ,      )

	if use_dynamo and mixed_precision == "no":
		print(
		    """Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts."""					)

	return SageMakerConfig(
	    image_uri=lowercase_    ,      compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER    ,      distributed_type=lowercase_    ,      use_cpu=lowercase_    ,      dynamo_config=lowercase_    ,      eca_instance_type=lowercase_    ,      profile=lowercase_    ,      region=lowercase_    ,      iam_role_name=lowercase_    ,      mixed_precision=lowercase_    ,      num_machines=lowercase_    ,      sagemaker_inputs_file=lowercase_    ,      sagemaker_metrics_file=lowercase_    ,      )






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


A_       : List[str]         =			logging.get_logger(__name__)

A_       : Tuple         =			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A_       : List[Any]         =			{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

A_       : Tuple         =			{
    'junnyu/roformer_chinese_small': 1536,
    'junnyu/roformer_chinese_base': 1536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


A_       : Union[str, Any]         =			{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}



class     _a						(__magic_name__				):


					'''simple docstring'''






					UpperCAmelCase__:   Dict  							=       VOCAB_FILES_NAMES
					UpperCAmelCase__:   Tuple  							=       PRETRAINED_VOCAB_FILES_MAP
					UpperCAmelCase__:   List[str]  							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCAmelCase__:   List[Any]  							=       PRETRAINED_INIT_CONFIGURATION
					UpperCAmelCase__:   Optional[int]  							=       RoFormerTokenizer






					def __init__( self    ,       A__=None    ,       A__=None    ,       A__=True    ,       A__="[UNK]"    ,       A__="[SEP]"    ,       A__="[PAD]"    ,       A__="[CLS]"    ,       A__="[MASK]"    ,       A__=True    ,       A__=None    ,       **A__    ,       ):
						super().__init__(
						    A__    ,       tokenizer_file=A__    ,       do_lower_case=A__    ,       unk_token=A__    ,       sep_token=A__    ,       pad_token=A__    ,       cls_token=A__    ,       mask_token=A__    ,       tokenize_chinese_chars=A__    ,       strip_accents=A__    ,       **A__    ,       )

						A__							:    Tuple        =		json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
						if (
						    pre_tok_state.get("""lowercase"""    ,       A__			) != do_lower_case
						    or pre_tok_state.get("""strip_accents"""    ,       A__			) != strip_accents
						):
							A__							:    List[Any]        =		getattr(A__    ,       pre_tok_state.pop("""type"""			)			)
							A__							:    Optional[int]        =		do_lower_case
							A__							:    int        =		strip_accents
							A__							:    str        =		pre_tok_class(**A__			)

						A__							:    Any        =		do_lower_case






					def __getstate__( self			):
						A__							:    int        =		self.__dict__.copy()
						A__							:    Union[str, Any]        =		BertPreTokenizer()
						return state






					def __setstate__( self    ,       A__			):
						A__							:    Union[str, Any]        =		d
						A__							:    Union[str, Any]        =		self.__dict__["""_tokenizer"""].get_vocab()
						A__							:    Dict        =		PreTokenizer.custom(JiebaPreTokenizer(A__			)			)






					def   __A			( self    ,       A__    ,       A__=None			):
						A__							:    Optional[Any]        =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

						return output






					def   __A			( self    ,       A__    ,       A__ = None			):
						A__							:    Tuple        =		[self.sep_token_id]
						A__							:    Tuple        =		[self.cls_token_id]
						if token_ids_a is None:
							return len(cls + token_ids_a + sep			) * [0]
						return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]






					def   __A			( self    ,       A__    ,       A__ = None			):
						A__							:    Any        =		self._tokenizer.model.save(A__    ,       name=A__			)
						return tuple(A__			)






					def   __A			( self    ,       A__    ,       A__=None    ,       A__=None    ,       A__=False    ,       **A__    ,       ):
						A__							:    str        =		BertPreTokenizer()
						return super().save_pretrained(A__    ,       A__    ,       A__    ,       A__    ,       **A__			)



