

'''simple docstring'''






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCAmelCase__											=	'''Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine'''


def 		_A     (    ):






 """simple docstring"""



 __lowercase							  = _ask_options(
     '''In which compute environment are you running?'''     ,				['''This machine''', '''AWS (Amazon SageMaker)''']     ,				_convert_compute_environment     ,				)
 if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
  __lowercase							  = get_sagemaker_input()
 else:
  __lowercase							  = get_cluster_input()
 return config


def 		_A     (    A__=None      ):






 """simple docstring"""



 if subparsers is not None:
  __lowercase							  = subparsers.add_parser('''config'''     ,				description=A__      )
 else:
  __lowercase							  = argparse.ArgumentParser('''Accelerate config command'''     ,				description=A__      )

 parser.add_argument(
     '''--config_file'''     ,				default=A__     ,				help=(
         '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
         '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
         '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
         '''with \'huggingface\'.'''
     )     ,				)

 if subparsers is not None:
  parser.set_defaults(func=A__      )
 return parser


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = get_user_input()
 if args.config_file is not None:
  __lowercase							  = args.config_file
 else:
  if not os.path.isdir(A__      ):
   os.makedirs(A__      )
  __lowercase							  = default_yaml_config_file

 if config_file.endswith('''.json'''      ):
  config.to_json_file(A__      )
 else:
  config.to_yaml_file(A__      )
 print(F"accelerate configuration saved at {config_file}"      )


def 		_A     (    ):






 """simple docstring"""



 __lowercase							  = config_command_parser()
 __lowercase							  = parser.parse_args()
 config_command(A__      )


if __name__ == "__main__":
 main()








"""simple docstring"""



import argparse
import json
from typing import List

from ltp import LTP

from transformers import BertTokenizer







def 		__lowercase     (   snake_case_  :  int						)   ->Tuple:



    '''simple docstring'''







    if (
        (cp >= 0x4e00 and cp <= 0x9fff)
        or (cp >= 0x3400 and cp <= 0x4dbf)  #
        or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
        or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
        or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
        or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
        or (cp >= 0xf900 and cp <= 0xfaff)
        or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
    ):  #
        return True

    return False







def 		__lowercase     (   snake_case_  :  str						)   ->Dict:



    '''simple docstring'''







    for char in word:
        __A    :						int						 =     ord(snake_case_						)
        if not _is_chinese_char(snake_case_						):
            return 0
    return 1







def 		__lowercase     (   snake_case_  :  List[str]						)   ->List[Any]:



    '''simple docstring'''







    __A    :						str						 =     set()

    for token in tokens:
        __A    :						List[Any]						 =     len(snake_case_						) > 1 and is_chinese(snake_case_						)
        if chinese_word:
            word_set.add(snake_case_						)
    __A    :						Any						 =     list(snake_case_						)
    return word_list







def 		__lowercase     (   snake_case_  :  List[str]				,snake_case_  :  set()						)   ->Any:



    '''simple docstring'''







    if not chinese_word_set:
        return bert_tokens
    __A    :						List[Any]						 =     max([len(snake_case_						) for w in chinese_word_set]						)

    __A    :						List[str]						 =     bert_tokens
    __A ,       __A    :						Any						 =     0, len(snake_case_						)
    while start < end:
        __A    :						str						 =     True
        if is_chinese(bert_word[start]						):
            __A    :						int						 =     min(end - start				,snake_case_						)
            for i in range(snake_case_				,1				,-1						):
                __A    :						Any						 =     ''''''.join(bert_word[start : start + i]						)
                if whole_word in chinese_word_set:
                    for j in range(start + 1				,start + i						):
                        __A    :						Any						 =     '''##''' + bert_word[j]
                    __A    :						Optional[int]						 =     start + i
                    __A    :						str						 =     False
                    break
        if single_word:
            start += 1
    return bert_word







def 		__lowercase     (   snake_case_  :  List[str]				,snake_case_  :  LTP				,snake_case_  :  BertTokenizer						)   ->Dict:



    '''simple docstring'''







    __A    :						Optional[Any]						 =     []

    for i in range(0				,len(snake_case_						)				,100						):
        __A    :						int						 =     ltp_tokenizer.seg(lines[i : i + 100]						)[0]
        __A    :						List[Any]						 =     [get_chinese_word(snake_case_						) for r in res]
        ltp_res.extend(snake_case_						)
    assert len(snake_case_						) == len(snake_case_						)

    __A    :						Any						 =     []
    for i in range(0				,len(snake_case_						)				,100						):
        __A    :						Tuple						 =     bert_tokenizer(lines[i : i + 100]				,add_special_tokens=snake_case_				,truncation=snake_case_				,max_length=512						)
        bert_res.extend(res['''input_ids''']						)
    assert len(snake_case_						) == len(snake_case_						)

    __A    :						Optional[int]						 =     []
    for input_ids, chinese_word in zip(snake_case_				,snake_case_						):
        __A    :						List[str]						 =     []
        for id in input_ids:
            __A    :						Tuple						 =     bert_tokenizer._convert_id_to_token(snake_case_						)
            input_tokens.append(snake_case_						)
        __A    :						Optional[int]						 =     add_sub_symbol(snake_case_				,snake_case_						)
        __A    :						Optional[Any]						 =     []
        # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
        for i, token in enumerate(snake_case_						):
            if token[:2] == "##":
                __A    :						Optional[Any]						 =     token[2:]
                # save chinese tokens' pos
                if len(snake_case_						) == 1 and _is_chinese_char(ord(snake_case_						)						):
                    ref_id.append(snake_case_						)
        ref_ids.append(snake_case_						)

    assert len(snake_case_						) == len(snake_case_						)

    return ref_ids







def 		__lowercase     (   snake_case_  :  int						)   ->List[Any]:



    '''simple docstring'''







    with open(args.file_name				,'''r'''				,encoding='''utf-8'''						) as f:
        __A    :						List[str]						 =     f.readlines()
    __A    :						Optional[Any]						 =     [line.strip() for line in data if len(snake_case_						) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
    __A    :						str						 =     LTP(args.ltp						)  # faster in GPU device
    __A    :						Optional[int]						 =     BertTokenizer.from_pretrained(args.bert						)

    __A    :						Optional[Any]						 =     prepare_ref(snake_case_				,snake_case_				,snake_case_						)

    with open(args.save_path				,'''w'''				,encoding='''utf-8'''						) as f:
        __A    :						int						 =     [json.dumps(snake_case_						) + '''\n''' for ref in ref_ids]
        f.writelines(snake_case_						)


if __name__ == "__main__":
       a_ =			argparse.ArgumentParser(description="""prepare_chinese_ref""")
       parser.add_argument(
           """--file_name""",
           type=str,
           default="""./resources/chinese-demo.txt""",
           help="""file need process, same as training data in lm""",
       )
       parser.add_argument(
           """--ltp""", type=str, default="""./resources/ltp""", help="""resources for LTP tokenizer, usually a path"""
       )
       parser.add_argument("""--bert""", type=str, default="""./resources/robert""", help="""resources for Bert tokenizer""")
       parser.add_argument("""--save_path""", type=str, default="""./resources/ref.txt""", help="""path to save res""")

       a_ =			parser.parse_args()
       main(args)




