"""simple docstring"""






import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
__lowerCAmelCase    :       List[Any]					     =logging.getLogger(__name__)

if __name__ == "__main__":
       __lowerCAmelCase    :       Optional[int]					     =argparse.ArgumentParser(
           description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
       )
       parser.add_argument(
           """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
       )
       parser.add_argument(
           """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
       )
       parser.add_argument("""--vocab_size""", default=3_0_5_2_2, type=int)
       __lowerCAmelCase    :       Optional[int]					     =parser.parse_args()

       logger.info(F"""Loading data from {args.data_file}""")
       with open(args.data_file, """rb""") as fp:
              __lowerCAmelCase    :       str					     =pickle.load(fp)

       logger.info("""Counting occurrences for MLM.""")
       __lowerCAmelCase    :       Tuple					     =Counter()
       for tk_ids in data:
              counter.update(tk_ids)
       __lowerCAmelCase    :       Any					     =[0] * args.vocab_size
       for k, v in counter.items():
              __lowerCAmelCase    :       Any					     =v

       logger.info(F"""Dump to {args.token_counts_dump}""")
       with open(args.token_counts_dump, """wb""") as handle:
              pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




"""simple docstring"""






from ..utils import DummyObject, requires_backends





class  _A						(       metaclass=lowerCAmelCase	):
      snake_case__			:   List[str]							  =	['onnx']

      def __init__(						self		,       *__lowerCAmelCase		,       **__lowerCAmelCase						):







            """simple docstring"""



            requires_backends(self		,       ["""onnx"""]						)

      @classmethod
      def 				A__       (						cls		,       *__lowerCAmelCase		,       **__lowerCAmelCase						):







            """simple docstring"""



            requires_backends(cls		,       ["""onnx"""]						)







      @classmethod
      def 				A__       (						cls		,       *__lowerCAmelCase		,       **__lowerCAmelCase						):







            """simple docstring"""



            requires_backends(cls		,       ["""onnx"""]						)



