



'''simple docstring'''




import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
a   :			int					  =    logging.getLogger(__name__)

if __name__ == "__main__":
							a   :			List[str]					  =    argparse.ArgumentParser(
							    description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
							)
							parser.add_argument(
							    """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
							)
							parser.add_argument(
							    """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
							)
							parser.add_argument("""--vocab_size""", default=3_0_5_2_2, type=int)
							a   :			List[Any]					  =    parser.parse_args()

							logger.info(F'''Loading data from {args.data_file}''')
							with open(args.data_file, """rb""") as fp:
														a   :			str					  =    pickle.load(fp)

							logger.info("""Counting occurrences for MLM.""")
							a   :			str					  =    Counter()
							for tk_ids in data:
														counter.update(tk_ids)
							a   :			int					  =    [0] * args.vocab_size
							for k, v in counter.items():
														a   :			Optional[int]					  =    v

							logger.info(F'''Dump to {args.token_counts_dump}''')
							with open(args.token_counts_dump, """wb""") as handle:
														pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)






# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import sys
import warnings
from os.path import abspath, dirname, join


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
A__								=						abspath(join(dirname(dirname(dirname(__file__))), '''src'''))
sys.path.insert(1, git_repo_path)


# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action='''ignore''', category=FutureWarning)



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					str:


				"""simple docstring"""

				from transformers.testing_utils import pytest_addoption_shared

				pytest_addoption_shared(__lowerCAmelCase      )





def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					List[Any]:


				"""simple docstring"""

				from transformers.testing_utils import pytest_terminal_summary_main

				snake_case__	:      Dict     		=	terminalreporter.config.getoption('''--make-reports'''      )
				if make_reports:
								pytest_terminal_summary_main(__lowerCAmelCase  ,  id=__lowerCAmelCase      )

