


"""simple docstring"""







import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
lowerCamelCase_   :   int									=  logging.getLogger(__name__)

if __name__ == "__main__":
       lowerCamelCase_   :   Any									=  argparse.ArgumentParser(
           description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
       )
       parser.add_argument(
           """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
       )
       parser.add_argument(
           """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
       )
       parser.add_argument("""--vocab_size""", default=3_0_5_2_2, type=int)
       lowerCamelCase_   :   int									=  parser.parse_args()

       logger.info(F'Loading data from {args.data_file}')
       with open(args.data_file, """rb""") as fp:
              lowerCamelCase_   :   Optional[int]									=  pickle.load(fp)

       logger.info("""Counting occurrences for MLM.""")
       lowerCamelCase_   :   Optional[int]									=  Counter()
       for tk_ids in data:
              counter.update(tk_ids)
       lowerCamelCase_   :   Dict									=  [0] * args.vocab_size
       for k, v in counter.items():
              lowerCamelCase_   :   Any									=  v

       logger.info(F'Dump to {args.token_counts_dump}')
       with open(args.token_counts_dump, """wb""") as handle:
              pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)
"""simple docstring"""






from __future__ import annotations

from typing import Any





class  _A						:

      def __init__(						self		,       __lowerCAmelCase = 6						):







            """simple docstring"""



            lowercase														=	None
            lowercase														=	None
            self.create_linked_list(__lowerCAmelCase						)

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	Node()
            lowercase														=	current_node
            lowercase														=	current_node
            lowercase														=	current_node
            for _ in range(1		,       __lowerCAmelCase						):
                  lowercase														=	Node()
                  lowercase														=	current_node
                  lowercase														=	previous_node
                  lowercase														=	current_node
            lowercase														=	self.front
            lowercase														=	previous_node

      def 				A__       (						self						):







            """simple docstring"""



            return (
                self.front == self.rear
                and self.front is not None
                and self.front.data is None
            )

      def 				A__       (						self						):







            """simple docstring"""



            self.check_can_perform_operation()
            return self.front.data if self.front else None

      def 				A__       (						self		,       __lowerCAmelCase						):







            """simple docstring"""



            if self.rear is None:
                  return

            self.check_is_full()
            if not self.is_empty():
                  lowercase														=	self.rear.next
            if self.rear:
                  lowercase														=	data

      def 				A__       (						self						):







            """simple docstring"""



            self.check_can_perform_operation()
            if self.rear is None or self.front is None:
                  return None
            if self.front == self.rear:
                  lowercase														=	self.front.data
                  lowercase														=	None
                  return data

            lowercase														=	self.front
            lowercase														=	old_front.next
            lowercase														=	old_front.data
            lowercase														=	None
            return data

      def 				A__       (						self						):







            """simple docstring"""



            if self.is_empty():
                  raise Exception("""Empty Queue"""						)







      def 				A__       (						self						):







            """simple docstring"""



            if self.rear and self.rear.next == self.front:
                  raise Exception("""Full Queue"""						)






class  _A						:

      def __init__(						self						):







            """simple docstring"""



            lowercase														=	None
            lowercase														=	None
            lowercase														=	None


if __name__ == "__main__":
       import doctest

       doctest.testmod()



