

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
snake_case_			  = logging.getLogger(__name__)

if __name__ == "__main__":
       snake_case_			  = argparse.ArgumentParser(
           description='''Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'''
       )
       parser.add_argument(
           '''--data_file''', type=str, default='''data/dump.bert-base-uncased.pickle''', help='''The binarized dataset.'''
       )
       parser.add_argument(
           '''--token_counts_dump''', type=str, default='''data/token_counts.bert-base-uncased.pickle''', help='''The dump file.'''
       )
       parser.add_argument('''--vocab_size''', default=30_522, type=int)
       snake_case_			  = parser.parse_args()

       logger.info(F'''Loading data from {args.data_file}''')
       with open(args.data_file, '''rb''') as fp:
              snake_case_			  = pickle.load(fp)

       logger.info('''Counting occurrences for MLM.''')
       snake_case_			  = Counter()
       for tk_ids in data:
              counter.update(tk_ids)
       snake_case_			  = [0] * args.vocab_size
       for k, v in counter.items():
              snake_case_			  = v

       logger.info(F'''Dump to {args.token_counts_dump}''')
       with open(args.token_counts_dump, '''wb''') as handle:
              pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)




import unittest

from transformers import DonutProcessor


snake_case_			  = '''naver-clova-ix/donut-base'''


class       SCREAMING_SNAKE_CASE__					(unittest.TestCase     ):





    def       snake_case_				(      self):
         lowercase__    :   Dict      =       DonutProcessor.from_pretrained(a)





    def       snake_case_				(      self):
         lowercase__    :   Tuple      =       {
             'name': 'John Doe',
             'age': '99',
             'city': 'Atlanta',
             'state': 'GA',
             'zip': '30301',
             'phone': '123-4567',
             'nicknames': [{'nickname': 'Johnny'}, {'nickname': 'JD'}],
         }

         lowercase__    :   Tuple      =       (
             '<s_name>John Doe</s_name><s_age>99</s_age><s_city>Atlanta</s_city>'
             '<s_state>GA</s_state><s_zip>30301</s_zip><s_phone>123-4567</s_phone>'
             '<s_nicknames><s_nickname>Johnny</s_nickname>'
             '<sep/><s_nickname>JD</s_nickname></s_nicknames>'
         )
         lowercase__    :   str      =       self.processor.tokenajson(a)

         self.assertDictEqual(a						,   a)

