


"""simple docstring"""







import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


a__				:							int						=     re.compile(r'''\s+''')




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					return {"hash": hashlib.mda(re.sub(lowerCAmelCase_  ,						""  ,						example["content"]   ).encode("utf-8"   )   ).hexdigest()}




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				[len(lowerCAmelCase_   ) for line in example["content"].splitlines()]
					return {"line_mean": np.mean(lowerCAmelCase_   ), "line_max": max(lowerCAmelCase_   )}




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				np.mean([c.isalnum() for c in example["content"]]   )
					return {"alpha_frac": alpha_frac}




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					if example["hash"] in uniques:
										uniques.remove(example["hash"]   )
										return True
					else:
										return False




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_=5   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				["auto-generated", "autogenerated", "automatically generated"]
					__SCREAMING_SNAKE_CASE						   =				example["content"].splitlines()
					for _, line in zip(range(lowerCAmelCase_   )  ,						lowerCAmelCase_   ):
										for keyword in keywords:
															if keyword in line.lower():
																				return {"autogenerated": True}
					else:
										return {"autogenerated": False}




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_=5  ,						lowerCAmelCase_=0.05   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				["unit tests", "test file", "configuration file"]
					__SCREAMING_SNAKE_CASE						   =				example["content"].splitlines()
					__SCREAMING_SNAKE_CASE						   =				0
					__SCREAMING_SNAKE_CASE						   =				0
					# first test
					for _, line in zip(range(lowerCAmelCase_   )  ,						lowerCAmelCase_   ):
										for keyword in keywords:
															if keyword in line.lower():
																				return {"config_or_test": True}
    # second test
					__SCREAMING_SNAKE_CASE						   =				example["content"].count("\n"   )
					__SCREAMING_SNAKE_CASE						   =				int(coeff * nlines   )
					for line in lines:
										count_config += line.lower().count("config"   )
										count_test += line.lower().count("test"   )
										if count_config > threshold or count_test > threshold:
															return {"config_or_test": True}
					return {"config_or_test": False}




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				["def ", "class ", "for ", "while "]
					__SCREAMING_SNAKE_CASE						   =				example["content"].splitlines()
					for line in lines:
										for keyword in keywords:
															if keyword in line.lower():
																				return {"has_no_keywords": False}
					return {"has_no_keywords": True}




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_=4   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				example["content"].splitlines()
					__SCREAMING_SNAKE_CASE						   =				0
					for line in lines:
										counter += line.lower().count("="   )
										if counter > minimum:
															return {"has_few_assignments": False}
					return {"has_few_assignments": True}




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				tokenizer(example["content"]  ,						truncation=lowerCAmelCase_   )["input_ids"]
					__SCREAMING_SNAKE_CASE						   =				len(example["content"]   ) / len(lowerCAmelCase_   )
					return {"ratio": ratio}




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE						   =				{}
					results.update(get_hash(lowerCAmelCase_   )   )
					results.update(line_stats(lowerCAmelCase_   )   )
					results.update(alpha_stats(lowerCAmelCase_   )   )
					results.update(char_token_ratio(lowerCAmelCase_   )   )
					results.update(is_autogenerated(lowerCAmelCase_   )   )
					results.update(is_config_or_test(lowerCAmelCase_   )   )
					results.update(has_no_keywords(lowerCAmelCase_   )   )
					results.update(has_few_assignments(lowerCAmelCase_   )   )
					return results




def    UpperCAmelCase__       (lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   ):




					'''simple docstring'''


					if not check_uniques(lowerCAmelCase_  ,						lowerCAmelCase_   ):
										return False
					elif example["autogenerated"]:
										return False
					elif example["line_max"] > args.line_max:
										return False
					elif example["line_mean"] > args.line_mean:
										return False
					elif example["alpha_frac"] < args.alpha_frac:
										return False
					elif example["ratio"] < args.min_token_ratio:
										return False
					elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
										return False
					elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
										return False
					elif example["has_few_assignments"]:
										return False
					else:
										return True




def    UpperCAmelCase__       (lowerCAmelCase_   ):




					'''simple docstring'''


					with open(lowerCAmelCase_  ,						"rb"   ) as f_in:
										with gzip.open(str(lowerCAmelCase_   ) + ".gz"  ,						"wb"  ,						compresslevel=6   ) as f_out:
															shutil.copyfileobj(lowerCAmelCase_  ,						lowerCAmelCase_   )
					os.unlink(lowerCAmelCase_   )


# Settings
a__				:							List[str]						=     HfArgumentParser(PreprocessingArguments)
a__				:							List[str]						=     parser.parse_args()
if args.num_workers is None:
		a__				:							Tuple						=     multiprocessing.cpu_count()
a__				:							int						=     AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
a__				:							Optional[int]						=     time.time()
a__				:							List[Any]						=     load_dataset(args.dataset_name, split='''train''')
print(F"Time to load dataset: {time.time()-t_start:.2f}")

# Run preprocessing
a__				:							Any						=     time.time()
a__				:							Dict						=     ds.map(preprocess, num_proc=args.num_workers)
print(F"Time to preprocess dataset: {time.time()-t_start:.2f}")

# Deduplicate hashes
a__				:							Tuple						=     set(ds.unique('''hash'''))
a__				:							Union[str, Any]						=     len(uniques) / len(ds)
print(F"Fraction of duplicates: {1-frac:.2%}")

# Deduplicate data and apply heuristics
a__				:							Optional[Any]						=     time.time()
a__				:							str						=     ds.filter(filter, fn_kwargs={'''uniques''': uniques, '''args''': args})
print(F"Time to filter dataset: {time.time()-t_start:.2f}")
print(F"Size of filtered dataset: {len(ds_filter)}")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
		a__				:							str						=     time.time()
		a__			,     a__				:							str						=     deduplicate_dataset(ds_filter, args.jaccard_threshold)
		print(F"Time to deduplicate dataset: {time.time()-t_start:.2f}")
		print(F"Size of deduplicate dataset: {len(ds_filter)}")

# Save data in batches of samples_per_file
a__				:							Dict						=     Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
		with open(output_dir / '''duplicate_clusters.json''', '''w''') as f:
				json.dump(duplicate_clusters, f)

a__				:							List[str]						=     output_dir / '''data'''
data_dir.mkdir(exist_ok=True)

a__				:							List[str]						=     time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
		a__				:							int						=     str(data_dir / F"file-{file_number+1:012}.json")
		a__				:							Union[str, Any]						=     min(len(ds_filter), index + args.samples_per_file)
		ds_filter.select(list(range(index, end_index))).to_json(file_path)
		compress_file(file_path)



print(F"Time to save dataset: {time.time()-t_start:.2f}")


import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
	from transformers.trainer import DEFAULT_CALLBACKS

	from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	def __init__(       self       ):
		lowercase       =    []





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_init_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_train_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_train_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_epoch_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_epoch_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_step_begin'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_step_end'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_evaluate'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_predict'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_save'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_log'       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    **snake_case       ):
		self.events.append('on_prediction_step'       )







@require_torch
class        A_   (			unittest.TestCase   ):




	'''simple docstring'''


	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    tempfile.mkdtemp()





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		shutil.rmtree(self.output_dir       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case=0		,    snake_case=0		,    snake_case=64		,    snake_case=64		,    snake_case=None		,    snake_case=False		,    **snake_case       ):
		# disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
		# its set to False since the tests later on depend on its value.
		lowercase       =    RegressionDataset(length=snake_case       )
		lowercase       =    RegressionDataset(length=snake_case       )
		lowercase       =    RegressionModelConfig(a=snake_case		,    b=snake_case       )
		lowercase       =    RegressionPreTrainedModel(snake_case       )

		lowercase       =    TrainingArguments(self.output_dir		,    disable_tqdm=snake_case		,    report_to=[]		,    **snake_case       )
		return Trainer(
		    snake_case		,    snake_case		,    train_dataset=snake_case		,    eval_dataset=snake_case		,    callbacks=snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case       ):
		self.assertEqual(len(snake_case       )		,    len(snake_case       )       )

		# Order doesn't matter
		lowercase       =    sorted(snake_case		,    key=lambda snake_case       : cb.__name__ if isinstance(snake_case		,    snake_case       ) else cb.__class__.__name__       )
		lowercase       =    sorted(snake_case		,    key=lambda snake_case       : cb.__name__ if isinstance(snake_case		,    snake_case       ) else cb.__class__.__name__       )

		for cba, cba in zip(snake_case		,    snake_case       ):
			if isinstance(snake_case		,    snake_case       ) and isinstance(snake_case		,    snake_case       ):
				self.assertEqual(snake_case		,    snake_case       )
			elif isinstance(snake_case		,    snake_case       ) and not isinstance(snake_case		,    snake_case       ):
				self.assertEqual(snake_case		,    cba.__class__       )
			elif not isinstance(snake_case		,    snake_case       ) and isinstance(snake_case		,    snake_case       ):
				self.assertEqual(cba.__class__		,    snake_case       )
			else:
				self.assertEqual(snake_case		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    ['on_init_end', 'on_train_begin']
		lowercase       =    0
		lowercase       =    len(trainer.get_eval_dataloader()       )
		lowercase       =    ['on_prediction_step'] * len(trainer.get_eval_dataloader()       ) + ['on_log', 'on_evaluate']
		for _ in range(trainer.state.num_train_epochs       ):
			expected_events.append('on_epoch_begin'       )
			for _ in range(snake_case       ):
				step += 1
				expected_events += ["on_step_begin", "on_step_end"]
				if step % trainer.args.logging_steps == 0:
					expected_events.append('on_log'       )
				if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
					expected_events += evaluation_events.copy()
				if step % trainer.args.save_steps == 0:
					expected_events.append('on_save'       )
			expected_events.append('on_epoch_end'       )
			if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
				expected_events += evaluation_events.copy()
		expected_events += ["on_log", "on_train_end"]
		return expected_events





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.get_trainer()
		lowercase       =    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# Callbacks passed at init are added to the default callbacks
		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]       )
		expected_callbacks.append(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
		lowercase       =    self.get_trainer(disable_tqdm=snake_case       )
		lowercase       =    DEFAULT_CALLBACKS.copy() + [PrinterCallback]
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    DEFAULT_CALLBACKS.copy() + [ProgressCallback]
		lowercase       =    self.get_trainer()

		# We can add, pop, or remove by class name
		trainer.remove_callback(snake_case       )
		expected_callbacks.remove(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		lowercase       =    self.get_trainer()
		lowercase       =    trainer.pop_callback(snake_case       )
		self.assertEqual(cb.__class__		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		trainer.add_callback(snake_case       )
		expected_callbacks.insert(0		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		# We can also add, pop, or remove by instance
		lowercase       =    self.get_trainer()
		lowercase       =    trainer.callback_handler.callbacks[0]
		trainer.remove_callback(snake_case       )
		expected_callbacks.remove(snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		lowercase       =    self.get_trainer()
		lowercase       =    trainer.callback_handler.callbacks[0]
		lowercase       =    trainer.pop_callback(snake_case       )
		self.assertEqual(snake_case		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )

		trainer.add_callback(snake_case       )
		expected_callbacks.insert(0		,    snake_case       )
		self.check_callbacks_equality(trainer.callback_handler.callbacks		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		import warnings

		# XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
		warnings.simplefilter(action='ignore'		,    category=snake_case       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# Independent log/save/eval
		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    logging_steps=5       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    save_steps=5       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    eval_steps=5		,    evaluation_strategy='steps'       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		lowercase       =    self.get_trainer(callbacks=[MyTestTrainerCallback]		,    evaluation_strategy='epoch'       )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# A bit of everything
		lowercase       =    self.get_trainer(
		    callbacks=[MyTestTrainerCallback]		,    logging_steps=3		,    save_steps=10		,    eval_steps=5		,    evaluation_strategy='steps'		,    )
		trainer.train()
		lowercase       =    trainer.callback_handler.callbacks[-2].events
		self.assertEqual(snake_case		,    self.get_expected_events(snake_case       )       )

		# warning should be emitted for duplicated callbacks
		with patch('transformers.trainer_callback.logger.warning'       ) as warn_mock:
			lowercase       =    self.get_trainer(
			    callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]		,    )
			assert str(snake_case       ) in warn_mock.call_args[0][0]

