



'''simple docstring'''
import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule






class 							lowercase__	(							lowercase   ):
	lowercase__  				=			42
	lowercase__  				=			jnp.floataa
	lowercase__  				=			True





	def 	UpperCamelCase_  ( self					:				Optional[Any] ):







							'''simple docstring'''
							super().setup()
							_UpperCamelCase					:		Optional[int] 				=   nn.Dense(5 ,dtype=self.dtype )





	def __call__( self					:				Optional[Any] ,*lowerCamelCase__					:				Tuple ,**lowerCamelCase__					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Any 				=   super().__call__(*lowerCamelCase__ ,**lowerCamelCase__ )
							_UpperCamelCase					:		Union[str, Any] 				=   self.cls(outputs[2] )
							return outputs[:2] + (cls_out,)






class 							lowercase__	(							lowercase   ):
	lowercase__  				=			FlaxBigBirdForNaturalQuestionsModule






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							):
						def cross_entropy(UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_=None							):
												_UpperCamelCase					:		Union[str, Any] 				=   logits.shape[-1]
												_UpperCamelCase					:		List[str] 				=   (labels[..., None] == jnp.arange(UpperCAmelCase_							)[None]).astype('f4'							)
												_UpperCamelCase					:		Optional[int] 				=   jax.nn.log_softmax(UpperCAmelCase_  ,  axis=-1							)
												_UpperCamelCase					:		List[str] 				=   -jnp.sum(labels * logits  ,  axis=-1							)
												if reduction is not None:
																		_UpperCamelCase					:		Optional[Any] 				=   reduction(UpperCAmelCase_							)
												return loss

						_UpperCamelCase					:		Dict 				=   partial(UpperCAmelCase_  ,  reduction=jnp.mean							)
						_UpperCamelCase					:		List[Any] 				=   cross_entropy(UpperCAmelCase_  ,  UpperCAmelCase_							)
						_UpperCamelCase					:		List[str] 				=   cross_entropy(UpperCAmelCase_  ,  UpperCAmelCase_							)
						_UpperCamelCase					:		Tuple 				=   cross_entropy(UpperCAmelCase_  ,  UpperCAmelCase_							)
						return (start_loss + end_loss + pooled_loss) / 3






@dataclass
class 							lowercase__	:
	lowercase__  				=			"google/bigbird-roberta-base"
	lowercase__  				=			30_00
	lowercase__  				=			1_05_00

	lowercase__  				=			1_28
	lowercase__  				=			3

	lowercase__  				=			1
	lowercase__  				=			5

	# tx_args
	lowercase__  				=			3E-5
	lowercase__  				=			0.0
	lowercase__  				=			2_00_00
	lowercase__  				=			0.00_95

	lowercase__  				=			"bigbird-roberta-natural-questions"
	lowercase__  				=			"training-expt"
	lowercase__  				=			"data/nq-training.jsonl"
	lowercase__  				=			"data/nq-validation.jsonl"





	def 	UpperCamelCase_  ( self					:				Dict ):







							'''simple docstring'''
							os.makedirs(self.base_dir ,exist_ok=lowerCamelCase__ )
							_UpperCamelCase					:		Tuple 				=   os.path.join(self.base_dir ,self.save_dir )
							_UpperCamelCase					:		Union[str, Any] 				=   self.batch_size_per_device * jax.device_count()






@dataclass
class 							lowercase__	:
	lowercase__  				=			42
	lowercase__  				=			40_96  # no dynamic padding on TPUs





	def __call__( self					:				Optional[int] ,lowerCamelCase__					:				str ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   self.collate_fn(lowerCamelCase__ )
							_UpperCamelCase					:		List[Any] 				=   jax.tree_util.tree_map(lowerCamelCase__ ,lowerCamelCase__ )
							return batch





	def 	UpperCamelCase_  ( self					:				Optional[int] ,lowerCamelCase__					:				int ):







							'''simple docstring'''
							_UpperCamelCase				,			_UpperCamelCase					:		Optional[int] 				=   self.fetch_inputs(features['input_ids'] )
							_UpperCamelCase					:		Tuple 				=   {
							    'input_ids': jnp.array(lowerCamelCase__ ,dtype=jnp.intaa ),
							    'attention_mask': jnp.array(lowerCamelCase__ ,dtype=jnp.intaa ),
							    'start_labels': jnp.array(features['start_token'] ,dtype=jnp.intaa ),
							    'end_labels': jnp.array(features['end_token'] ,dtype=jnp.intaa ),
							    'pooled_labels': jnp.array(features['category'] ,dtype=jnp.intaa ),
							}
							return batch





	def 	UpperCamelCase_  ( self					:				Union[str, Any] ,lowerCamelCase__					:				list ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   [self._fetch_inputs(lowerCamelCase__ ) for ids in input_ids]
							return zip(*lowerCamelCase__ )





	def 	UpperCamelCase_  ( self					:				Tuple ,lowerCamelCase__					:				list ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   [1 for _ in range(len(lowerCamelCase__ ) )]
							while len(lowerCamelCase__ ) < self.max_length:
													input_ids.append(self.pad_id )
													attention_mask.append(0 )
							return input_ids, attention_mask






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_=None							):
						if seed is not None:
												_UpperCamelCase					:		int 				=   dataset.shuffle(seed=UpperCAmelCase_							)
						for i in range(len(UpperCAmelCase_							) // batch_size							):
												_UpperCamelCase					:		Union[str, Any] 				=   dataset[i * batch_size : (i + 1) * batch_size]
												yield dict(UpperCAmelCase_							)






@partial(jax.pmap  ,  axis_name='batch'							)
def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_  ,  **UpperCAmelCase_							):
						def loss_fn(UpperCAmelCase_							):
												_UpperCamelCase					:		int 				=   model_inputs.pop('start_labels'							)
												_UpperCamelCase					:		Tuple 				=   model_inputs.pop('end_labels'							)
												_UpperCamelCase					:		Optional[int] 				=   model_inputs.pop('pooled_labels'							)

												_UpperCamelCase					:		str 				=   state.apply_fn(**UpperCAmelCase_  ,  params=UpperCAmelCase_  ,  dropout_rng=UpperCAmelCase_  ,  train=UpperCAmelCase_							)
												_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase					:		Any 				=   outputs

												return state.loss_fn(
												    UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  )

						_UpperCamelCase				,			_UpperCamelCase					:		List[Any] 				=   jax.random.split(UpperCAmelCase_							)
						_UpperCamelCase					:		Any 				=   jax.value_and_grad(UpperCAmelCase_							)
						_UpperCamelCase				,			_UpperCamelCase					:		Union[str, Any] 				=   grad_fn(state.params							)
						_UpperCamelCase					:		List[str] 				=   jax.lax.pmean({'loss': loss}  ,  axis_name='batch'							)
						_UpperCamelCase					:		str 				=   jax.lax.pmean(UpperCAmelCase_  ,  'batch'							)

						_UpperCamelCase					:		Optional[int] 				=   state.apply_gradients(grads=UpperCAmelCase_							)
						return state, metrics, new_drp_rng






@partial(jax.pmap  ,  axis_name='batch'							)
def A__    (     UpperCAmelCase_  ,  **UpperCAmelCase_							):
						_UpperCamelCase					:		Optional[int] 				=   model_inputs.pop('start_labels'							)
						_UpperCamelCase					:		Optional[Any] 				=   model_inputs.pop('end_labels'							)
						_UpperCamelCase					:		List[str] 				=   model_inputs.pop('pooled_labels'							)

						_UpperCamelCase					:		Tuple 				=   state.apply_fn(**UpperCAmelCase_  ,  params=state.params  ,  train=UpperCAmelCase_							)
						_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase					:		Optional[int] 				=   outputs

						_UpperCamelCase					:		int 				=   state.loss_fn(UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							)
						_UpperCamelCase					:		Dict 				=   jax.lax.pmean({'loss': loss}  ,  axis_name='batch'							)
						return metrics






class 							lowercase__	(							train_state.TrainState   ):
	lowercase__  				=			struct.field(pytree_node=lowercase   )






@dataclass
class 							lowercase__	:
	lowercase__  				=			42
	lowercase__  				=			42
	lowercase__  				=			42
	lowercase__  				=			42
	lowercase__  				=			42
	lowercase__  				=			42
	lowercase__  				=			None





	def 	UpperCamelCase_  ( self					:				List[Any] ,lowerCamelCase__					:				Any ,lowerCamelCase__					:				List[str] ,lowerCamelCase__					:				Union[str, Any] ,lowerCamelCase__					:				Dict=None ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   model.params
							_UpperCamelCase					:		Any 				=   TrainState.create(
							    apply_fn=model.__call__ ,params=lowerCamelCase__ ,tx=lowerCamelCase__ ,loss_fn=lowerCamelCase__ ,)
							if ckpt_dir is not None:
													_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase					:		Union[str, Any] 				=   restore_checkpoint(lowerCamelCase__ ,lowerCamelCase__ )
													_UpperCamelCase					:		int 				=   {
													    'lr': args.lr,
													    'init_lr': args.init_lr,
													    'warmup_steps': args.warmup_steps,
													    'num_train_steps': num_train_steps,
													    'weight_decay': args.weight_decay,
													}
													_UpperCamelCase				,			_UpperCamelCase					:		Union[str, Any] 				=   build_tx(**lowerCamelCase__ )
													_UpperCamelCase					:		Tuple 				=   train_state.TrainState(
													    step=lowerCamelCase__ ,apply_fn=model.__call__ ,params=lowerCamelCase__ ,tx=lowerCamelCase__ ,opt_state=lowerCamelCase__ ,)
													_UpperCamelCase					:		Tuple 				=   args
													_UpperCamelCase					:		Any 				=   data_collator
													_UpperCamelCase					:		Optional[int] 				=   lr
													_UpperCamelCase					:		str 				=   params
							_UpperCamelCase					:		List[Any] 				=   jax_utils.replicate(lowerCamelCase__ )
							return state





	def 	UpperCamelCase_  ( self					:				Optional[Any] ,lowerCamelCase__					:				List[str] ,lowerCamelCase__					:				Optional[Any] ,lowerCamelCase__					:				Optional[Any] ):







							'''simple docstring'''
							_UpperCamelCase					:		Union[str, Any] 				=   self.args
							_UpperCamelCase					:		Dict 				=   len(lowerCamelCase__ ) // args.batch_size

							_UpperCamelCase					:		Dict 				=   jax.random.PRNGKey(0 )
							_UpperCamelCase					:		List[str] 				=   jax.random.split(lowerCamelCase__ ,jax.device_count() )
							for epoch in range(args.max_epochs ):
													_UpperCamelCase					:		Union[str, Any] 				=   jnp.array(0 ,dtype=jnp.floataa )
													_UpperCamelCase					:		Optional[Any] 				=   get_batched_dataset(lowerCamelCase__ ,args.batch_size ,seed=lowerCamelCase__ )
													_UpperCamelCase					:		Tuple 				=   0
													for batch in tqdm(lowerCamelCase__ ,total=lowerCamelCase__ ,desc=F'Running EPOCH-{epoch}' ):
																			_UpperCamelCase					:		Optional[Any] 				=   self.data_collator(lowerCamelCase__ )
																			_UpperCamelCase				,			_UpperCamelCase				,			_UpperCamelCase					:		List[Any] 				=   self.train_step_fn(lowerCamelCase__ ,lowerCamelCase__ ,**lowerCamelCase__ )
																			running_loss += jax_utils.unreplicate(metrics['loss'] )
																			i += 1
																			if i % args.logging_steps == 0:
																									_UpperCamelCase					:		str 				=   jax_utils.unreplicate(state.step )
																									_UpperCamelCase					:		str 				=   running_loss.item() / i
																									_UpperCamelCase					:		str 				=   self.scheduler_fn(state_step - 1 )

																									_UpperCamelCase					:		List[Any] 				=   self.evaluate(lowerCamelCase__ ,lowerCamelCase__ )
																									_UpperCamelCase					:		Union[str, Any] 				=   {
																									    'step': state_step.item(),
																									    'eval_loss': eval_loss.item(),
																									    'tr_loss': tr_loss,
																									    'lr': lr.item(),
																									}
																									tqdm.write(str(lowerCamelCase__ ) )
																									self.logger.log(lowerCamelCase__ ,commit=lowerCamelCase__ )

																			if i % args.save_steps == 0:
																									self.save_checkpoint(args.save_dir + F'-e{epoch}-s{i}' ,state=lowerCamelCase__ )





	def 	UpperCamelCase_  ( self					:				str ,lowerCamelCase__					:				List[str] ,lowerCamelCase__					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[int] 				=   get_batched_dataset(lowerCamelCase__ ,self.args.batch_size )
							_UpperCamelCase					:		Union[str, Any] 				=   len(lowerCamelCase__ ) // self.args.batch_size
							_UpperCamelCase					:		Union[str, Any] 				=   jnp.array(0 ,dtype=jnp.floataa )
							_UpperCamelCase					:		Any 				=   0
							for batch in tqdm(lowerCamelCase__ ,total=lowerCamelCase__ ,desc='Evaluating ... ' ):
													_UpperCamelCase					:		str 				=   self.data_collator(lowerCamelCase__ )
													_UpperCamelCase					:		Any 				=   self.val_step_fn(lowerCamelCase__ ,**lowerCamelCase__ )
													running_loss += jax_utils.unreplicate(metrics['loss'] )
													i += 1
							return running_loss / i





	def 	UpperCamelCase_  ( self					:				List[str] ,lowerCamelCase__					:				Optional[Any] ,lowerCamelCase__					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		List[Any] 				=   jax_utils.unreplicate(lowerCamelCase__ )
							print(F'SAVING CHECKPOINT IN {save_dir}' ,end=' ... ' )
							self.model_save_fn(lowerCamelCase__ ,params=state.params )
							with open(os.path.join(lowerCamelCase__ ,'opt_state.msgpack' ) ,'wb' ) as f:
													f.write(to_bytes(state.opt_state ) )
							joblib.dump(self.args ,os.path.join(lowerCamelCase__ ,'args.joblib' ) )
							joblib.dump(self.data_collator ,os.path.join(lowerCamelCase__ ,'data_collator.joblib' ) )
							with open(os.path.join(lowerCamelCase__ ,'training_state.json' ) ,'w' ) as f:
													json.dump({'step': state.step.item()} ,lowerCamelCase__ )
							print('DONE' )






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_							):
						print(f'RESTORING CHECKPOINT FROM {save_dir}'  ,  end=' ... '							)
						with open(os.path.join(UpperCAmelCase_  ,  'flax_model.msgpack'							)  ,  'rb'							) as f:
												_UpperCamelCase					:		Optional[int] 				=   from_bytes(state.params  ,  f.read()							)

						with open(os.path.join(UpperCAmelCase_  ,  'opt_state.msgpack'							)  ,  'rb'							) as f:
												_UpperCamelCase					:		Optional[int] 				=   from_bytes(state.opt_state  ,  f.read()							)

						_UpperCamelCase					:		Optional[Any] 				=   joblib.load(os.path.join(UpperCAmelCase_  ,  'args.joblib'							)							)
						_UpperCamelCase					:		Dict 				=   joblib.load(os.path.join(UpperCAmelCase_  ,  'data_collator.joblib'							)							)

						with open(os.path.join(UpperCAmelCase_  ,  'training_state.json'							)  ,  'r'							) as f:
												_UpperCamelCase					:		Optional[Any] 				=   json.load(UpperCAmelCase_							)
						_UpperCamelCase					:		Optional[Any] 				=   training_state['step']

						print('DONE'							)
						return params, opt_state, step, args, data_collator






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							):
						_UpperCamelCase					:		Optional[int] 				=   num_train_steps - warmup_steps
						_UpperCamelCase					:		int 				=   optax.linear_schedule(init_value=UpperCAmelCase_  ,  end_value=UpperCAmelCase_  ,  transition_steps=UpperCAmelCase_							)
						_UpperCamelCase					:		Optional[int] 				=   optax.linear_schedule(init_value=UpperCAmelCase_  ,  end_value=1E-7  ,  transition_steps=UpperCAmelCase_							)
						_UpperCamelCase					:		List[str] 				=   optax.join_schedules(schedules=[warmup_fn, decay_fn]  ,  boundaries=[warmup_steps]							)
						return lr






def A__    (     UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							):
						def weight_decay_mask(UpperCAmelCase_							):
												_UpperCamelCase					:		Optional[Any] 				=   traverse_util.flatten_dict(UpperCAmelCase_							)
												_UpperCamelCase					:		List[Any] 				=   {k: (v[-1] != 'bias' and v[-2:] != ('LayerNorm', 'scale')) for k, v in params.items()}
												return traverse_util.unflatten_dict(UpperCAmelCase_							)

						_UpperCamelCase					:		str 				=   scheduler_fn(UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_  ,  UpperCAmelCase_							)

						_UpperCamelCase					:		int 				=   optax.adamw(learning_rate=UpperCAmelCase_  ,  weight_decay=UpperCAmelCase_  ,  mask=UpperCAmelCase_							)
						return tx, lr









'''simple docstring'''




from sympy import diff, lambdify, symbols
from sympy.functions import *  # noqa: F403






def        _lowerCAmelCase			(  __snake_case  : str ,							__snake_case  : complex ,							__snake_case  : str = "x" ,							__snake_case  : float = 10**-10 ,							__snake_case  : int = 1 ,							)     ->     complex:
     __A						: int      =       symbols(__snake_case			)
     __A						: Tuple      =       lambdify(__snake_case ,							__snake_case			)
     __A						: Any      =       lambdify(__snake_case ,							diff(__snake_case ,							__snake_case			)			)

     __A						: str      =       starting_point

     while True:
          if diff_function(__snake_case			) != 0:
               __A						: Optional[Any]      =       prev_guess - multiplicity * func(__snake_case			) / diff_function(
                   __snake_case			)
          else:
               raise ZeroDivisionError('Could not find root'			) from None

          # Precision is checked by comparing the difference of consecutive guesses
          if abs(next_guess - prev_guess			) < precision:
               return next_guess

          __A						: Dict      =       next_guess


# Let's Execute
if __name__ == "__main__":
  # Find root of trigonometric function
  # Find value of pi
  print(f"""The root of sin(x) = 0 is {newton_raphson("sin(x)", 2)}""")

  # Find root of polynomial
  # Find fourth Root of 5
  print(f"""The root of x**4 - 5 = 0 is {newton_raphson("x**4 -5", 0.4 +5j)}""")

  # Find value of e
  print(
      '''The root of log(y) - 1 = 0 is ''',
      f"""{newton_raphson("log(y) - 1", 2, variable="y")}""",
  )

  # Exponential Roots
  print(
      '''The root of exp(x) - 1 = 0 is''',
      f"""{newton_raphson("exp(x) - 1", 10, precision=0.005)}""",
  )

  # Find root of cos(x)
  print(f"""The root of cos(x) = 0 is {newton_raphson("cos(x)", 0)}""")