'''simple docstring'''




import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__=True				,  UpperCamelCase__="pt"				)	->    List[Any]:
    __lowerCamelCase		=  {'''add_prefix_space''': True} if isinstance(UpperCamelCase__				,  UpperCamelCase__				) and not line.startswith(''' '''				) else {}
    __lowerCamelCase		=  padding_side
    return tokenizer(
        [line]				,  max_length=UpperCamelCase__				,  padding='''max_length''' if pad_to_max_length else None				,  truncation=UpperCamelCase__				,  return_tensors=UpperCamelCase__				,  add_special_tokens=UpperCamelCase__				,  **UpperCamelCase__				,  )






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__=None				,  )	->    Tuple:
    __lowerCamelCase		=  input_ids.ne(UpperCamelCase__				).any(dim=0				)
    if attention_mask is None:
        return input_ids[:, keep_column_mask]
    else:
        return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])




class     a__      (   UpperCAmelCase__ ):






     def __init__( self		:		Dict  ,				a		:		Optional[Any]  ,				a		:		int  ,				a		:		Optional[int]  ,				a		:		Union[str, Any]  ,				a		:		Optional[int]="train"  ,				a		:		List[str]=None  ,				a		:		List[str]=None  ,				a		:		List[str]=None  ,				a		:		Dict=""  ,				):





         """simple docstring"""



         super().__init__()
         __lowerCamelCase		=  Path(a    ).joinpath(type_path + '''.source'''    )
         __lowerCamelCase		=  Path(a    ).joinpath(type_path + '''.target'''    )
         __lowerCamelCase		=  self.get_char_lens(self.src_file    )
         __lowerCamelCase		=  max_source_length
         __lowerCamelCase		=  max_target_length
         assert min(self.src_lens    ) > 0, f"""found empty line in {self.src_file}"""
         __lowerCamelCase		=  tokenizer
         __lowerCamelCase		=  prefix
         if n_obs is not None:
             __lowerCamelCase		=  self.src_lens[:n_obs]
         __lowerCamelCase		=  src_lang
         __lowerCamelCase		=  tgt_lang






     def __len__( self		:		Dict    ):





         """simple docstring"""



         return len(self.src_lens    )






     def __getitem__( self		:		int  ,				a		:		Any    ):





         """simple docstring"""



         __lowerCamelCase		=  index + 1  # linecache starts at 1
         __lowerCamelCase		=  self.prefix + linecache.getline(str(self.src_file    )  ,				a    ).rstrip('''\n'''    )
         __lowerCamelCase		=  linecache.getline(str(self.tgt_file    )  ,				a    ).rstrip('''\n'''    )
         assert source_line, f"""empty source line for index {index}"""
         assert tgt_line, f"""empty tgt line for index {index}"""

         # Need to add eos token manually for T5
         if isinstance(self.tokenizer  ,				a    ):
             source_line += self.tokenizer.eos_token
             tgt_line += self.tokenizer.eos_token

         # Pad source and target to the right
         __lowerCamelCase		=  (
             self.tokenizer.question_encoder if isinstance(self.tokenizer  ,				a    ) else self.tokenizer
         )
         __lowerCamelCase		=  self.tokenizer.generator if isinstance(self.tokenizer  ,				a    ) else self.tokenizer

         __lowerCamelCase		=  encode_line(a  ,				a  ,				self.max_source_length  ,				'''right'''    )
         __lowerCamelCase		=  encode_line(a  ,				a  ,				self.max_target_length  ,				'''right'''    )

         __lowerCamelCase		=  source_inputs['''input_ids'''].squeeze()
         __lowerCamelCase		=  target_inputs['''input_ids'''].squeeze()
         __lowerCamelCase		=  source_inputs['''attention_mask'''].squeeze()
         return {
             "input_ids": source_ids,
             "attention_mask": src_mask,
             "decoder_input_ids": target_ids,
         }






     @staticmethod
     def    SCREAMING_SNAKE_CASE__   ( a		:		Any    ):





         """simple docstring"""



         return [len(a    ) for x in Path(a    ).open().readlines()]






     def    SCREAMING_SNAKE_CASE__   ( self		:		Tuple  ,				a		:		Optional[Any]    ):





         """simple docstring"""



         __lowerCamelCase		=  torch.stack([x['''input_ids'''] for x in batch]    )
         __lowerCamelCase		=  torch.stack([x['''attention_mask'''] for x in batch]    )
         __lowerCamelCase		=  torch.stack([x['''decoder_input_ids'''] for x in batch]    )
         __lowerCamelCase		=  (
             self.tokenizer.generator.pad_token_id
             if isinstance(self.tokenizer  ,				a    )
             else self.tokenizer.pad_token_id
         )
         __lowerCamelCase		=  (
             self.tokenizer.question_encoder.pad_token_id
             if isinstance(self.tokenizer  ,				a    )
             else self.tokenizer.pad_token_id
         )
         __lowerCamelCase		=  trim_batch(a  ,				a    )
         __lowerCamelCase	,  __lowerCamelCase		=  trim_batch(a  ,				a  ,				attention_mask=a    )
         __lowerCamelCase		=  {
             '''input_ids''': source_ids,
             '''attention_mask''': source_mask,
             '''decoder_input_ids''': y,
         }
         return batch
__UpperCAmelCase     							=getLogger(__name__)






def __lowerCAmelCase (							UpperCamelCase__				)	->    Dict:
    return list(itertools.chain.from_iterable(UpperCamelCase__				)				)






def __lowerCAmelCase (							UpperCamelCase__				)	->    None:
    __lowerCamelCase		=  get_git_info()
    save_json(UpperCamelCase__				,  os.path.join(UpperCamelCase__				,  '''git_log.json'''				)				)






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__=4				,  **UpperCamelCase__				)	->    Tuple:
    with open(UpperCamelCase__				,  '''w'''				) as f:
        json.dump(UpperCamelCase__				,  UpperCamelCase__				,  indent=UpperCamelCase__				,  **UpperCamelCase__				)






def __lowerCAmelCase (							UpperCamelCase__				)	->    Optional[Any]:
    with open(UpperCamelCase__				) as f:
        return json.load(UpperCamelCase__				)






def __lowerCAmelCase (							)	->    Union[str, Any]:
    __lowerCamelCase		=  git.Repo(search_parent_directories=UpperCamelCase__				)
    __lowerCamelCase		=  {
        '''repo_id''': str(UpperCamelCase__				),
        '''repo_sha''': str(repo.head.object.hexsha				),
        '''repo_branch''': str(repo.active_branch				),
        '''hostname''': str(socket.gethostname()				),
    }
    return repo_infos






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				)	->    List:
    return list(map(UpperCamelCase__				,  UpperCamelCase__				)				)






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				)	->    Dict:
    with open(UpperCamelCase__				,  '''wb'''				) as f:
        return pickle.dump(UpperCamelCase__				,  UpperCamelCase__				)






def __lowerCAmelCase (							UpperCamelCase__				)	->    Dict:

    def remove_articles(UpperCamelCase__				):
        return re.sub(r'''\b(a|an|the)\b'''				,  ''' '''				,  UpperCamelCase__				)

    def white_space_fix(UpperCamelCase__				):
        return " ".join(text.split()				)

    def remove_punc(UpperCamelCase__				):
        __lowerCamelCase		=  set(string.punctuation				)
        return "".join(ch for ch in text if ch not in exclude				)

    def lower(UpperCamelCase__				):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(UpperCamelCase__				)				)				)				)






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				)	->    str:
    __lowerCamelCase		=  normalize_answer(UpperCamelCase__				).split()
    __lowerCamelCase		=  normalize_answer(UpperCamelCase__				).split()
    __lowerCamelCase		=  Counter(UpperCamelCase__				) & Counter(UpperCamelCase__				)
    __lowerCamelCase		=  sum(common.values()				)
    if num_same == 0:
        return 0
    __lowerCamelCase		=  1.0 * num_same / len(UpperCamelCase__				)
    __lowerCamelCase		=  1.0 * num_same / len(UpperCamelCase__				)
    __lowerCamelCase		=  (2 * precision * recall) / (precision + recall)
    return fa






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				)	->    Optional[Any]:
    return normalize_answer(UpperCamelCase__				) == normalize_answer(UpperCamelCase__				)






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				)	->    Dict:
    assert len(UpperCamelCase__				) == len(UpperCamelCase__				)
    __lowerCamelCase		=  0
    for hypo, pred in zip(UpperCamelCase__				,  UpperCamelCase__				):
        em += exact_match_score(UpperCamelCase__				,  UpperCamelCase__				)
    if len(UpperCamelCase__				) > 0:
        em /= len(UpperCamelCase__				)
    return {"em": em}






def __lowerCAmelCase (							UpperCamelCase__				)	->    Tuple:
    return model_prefix.startswith('''rag'''				)






def __lowerCAmelCase (							UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__				)	->    Union[str, Any]:
    __lowerCamelCase		=  {p: p for p in extra_params}
    # T5 models don't have `dropout` param, they have `dropout_rate` instead
    __lowerCamelCase		=  '''dropout_rate'''
    for p in extra_params:
        if getattr(UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__				):
            if not hasattr(UpperCamelCase__				,  UpperCamelCase__				) and not hasattr(UpperCamelCase__				,  equivalent_param[p]				):
                logger.info('''config doesn\'t have a `{}` attribute'''.format(UpperCamelCase__				)				)
                delattr(UpperCamelCase__				,  UpperCamelCase__				)
                continue
            __lowerCamelCase		=  p if hasattr(UpperCamelCase__				,  UpperCamelCase__				) else equivalent_param[p]
            setattr(UpperCamelCase__				,  UpperCamelCase__				,  getattr(UpperCamelCase__				,  UpperCamelCase__				)				)
            delattr(UpperCamelCase__				,  UpperCamelCase__				)
    return hparams, config







"""simple docstring"""



import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
       import tensorflow as tf

if is_torch_available():
       import torch


a_ =			logging.get_logger(__name__)






class 		__snake_case					:





 """simple docstring"""


 def __init__(    self					,						__lowerCamelCase = None					,						__lowerCamelCase = None					,						__lowerCamelCase=None					,						__lowerCamelCase=None  ):



     '''simple docstring'''




     if not conversation_id:
         __A    :						List[Any]						 =     uuid.uuida()
     if past_user_inputs is None:
         __A    :						List[str]						 =     []
     if generated_responses is None:
         __A    :						Tuple						 =     []

     __A    :						uuid.UUID						 =     conversation_id
     __A    :						List[str]						 =     past_user_inputs
     __A    :						List[str]						 =     generated_responses
     __A    :						Optional[str]						 =     text

 def __eq__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     if not isinstance(__lowerCamelCase					,						__lowerCamelCase  ):
         return False
     if self.uuid == other.uuid:
         return True
     return (
         self.new_user_input == other.new_user_input
         and self.past_user_inputs == other.past_user_inputs
         and self.generated_responses == other.generated_responses
     )

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase = False  ):



     '''simple docstring'''




     if self.new_user_input:
         if overwrite:
             logger.warning(
                 F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" was overwritten """
                 F"""with: \"{text}\"."""  )
             __A    :						str						 =     text
         else:
             logger.warning(
                 F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" new input """
                 F"""ignored: \"{text}\". Set `overwrite` to True to overwrite unprocessed user input"""  )
     else:
         __A    :						Union[str, Any]						 =     text

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     if self.new_user_input:
         self.past_user_inputs.append(self.new_user_input  )
     __A    :						List[Any]						 =     None

 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     self.generated_responses.append(__lowerCamelCase  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     for user_input, generated_response in zip(self.past_user_inputs					,						self.generated_responses  ):
         yield True, user_input
         yield False, generated_response
     if self.new_user_input:
         yield True, self.new_user_input







 def __repr__(    self  ):



     '''simple docstring'''




     __A    :						Optional[Any]						 =     F"""Conversation id: {self.uuid} \n"""
     for is_user, text in self.iter_texts():
         __A    :						Tuple						 =     '''user''' if is_user else '''bot'''
         output += F"""{name} >> {text} \n"""
     return output






@add_end_docstrings(
    SCREAMING_SNAKE_CASE__							,							R"""
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    """							,							)
class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 def __init__(    self					,						*__lowerCamelCase					,						**__lowerCamelCase  ):



     '''simple docstring'''




     super().__init__(*__lowerCamelCase					,						**__lowerCamelCase  )
     if self.tokenizer.pad_token_id is None:
         __A    :						Union[str, Any]						 =     self.tokenizer.eos_token

 def 					UpperCamelCase__(    self					,						__lowerCamelCase=None					,						__lowerCamelCase=None					,						__lowerCamelCase=None					,						**__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						str						 =     {}
     __A    :						List[str]						 =     {}
     __A    :						Any						 =     {}

     if min_length_for_response is not None:
         __A    :						int						 =     min_length_for_response
     if minimum_tokens is not None:
         __A    :						Any						 =     minimum_tokens

     if "max_length" in generate_kwargs:
         __A    :						List[Any]						 =     generate_kwargs['''max_length''']
         # self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
     if clean_up_tokenization_spaces is not None:
         __A    :						str						 =     clean_up_tokenization_spaces

     if generate_kwargs:
         forward_params.update(__lowerCamelCase  )
     return preprocess_params, forward_params, postprocess_params

 def __call__(    self					,						__lowerCamelCase					,						__lowerCamelCase=0					,						**__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						Any						 =     super().__call__(__lowerCamelCase					,						num_workers=__lowerCamelCase					,						**__lowerCamelCase  )
     if isinstance(__lowerCamelCase					,						__lowerCamelCase  ) and len(__lowerCamelCase  ) == 1:
         return outputs[0]
     return outputs

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase=32  ):



     '''simple docstring'''




     if not isinstance(__lowerCamelCase					,						__lowerCamelCase  ):
         raise ValueError('''ConversationalPipeline, expects Conversation as inputs'''  )
     if conversation.new_user_input is None:
         raise ValueError(
             F"""Conversation with UUID {type(conversation.uuid  )} does not contain new user input to process. """
             '''Add user inputs with the conversation\'s `add_user_input` method'''  )
     if hasattr(self.tokenizer					,						'''_build_conversation_input_ids'''  ):
         __A    :						List[Any]						 =     self.tokenizer._build_conversation_input_ids(__lowerCamelCase  )
     else:
         # If the tokenizer cannot handle conversations, we default to only the old version
         __A    :						int						 =     self._legacy_parse_and_tokenize(__lowerCamelCase  )

     if self.framework == "pt":
         __A    :						Union[str, Any]						 =     torch.LongTensor([input_ids]  )
     elif self.framework == "tf":
         __A    :						int						 =     tf.constant([input_ids]  )
     return {"input_ids": input_ids, "conversation": conversation}

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase=10					,						**__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						Tuple						 =     generate_kwargs.get('''max_length'''					,						self.model.config.max_length  )

     __A    :						str						 =     model_inputs['''input_ids'''].shape[1]
     if max_length - minimum_tokens < n:
         logger.warning(F"""Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})"""  )
         __A    :						str						 =     max_length - minimum_tokens
         __A    :						Any						 =     model_inputs['''input_ids'''][:, -trim:]
         if "attention_mask" in model_inputs:
             __A    :						Union[str, Any]						 =     model_inputs['''attention_mask'''][:, -trim:]
     __A    :						Dict						 =     model_inputs.pop('''conversation'''  )
     __A    :						List[str]						 =     max_length
     __A    :						Dict						 =     self.model.generate(**__lowerCamelCase					,						**__lowerCamelCase  )
     if self.model.config.is_encoder_decoder:
         __A    :						Any						 =     1
     else:
         __A    :						List[Any]						 =     n
     return {"output_ids": output_ids[:, start_position:], "conversation": conversation}

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase=True  ):



     '''simple docstring'''




     __A    :						int						 =     model_outputs['''output_ids''']
     __A    :						Optional[int]						 =     self.tokenizer.decode(
         output_ids[0]					,						skip_special_tokens=__lowerCamelCase					,						clean_up_tokenization_spaces=__lowerCamelCase					,						)
     __A    :						Dict						 =     model_outputs['''conversation''']
     conversation.mark_processed()
     conversation.append_response(__lowerCamelCase  )
     return conversation







 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						Tuple						 =     self.tokenizer.eos_token_id
     __A    :						List[str]						 =     []
     for is_user, text in conversation.iter_texts():
         if eos_token_id is not None:
             input_ids.extend(self.tokenizer.encode(__lowerCamelCase					,						add_special_tokens=__lowerCamelCase  ) + [eos_token_id]  )
         else:
             input_ids.extend(self.tokenizer.encode(__lowerCamelCase					,						add_special_tokens=__lowerCamelCase  )  )

     if len(__lowerCamelCase  ) > self.tokenizer.model_max_length:
         __A    :						List[str]						 =     input_ids[-self.tokenizer.model_max_length :]
     return input_ids




