





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A    :					Optional[int] =      logging.get_logger(__name__)

A    :					str =      {'''vocab_file''': '''sentencepiece.bpe.model'''}

A    :					Union[str, Any] =      {
    '''vocab_file''': {
        '''camembert-base''': '''https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model''',
    }
}

A    :					str =      {
    '''camembert-base''': 5_1_2,
}

A    :					Union[str, Any] =      '''▁'''


class   A			(__UpperCamelCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       int							=    VOCAB_FILES_NAMES
			__lowerCamelCase	:       Dict							=    PRETRAINED_VOCAB_FILES_MAP
			__lowerCamelCase	:       List[Any]							=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__lowerCamelCase	:       Union[str, Any]							=    ['input_ids', 'attention_mask']






			def __init__(			self   :    Dict							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Union[str, Any]="<s>"							,	__lowerCAmelCase   :    str="</s>"							,	__lowerCAmelCase   :    Tuple="</s>"							,	__lowerCAmelCase   :    Any="<s>"							,	__lowerCAmelCase   :    Dict="<unk>"							,	__lowerCAmelCase   :    Any="<pad>"							,	__lowerCAmelCase   :    str="<mask>"							,	__lowerCAmelCase   :    List[Any]=["<s>NOTUSED", "</s>NOTUSED"]							,	__lowerCAmelCase   :    List[Any] = None							,	**__lowerCAmelCase   :    Tuple							,	)	->	None:







					"""simple docstring"""






					A__        =   AddedToken(lowerCAmelCase__							,	lstrip=lowerCAmelCase__							,	rstrip=lowerCAmelCase__   ) if isinstance(lowerCAmelCase__							,	lowerCAmelCase__   ) else mask_token

					A__        =   {} if sp_model_kwargs is None else sp_model_kwargs

					super().__init__(
					    bos_token=lowerCAmelCase__							,	eos_token=lowerCAmelCase__							,	unk_token=lowerCAmelCase__							,	sep_token=lowerCAmelCase__							,	cls_token=lowerCAmelCase__							,	pad_token=lowerCAmelCase__							,	mask_token=lowerCAmelCase__							,	additional_special_tokens=lowerCAmelCase__							,	sp_model_kwargs=self.sp_model_kwargs							,	**lowerCAmelCase__							,	)
					A__        =   spm.SentencePieceProcessor(**self.sp_model_kwargs   )
					self.sp_model.Load(str(lowerCAmelCase__   )   )
					A__        =   vocab_file
					# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
					# sentencepiece vocabulary (this is the case for <s> and </s>
					A__        =   {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
					A__        =   len(self.fairseq_tokens_to_ids   )
					A__        =   len(self.sp_model   ) + len(self.fairseq_tokens_to_ids   )
					A__        =   {v: k for k, v in self.fairseq_tokens_to_ids.items()}






			def    a_       (			self   :    Union[str, Any]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Any = None   )	->	List[int]:







					"""simple docstring"""






					if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
					A__        =   [self.cls_token_id]
					A__        =   [self.sep_token_id]
					return cls + token_ids_a + sep + sep + token_ids_a + sep






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    Optional[int] = None							,	__lowerCAmelCase   :    List[str] = False   )	->	List[int]:







					"""simple docstring"""






					if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowerCAmelCase__							,	token_ids_a=lowerCAmelCase__							,	already_has_special_tokens=lowerCAmelCase__   )

					if token_ids_a is None:
							return [1] + ([0] * len(lowerCAmelCase__   )) + [1]
					return [1] + ([0] * len(lowerCAmelCase__   )) + [1, 1] + ([0] * len(lowerCAmelCase__   )) + [1]






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    Tuple = None   )	->	List[int]:







					"""simple docstring"""






					A__        =   [self.sep_token_id]
					A__        =   [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep   ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






			@property
			def    a_       (			self   :    Any   )	->	List[str]:







					"""simple docstring"""






					return len(self.fairseq_tokens_to_ids   ) + len(self.sp_model   )






			def    a_       (			self   :    Optional[Any]   )	->	str:







					"""simple docstring"""






					A__        =   {self.convert_ids_to_tokens(lowerCAmelCase__   ): i for i in range(self.vocab_size   )}
					vocab.update(self.added_tokens_encoder   )
					return vocab






			def    a_       (			self   :    str							,	__lowerCAmelCase   :    Dict   )	->	List[str]:







					"""simple docstring"""






					return self.sp_model.encode(lowerCAmelCase__							,	out_type=lowerCAmelCase__   )






			def    a_       (			self   :    str							,	__lowerCAmelCase   :    int   )	->	int:







					"""simple docstring"""






					if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
					elif self.sp_model.PieceToId(lowerCAmelCase__   ) == 0:
							# Convert sentence piece unk token to fairseq unk token index
							return self.unk_token_id
					return self.fairseq_offset + self.sp_model.PieceToId(lowerCAmelCase__   )






			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    List[str]   )	->	Dict:







					"""simple docstring"""






					if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset   )






			def    a_       (			self   :    Optional[int]							,	__lowerCAmelCase   :    Dict   )	->	Any:







					"""simple docstring"""






					A__        =   []
					A__        =   ''
					A__        =   False
					for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
									if not prev_is_special:
											out_string += " "
									out_string += self.sp_model.decode(lowerCAmelCase__   ) + token
									A__        =   True
									A__        =   []
							else:
									current_sub_tokens.append(lowerCAmelCase__   )
									A__        =   False
					out_string += self.sp_model.decode(lowerCAmelCase__   )
					return out_string.strip()






			def __getstate__(			self   :    List[str]   )	->	Optional[Any]:







					"""simple docstring"""






					A__        =   self.__dict__.copy()
					A__        =   None
					return state






			def __setstate__(			self   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]   )	->	Any:







					"""simple docstring"""






					A__        =   d

					# for backward compatibility
					if not hasattr(self							,	"""sp_model_kwargs"""   ):
							A__        =   {}

					A__        =   spm.SentencePieceProcessor(**self.sp_model_kwargs   )
					self.sp_model.Load(self.vocab_file   )







			def    a_       (			self   :    List[Any]							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    List[Any] = None   )	->	Tuple[str]:







					"""simple docstring"""






					if not os.path.isdir(lowerCAmelCase__   ):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
							return
					A__        =   os.path.join(
					    lowerCAmelCase__							,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

					if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCAmelCase__   ) and os.path.isfile(self.vocab_file   ):
							copyfile(self.vocab_file							,	lowerCAmelCase__   )
					elif not os.path.isfile(self.vocab_file   ):
							with open(lowerCAmelCase__							,	"""wb"""   ) as fi:
									A__        =   self.sp_model.serialized_model_proto()
									fi.write(lowerCAmelCase__   )

					return (out_vocab_file,)







import unittest

from transformers import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING, AutoTokenizer, is_vision_available
from transformers.pipelines import pipeline
from transformers.pipelines.document_question_answering import apply_tesseract
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_detectrona,
    require_pytesseract,
    require_tf,
    require_torch,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
 from PIL import Image

 from transformers.image_utils import load_image
else:





 class   __a			:
  @staticmethod
  def 		SCREAMING_SNAKE_CASE__   (   *lowerCAmelCase__				, **lowerCAmelCase__ )					->				List[str]:




         '''simple docstring'''
         pass





 def    snake_case_     (  snake_case     )  ->				Optional[Any]:
        return None


# This is a pinned image from a specific revision of a document question answering space, hosted by HuggingFace,
# so we can expect it to be available.
__lowerCAmelCase		       =							(
    '''https://huggingface.co/spaces/impira/docquery/resolve/2f6c96314dc84dfda62d40de9da55f2f5165d403/invoice.png'''
)





@is_pipeline_test
@require_torch
@require_vision
class   __a			(    unittest.TestCase      ):
 __lowercase :							Dict        = MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING
 @require_pytesseract
 @require_vision
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__				, lowerCAmelCase__ )					->				int:




        '''simple docstring'''
        lowercase__:   Optional[Any]  			=   pipeline(
            'document-question-answering'				, model=lowerCAmelCase__				, tokenizer=lowerCAmelCase__				, image_processor=lowerCAmelCase__ )

        lowercase__:   int  			=   INVOICE_URL
        lowercase__:   Dict  			=   list(zip(*apply_tesseract(load_image(lowerCAmelCase__ )				, lowerCAmelCase__				, '' ) ) )
        lowercase__:   str  			=   'What is the placebo?'
        lowercase__:   Any  			=   [
            {
                'image': load_image(lowerCAmelCase__ ),
                'question': question,
            },
            {
                'image': image,
                'question': question,
            },
            {
                'image': image,
                'question': question,
                'word_boxes': word_boxes,
            },
        ]
        return dqa_pipeline, examples
 def 		SCREAMING_SNAKE_CASE__   (   self				, lowerCAmelCase__				, lowerCAmelCase__ )					->				int:




        '''simple docstring'''
        lowercase__:   str  			=   dqa_pipeline(lowerCAmelCase__				, top_k=2 )
        self.assertEqual(
            lowerCAmelCase__				, [
                [
                    {'score': ANY(lowerCAmelCase__ ), 'answer': ANY(lowerCAmelCase__ ), 'start': ANY(lowerCAmelCase__ ), 'end': ANY(lowerCAmelCase__ )},
                    {'score': ANY(lowerCAmelCase__ ), 'answer': ANY(lowerCAmelCase__ ), 'start': ANY(lowerCAmelCase__ ), 'end': ANY(lowerCAmelCase__ )},
                ]
            ]
            * 3				, )
 @require_torch
 @require_detectrona
 @require_pytesseract
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Dict:




        '''simple docstring'''
        lowercase__:   Union[str, Any]  			=   pipeline('document-question-answering'				, model='hf-internal-testing/tiny-random-layoutlmv2' )
        lowercase__:   Optional[Any]  			=   INVOICE_URL
        lowercase__:   int  			=   'How many cats are there?'

        lowercase__:   List[str]  			=   [
            {'score': 0.0_0_0_1, 'answer': 'oy 2312/2019', 'start': 38, 'end': 39},
            {'score': 0.0_0_0_1, 'answer': 'oy 2312/2019 DUE', 'start': 38, 'end': 40},
        ]
        lowercase__:   Dict  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(nested_simplify(lowerCAmelCase__				, decimals=4 )				, lowerCAmelCase__ )

        lowercase__:   Tuple  			=   dqa_pipeline({'image': image, 'question': question}				, top_k=2 )
        self.assertEqual(nested_simplify(lowerCAmelCase__				, decimals=4 )				, lowerCAmelCase__ )

        # This image does not detect ANY text in it, meaning layoutlmv2 should fail.
        # Empty answer probably
        lowercase__:   str  			=   './tests/fixtures/tests_samples/COCO/000000039769.png'
        lowercase__:   Tuple  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(lowerCAmelCase__				, [] )

        # We can optionnally pass directly the words and bounding boxes
        lowercase__:   int  			=   './tests/fixtures/tests_samples/COCO/000000039769.png'
        lowercase__:   List[Any]  			=   []
        lowercase__:   Optional[int]  			=   []
        lowercase__:   Any  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, words=lowerCAmelCase__				, boxes=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(lowerCAmelCase__				, [] )
 @slow
 @require_torch
 @require_detectrona
 @require_pytesseract
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Any:




        '''simple docstring'''
        lowercase__:   List[str]  			=   pipeline(
            'document-question-answering'				, model='tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'				, revision='9977165'				, )
        lowercase__:   int  			=   INVOICE_URL
        lowercase__:   str  			=   'What is the invoice number?'

        lowercase__:   Union[str, Any]  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )

        lowercase__:   Any  			=   dqa_pipeline({'image': image, 'question': question}				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )

        lowercase__:   Optional[int]  			=   dqa_pipeline(
            [{'image': image, 'question': question}, {'image': image, 'question': question}]				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                [
                    {'score': 0.9_9_4_4, 'answer': 'us-001', 'start': 16, 'end': 16},
                    {'score': 0.0_0_0_9, 'answer': 'us-001', 'start': 16, 'end': 16},
                ],
            ]
            * 2				, )
 @slow
 @require_torch
 @require_detectrona
 @require_pytesseract
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				str:




        '''simple docstring'''
        lowercase__:   Any  			=   pipeline(
            'document-question-answering'				, model='tiennvcs/layoutlmv2-base-uncased-finetuned-docvqa'				, revision='9977165'				, max_seq_len=50				, )
        lowercase__:   Optional[int]  			=   INVOICE_URL
        lowercase__:   Union[str, Any]  			=   'What is the invoice number?'

        lowercase__:   Optional[Any]  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
                {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )

        lowercase__:   Tuple  			=   dqa_pipeline({'image': image, 'question': question}				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
                {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )

        lowercase__:   Dict  			=   dqa_pipeline(
            [{'image': image, 'question': question}, {'image': image, 'question': question}]				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                [
                    {'score': 0.9_9_7_4, 'answer': '1110212019', 'start': 23, 'end': 23},
                    {'score': 0.9_9_4_8, 'answer': 'us-001', 'start': 16, 'end': 16},
                ]
            ]
            * 2				, )
 @slow
 @require_torch
 @require_pytesseract
 @require_vision
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				str:




        '''simple docstring'''
        lowercase__:   Optional[Any]  			=   AutoTokenizer.from_pretrained(
            'impira/layoutlm-document-qa'				, revision='3dc6de3'				, add_prefix_space=lowerCAmelCase__ )
        lowercase__:   Optional[Any]  			=   pipeline(
            'document-question-answering'				, model='impira/layoutlm-document-qa'				, tokenizer=lowerCAmelCase__				, revision='3dc6de3'				, )
        lowercase__:   List[str]  			=   INVOICE_URL
        lowercase__:   Union[str, Any]  			=   'What is the invoice number?'

        lowercase__:   Dict  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
            ]				, )

        lowercase__:   List[str]  			=   dqa_pipeline({'image': image, 'question': question}				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
            ]				, )

        lowercase__:   int  			=   dqa_pipeline(
            [{'image': image, 'question': question}, {'image': image, 'question': question}]				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                [
                    {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
                    {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
                ]
            ]
            * 2				, )

        lowercase__:   Any  			=   list(zip(*apply_tesseract(load_image(lowerCAmelCase__ )				, lowerCAmelCase__				, '' ) ) )

        # This model should also work if `image` is set to None
        lowercase__:   List[Any]  			=   dqa_pipeline({'image': None, 'word_boxes': word_boxes, 'question': question}				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.4_2_5_1, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.0_8_1_9, 'answer': '1110212019', 'start': 23, 'end': 23},
            ]				, )
 @slow
 @require_torch
 @require_pytesseract
 @require_vision
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Dict:




        '''simple docstring'''
        lowercase__:   Any  			=   AutoTokenizer.from_pretrained(
            'impira/layoutlm-document-qa'				, revision='3dc6de3'				, add_prefix_space=lowerCAmelCase__ )
        lowercase__:   str  			=   pipeline(
            'document-question-answering'				, model='impira/layoutlm-document-qa'				, tokenizer=lowerCAmelCase__				, revision='3dc6de3'				, max_seq_len=50				, )
        lowercase__:   Optional[Any]  			=   INVOICE_URL
        lowercase__:   Optional[Any]  			=   'What is the invoice number?'

        lowercase__:   Optional[int]  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )

        lowercase__:   Any  			=   dqa_pipeline(
            [{'image': image, 'question': question}, {'image': image, 'question': question}]				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                [
                    {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
                    {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
                ]
            ]
            * 2				, )

        lowercase__:   Optional[int]  			=   list(zip(*apply_tesseract(load_image(lowerCAmelCase__ )				, lowerCAmelCase__				, '' ) ) )

        # This model should also work if `image` is set to None
        lowercase__:   Tuple  			=   dqa_pipeline({'image': None, 'word_boxes': word_boxes, 'question': question}				, top_k=2 )
        self.assertEqual(
            nested_simplify(lowerCAmelCase__				, decimals=4 )				, [
                {'score': 0.9_9_9_9, 'answer': 'us-001', 'start': 16, 'end': 16},
                {'score': 0.9_9_9_8, 'answer': 'us-001', 'start': 16, 'end': 16},
            ]				, )
 @slow
 @require_torch
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				Optional[Any]:




        '''simple docstring'''
        lowercase__:   List[Any]  			=   pipeline(
            'document-question-answering'				, model='naver-clova-ix/donut-base-finetuned-docvqa'				, tokenizer=AutoTokenizer.from_pretrained('naver-clova-ix/donut-base-finetuned-docvqa' )				, feature_extractor='naver-clova-ix/donut-base-finetuned-docvqa'				, )

        lowercase__:   int  			=   INVOICE_URL
        lowercase__:   int  			=   'What is the invoice number?'
        lowercase__:   Union[str, Any]  			=   dqa_pipeline(image=lowerCAmelCase__				, question=lowerCAmelCase__				, top_k=2 )
        self.assertEqual(nested_simplify(lowerCAmelCase__				, decimals=4 )				, [{'answer': 'us-001'}] )
 @require_tf
 @unittest.skip('Document question answering not implemented in TF' )
 def 		SCREAMING_SNAKE_CASE__   (   self )					->				List[str]:




        '''simple docstring'''
        pass


