

import re
import tempfile
from pathlib import Path

import pytest
import yaml

from datasets.utils.readme import ReadMe


# @pytest.fixture
# def example_yaml_structure():

a_     =			yaml.safe_load(
    '\\nname: ""\nallow_empty: false\nallow_empty_text: true\nsubsections:\n  - name: "Dataset Card for X" # First-level markdown heading\n    allow_empty: false\n    allow_empty_text: true\n    subsections:\n      - name: "Table of Contents"\n        allow_empty: false\n        allow_empty_text: false\n        subsections: null\n      - name: "Dataset Description"\n        allow_empty: false\n        allow_empty_text: false\n        subsections:\n          - name: "Dataset Summary"\n            allow_empty: false\n            allow_empty_text: false\n            subsections: null\n          - name: "Supported Tasks and Leaderboards"\n            allow_empty: true\n            allow_empty_text: true\n            subsections: null\n          - name: Languages\n            allow_empty: false\n            allow_empty_text: true\n            subsections: null\n'
)


a_     =			{
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}


a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'


a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n#### Extra Ignored Subsection\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			{
    'name': 'root',
    'text': '',
    'is_empty_text': True,
    'subsections': [
        {
            'name': 'Dataset Card for My Dataset',
            'text': '',
            'is_empty_text': True,
            'subsections': [
                {'name': 'Table of Contents', 'text': 'Some text here.', 'is_empty_text': False, 'subsections': []},
                {
                    'name': 'Dataset Description',
                    'text': 'Some text here.',
                    'is_empty_text': False,
                    'subsections': [
                        {
                            'name': 'Dataset Summary',
                            'text': 'Some text here.',
                            'is_empty_text': False,
                            'subsections': [
                                {
                                    'name': 'Extra Ignored Subsection',
                                    'text': '',
                                    'is_empty_text': True,
                                    'subsections': [],
                                }
                            ],
                        },
                        {
                            'name': 'Supported Tasks and Leaderboards',
                            'text': '',
                            'is_empty_text': True,
                            'subsections': [],
                        },
                        {'name': 'Languages', 'text': 'Language Text', 'is_empty_text': False, 'subsections': []},
                    ],
                },
            ],
        }
    ],
}

a_     =			'\\n---\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			(
    'The following issues were found for the README at `{path}`:\n-\tEmpty YAML markers are present in the README.'
)

a_     =			'\\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			(
    'The following issues were found for the README at `{path}`:\n-\tNo YAML markers are present in the README.'
)

a_     =			'\\n---\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			'The following issues were found for the README at `{path}`:\n-\tOnly the start of YAML tags present in the README.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
a_     =			'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Summary` but it is empty.\n-\tExpected some text in section `Dataset Summary` but it is empty (text in subsections are ignored).'


a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n'
a_     =			'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Dataset Card for My Dataset` but it is empty.\n-\tSection `Dataset Card for My Dataset` expected the following subsections: `Table of Contents`, `Dataset Description`. Found \'None\'.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Languages\nLanguage Text\n'

a_     =			'The following issues were found for the README at `{path}`:\n-\tSection `Dataset Description` is missing subsection: `Supported Tasks and Leaderboards`.'


a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\n'

a_     =			'The following issues were found for the README at `{path}`:\n-\tExpected some content in section `Languages` but it is empty.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'
a_     =			'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n# Dataset Card My Dataset\n'

a_     =			'The following issues were found for the README at `{path}`:\n-\tThe README has several first-level headings: `Dataset Card for My Dataset`, `Dataset Card My Dataset`. Only one heading is expected. Skipping further validation for this README.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			'The following issues were found for the README at `{path}`:\n-\tNo first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'

a_     =			''

a_     =			'The following issues were found for the README at `{path}`:\n-\tThe README has no first-level headings. One heading is expected. Skipping further validation for this README.\n-\tNo YAML markers are present in the README.'

a_     =			'\\n---\nlanguage:\n- zh\n- en\n---\n\n# Dataset Card for My Dataset\n# Dataset Card for My Dataset\n## Table of Contents\nSome text here.\n## Dataset Description\nSome text here.\n### Dataset Summary\nSome text here.\n### Supported Tasks and Leaderboards\n### Languages\nLanguage Text\n'

a_     =			'The following issues were found while parsing the README at `{path}`:\n-\tMultiple sections with the same heading `Dataset Card for My Dataset` have been found. Please keep only one of these sections.'

@pytest.mark.parametrize(
    "readme_md, expected_dict"							,	[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	assert ReadMe.from_string(_a							,	_a).to_dict() == expected_dict

@pytest.mark.parametrize(
    "readme_md, expected_error"							,	[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	with pytest.raises(_a							,	match=re.escape(expected_error.format(path="root"))):
		SCREAMING_SNAKE_CASE      :					str    =				ReadMe.from_string(_a							,	_a)
		readme.validate()

@pytest.mark.parametrize(
    "readme_md, expected_error"							,	[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	with pytest.raises(_a							,	match=re.escape(expected_error.format(path="root"))):
		ReadMe.from_string(_a							,	_a)

@pytest.mark.parametrize(
    "readme_md,"							,	[
        (README_MULTIPLE_SAME_HEADING_1),
    ]							,	)
def        lowerCamelCase__       (    _a):
	ReadMe.from_string(_a							,	_a							,	suppress_parsing_errors=_a)

@pytest.mark.parametrize(
    "readme_md, expected_dict"							,	[
        (README_CORRECT, CORRECT_DICT),
        (README_CORRECT_FOUR_LEVEL, CORRECT_DICT_FOUR_LEVEL),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	with tempfile.TemporaryDirectory() as tmp_dir:
		SCREAMING_SNAKE_CASE      :					Optional[int]    =				Path(_a) / "README.md"
		with open(_a							,	"w+") as readme_file:
			readme_file.write(_a)
		SCREAMING_SNAKE_CASE      :					Optional[Any]    =				ReadMe.from_readme(_a							,	_a).to_dict()
		assert out["name"] == path
		assert out["text"] == ""
		assert out["is_empty_text"]
		assert out["subsections"] == expected_dict["subsections"]

@pytest.mark.parametrize(
    "readme_md, expected_error"							,	[
        (README_NO_YAML, EXPECTED_ERROR_README_NO_YAML),
        (README_EMPTY_YAML, EXPECTED_ERROR_README_EMPTY_YAML),
        (README_INCORRECT_YAML, EXPECTED_ERROR_README_INCORRECT_YAML),
        (README_EMPTY, EXPECTED_ERROR_README_EMPTY),
        (README_NONE_SUBSECTION, EXPECTED_ERROR_README_NONE_SUBSECTION),
        (README_MISSING_FIRST_LEVEL, EXPECTED_ERROR_README_MISSING_FIRST_LEVEL),
        (README_MISSING_SUBSECTION, EXPECTED_ERROR_README_MISSING_SUBSECTION),
        (README_MISSING_TEXT, EXPECTED_ERROR_README_MISSING_TEXT),
        (README_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_WRONG_FIRST_LEVEL),
        (README_MULTIPLE_WRONG_FIRST_LEVEL, EXPECTED_ERROR_README_MULTIPLE_WRONG_FIRST_LEVEL),
        (README_MISSING_CONTENT, EXPECTED_ERROR_README_MISSING_CONTENT),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	with tempfile.TemporaryDirectory() as tmp_dir:
		SCREAMING_SNAKE_CASE      :					List[str]    =				Path(_a) / "README.md"
		with open(_a							,	"w+") as readme_file:
			readme_file.write(_a)
		SCREAMING_SNAKE_CASE      :					Optional[Any]    =				expected_error.format(path=_a)
		with pytest.raises(_a							,	match=re.escape(_a)):
			SCREAMING_SNAKE_CASE      :					Optional[int]    =				ReadMe.from_readme(_a							,	_a)
			readme.validate()

@pytest.mark.parametrize(
    "readme_md, expected_error"							,	[
        (README_MULTIPLE_SAME_HEADING_1, EXPECTED_ERROR_README_MULTIPLE_SAME_HEADING_1),
    ]							,	)
def        lowerCamelCase__       (    _a							,	_a):
	with tempfile.TemporaryDirectory() as tmp_dir:
		SCREAMING_SNAKE_CASE      :					Any    =				Path(_a) / "README.md"
		with open(_a							,	"w+") as readme_file:
			readme_file.write(_a)
		SCREAMING_SNAKE_CASE      :					Dict    =				expected_error.format(path=_a)
		with pytest.raises(_a							,	match=re.escape(_a)):
			ReadMe.from_readme(_a							,	_a)


@pytest.mark.parametrize(
    "readme_md,"							,	[
        (README_MULTIPLE_SAME_HEADING_1),
    ]							,	)
def        lowerCamelCase__       (    _a):
	with tempfile.TemporaryDirectory() as tmp_dir:
		SCREAMING_SNAKE_CASE      :					int    =				Path(_a) / "README.md"
		with open(_a							,	"w+") as readme_file:
			readme_file.write(_a)
		ReadMe.from_readme(_a							,	_a							,	suppress_parsing_errors=_a)

import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case								=  get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
class 						lowercase__  (						_UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   List[str]				       =XLMProphetNetTokenizer
	A__    :   List[Any]				       =False
	A__    :   Tuple				       =True
	def      A_				(			self							:   str							):
						super().setUp()

						# We have a SentencePiece fixture for testing
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)
						tokenizer.save_pretrained(self.tmpdirname							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= '[PAD]'
						SCREAMING_SNAKE_CASE__													= 0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= list(self.get_tokenizer().get_vocab().keys()							)

						self.assertEqual(vocab_keys[0]       ,   '[PAD]'							)
						self.assertEqual(vocab_keys[1]       ,   '[CLS]'							)
						self.assertEqual(vocab_keys[-1]       ,   'j'							)
						self.assertEqual(len(UpperCAmelCase_							)       ,   1012							)
	def      A_				(			self							:   List[Any]							):
						self.assertEqual(self.get_tokenizer().vocab_size       ,   1012							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('This is a test'							)
						self.assertListEqual(UpperCAmelCase_       ,   ['▁This', '▁is', '▁a', '▁t', 'est']							)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)       ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]       ,   )
						SCREAMING_SNAKE_CASE__													= tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.convert_ids_to_tokens(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '[UNK]',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '[UNK]',
						        '.',
						    ]       ,   )
	@cached_property
	def      A_				(			self							:   Optional[Any]							):
						return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'							)
	@slow
	def      A_				(			self							:   Any							):
						SCREAMING_SNAKE_CASE__													= 'Hello World!'
						SCREAMING_SNAKE_CASE__													= [35389, 6672, 49, 2]
						self.assertListEqual(UpperCAmelCase_       ,   self.big_tokenizer.encode(UpperCAmelCase_							)							)





	@slow
	def      A_				(			self							:   Tuple							):
						# fmt: off
						SCREAMING_SNAKE_CASE__													= {'input_ids': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=UpperCAmelCase_       ,   model_name='microsoft/xprophetnet-large-wiki100-cased'       ,   revision='1acad1643ddd54a44df6a1b797ada8373685d90e'       ,   )





