code: 
  
  
  '''simple docstring'''
  
  import os
  import pickle
  import tempfile
  import unittest
  from typing import Callable, Optional
  
  import numpy as np
  
  from transformers import (
      BatchEncoding,
      BertTokenizer,
      BertTokenizerFast,
      PreTrainedTokenizer,
      PreTrainedTokenizerFast,
      TensorType,
      TokenSpan,
      is_tokenizers_available,
  )
  from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
  from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow
  
  
  if is_tokenizers_available():
  							from tokenizers import Tokenizer
  							from tokenizers.models import WordPiece
  
  
  
  
  
  
  
  class  UpperCAmelCase (       unittest.TestCase):
  
  	def 		lowercase_						( self :	Tuple,					a_ :	Union[str, Any]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						list(tokenizer_class.max_model_input_sizes.keys()					)
  				for model_name in sa_models[:1]:
  							UpperCamelCase__ 						=						tokenizer_class.from_pretrained(a_					)
  							self.assertIsNotNone(a_					)
  							self.assertIsInstance(a_,					a_					)
  							self.assertIsInstance(a_,					a_					)
  
  							for special_tok in tokenizer.all_special_tokens:
  										self.assertIsInstance(a_,					a_					)
  										UpperCamelCase__ 						=						tokenizer.convert_tokens_to_ids(a_					)
  										self.assertIsInstance(a_,					a_					)
  
  	def 		lowercase_						( self :	Union[str, Any],					a_ :	BatchEncoding,					a_ :	Optional[Callable] = None					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						pickle.dumps(a_					)
  				self.assertIsNotNone(a_					)
  
  				UpperCamelCase__ 						=						pickle.loads(a_					)
  
  				# Ensure is_fast is correctly restored
  				self.assertEqual(be_restored.is_fast,					be_original.is_fast					)
  
  				# Ensure encodings are potentially correctly restored
  				if be_original.is_fast:
  							self.assertIsNotNone(be_restored.encodings					)
  				else:
  							self.assertIsNone(be_restored.encodings					)
  
  				# Ensure the keys are the same
  				for original_v, restored_v in zip(be_original.values(),					be_restored.values()					):
  							if equal_op:
  										self.assertTrue(equal_op(a_,					a_					)					)
  							else:
  										self.assertEqual(a_,					a_					)
  
  	@slow
  	def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				self.check_tokenizer_from_pretrained(a_					)
  
  	def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				self.assertEqual(TensorType("tf"					),					TensorType.TENSORFLOW					)
  				self.assertEqual(TensorType("pt"					),					TensorType.PYTORCH					)
  				self.assertEqual(TensorType("np"					),					TensorType.NUMPY					)
  
  	@require_tokenizers
  	def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				import numpy as np
  
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  				UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
  				# Python no tensor
  				with self.subTest("BatchEncoding (Python, return_tensors=None)"					):
  							self.assert_dump_and_restore(tokenizer_p("Small example to encode"					)					)
  
  				with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_p("Small example to encode",					return_tensors=TensorType.NUMPY					),					np.array_equal					)
  
  				with self.subTest("BatchEncoding (Rust, return_tensors=None)"					):
  							self.assert_dump_and_restore(tokenizer_r("Small example to encode"					)					)
  
  				with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_r("Small example to encode",					return_tensors=TensorType.NUMPY					),					np.array_equal					)
  
  	@require_tf
  	@require_tokenizers
  	def 		lowercase_						( self :	str					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				import tensorflow as tf
  
  				def tf_array_equals(a_ :	Tuple,					a_ :	str					):
  							return tf.reduce_all(tf.equal(a_,					a_					)					)
  
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  				UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
  				with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_p("Small example to encode",					return_tensors=TensorType.TENSORFLOW					),					a_					)
  
  				with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_r("Small example to encode",					return_tensors=TensorType.TENSORFLOW					),					a_					)
  
  	@require_torch
  	@require_tokenizers
  	def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				import torch
  
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  				UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
  				with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_p("Small example to encode",					return_tensors=TensorType.PYTORCH					),					torch.equal					)
  
  				with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"					):
  							self.assert_dump_and_restore(
  							    tokenizer_r("Small example to encode",					return_tensors=TensorType.PYTORCH					),					torch.equal					)
  
  	@require_tokenizers
  	def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  				UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  
  				with self.subTest("Python Tokenizer"					):
  							self.assertFalse(tokenizer_p("Small example to_encode"					).is_fast					)
  
  				with self.subTest("Rust Tokenizer"					):
  							self.assertTrue(tokenizer_r("Small example to_encode"					).is_fast					)
  
  	@require_tokenizers
  	def 		lowercase_						( self :	str					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BertTokenizerFast.from_pretrained("bert-base-cased"					)
  				UpperCamelCase__ 						=						tokenizer_r(["Test", "\xad", "test"],					is_split_into_words=a_					)
  
  				self.assertEqual(encoded.word_to_tokens(0					),					TokenSpan(start=1,					end=2					)					)
  				self.assertEqual(encoded.word_to_tokens(1					),					a_					)
  				self.assertEqual(encoded.word_to_tokens(2					),					TokenSpan(start=2,					end=3					)					)
  
  	def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np"					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
  				# test converting the converted
  				with CaptureStderr() as cs:
  							UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np"					)
  				self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="np",					prepend_batch_axis=a_					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
  	@require_torch
  	def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt"					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
  				# test converting the converted
  				with CaptureStderr() as cs:
  							UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt"					)
  				self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="pt",					prepend_batch_axis=a_					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
  	@require_tf
  	def 		lowercase_						( self :	Optional[Any]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf"					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
  				# test converting the converted
  				with CaptureStderr() as cs:
  							UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf"					)
  				self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="tf",					prepend_batch_axis=a_					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
  	@require_flax
  	def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax"					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(2, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(2,)					)
  				# test converting the converted
  				with CaptureStderr() as cs:
  							UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax"					)
  				self.assertFalse(len(cs.err					),					msg=f'should have no warning, but got {cs.err}'					)
  
  				UpperCamelCase__ 						=						BatchEncoding({"inputs": [1, 2, 3], "labels": 0}					)
  				UpperCamelCase__ 						=						batch.convert_to_tensors(tensor_type="jax",					prepend_batch_axis=a_					)
  				self.assertEqual(tensor_batch["inputs"].shape,					(1, 3)					)
  				self.assertEqual(tensor_batch["labels"].shape,					(1,)					)
  
  	def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						[{"input_ids": np.array([0, 1, 2]					)}, {"input_ids": np.array([0, 1, 2, 3]					)}]
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
  				self.assertTrue(isinstance(batch["input_ids"],					np.ndarray					)					)
  				self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="np"					)
  				self.assertTrue(isinstance(batch["input_ids"],					np.ndarray					)					)
  				self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
  	@require_torch
  	def 		lowercase_						( self :	List[Any]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				import torch
  
  				UpperCamelCase__ 						=						[{"input_ids": torch.tensor([0, 1, 2]					)}, {"input_ids": torch.tensor([0, 1, 2, 3]					)}]
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
  				self.assertTrue(isinstance(batch["input_ids"],					torch.Tensor					)					)
  				self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="pt"					)
  				self.assertTrue(isinstance(batch["input_ids"],					torch.Tensor					)					)
  				self.assertEqual(batch["input_ids"].tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
  	@require_tf
  	def 		lowercase_						( self :	int					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				import tensorflow as tf
  
  				UpperCamelCase__ 						=						[{"input_ids": tf.constant([0, 1, 2]					)}, {"input_ids": tf.constant([0, 1, 2, 3]					)}]
  				UpperCamelCase__ 						=						BertTokenizer.from_pretrained("bert-base-cased"					)
  
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_					)
  				self.assertTrue(isinstance(batch["input_ids"],					tf.Tensor					)					)
  				self.assertEqual(batch["input_ids"].numpy().tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  				UpperCamelCase__ 						=						tokenizer.pad(a_,					padding=a_,					return_tensors="tf"					)
  				self.assertTrue(isinstance(batch["input_ids"],					tf.Tensor					)					)
  				self.assertEqual(batch["input_ids"].numpy().tolist(),					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]					)
  
  	@require_tokenizers
  	def 		lowercase_						( self :	List[str]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						Tokenizer(WordPiece(unk_token="[UNK]"					)					)
  				PreTrainedTokenizerFast(tokenizer_object=a_					)
  
  	@require_tokenizers
  	def 		lowercase_						( self :	Optional[int]					):
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  				UpperCamelCase__ 						=						Tokenizer(WordPiece(unk_token="[UNK]"					)					)
  				with tempfile.TemporaryDirectory() as tmpdirname:
  							bert_tokenizer.save(os.path.join(a_,					"tokenizer.json"					)					)
  							PreTrainedTokenizerFast(tokenizer_file=os.path.join(a_,					"tokenizer.json"					)					)
style_context: 
  
  
  '''simple docstring'''
  
  from dataclasses import dataclass
  
  __lowercase:		List[str]							=     "0.1"
  __lowercase:		Union[str, Any]							=     "Lucia Harcekova"
  
  
  
  
  
  
  
  @dataclass
  class  UpperCAmelCase :
   _lowerCamelCase :					int
   _lowerCamelCase :					int
   _lowerCamelCase :					str
  
   def __repr__( self :	List[str]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      return f'({self.offset}, {self.length}, {self.indicator})'
  
  
  
  
  
  
  
  class  UpperCAmelCase :
  
   def __init__( self :	Optional[int],					a_ :	int = 13,					a_ :	int = 6					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						window_size
      UpperCamelCase__ 						=						lookahead_buffer_size
      UpperCamelCase__ 						=						self.window_size - self.lookahead_buffer_size
  
   def 		lowercase_						( self :	Optional[int],					a_ :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						[]
      UpperCamelCase__ 						=						""
  
      # while there are still characters in text to compress
      while text:
         # find the next encoding phrase
         # - triplet with offset, length, indicator (the next encoding character)
         UpperCamelCase__ 						=						self._find_encoding_token(a_,					a_					)
  
         # update the search buffer:
         # - add new characters from text into it
         # - check if size exceed the max search buffer size, if so, drop the
         #   oldest elements
         search_buffer += text[: token.length + 1]
         if len(a_					) > self.search_buffer_size:
            UpperCamelCase__ 						=						search_buffer[-self.search_buffer_size :]
  
         # update the text
         UpperCamelCase__ 						=						text[token.length + 1 :]
  
         # append the token to output
         output.append(a_					)
  
      return output
  
   def 		lowercase_						( self :	List[Any],					a_ :	list[Token]					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      UpperCamelCase__ 						=						""
  
      for token in tokens:
         for _ in range(token.length					):
            output += output[-token.offset]
         output += token.indicator
  
      return output
  
   def 		lowercase_						( self :	Any,					a_ :	str,					a_ :	str					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      if not text:
         raise ValueError("We need some text to work with."					)
  
      # Initialise result parameters to default values
      UpperCamelCase__						,		UpperCamelCase__ 						=						0, 0
  
      if not search_buffer:
         return Token(a_,					a_,					text[length]					)
  
      for i, character in enumerate(a_					):
         UpperCamelCase__ 						=						len(a_					) - i
         if character == text[0]:
            UpperCamelCase__ 						=						self._match_length_from_index(a_,					a_,					0,					a_					)
            # if the found length is bigger than the current or if it's equal,
            # which means it's offset is smaller: update offset and length
            if found_length >= length:
               UpperCamelCase__						,		UpperCamelCase__ 						=						found_offset, found_length
  
      return Token(a_,					a_,					text[length]					)
  
   def 		lowercase_						( self :	Optional[int],					a_ :	str,					a_ :	str,					a_ :	int,					a_ :	int					):
  
  
  
  
  
      """simple docstring"""
  
  
  
      if not text or text[text_index] != window[window_index]:
         return 0
      return 1 + self._match_length_from_index(
          a_,					window + text[text_index],					text_index + 1,					window_index + 1					)
  
  
  if __name__ == "__main__":
         from doctest import testmod
  
         testmod()
         # Initialize compressor class
         __lowercase:		Optional[Any]							=     LZaaCompressor(window_size=13, lookahead_buffer_size=6)
  
         # Example
         __lowercase:		Union[str, Any]							=     "cabracadabrarrarrad"
         __lowercase:		Optional[int]							=     lzaa_compressor.compress(TEXT)
         print(lzaa_compressor.compress("ababcbababaa"))
         __lowercase:		Optional[int]							=     lzaa_compressor.decompress(compressed_text)
         assert decompressed_text == TEXT, "The LZ77 algorithm returned the invalid result."