code: 
  
  
  """simple docstring"""
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  
  
  
  
  class   lowercase__							(				tf.keras.layers.Layer							):
  
  
  
  
  
     def __init__(       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None			,      SCREAMING_SNAKE_CASE = None			,      SCREAMING_SNAKE_CASE = None			,      SCREAMING_SNAKE_CASE = "longest"			,      SCREAMING_SNAKE_CASE = True			,      SCREAMING_SNAKE_CASE = 512			,      SCREAMING_SNAKE_CASE = None			,      SCREAMING_SNAKE_CASE = True			,      SCREAMING_SNAKE_CASE = True			,      SCREAMING_SNAKE_CASE = True			,      **SCREAMING_SNAKE_CASE			,      )     ->     List[str]:
        super().__init__()
        if use_fast_bert_tokenizer:
           _lowerCamelCase  :				Tuple      =		FastBertTokenizer(
               SCREAMING_SNAKE_CASE			,      token_out_type=tf.intaa			,      lower_case_nfd_strip_accents=SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
        else:
           _lowerCamelCase  :				Tuple      =		tf.lookup.StaticVocabularyTable(
               tf.lookup.KeyValueTensorInitializer(
                   keys=SCREAMING_SNAKE_CASE			,      key_dtype=tf.string			,      values=tf.range(tf.size(SCREAMING_SNAKE_CASE			,      out_type=tf.intaa)			,      dtype=tf.intaa)			,      value_dtype=tf.intaa			,      )			,      num_oov_buckets=1			,      )
           _lowerCamelCase  :				List[str]      =		BertTokenizerLayer(
               SCREAMING_SNAKE_CASE			,      token_out_type=tf.intaa			,      lower_case=SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
  
        _lowerCamelCase  :				str      =		vocab_list
        _lowerCamelCase  :				List[str]      =		do_lower_case
        _lowerCamelCase  :				Optional[int]      =		cls_token_id or vocab_list.index("""[CLS]""")
        _lowerCamelCase  :				Dict      =		sep_token_id or vocab_list.index("""[SEP]""")
        _lowerCamelCase  :				List[str]      =		pad_token_id or vocab_list.index("""[PAD]""")
        _lowerCamelCase  :				str      =		ShrinkLongestTrimmer(max_length - 3			,      axis=1)  # Allow room for special tokens
        _lowerCamelCase  :				str      =		max_length
        _lowerCamelCase  :				str      =		padding
        _lowerCamelCase  :				List[str]      =		truncation
        _lowerCamelCase  :				str      =		pad_to_multiple_of
        _lowerCamelCase  :				List[Any]      =		return_token_type_ids
        _lowerCamelCase  :				List[str]      =		return_attention_mask
  
  
  
  
  
     @classmethod
     def 		UpperCamelCase_     (       cls			,      SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)     ->     List[str]:  # noqa: F821
        _lowerCamelCase  :				List[Any]      =		kwargs.pop("""do_lower_case"""			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Dict      =		tokenizer.do_lower_case if do_lower_case is None else do_lower_case
        _lowerCamelCase  :				List[Any]      =		kwargs.pop("""cls_token_id"""			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Dict      =		tokenizer.cls_token_id if cls_token_id is None else cls_token_id
        _lowerCamelCase  :				Optional[int]      =		kwargs.pop("""sep_token_id"""			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Tuple      =		tokenizer.sep_token_id if sep_token_id is None else sep_token_id
        _lowerCamelCase  :				int      =		kwargs.pop("""pad_token_id"""			,      SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Dict      =		tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
        _lowerCamelCase  :				Optional[Any]      =		tokenizer.get_vocab()
        _lowerCamelCase  :				Any      =		sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]			,      key=lambda SCREAMING_SNAKE_CASE: x[1])
        _lowerCamelCase  :				Dict      =		[entry[0] for entry in vocab]
        return cls(
            vocab_list=SCREAMING_SNAKE_CASE			,      do_lower_case=SCREAMING_SNAKE_CASE			,      cls_token_id=SCREAMING_SNAKE_CASE			,      sep_token_id=SCREAMING_SNAKE_CASE			,      pad_token_id=SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE			,      )
  
  
  
  
  
     @classmethod
     def 		UpperCamelCase_     (       cls			,      SCREAMING_SNAKE_CASE			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)     ->     Dict:
        try:
           _lowerCamelCase  :				Dict      =		BertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
        except:  # noqa: E722
           from .tokenization_bert_fast import BertTokenizerFast
  
           _lowerCamelCase  :				Union[str, Any]      =		BertTokenizerFast.from_pretrained(SCREAMING_SNAKE_CASE			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
        return cls.from_tokenizer(SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Any:
        if self.do_lower_case:
           _lowerCamelCase  :				List[Any]      =		case_fold_utfa(SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Any      =		self.tf_tokenizer.tokenize(SCREAMING_SNAKE_CASE)
        return tokens.merge_dims(1			,      -1)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      )     ->     Optional[int]:
        if padding is None:
           _lowerCamelCase  :				Union[str, Any]      =		self.padding
        if padding not in ("longest", "max_length"):
           raise ValueError("""Padding must be either 'longest' or 'max_length'!""")
        if max_length is not None and text_pair is not None:
           # Because we have to instantiate a Trimmer to do it properly
           raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!""")
        if max_length is None:
           _lowerCamelCase  :				Optional[int]      =		self.max_length
        if truncation is None:
           _lowerCamelCase  :				Any      =		self.truncation
        if pad_to_multiple_of is None:
           _lowerCamelCase  :				str      =		self.pad_to_multiple_of
        if return_token_type_ids is None:
           _lowerCamelCase  :				Union[str, Any]      =		self.return_token_type_ids
        if return_attention_mask is None:
           _lowerCamelCase  :				Any      =		self.return_attention_mask
        if not isinstance(SCREAMING_SNAKE_CASE			,      tf.Tensor):
           _lowerCamelCase  :				Dict      =		tf.convert_to_tensor(SCREAMING_SNAKE_CASE)
        if text_pair is not None and not isinstance(SCREAMING_SNAKE_CASE			,      tf.Tensor):
           _lowerCamelCase  :				List[str]      =		tf.convert_to_tensor(SCREAMING_SNAKE_CASE)
        if text_pair is not None:
           if text.shape.rank > 1:
              raise ValueError("""text argument should not be multidimensional when a text pair is supplied!""")
           if text_pair.shape.rank > 1:
              raise ValueError("""text_pair should not be multidimensional!""")
        if text.shape.rank == 2:
           _lowerCamelCase      ,						_lowerCamelCase  :				str      =		text[:, 0], text[:, 1]
        _lowerCamelCase  :				Union[str, Any]      =		self.unpaired_tokenize(SCREAMING_SNAKE_CASE)
        if text_pair is None:  # Unpaired text
           if truncation:
              _lowerCamelCase  :				List[str]      =		text[:, : max_length - 2]  # Allow room for special tokens
           _lowerCamelCase      ,						_lowerCamelCase  :				int      =		combine_segments(
               (text,)			,      start_of_sequence_id=self.cls_token_id			,      end_of_segment_id=self.sep_token_id)
        else:  # Paired text
           _lowerCamelCase  :				Any      =		self.unpaired_tokenize(SCREAMING_SNAKE_CASE)
           if truncation:
              _lowerCamelCase      ,						_lowerCamelCase  :				Dict      =		self.paired_trimmer.trim([text, text_pair])
           _lowerCamelCase      ,						_lowerCamelCase  :				Union[str, Any]      =		combine_segments(
               (text, text_pair)			,      start_of_sequence_id=self.cls_token_id			,      end_of_segment_id=self.sep_token_id)
        if padding == "longest":
           _lowerCamelCase  :				Optional[int]      =		input_ids.bounding_shape(axis=1)
           if pad_to_multiple_of is not None:
              # No ceiling division in tensorflow, so we negate floordiv instead
              _lowerCamelCase  :				int      =		pad_to_multiple_of * (-tf.math.floordiv(-pad_length			,      SCREAMING_SNAKE_CASE))
        else:
           _lowerCamelCase  :				Dict      =		max_length
  
        _lowerCamelCase      ,						_lowerCamelCase  :				Optional[int]      =		pad_model_inputs(SCREAMING_SNAKE_CASE			,      max_seq_length=SCREAMING_SNAKE_CASE			,      pad_value=self.pad_token_id)
        _lowerCamelCase  :				Optional[Any]      =		{"""input_ids""": input_ids}
        if return_attention_mask:
           _lowerCamelCase  :				str      =		attention_mask
        if return_token_type_ids:
           _lowerCamelCase      ,						_lowerCamelCase  :				Union[str, Any]      =		pad_model_inputs(
               SCREAMING_SNAKE_CASE			,      max_seq_length=SCREAMING_SNAKE_CASE			,      pad_value=self.pad_token_id)
           _lowerCamelCase  :				List[str]      =		token_type_ids
        return output
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     Union[str, Any]:
        return {
            "vocab_list": self.vocab_list,
            "do_lower_case": self.do_lower_case,
            "cls_token_id": self.cls_token_id,
            "sep_token_id": self.sep_token_id,
            "pad_token_id": self.pad_token_id,
        }
  
  
style_context: 
  
  
  """simple docstring"""
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  UpperCAmelCase   =	datasets.utils.logging.get_logger(__name__)
  
  
  
  
  @dataclass
  class   lowercase__							(				datasets.BuilderConfig							):
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				"utf-8"
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				10 << 20  # 10MB
     __UpperCAmelCase       					=				False
     __UpperCAmelCase       					=				"line"
  
  
  
  
  
  
  
  class   lowercase__							(				datasets.ArrowBasedBuilder							):
     __UpperCAmelCase       					=				TextConfig
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     int:
        return datasets.DatasetInfo(features=self.config.features)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Optional[int]:
        if not self.config.data_files:
           raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
        _lowerCamelCase  :				Dict      =		dl_manager.download_and_extract(self.config.data_files)
        if isinstance(SCREAMING_SNAKE_CASE			,      (str, list, tuple)):
           _lowerCamelCase  :				Optional[Any]      =		data_files
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				List[Any]      =		[files]
           _lowerCamelCase  :				List[str]      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,      gen_kwargs={"""files""": files})]
        _lowerCamelCase  :				int      =		[]
        for split_name, files in data_files.items():
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				Any      =		[files]
           _lowerCamelCase  :				Any      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE			,      gen_kwargs={"""files""": files}))
        return splits
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     pa.Table:
        if self.config.features is not None:
           _lowerCamelCase  :				Union[str, Any]      =		self.config.features.arrow_schema
           if all(not require_storage_cast(SCREAMING_SNAKE_CASE) for feature in self.config.features.values()):
              # cheaper cast
              _lowerCamelCase  :				Any      =		pa_table.cast(SCREAMING_SNAKE_CASE)
           else:
              # more expensive cast; allows str <-> int/float or str to Audio for example
              _lowerCamelCase  :				Any      =		table_cast(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
           return pa_table
        else:
           return pa_table.cast(pa.schema({"""text""": pa.string()}))
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Any:
        _lowerCamelCase  :				Optional[Any]      =		list(self.config.features) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE)):
           # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
           with open(SCREAMING_SNAKE_CASE			,      encoding=self.config.encoding			,      errors=self.config.errors) as f:
              if self.config.sample_by == "line":
                 _lowerCamelCase  :				Union[str, Any]      =		0
                 while True:
                    _lowerCamelCase  :				int      =		f.read(self.config.chunksize)
                    if not batch:
                       break
                    batch += f.readline()  # finish current line
                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                    _lowerCamelCase  :				List[str]      =		StringIO(SCREAMING_SNAKE_CASE).readlines()
                    if not self.config.keep_linebreaks:
                       _lowerCamelCase  :				Optional[Any]      =		[line.rstrip("""\n""") for line in batch]
                    _lowerCamelCase  :				Any      =		pa.Table.from_arrays([pa.array(SCREAMING_SNAKE_CASE)]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
              elif self.config.sample_by == "paragraph":
                 _lowerCamelCase  :				Optional[Any]      =		0
                 _lowerCamelCase  :				Dict      =		""""""
                 while True:
                    _lowerCamelCase  :				Union[str, Any]      =		f.read(self.config.chunksize)
                    if not new_batch:
                       break
                    batch += new_batch
                    batch += f.readline()  # finish current line
                    _lowerCamelCase  :				Any      =		batch.split("""\n\n""")
                    _lowerCamelCase  :				Dict      =		pa.Table.from_arrays(
                        [pa.array([example for example in batch[:-1] if example])]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
                    _lowerCamelCase  :				Union[str, Any]      =		batch[-1]
                 if batch:
                    _lowerCamelCase  :				Optional[int]      =		pa.Table.from_arrays([pa.array([batch])]			,      names=SCREAMING_SNAKE_CASE)
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
              elif self.config.sample_by == "document":
                 _lowerCamelCase  :				Tuple      =		f.read()
                 _lowerCamelCase  :				List[Any]      =		pa.Table.from_arrays([pa.array([text])]			,      names=SCREAMING_SNAKE_CASE)
                 yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE)
  
  