code: 
  
  
  """simple docstring"""
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
  from ...utils import logging
  
  
  UpperCAmelCase   =	logging.get_logger(__name__)
  
  UpperCAmelCase   =	{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}
  
  UpperCAmelCase   =	{
      """vocab_file""": {
          """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model""",
      },
      """tokenizer_file""": {
          """TsinghuaAI/CPM-Generate""": """https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json""",
      },
  }
  
  
  
  
  class   lowercase__							(				A_							):
  
  
  
  
  
     def __init__(       self			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=None			,      SCREAMING_SNAKE_CASE=False			,      SCREAMING_SNAKE_CASE=True			,      SCREAMING_SNAKE_CASE=False			,      SCREAMING_SNAKE_CASE="<s>"			,      SCREAMING_SNAKE_CASE="</s>"			,      SCREAMING_SNAKE_CASE="<unk>"			,      SCREAMING_SNAKE_CASE="<sep>"			,      SCREAMING_SNAKE_CASE="<pad>"			,      SCREAMING_SNAKE_CASE="<cls>"			,      SCREAMING_SNAKE_CASE="<mask>"			,      SCREAMING_SNAKE_CASE=["<eop>", "<eod>"]			,      **SCREAMING_SNAKE_CASE			,      )     ->     Optional[Any]:
        _lowerCamelCase  :				Tuple      =		AddedToken(SCREAMING_SNAKE_CASE			,      lstrip=SCREAMING_SNAKE_CASE			,      rstrip=SCREAMING_SNAKE_CASE) if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE) else mask_token
  
        super().__init__(
            vocab_file=SCREAMING_SNAKE_CASE			,      tokenizer_file=SCREAMING_SNAKE_CASE			,      do_lower_case=SCREAMING_SNAKE_CASE			,      remove_space=SCREAMING_SNAKE_CASE			,      keep_accents=SCREAMING_SNAKE_CASE			,      bos_token=SCREAMING_SNAKE_CASE			,      eos_token=SCREAMING_SNAKE_CASE			,      unk_token=SCREAMING_SNAKE_CASE			,      sep_token=SCREAMING_SNAKE_CASE			,      pad_token=SCREAMING_SNAKE_CASE			,      cls_token=SCREAMING_SNAKE_CASE			,      mask_token=SCREAMING_SNAKE_CASE			,      additional_special_tokens=SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE			,      )
  
        _lowerCamelCase  :				List[Any]      =		3
        _lowerCamelCase  :				str      =		do_lower_case
        _lowerCamelCase  :				str      =		remove_space
        _lowerCamelCase  :				Optional[int]      =		keep_accents
        _lowerCamelCase  :				Dict      =		vocab_file
        _lowerCamelCase  :				List[str]      =		False if not self.vocab_file else True
  
        try:
           import jieba
        except ModuleNotFoundError as error:
           raise error.__class__(
               """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
               """See https://pypi.org/project/jieba/ for installation.""")
        _lowerCamelCase  :				Optional[int]      =		jieba
        _lowerCamelCase  :				Optional[Any]      =		str.maketrans(""" \n"""			,      """\u2582\u2583""")
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     List[int]:
        _lowerCamelCase  :				List[str]      =		[self.sep_token_id]
        _lowerCamelCase  :				Union[str, Any]      =		[self.cls_token_id]
        if token_ids_a is None:
           return token_ids_a + sep + cls
        return token_ids_a + sep + token_ids_a + sep + cls
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     List[int]:
        _lowerCamelCase  :				Tuple      =		[self.sep_token_id]
        _lowerCamelCase  :				Union[str, Any]      =		[2]
  
        if token_ids_a is None:
           return len(token_ids_a + sep) * [0] + cls_segment_id
        return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE = None)     ->     Tuple[str]:
        if not self.can_save_slow_tokenizer:
           raise ValueError(
               """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
               """tokenizer.""")
  
        if not os.path.isdir(SCREAMING_SNAKE_CASE):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory')
           return
        _lowerCamelCase  :				Optional[Any]      =		os.path.join(
            SCREAMING_SNAKE_CASE			,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
  
        if os.path.abspath(self.vocab_file) != os.path.abspath(SCREAMING_SNAKE_CASE):
           copyfile(self.vocab_file			,      SCREAMING_SNAKE_CASE)
  
        return (out_vocab_file,)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)     ->     Optional[int]:
        _lowerCamelCase  :				List[Any]      =		[
            """ """.join([x.translate(self.translator) for x in self.jieba.cut(SCREAMING_SNAKE_CASE			,      cut_all=SCREAMING_SNAKE_CASE)])
            for text in batch_text_or_text_pairs
        ]
        return super()._batch_encode_plus(SCREAMING_SNAKE_CASE			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      *SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)     ->     Union[str, Any]:
        _lowerCamelCase  :				Optional[int]      =		super()._decode(*SCREAMING_SNAKE_CASE			,      **SCREAMING_SNAKE_CASE)
        _lowerCamelCase  :				Tuple      =		text.replace(""" """			,      """""").replace("""\u2582"""			,      """ """).replace("""\u2583"""			,      """\n""")
        return text
  
style_context: 
  
  
  """simple docstring"""
  import itertools
  from dataclasses import dataclass
  from io import StringIO
  from typing import Optional
  
  import pyarrow as pa
  
  import datasets
  from datasets.features.features import require_storage_cast
  from datasets.table import table_cast
  
  
  UpperCAmelCase   =	datasets.utils.logging.get_logger(__name__)
  
  
  
  
  @dataclass
  class   lowercase__							(				datasets.BuilderConfig							):
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				"utf-8"
     __UpperCAmelCase       					=				None
     __UpperCAmelCase       					=				10 << 20  # 10MB
     __UpperCAmelCase       					=				False
     __UpperCAmelCase       					=				"line"
  
  
  
  
  
  
  
  class   lowercase__							(				datasets.ArrowBasedBuilder							):
     __UpperCAmelCase       					=				TextConfig
  
  
  
  
  
     def 		UpperCamelCase_     (       self)     ->     int:
        return datasets.DatasetInfo(features=self.config.features)
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Optional[int]:
        if not self.config.data_files:
           raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
        _lowerCamelCase  :				Dict      =		dl_manager.download_and_extract(self.config.data_files)
        if isinstance(SCREAMING_SNAKE_CASE			,      (str, list, tuple)):
           _lowerCamelCase  :				Optional[Any]      =		data_files
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				List[Any]      =		[files]
           _lowerCamelCase  :				List[str]      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,      gen_kwargs={"""files""": files})]
        _lowerCamelCase  :				int      =		[]
        for split_name, files in data_files.items():
           if isinstance(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE):
              _lowerCamelCase  :				Any      =		[files]
           _lowerCamelCase  :				Any      =		[dl_manager.iter_files(SCREAMING_SNAKE_CASE) for file in files]
           splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE			,      gen_kwargs={"""files""": files}))
        return splits
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     pa.Table:
        if self.config.features is not None:
           _lowerCamelCase  :				Union[str, Any]      =		self.config.features.arrow_schema
           if all(not require_storage_cast(SCREAMING_SNAKE_CASE) for feature in self.config.features.values()):
              # cheaper cast
              _lowerCamelCase  :				Any      =		pa_table.cast(SCREAMING_SNAKE_CASE)
           else:
              # more expensive cast; allows str <-> int/float or str to Audio for example
              _lowerCamelCase  :				Any      =		table_cast(SCREAMING_SNAKE_CASE			,      SCREAMING_SNAKE_CASE)
           return pa_table
        else:
           return pa_table.cast(pa.schema({"""text""": pa.string()}))
  
  
  
  
  
     def 		UpperCamelCase_     (       self			,      SCREAMING_SNAKE_CASE)     ->     Any:
        _lowerCamelCase  :				Optional[Any]      =		list(self.config.features) if self.config.features is not None else ["""text"""]
        for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE)):
           # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
           with open(SCREAMING_SNAKE_CASE			,      encoding=self.config.encoding			,      errors=self.config.errors) as f:
              if self.config.sample_by == "line":
                 _lowerCamelCase  :				Union[str, Any]      =		0
                 while True:
                    _lowerCamelCase  :				int      =		f.read(self.config.chunksize)
                    if not batch:
                       break
                    batch += f.readline()  # finish current line
                    # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                    _lowerCamelCase  :				List[str]      =		StringIO(SCREAMING_SNAKE_CASE).readlines()
                    if not self.config.keep_linebreaks:
                       _lowerCamelCase  :				Optional[Any]      =		[line.rstrip("""\n""") for line in batch]
                    _lowerCamelCase  :				Any      =		pa.Table.from_arrays([pa.array(SCREAMING_SNAKE_CASE)]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
              elif self.config.sample_by == "paragraph":
                 _lowerCamelCase  :				Optional[Any]      =		0
                 _lowerCamelCase  :				Dict      =		""""""
                 while True:
                    _lowerCamelCase  :				Union[str, Any]      =		f.read(self.config.chunksize)
                    if not new_batch:
                       break
                    batch += new_batch
                    batch += f.readline()  # finish current line
                    _lowerCamelCase  :				Any      =		batch.split("""\n\n""")
                    _lowerCamelCase  :				Dict      =		pa.Table.from_arrays(
                        [pa.array([example for example in batch[:-1] if example])]			,      names=SCREAMING_SNAKE_CASE)
                    # Uncomment for debugging (will print the Arrow table size and elements)
                    # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                    # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
                    batch_idx += 1
                    _lowerCamelCase  :				Union[str, Any]      =		batch[-1]
                 if batch:
                    _lowerCamelCase  :				Optional[int]      =		pa.Table.from_arrays([pa.array([batch])]			,      names=SCREAMING_SNAKE_CASE)
                    yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE)
              elif self.config.sample_by == "document":
                 _lowerCamelCase  :				Tuple      =		f.read()
                 _lowerCamelCase  :				List[Any]      =		pa.Table.from_arrays([pa.array([text])]			,      names=SCREAMING_SNAKE_CASE)
                 yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE)
  
  