






"""simple docstring"""
import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


A_     :					Tuple							=     datasets.utils.logging.get_logger(__name__)



@dataclass
class 						a_ (      datasets.BuilderConfig					):







      '''simple docstring'''

      lowerCamelCase__			:							Optional[datasets.Features]					=      None
      lowerCamelCase__			:							str					=      "utf-8"
      lowerCamelCase__			:							Optional[str]					=      None
      lowerCamelCase__			:							int					=      10 << 20  # 10MB
      lowerCamelCase__			:							bool					=      False
      lowerCamelCase__			:							str					=      "line"



class 						a_ (      datasets.ArrowBasedBuilder					):







      '''simple docstring'''

      lowerCamelCase__			:							Tuple					=      TextConfig




      def      a__				(self   ):

       '''simple docstring'''



       return datasets.DatasetInfo(features=self.config.features   )




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       if not self.config.data_files:
        raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''   )
       lowerCamelCase__	:     Union[str, Any]								=		dl_manager.download_and_extract(self.config.data_files   )
       if isinstance(lowerCamelCase_,	(str, list, tuple)   ):
        lowerCamelCase__	:     List[str]								=		data_files
        if isinstance(lowerCamelCase_,	lowerCamelCase_   ):
         lowerCamelCase__	:     Dict								=		[files]
        lowerCamelCase__	:     Dict								=		[dl_manager.iter_files(lowerCamelCase_   ) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN,	gen_kwargs={'files': files}   )]
       lowerCamelCase__	:     List[str]								=		[]
       for split_name, files in data_files.items():
        if isinstance(lowerCamelCase_,	lowerCamelCase_   ):
         lowerCamelCase__	:     List[Any]								=		[files]
        lowerCamelCase__	:     int								=		[dl_manager.iter_files(lowerCamelCase_   ) for file in files]
        splits.append(datasets.SplitGenerator(name=lowerCamelCase_,	gen_kwargs={'files': files}   )   )
       return splits




      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       if self.config.features is not None:
        lowerCamelCase__	:     List[str]								=		self.config.features.arrow_schema
        if all(not require_storage_cast(lowerCamelCase_   ) for feature in self.config.features.values()   ):
         # cheaper cast
         lowerCamelCase__	:     Optional[Any]								=		pa_table.cast(lowerCamelCase_   )
        else:
         # more expensive cast; allows str <-> int/float or str to Audio for example
         lowerCamelCase__	:     List[Any]								=		table_cast(lowerCamelCase_,	lowerCamelCase_   )
        return pa_table
       else:
        return pa_table.cast(pa.schema({'text': pa.string()}   )   )





      def      a__				(self,	lowerCamelCase_   ):

       '''simple docstring'''



       lowerCamelCase__	:     str								=		list(self.config.features   ) if self.config.features is not None else ['text']
       for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase_   )   ):
        # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
        with open(lowerCamelCase_,	encoding=self.config.encoding,	errors=self.config.errors   ) as f:
         if self.config.sample_by == "line":
          lowerCamelCase__	:     List[Any]								=		0
          while True:
           lowerCamelCase__	:     Any								=		f.read(self.config.chunksize   )
           if not batch:
            break
           batch += f.readline()  # finish current line
           # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
           lowerCamelCase__	:     List[Any]								=		StringIO(lowerCamelCase_   ).readlines()
           if not self.config.keep_linebreaks:
            lowerCamelCase__	:     Union[str, Any]								=		[line.rstrip('\n'   ) for line in batch]
           lowerCamelCase__	:     Union[str, Any]								=		pa.Table.from_arrays([pa.array(lowerCamelCase_   )],	names=lowerCamelCase_   )
           # Uncomment for debugging (will print the Arrow table size and elements)
           # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
           # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
           yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
           batch_idx += 1
         elif self.config.sample_by == "paragraph":
          lowerCamelCase__	:     List[Any]								=		0
          lowerCamelCase__	:     Any								=		''
          while True:
           lowerCamelCase__	:     Optional[int]								=		f.read(self.config.chunksize   )
           if not new_batch:
            break
           batch += new_batch
           batch += f.readline()  # finish current line
           lowerCamelCase__	:     int								=		batch.split('\n\n'   )
           lowerCamelCase__	:     Dict								=		pa.Table.from_arrays(
               [pa.array([example for example in batch[:-1] if example]   )],	names=lowerCamelCase_   )
           # Uncomment for debugging (will print the Arrow table size and elements)
           # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
           # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
           yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
           batch_idx += 1
           lowerCamelCase__	:     Dict								=		batch[-1]
          if batch:
           lowerCamelCase__	:     List[Any]								=		pa.Table.from_arrays([pa.array([batch]   )],	names=lowerCamelCase_   )
           yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_   )
         elif self.config.sample_by == "document":
          lowerCamelCase__	:     str								=		f.read()
          lowerCamelCase__	:     Tuple								=		pa.Table.from_arrays([pa.array([text]   )],	names=lowerCamelCase_   )
          yield file_idx, self._cast_table(lowerCamelCase_   )









"""simple docstring"""
import unittest

from cer import CER


A_     :					Union[str, Any]							=     CER()



class 						a_ (      unittest.TestCase					):







      '''simple docstring'''

      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		['White House']
       lowerCamelCase__	:     List[str]								=		['white house']
       # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11
       lowerCamelCase__	:     Dict								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.1_818_181_818   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Union[str, Any]								=		['were wolf']
       lowerCamelCase__	:     Union[str, Any]								=		['werewolf']
       # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9
       lowerCamelCase__	:     Dict								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.1_111_111   ) < 1e-6   )

       lowerCamelCase__	:     List[Any]								=		['werewolf']
       lowerCamelCase__	:     Optional[Any]								=		['weae     wolf']
       # S = 1, D = 1, I = 0, N = 8, CER = 0.25
       lowerCamelCase__	:     List[str]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.25   ) < 1e-6   )

       # consecutive whitespaces case 1
       lowerCamelCase__	:     Optional[Any]								=		['were wolf']
       lowerCamelCase__	:     Tuple								=		['were               wolf']
       # S = 0, D = 0, I = 0, N = 9, CER = 0
       lowerCamelCase__	:     Tuple								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.0   ) < 1e-6   )

       # consecutive whitespaces case 2
       lowerCamelCase__	:     Any								=		['were   wolf']
       lowerCamelCase__	:     int								=		['were               wolf']
       # S = 0, D = 0, I = 0, N = 9, CER = 0
       lowerCamelCase__	:     Optional[int]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.0   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		['werewolf']
       lowerCamelCase__	:     Optional[int]								=		['weaewolf']
       # S = 1, D = 0, I = 0, N = 8, CER = 0.125
       lowerCamelCase__	:     Union[str, Any]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.125   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     int								=		['werewolf']
       lowerCamelCase__	:     Union[str, Any]								=		['wereawolf']
       # S = 0, D = 1, I = 0, N = 8, CER = 0.125
       lowerCamelCase__	:     Any								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.125   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[Any]								=		['werewolf']
       lowerCamelCase__	:     Dict								=		['wereolf']
       # S = 0, D = 0, I = 1, N = 8, CER = 0.125
       lowerCamelCase__	:     int								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.125   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[int]								=		['werewolf']
       lowerCamelCase__	:     Tuple								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertEqual(lowerCamelCase_,	0.0   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     List[str]								=		['werewolf', 'I am your father']
       lowerCamelCase__	:     List[Any]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertEqual(lowerCamelCase_,	0.0   )

       lowerCamelCase__	:     Dict								=		['werewolf', 'I am your father', 'doge']
       lowerCamelCase__	:     int								=		['werxwolf', 'I       am your father', 'doge']
       # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28
       lowerCamelCase__	:     Optional[int]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.03_571_428   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Union[str, Any]								=		['My hovercraft', 'is full of eels']
       lowerCamelCase__	:     List[Any]								=		['My hovercraft is full', ' of eels']
       # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28
       # whitespace at the front of " of eels" will be strip during preporcessing
       # so need to insert 2 whitespaces
       lowerCamelCase__	:     Dict								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_,	concatenate_texts=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.071_428   ) < 1e-6   )




      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[Any]								=		['我能吞下玻璃而不伤身体']
       lowerCamelCase__	:     Any								=		[' 能吞虾玻璃而 不霜身体啦']
       # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11
       lowerCamelCase__	:     List[str]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.4_545_454_545   ) < 1e-6   )

       lowerCamelCase__	:     Optional[Any]								=		['我能吞下玻璃', '而不伤身体']
       lowerCamelCase__	:     Optional[Any]								=		['我    能 吞 下 玻 璃', '而不伤身体']
       # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11
       lowerCamelCase__	:     Dict								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertTrue(abs(char_error_rate - 0.454_545_454_545   ) < 1e-6   )

       lowerCamelCase__	:     Optional[int]								=		['我能吞下玻璃而不伤身体']
       lowerCamelCase__	:     Union[str, Any]								=		cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )
       self.assertFalse(lowerCamelCase_,	0.0   )





      def      a__				(self   ):

       '''simple docstring'''



       lowerCamelCase__	:     Optional[int]								=		['']
       lowerCamelCase__	:     Optional[int]								=		['Hypothesis']
       with self.assertRaises(lowerCamelCase_   ):
        cer.compute(predictions=lowerCamelCase_,	references=lowerCamelCase_   )


if __name__ == "__main__":
      unittest.main()

