from typing import Sequence
import re


def _tokenize_for_bleu_eval(code: str) -> Sequence[str]:
  """Tokenize the input for computing the bleu score.

  We perform the same tokenizing step as the retrieve and edit dataset. It is
  the 'def tokenize_for_bleu_eval(code)' function from 'github_eval.py' in the
  published retrieve and edit source code.

  Args:
    code: A string representing the code.

  Returns:
    The tokenized code as a list.
  """
  code = re.sub(r'([^A-Za-z0-9_])', r' \1 ', code)
  code = re.sub(r'([a-z])([A-Z])', r'\1 \2', code)
  code = re.sub(r'\s+', ' ', code)
  code = code.replace('"', '`')
  code = code.replace('\'', '`')
  tokens = [t for t in code.split(' ') if t]
  return tokens