import re


def encode(text):
  """Encode a unicode string as a list of tokens.

  Args:
    text: a unicode string
    
  Returns:
    a list of tokens as Unicode strings
  """
  if not text:
    return []

  # Limit more than 4 consecutive digits. 
  text = re.sub(r"\d{5,}", "#", text.lower().strip())

  ret = []
  token_start = 0
  for pos in range(1, len(text)):
    if text[pos].isalnum() != text[pos - 1].isalnum() or (
        text[pos].strip() == ""):
      token = text[token_start:pos]
      if token.strip() != "":
        ret.append(token.strip())
      token_start = pos

  if token_start < len(text):
    final_token = text[token_start:]
    ret.append(final_token.strip())

  return ret