import unicodedata


def _should_remove(char):
  """Checks whether a character should be removed when tokenizing"""

  # These are technically control characters but BERT treats these as
  # whitespaces

  if char == "\t" or char == "\n" or char == "\r":
    return False

  cp = ord(char)
  if cp == 0 or cp == 0xfffd:
    return True

  cat = unicodedata.category(char)
  if cat in ("Cc", "Cf"):
    return True

  return False