import re


def normalise_text(s):
    """Normalise text for duplicate detection"""

    # simplistic pattern for URLs (doesn't have to be perfect)
    # the goal is to filter out duplicate spam messages that differ only in the URL
    s = re.sub(r'\bhttps?://[/?&%=@a-z0-9_.~-]+\b', ' URL ', s, flags=re.I)

    # simplistic pattern for user tags (doesn't have to be perfect)
    # the goal is to filter out
    s = re.sub(r'\b@[a-z0-9_]+\b', ' USER ', s, flags=re.I)

    # remove all non-word characters
    return ' '.join(re.split(r'\W+', s)).strip().lower()