from collections import defaultdict
from unidecode import unidecode
import re

_sep_re = re.compile(r'[/_\-]+')
_alnumspace_re = re.compile(r'[^0-9A-Za-z ]')
_spaces_re = re.compile(r'\s+')

def normalize_alnum(s):
    s = unidecode(s).upper()
    s = s.replace('&',' AND ')
    s = _sep_re.sub(' ',s)
    s = _alnumspace_re.sub('',s)
    s = _spaces_re.sub(' ',s).strip()
    return s

class WordlistGenerator:
    def __init__(self):
        self.words = defaultdict(lambda : 0)

    def add(self,name,freq,func=lambda x,y: x+y):
        name = normalize_alnum(name)
        freq = int(freq)
        self.words[name]=func(self.words[name],freq)

    def write(self,filename,cutoff=0):
        l = [(v,k) for k,v in self.words.items() if v>=cutoff]
        l.sort(reverse=True)
        with open(filename,'w') as f:
            for v,k in l:
                print("%s\t%d"%(k,v),file=f)

@@ -182,16 +182,53 @@
 (r'%5Cs+')
+%0A_nonalpha_re = re.compile(r'%5B%5EA-Z%5D')
 %0A%0Adef no
@@ -690,36 +690,62 @@
 def 
-write(self,filename,cutoff=0
+make_list(self,cutoff=0,eliminate_duplicate_slugs=True
 ):%0A 
@@ -835,16 +835,582 @@
 e=True)%0A
+        if eliminate_duplicate_slugs:%0A            # solvertools doesn't like it when wordlists have entries%0A            # that differ by only non-alphabetic characters%0A            oldl = l%0A            slugs = set()%0A            l = %5B%5D%0A            for v,k in oldl:%0A                slug=_nonalpha_re.sub('',k)%0A                if slug not in slugs:%0A                    l.append((v,k))%0A                    slugs.add(slug)%0A        return l%0A%0A    def write(self,filename,cutoff=0,eliminate_duplicate_slugs=True):%0A        l = self.make_list(cutoff,eliminate_duplicate_slugs)%0A
         
@@ -1494,10 +1494,9 @@
 (%22%25s
-%5Ct
+,
 %25d%22%25

Make wordlists solvertools compatible