#!/usr/bin/env python
import os
import urllib2 as urllib
import anyjson as json


URL_LIST = "http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/src/effective_tld_names.dat?raw=1"

# generate json
print 'downloading suffix list..'
rules = {}
lst = urllib.urlopen(URL_LIST).read()
print 'processing list..'
lines = lst.split('\n')
for i,line in enumerate(lines):
    if line[:2] == '//' or len(line) == 0:
        continue # skip comments
    EXCEPT = line[0] == '!'
    if EXCEPT: # exception rule
        line = line[1:]
    doms = line.split('.')
    lst = rules
    # find node to update
    for d in reversed(doms):
        node = lst.get(d, None)
        if not node:
            node = {}
            lst[d] = node
        lst = node
    if EXCEPT:
        lst['!'] = 1;
json = json.serialize(rules).replace(' ','')

# functions for checking domains
def get_reg_domain(rules, doms):
    node = rules.get(doms[0],None)
    if node == None: node = rules.get('*',None)
    if node == None or (len(node) == 1 and node['!'] == 1):
        return doms[0]
    elif len(doms) == 1:
        return None
    reg = get_reg_domain(node, doms[1:])
    if(reg != None):
        return '%s.%s' % (reg, doms[0])

def get_host(domain):
    doms = list(reversed(domain.split('.')))
    return get_reg_domain(rules, doms)

# test the list
print 'testing list..'
tests = {'qwe.parliament.co.uk': 'parliament.co.uk',
         'foo.bar.version2.dk': 'version2.dk',
         'ecs.soton.ac.uk': 'soton.ac.uk'}
for (test,res) in tests.items():
    assert get_host(test) == res

# output new list as javascript
print 'writing list..'
file('../data/suffix-list.js','w').write('suffix_list=%s;' % json);

print 'done.'

@@ -778,55 +778,11 @@
 %5D = 
-1;%0Ajson = json.serialize(rules).replace(' ','')
+%7B%7D;
 %0A%0A# 
@@ -1158,17 +1158,16 @@
 oms%5B0%5D)%0A
-%0A
 def get_
@@ -1514,16 +1514,337 @@
 == res%0A%0A
+# convert the dictionary into a list%0Adef build_string(rules):%0A    lst = %5B%5D%0A    for key in rules:%0A        c = build_string(rules%5Bkey%5D)%0A        s = key%0A        if len(c) != 0: s += '(%25s)' %25 c%0A        lst.append(s)%0A    return ','.join(lst)%0Adef set_length(d):%0A    for key in d:%0A        set_length(d%5Bkey%5D)%0A    d%5B'L'%5D = len(d)%0A
 # output
@@ -1867,16 +1867,34 @@
 ascript%0A
+set_length(rules)%0A
 print 'w
@@ -1912,17 +1912,59 @@
 ..'%0A
+json = json.serialize(rules).replace(' ','')%0A
 file('
-../
 data
@@ -1996,19 +1996,23 @@
 te('
-suffix_list
+var SUFFIX_LIST
 =%25s;

Update suffix list updater