#!/usr/bin/env python
import os, shutil, sys
import urllib2 as urllib
import anyjson as json


URL_LIST = "http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/src/effective_tld_names.dat?raw=1"
SUFFIX_FILE = "data/suffix-list.js"

# generate json
sys.stdout.write('Updating suffix list.. ')
sys.stdout.flush()

rules = {}
lst = urllib.urlopen(URL_LIST).read()

lines = lst.split('\n')
for i,line in enumerate(lines):
    if line[:2] == '//' or len(line) == 0:
        continue # skip comments
    EXCEPT = line[0] == '!'
    if EXCEPT: # exception rule
        line = line[1:]
    doms = line.split('.')
    lst = rules
    # find node to update
    for d in reversed(doms):
        node = lst.get(d, None)
        if not node:
            node = {}
            lst[d] = node
        lst = node
    if EXCEPT:
        lst['!'] = {};

# functions for checking domains
def get_reg_domain(rules, doms):
    node = rules.get(doms[0],None)
    if node == None: node = rules.get('*',None)
    if node == None or (len(node) == 1 and node['!'] == 1):
        return doms[0]
    elif len(doms) == 1:
        return None
    reg = get_reg_domain(node, doms[1:])
    if(reg != None):
        return '%s.%s' % (reg, doms[0])
def get_host(domain):
    doms = list(reversed(domain.split('.')))
    return get_reg_domain(rules, doms)

# test the list
tests = {'qwe.parliament.co.uk': 'parliament.co.uk',
         'foo.bar.version2.dk': 'version2.dk',
         'ecs.soton.ac.uk': 'soton.ac.uk'}
for (test,res) in tests.items():
    assert get_host(test) == res

# convert the dictionary into a list
def build_string(rules):
    lst = []
    for key in rules:
        c = build_string(rules[key])
        s = key
        if len(c) != 0: s += '(%s)' % c
        lst.append(s)
    return ','.join(lst)
def set_length(d):
    for key in d:
        set_length(d[key])
    d['L'] = len(d)
# output new list as javascript
set_length(rules)
json = json.serialize(rules).replace(' ','')
js = 'var SUFFIX_LIST=%s;' % json
if not os.path.isfile(SUFFIX_FILE) or file(SUFFIX_FILE).read() != js:
    file('%s.new' % SUFFIX_FILE,'w').write(js);
    shutil.move('%s.new' % SUFFIX_FILE, SUFFIX_FILE)
    print 'Updated!'
else:
    print 'Already up to date.'


@@ -86,17 +86,91 @@
 as json%0A
-%0A
+from datetime import datetime%0Aimport httplib%0A%0A%0ADOM_LIST = %22mxr.mozilla.org%22
 %0AURL_LIS
@@ -308,24 +308,8 @@
 s%22%0A%0A
-# generate json%0A
 sys.
@@ -368,16 +368,554 @@
 lush()%0A%0A
+def get_modified_url():%0A    conn = httplib.HTTPConnection(DOM_LIST)%0A    conn.request(%22HEAD%22, URL_LIST)%0A    res = conn.getresponse()%0A    s = dict(res.getheaders())%5B'last-modified'%5D%0A    return datetime.strptime(s, '%25a, %25d %25b %25Y %25H:%25M:%25S %25Z')%0A%0Adef get_modified_file():%0A    try:%0A        return datetime.fromtimestamp(os.stat(SUFFIX_FILE).st_mtime)%0A    except: pass%0A    return datetime.fromtimestamp(0)%0A%0A# Compare modification dates%0Aif get_modified_url() %3C get_modified_file():%0A    print %22Already up to date.%22%0A    sys.exit(0)%0A%0A# generate json%0A
 rules = 
@@ -2462,16 +2462,17 @@
  len(d)%0A
+%0A
 # output
@@ -2596,82 +2596,8 @@
 son%0A
-if not os.path.isfile(SUFFIX_FILE) or file(SUFFIX_FILE).read() != js:%0A    
 file
@@ -2636,20 +2636,16 @@
 te(js);%0A
-    
 shutil.m
@@ -2685,20 +2685,16 @@
 X_FILE)%0A
-    
 print 'U
@@ -2705,44 +2705,6 @@
 ed!'
-%0Aelse:%0A    print 'Already up to date.'
 %0A%0A

Use modification dates to check version.