import requests
from pattern import web
import re
import csv

def get_dom(url):
    html = requests.get(url).text
    dom = web.Element(html)
    return dom

def get_taxons_from_CRISPRdb():
    url = "http://crispr.u-psud.fr/crispr/"
    dom_homepage = get_dom(url)
    container = dom_homepage('div[class="strainlist"]')[0]
    crisprs = []
    for link in container('a'):
        crispr = {}
        crispr['Name'] = link.content.encode('ascii','ignore')
        crispr['Taxon_id'] = link.href.encode('ascii','ignore')[46:]
        crisprs.append(crispr)
    return crisprs

def get_genome_properties(crispr):
    url = "http://crispr.u-psud.fr/cgi-bin/crispr/SpecieProperties.cgi?Taxon_id=" + crispr['Taxon_id']
    dom = get_dom(url)
    table = dom('table[class="primary_table"]')[1]
    crispr['Ref_seqs']={}
    for sequence in table('tr'):
        if sequence('th')==[]:
            crispr['Ref_seqs'][sequence('td')[1].content.encode('ascii','ignore')]=[]
    return crispr

def get_CRISPR_properties(crispr):
    for ref_seq in crispr['Ref_seqs'].keys():
        url = "http://crispr.u-psud.fr/crispr/CRISPRProperties.php?RefSeq=" + ref_seq + "&Taxon=" + crispr['Taxon_id']
        dom = get_dom(url)
        div = dom('div[class="rightcontent"]')[1]
        table = div('table[class="primary_table"]')[0]
        tbody = table('tbody')[0]
        for crispr_id in tbody('tr'):
            cell = crispr_id('td')[1]
            crispr_id_value = cell('font')[0].content.encode('ascii','ignore').replace('<br/n/>','')
            crispr_id_value = crispr_id_value.replace('\n','')
            crispr_id_value = crispr_id_value.replace('\t','')
            crispr_id_value = crispr_id_value.replace('<br />','')
            crispr['Ref_seqs'][ref_seq].append(crispr_id_value)
    return crispr

def get_begin(source):
    begin = re.search('(?<=Crispr_begin_position: )\d+', source)
    begin = begin.group(0)
    return begin

def get_end(source):
    end = re.search('(?<=Crispr_end_position: )\d+', source)
    end = end.group(0)
    return end

def get_loc_id(source):
    loc_id = re.search('(?<=Crispr Rank in the sequence: )\d+', source)
    loc_id = loc_id.group(0)
    return loc_id

def download(source,a,b):
    directory = "files/"
    namefile =  a + "_" + b
    if not os.path.exists(directory):
        os.makedirs(directory)
    f = open(directory + namefile, 'w')
    f.write(source)
    f.close()

def get_results():
    crisprs =  get_taxons_from_CRISPRdb()
    n = len(crisprs)
    i = 0.
    print "Getting genome and CRISPR properties..."
    start = time.time()
    for crispr in crisprs:
        print str(i/n*100) + "%"
        crispr = get_genome_properties(crispr)
        crispr = get_CRISPR_properties(crispr)
        i += 1
    end = time.time()
    print "Getting properties required " + end-start " seconds."
    results = []
    i = 0.
    print "Downloading files..."
    start = time.time()
    for crispr in crisprs:
        print str(i/n*100) + "%"
        for ref_seq in crispr['Ref_seqs'].keys():
            for crispr_id in crispr['Ref_seqs'][ref_seq]:
                params = {'checked[]': crispr_id, 'Taxon': crispr['Taxon_id']}
                r = requests.post("http://crispr.u-psud.fr/crispr/crispr.php", data=params)
                file_crispr_seq = web.Element(r.text)
                table = file_crispr_seq('table[class="crisprs_table"]')[0]
                url = "http://crispr.u-psud.fr" + table('form')[-2].action
                source = requests.get(url).text
                begin = get_begin(source)
                end = get_end(source)
                main_accession_number = ref_seq
                loc_id = get_loc_id(source)
                download(source,main_accession_number,loc_id)
                results.append([main_accession_number,loc_id,begin,end])
        i += 1
    end = time.time()
    print "Downloading files required " + end-start + " seconds."

    print "Writing .csv file..."
    start = time.time()
    with open('results.csv', 'wb') as csvfile:
        writer = csv.writer(csvfile, delimiter=',')
        for result in results:
            writer.writerow(result)
    end = time.time()
    print "Writing .csv file required " + end-start + " seconds."

start = time.time() 
get_results()
end = time.time()
print = "The complete process required " + end-start + " seconds."

@@ -53,16 +53,28 @@
 port csv
+%0Aimport time
 %0A%0Adef ge
@@ -2639,34 +2639,36 @@
   print 
-str(i/n*100) + %22%25%22
+%22%7B:.2%25%7D%22.format(i/n)
 %0A       
@@ -2826,32 +2826,36 @@
 es required %22 + 
+str(
 end-start %22 seco
@@ -2847,16 +2847,19 @@
 nd-start
+) +
  %22 secon
@@ -2993,26 +2993,28 @@
 int 
-str(i/n*100) + %22%25%22
+%22%7B:.2%25%7D%22.format(i/n)
 %0A   
@@ -3919,32 +3919,36 @@
 es required %22 + 
+str(
 end-start + %22 se
@@ -3932,32 +3932,33 @@
  + str(end-start
+)
  + %22 seconds.%22%0A%0A
@@ -4236,32 +4236,36 @@
 le required %22 + 
+str(
 end-start + %22 se
@@ -4249,32 +4249,33 @@
  + str(end-start
+)
  + %22 seconds.%22%0A%0A
@@ -4332,18 +4332,16 @@
 ()%0Aprint
- =
  %22The co
@@ -4368,16 +4368,20 @@
 red %22 + 
+str(
 end-star
@@ -4381,16 +4381,17 @@
 nd-start
+)
  + %22 sec

Fix some bugs and improve the percentage indicator.