import csv
import os
import re

import requests
from pattern import web


def get_dom(url):
    html = requests.get(url).text
    dom = web.Element(html)
    return dom


def get_taxons_from_crispr_db():
    taxon_ids = []
    if os.path.exists('taxon_ids.csv'):
        with open('taxon_ids.csv', 'r') as csvfile:
            reader = csv.reader(csvfile, delimiter=',')
            for row in reader:
                taxon_ids.append(row[0])
            return taxon_ids
    url = "http://crispr.u-psud.fr/crispr/"
    dom_homepage = get_dom(url)
    container = dom_homepage('div[class="strainlist"]')[0]
    for link in container('a'):
        taxon_id = link.href.encode('ascii', 'ignore')[46:]
        taxon_ids.append(taxon_id)
        with open('taxon_ids.csv', 'a') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([taxon_id])
    return taxon_ids


def get_sequences(taxon_id):
    url = ("http://crispr.u-psud.fr/cgi-bin/crispr/SpecieProperties.cgi?"
           "Taxon_id=" + taxon_id)
    dom = get_dom(url)
    table = dom('table[class="primary_table"]')[1]
    sequences = []
    for sequence in table('tr'):
        if not sequence('th'):
            seq = {
                'Taxon_id': taxon_id,
                'RefSeq': sequence('td')[1].content.encode('ascii', 'ignore')
            }
            sequences.append(seq)
    return sequences


def get_loc_ids(sequence):
    url = ("http://crispr.u-psud.fr/crispr/CRISPRProperties.php?RefSeq="
           + sequence['RefSeq'])
    dom = get_dom(url)
    div = dom('div[class="rightcontent"]')[1]
    table = div('table[class="primary_table"]')[0]
    tbody = table('tbody')[0]
    loc_ids = []
    for crispr_id in tbody('tr'):
        cell = crispr_id('td')[1]
        crispr_id_value = (cell('font')[0]
                           .content
                           .encode('ascii', 'ignore')
                           .replace('<br/n/>', ''))
        crispr_id_value = crispr_id_value.replace('\n', '')
        crispr_id_value = crispr_id_value.replace('\t', '')
        crispr_id_value = crispr_id_value.replace('<br />', '')
        loc_ids.append(crispr_id_value.split('_')[-1])
    return loc_ids


def get_positions(sequence, loc_id):
    params = {'Taxon': sequence['Taxon_id']}
    crispr_id = sequence['RefSeq'] + "_" + loc_id
    params['checked[]'] = crispr_id
    r = requests.post(
        "http://crispr.u-psud.fr/crispr/crispr.php",
        data=params
    )
    source = web.Element(r.text)
    print("Crispr id: " + crispr_id)
    table = source('table[class="crispr"]')[0]
    tr = table('tr')[3]
    td = tr('td')
    begin = re.search('(?<=</span>) +\d+', td[0].content)
    begin = begin.group(0).split(' ')[-1]
    end = re.search('(?<=</span>) +\d+', td[1].content)
    end = end.group(0).split(' ')[-1]
    return [begin, end]


def get_results():
    print("Getting taxon ids...")
    taxon_ids = get_taxons_from_crispr_db()
    n = len(taxon_ids)
    i = 0.
    print("Getting positions...")
    restart = False
    var = False
    if os.path.exists('last'):
        with open('last', 'r') as f:
            last = f.read()
        [last_taxon_id, last_refseq, last_loc_id] = last.split(',')
        restart = True
    for taxon_id in taxon_ids:
        print("{:.2%}".format(i / n))
        print("Taxon id: " + taxon_id)
        if restart and last_taxon_id == taxon_id:
            var = True
            continue
        if var or not restart:
            sequences = get_sequences(taxon_id)
            for sequence in sequences:
                loc_ids = get_loc_ids(sequence)
                sequence['Loc_ids'] = loc_ids
                for loc_id in sequence['Loc_ids']:
                    [begin, end] = get_positions(sequence, loc_id)
                    result = [sequence['RefSeq'], loc_id, begin, end]
                    with open('results.csv', 'a') as csvfile:
                        writer = csv.writer(csvfile, delimiter=',')
                        writer.writerow(result)
                    with open('last', 'w') as f:
                        f.write(','.join((
                            sequence['Taxon_id'],
                            sequence['RefSeq'],
                            loc_id
                        )))

        i += 1


get_results()

@@ -1936,17 +1936,16 @@
 /%3E', '')
-)
 %0A       
@@ -1949,161 +1949,119 @@
     
-crispr_id_value = crispr_id_value.replace('%5Cn', '')%0A        crispr_id_value = crispr_id_value.replace('%5Ct', '')%0A        crispr_id_value = crispr_id_value
+                   .replace('%5Cn', '')%0A                           .replace('%5Ct', '')%0A                           
 .rep
@@ -2070,32 +2070,33 @@
 ce('%3Cbr /%3E', '')
+)
 %0A        loc_ids

Chain the rest of the `.replace`'s