# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from lxml import etree
from colyzer import ShingleBased, SelkowTED, ClusterAlgorithms


class CobotPipeline(object):
    def __init__(self):
        self.doc_list = []
        self.ted_dist = []

    def open_spider(self, spider):
        print('OPENED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!111111')
        print('Algorithm -->>' + spider.algorithm.which)

    def process_item(self, item, spider):
        print('page name -->' + item['page_name'])
        print('Processing....')
        self.__create_doc_list(item, spider.algorithm.which)
        return item

    def close_spider(self, spider):
        print('CLOSED!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1111111111')
        if spider.algorithm.which == 'shingle':
            distance_matrix = ShingleBased.get_distance_matrix(self.doc_list)
            print(distance_matrix)
            # clustering = ClusterAlgorithms.Clustering(0.755, 500, 5)
            clustering = ClusterAlgorithms.Clustering(spider.algorithm.shingle.threshold,
                                                      spider.algorithm.shingle.iteration,
                                                      spider.algorithm.shingle.cluster_size,
                                                      spider.initialize.site_name)
            clustering.process(self.doc_list, distance_matrix)
            clustering.pretty_print()
        elif spider.algorithm.which == 'ted':
            distance_matrix = SelkowTED.get_distance_matrix(self.doc_list)
            print(distance_matrix)
            # clustering = ClusterAlgorithms.Clustering(0.755, 500, 5)
            clustering = ClusterAlgorithms.Clustering(spider.algorithm.ted.threshold, spider.algorithm.ted.iteration,
                                                      spider.algorithm.ted.cluster_size, spider.initialize.site_name)
            clustering.process(self.doc_list, distance_matrix)
            clustering.pretty_print()

    def __create_doc_list(self, item, which):
        tree = etree.parse(item['page_full_path'], parser=etree.HTMLParser(remove_comments=True))
        root_tag = tree.getroot()[1]
        if which == 'shingle':
            doc = ShingleBased.Document(doc_name=item['page_name'], doc_link=item['page_url'])
            doc.find_all_real_paths(root_tag, -1)
            doc.find_all_virtual_paths(root_tag, 0)
            self.doc_list.append(doc)
        elif which == 'ted':
            doc = SelkowTED.Nodes(root_tag, doc_name=item['page_name'], doc_link=item['page_url'])
            self.doc_list.append(doc)
@@ -272,16 +272,100 @@
 orithms%0A
+import json%0Aimport os%0Afrom cobot.spiders import SITES%0Afrom cobot import RawDocuments
 %0A%0Aclass 
@@ -441,35 +441,8 @@
 = %5B%5D
-%0A        self.ted_dist = %5B%5D
 %0A%0A  
@@ -493,16 +493,23 @@
 nt('
+SPIDER 
 OPENED!!
 !!!!
@@ -508,61 +508,8 @@
 ED!!
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!111111
 ')%0A 
@@ -662,40 +662,8 @@
 '%5D)%0A
-        print('Processing....')%0A
     
@@ -795,16 +795,23 @@
 nt('
+SPIDER 
 CLOSED!!
 !!!!
@@ -810,305 +810,868 @@
 ED!!
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1111111111')%0A        if spider.algorithm.which == 'shingle':%0A            distance_matrix = ShingleBased.get_distance_matrix(self.doc_list)%0A            print(distance_matrix)%0A            # clustering = ClusterAlgorithms.Clustering(0.755, 500, 5)
+')%0A        raw_doc_list = %5B%5D%0A        for doc in self.doc_list:%0A            raw_doc = ClusterAlgorithms.Document(doc.doc_name, doc.doc_link)%0A            raw_doc_list.append(raw_doc.__dict__)%0A        result = RawDocuments(raw_doc_list)%0A%0A        json_txt = json.dumps(result.__dict__)%0A        with open(os.path.join(SITES, spider.initialize.site_name + '_raw.json'), 'w') as f:%0A            f.write(json_txt)%0A%0A        if spider.algorithm.which == 'shingle':%0A            distance_matrix = ShingleBased.get_distance_matrix(self.doc_list)%0A            print(distance_matrix)%0A        elif spider.algorithm.which == 'ted':%0A            distance_matrix = SelkowTED.get_distance_matrix(self.doc_list)%0A            print(distance_matrix)%0A        else:%0A            print('There is no algorithm!!')%0A            return%0A%0A        if spider.algorithm.which_clustering == 'k_means':
 %0A   
@@ -1725,42 +1725,12 @@
 ing(
-spider.algorithm.shingle.threshold
+None
 ,%0A  
@@ -1798,23 +1798,23 @@
 gorithm.
-shingle
+k_means
 .iterati
@@ -1888,23 +1888,23 @@
 gorithm.
-shingle
+k_means
 .cluster
@@ -2008,32 +2008,40 @@
      clustering.
+k_means_
 process(self.doc
@@ -2076,264 +2076,66 @@
     
-    clustering.pretty_print()%0A        elif spider.algorithm.which == 'ted':%0A            distance_matrix = SelkowTED.get_distance_matrix(self.doc_list)%0A            print(distance_matrix)%0A            # clustering = ClusterAlgorithms.Clustering(0.755, 500, 5)
+elif spider.algorithm.which_clustering == 'shingle_based':
 %0A   
@@ -2202,17 +2202,27 @@
 gorithm.
-t
+shingle_bas
 ed.thres
@@ -2226,16 +2226,70 @@
 reshold,
+%0A                                                     
  spider.
@@ -2298,17 +2298,27 @@
 gorithm.
-t
+shingle_bas
 ed.itera
@@ -2394,17 +2394,27 @@
 gorithm.
-t
+shingle_bas
 ed.clust
@@ -2421,16 +2421,70 @@
 er_size,
+%0A                                                     
  spider.
@@ -2568,24 +2568,109 @@
 matrix)%0A    
+    else:%0A            print('There is no clustering algorithm!!')%0A            return%0A
         clus

save raw link and name