#!/usr/bin/python
# encoding=utf-8
#check duplicate picture from web
import urllib2
import urlparse
import gevent
from gevent import monkey
import json
from cStringIO import StringIO
import imagehash
import Image

monkey.patch_all()

def checkImage(url):
        buffer = []
        try:
                data = StringIO(urllib2.urlopen(url).read())
                hash = imagehash.average_hash(Image.open(data))
                print hash,url
                buffer.append((str(hash),url.strip()))
                data.close()
        except urllib2.URLError, e:
            print e
        except IOError,e:
            print e
        except:
            pass

        #sort buffer ordered by phash
        sorted(buffer,key=lambda item:item[1])
        result = open('result','w',2048)
        for val in buffer:
            result.write(val[0] + ' ' + val[1] + '\n')
        result.close()

def checkImg():
        f = file('imageLists','r')
        lines = f.readlines()
        f.close()
#        for imgUrl in jsonData[0::2]:
#                checkImage(imgUrl)        
        jobs = [gevent.spawn(checkImage,imgUrl) for imgUrl in lines]
        gevent.joinall(jobs)

def checkSimilar():
    f = file('result')
    outfile = file('output','w')
    delfile = file('toBeDel','w')
    lines = f.readlines()
    f.close()

    # if toBeDel.old exists, remove deleted images from result
    import os 
    if os.path.isfile('toBeDel.old'):
        deledfile = file('toBeDel.old')
        deledLines = deledfile.readlines()
        deledfile.close()
        lines = [line for line in lines if line.split(' ')[1] not in deledLines]
    lasthash = 0
    similar = []
    similar.append(lines[0].strip())
    for line in lines:
        data = line.split(' ')
        hash = int(data[0],16)
        print hash
        if hash - lasthash < 500 :
           similar.append(line.strip())
        else:
            if len(similar) > 1 :
                outfile.write(",".join(similar) + '\n')
                for item in similar[1:]:
                    delfile.write(item.split(' ')[1] + "\n")
                similar = []
                similar.append(line.strip())
            elif len(similar) == 1:
                similar[0] = line.strip()
        lasthash = hash
    outfile.close()

if __name__ == '__main__':
    import os
    #if result has been caculated, donot process again
    if not os.path.isfile('result'): 
       checkImg()
    checkSimilar()

@@ -406,16 +406,17 @@
 (data))%0A
+#
         
@@ -1188,16 +1188,29 @@
 Similar(
+differ = 5000
 ):%0A    f
@@ -1253,20 +1253,59 @@
 ('output
-','w
+.html','w')%0A    outfile.write('%3Chtml%3E%3Cbody%3E
 ')%0A    d
@@ -1373,17 +1373,16 @@
 close()%0A
-%0A
     # if
@@ -1839,27 +1839,8 @@
 16)%0A
-        print hash%0A
     
@@ -1864,19 +1864,22 @@
 thash %3C 
-500
+differ
  :%0A     
@@ -1995,29 +1995,253 @@
 ite(
-%22,%22.join(
+'%3Cdiv%3E')%0A                for item in 
 similar
-) + '
+:%0A                    imginfo = item.split(' ')%0A                    outfile.write(imginfo%5B0%5D)%0A                    outfile.write('%3Cimg src=%22' + imginfo%5B1%5D + '%22 /%3E')%0A                outfile.write('%3C/div%3E
 %5Cn')
@@ -2515,24 +2515,60 @@
 hash = hash%0A
+    outfile.write('%3C/body%3E%3C/html%3E')%0A
     outfile.
@@ -2617,16 +2617,125 @@
 port os%0A
+    import sys%0A    args = sys.argv%5B1:%5D%0A    %0A    differ = 5000%0A    if len(args) %3E 0:%0A        differ = args%5B0%5D%0A
     #if 
@@ -2854,10 +2854,21 @@
 Similar(
+int(differ)
 )%0A

change the output format to HTML; get the args of differ