from rdflib.graph import Graph, ConjunctiveGraph, Dataset, URIRef
from rdflib.plugins.sparql import prepareQuery
import argparse
import warnings
import os


def read_by_chunk(f, chunk_size):
    i = 0
    buff = ''

    for line in f:
        if chunk_size != 0:
            if i >= chunk_size:
                yield buff
                i = 0
                buff = ''
        buff += line + '\n'
        i += 1
    yield buff


def add_arguments(parser):
    """
    Parse arguments
    Args:
        parser (argparse.ArgumentParser)
    """
    parser.description = 'Triple re-ontologization'
    parser.add_argument('-i', '--input-file', type=argparse.FileType('r'), dest='input_file')
    parser.add_argument('--input-type', action='store', dest='input_type', default='nt')
    parser.add_argument('-o', '--output-file', type=argparse.FileType('w'), dest='output_file')
    parser.add_argument('--output-type', action='store', dest='output_type', default='nt')
    parser.add_argument('-q', '--query-file', type=argparse.FileType('r'), dest='query_file')
    parser.add_argument('-s', '--chunk-size', action='store', dest='chunk_size', default=0)
    parser.add_argument('-g', '--graph', type=argparse.FileType('r'), dest='graphs', nargs='+', default=[])


def run(args):
    """
    Args:
        args (argparse.Namespace)
    """

    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        query = prepareQuery(args.query_file.read())

        ds = Dataset()
        for f in args.graphs:
            g = Graph(identifier=os.path.basename(f.name))
            g.parse(data=f.read(), format='n3')
            ds.add_graph(g)

        for data in read_by_chunk(args.input_file, int(args.chunk_size)):
            g = Graph(identifier='data')
            g.parse(data=data, format=args.input_type)
            ds.add_graph(g)
            res = ds.query(query)
            res = res.serialize(format=args.output_type)
            if res == b'\n':
                continue
            args.output_file.write(res.decode('utf-8'))
            ds.remove_graph(g)

@@ -106,16 +106,73 @@
 reQuery%0A
+from rdflib.plugins.sparql.processor import SPARQLResult%0A
 import a
@@ -205,16 +205,31 @@
 port os%0A
+import hashlib%0A
 %0A%0Adef re
@@ -262,24 +262,81 @@
 :%0A    i = 0%0A
+    # buff_prev + buff = slide window%0A    buff_prev = ''%0A
     buff = '
@@ -443,16 +443,28 @@
 eld buff
+_prev + buff
 %0A       
@@ -474,24 +474,57 @@
       i = 0%0A
+                buff_prev = buff%0A
             
@@ -594,16 +594,195 @@
 eld buff
+_prev + buff%0A%0A%0Adef generate_index(triple):%0A    s, p, o = triple%0A    content = '%7B%7D %7B%7D %7B%7D'.format(str(s), str(p), str(o))%0A    return hashlib.md5(content.encode('utf-8')).hexdigest()
 %0A%0A%0Adef a
@@ -1197,16 +1197,17 @@
 eType('w
+b
 '), dest
@@ -1815,17 +1815,16 @@
 read())%0A
-%0A
         
@@ -1837,16 +1837,125 @@
 ataset()
+%0A%0A        res_indices_prev = set()  # de-duplication%0A        res_indices = set()%0A%0A        # create sub graphs
 %0A       
@@ -2113,16 +2113,54 @@
 aph(g)%0A%0A
+        # create and query data graph%0A
         
@@ -2383,16 +2383,17 @@
 (query)%0A
+%0A
         
@@ -2400,51 +2400,177 @@
     
-res = res.serialize(format=args.output_type
+dedup_res_graph = Graph()%0A            if len(res) != 0:%0A                for r in res:%0A                    tid = generate_index(r)%0A                    res_indices.add(tid
 )%0A  
@@ -2583,25 +2583,66 @@
     
+ 
-if res == b'%5Cn':%0A
+       if tid in res_indices_prev:  # duplicated%0A        
     
@@ -2678,79 +2678,278 @@
     
+ 
-args.output_file.write(res.decode('utf-8'))%0A            ds.remove_graph(g
+       dedup_res_graph.add(r)%0A%0A                ret = dedup_res_graph.serialize(format=args.output_type, encoding='utf-8')%0A                args.output_file.write(ret)%0A%0A            ds.remove_graph(g)%0A            res_indices_prev = res_indices%0A            res_indices = set(
 )%0A

use slide window to process stream rdf