#
# This file is part of MLDB. Copyright 2016 Datacratic. All rights reserved.
#

import unittest

mldb = mldb_wrapper.wrap(mldb) # noqa

class RedditTest(MldbUnitTest):  

    def test_pipeline(self):

        mldb.put('/v1/procedures/import_reddit', { 
            "type": "import.text",  
            "params": { 
                "dataFileUrl": "http://public.mldb.ai/reddit.csv.gz",
                'delimiter':'', 
                'quotechar':'',
                'outputDataset': 'reddit_raw',
                'limit': 5000,
                'runOnCreation': True
            } 
        })
        mldb.query("select * from reddit_raw limit 5")

        mldb.put('/v1/procedures/reddit_import', {
            "type": "transform",
            "params": {
                "inputData": """select tokenize(lineText, {offset: 1, value: 1}) as * 
                from reddit_raw""",
                "outputDataset": "reddit_dataset",
                "runOnCreation": True
            }
        })
        mldb.query("select * from reddit_dataset limit 5")


        mldb.put('/v1/procedures/reddit_svd', {
            "type" : "svd.train",
            "params" : {
                "trainingData" : """
                    SELECT 
                        COLUMN EXPR (AS columnName() ORDER BY rowCount() DESC, 
                            columnName() LIMIT 4000) 
                    FROM reddit_dataset
                """,
                "columnOutputDataset" : "reddit_svd_embedding",
                "modelFileUrl": "file://tmp/MLDB-466.svd",
                "functionName": "embedder",
                "runOnCreation": True
            }
        })
        mldb.query("select * from reddit_svd_embedding limit 5")

        mldb.query("""
            select rowName(), rowPath(), rowPathElement(0) 
            from reddit_svd_embedding 
            limit 5
            """)
        
        mldb.query("""
            select embedder({ row: {HongKong: 1} })
            """)

        mldb.put("/v1/functions/nearest_subreddit", {
            "type": "embedding.neighbors",
            "params": {
                "dataset": "reddit_svd_embedding",
                "defaultNumNeighbors": 1,
                "columnName": "embedding"
            }
        })

        mldb.log("querying for nearest subreddit")
        
        mldb.query("""
            select nearest_subreddit({ coords: 
                embedder({ row: {HongKong: 1} })[embedding]
            })
        """)

        mldb.put('/v1/procedures/reddit_kmeans', {
            "type" : "kmeans.train",
            "params" : {
                "trainingData" : "select * from reddit_svd_embedding",
                "outputDataset" : "reddit_kmeans_clusters",
                "numClusters" : 20,
                "runOnCreation": True
            }
        })
        mldb.query("select * from reddit_kmeans_clusters limit 5")


        mldb.put('/v1/procedures/reddit_tsne', {
            "type" : "tsne.train",
            "params" : {
                "trainingData" : "select * from reddit_svd_embedding",
                "rowOutputDataset" : "reddit_tsne_embedding",
                "runOnCreation": True
            }
        })
        mldb.query("select * from reddit_tsne_embedding limit 5")


        mldb.put('/v1/procedures/reddit_count_users', {
            "type": "transform",
            "params": {
                "inputData": """select columnCount() as numUsers 
                    from transpose(reddit_dataset)""",
                "outputDataset": "reddit_user_counts",
                "runOnCreation": True
            }
        })
        mldb.query("select * from reddit_user_counts limit 5")

        mldb.query("""
            select *, quantize(x, 7) as grid_x, quantize(y, 7) as grid_y 
            named regex_replace(rowName(), '\|1', '') 
            from merge(reddit_tsne_embedding, reddit_kmeans_clusters) as r
                join reddit_user_counts as c on c.rowName() = r.rowPathElement(0)
            where cluster is not null 
            order by numUsers desc
        """)

mldb.run_tests()

@@ -2319,39 +2319,260 @@
    %0A        
-mldb.query(%22%22%22%0A
+result = mldb.get(%22/v1/query%22, format=%22aos%22, q=%22%22%22%0A            select %0A                embedder(%7B row: %7Bsoccer: 1%7D %7D)%5Bembedding%5D as soccer,%0A                neighbour.* as *, %0A                x.distance%0A            from transpose(( %0A    
             
@@ -2615,32 +2615,36 @@
                 
+    
 embedder(%7B row: 
@@ -2640,32 +2640,30 @@
 der(%7B row: %7B
-HongKong
+soccer
 : 1%7D %7D)%5Bembe
@@ -2685,22 +2685,670 @@
     
-%7D)%0A        %22%22%22
+    %7D)%5Bneighbors%5D as *%0A                named 'distance'%0A            )) as x%0A            join reddit_svd_embedding as neighbour%0A            on (x.rowName() = neighbour.rowName())%0A        %22%22%22).json()%5B0%5D%0A%0A        #result now contains a vector and its nearest neighbour%0A        #plus distance from the nearest-neighbour function%0A        #let's independently reconfirm that the distance is correct%0A%0A        dist_accum = 0%0A        for i in range(100):%0A            diff = result%5B%22embedding.%25d%22 %25 i%5D-result%5B%22soccer.%25d%22 %25 i%5D%0A            dist_accum += diff*diff%0A%0A        import math%0A        # MLDB-1677%0A        #self.assertEqual(math.sqrt(dist_accum), result%5B%22x.distance%22%5D
 )%0A%0A 

test case for MLDB-1677