package org.deeplearning4j.eval;

import org.deeplearning4j.datasets.iterator.impl.IrisDataSetIterator;
import org.deeplearning4j.evaluation.EvaluationTools;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.layers.DenseLayer;
import org.deeplearning4j.nn.conf.layers.OutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.junit.Test;
import org.nd4j.linalg.activations.Activation;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.api.DataSet;
import org.nd4j.linalg.dataset.api.iterator.DataSetIterator;
import org.nd4j.linalg.dataset.api.preprocessor.NormalizerStandardize;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions;

import java.util.Arrays;

/**
 * Created by Alex on 07/01/2017.
 */
public class EvaluationToolsTests {

    @Test
    public void testRocHtml() throws Exception {

        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                        .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1,
                                        new OutputLayer.Builder().nIn(4).nOut(2).activation(Activation.SOFTMAX)
                                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                        .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        NormalizerStandardize ns = new NormalizerStandardize();
        DataSet ds = iter.next();
        ns.fit(ds);
        ns.transform(ds);

        INDArray newLabels = Nd4j.create(150, 2);
        newLabels.getColumn(0).assign(ds.getLabels().getColumn(0));
        newLabels.getColumn(0).addi(ds.getLabels().getColumn(1));
        newLabels.getColumn(1).assign(ds.getLabels().getColumn(2));
        ds.setLabels(newLabels);

        for (int i = 0; i < 30; i++) {
            net.fit(ds);
        }

        ROC roc = new ROC(20);
        iter.reset();

        INDArray f = ds.getFeatures();
        INDArray l = ds.getLabels();
        INDArray out = net.output(f);
        roc.eval(l, out);


        String str = EvaluationTools.rocChartToHtml(roc);
        //        System.out.println(str);
    }

    @Test
    public void testRocMultiToHtml() throws Exception {
        DataSetIterator iter = new IrisDataSetIterator(150, 150);

        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder().weightInit(WeightInit.XAVIER).list()
                        .layer(0, new DenseLayer.Builder().nIn(4).nOut(4).activation(Activation.TANH).build()).layer(1,
                                        new OutputLayer.Builder().nIn(4).nOut(3).activation(Activation.SOFTMAX)
                                                        .lossFunction(LossFunctions.LossFunction.MCXENT).build())
                        .build();
        MultiLayerNetwork net = new MultiLayerNetwork(conf);
        net.init();

        NormalizerStandardize ns = new NormalizerStandardize();
        DataSet ds = iter.next();
        ns.fit(ds);
        ns.transform(ds);

        for (int i = 0; i < 30; i++) {
            net.fit(ds);
        }

        ROCMultiClass roc = new ROCMultiClass(20);
        iter.reset();

        INDArray f = ds.getFeatures();
        INDArray l = ds.getLabels();
        INDArray out = net.output(f);
        roc.eval(l, out);


        String str = EvaluationTools.rocChartToHtml(roc, Arrays.asList("setosa", "versicolor", "virginica"));
        //        System.out.println(str);
    }

}

@@ -2169,32 +2169,83 @@
 ds);%0A        %7D%0A%0A
+        for(int numSteps : new int%5B%5D%7B20, 0%7D) %7B%0A    
         ROC roc 
@@ -2254,21 +2254,31 @@
 new ROC(
-20);%0A
+numSteps);%0A    
         
@@ -2292,32 +2292,36 @@
 set();%0A%0A        
+    
 INDArray f = ds.
@@ -2335,32 +2335,36 @@
 ures();%0A        
+   
+ 
 INDArray l = ds.
@@ -2368,32 +2368,36 @@
 ds.getLabels();%0A
+    
         INDArray
@@ -2418,32 +2418,36 @@
 put(f);%0A        
+ 
+   
 roc.eval(l, out)
@@ -2442,32 +2442,36 @@
 eval(l, out);%0A%0A%0A
+    
         String s
@@ -2512,34 +2512,30 @@
 l(roc);%0A
-  
+//
       
-//
       
-  
 System.o
@@ -2547,24 +2547,34 @@
 intln(str);%0A
+        %7D%0A
     %7D%0A%0A    @
@@ -3488,32 +3488,83 @@
 ds);%0A        %7D%0A%0A
+        for(int numSteps : new int%5B%5D%7B20, 0%7D) %7B%0A    
         ROCMulti
@@ -3597,21 +3597,30 @@
 ass(
-20);%0A
+numSteps);%0A   
         
 iter
@@ -3607,32 +3607,33 @@
 ps);%0A           
+ 
 iter.reset();%0A%0A 
@@ -3623,32 +3623,36 @@
  iter.reset();%0A%0A
+    
         INDArray
@@ -3674,32 +3674,36 @@
 ures();%0A        
+ 
+   
 INDArray l = ds.
@@ -3715,32 +3715,36 @@
 bels();%0A        
+   
+ 
 INDArray out = n
@@ -3757,32 +3757,36 @@
 put(f);%0A        
+    
 roc.eval(l, out)
@@ -3781,32 +3781,36 @@
 eval(l, out);%0A%0A%0A
+    
         String s
@@ -3911,14 +3911,8 @@
     
-    //
     
@@ -3936,21 +3936,31 @@
 intln(str);%0A
+        %7D%0A
     %7D%0A%0A%7D%0A

Test exact ROC/P-R curves for HTML export (EvaluationTools)