import unittest


class TestMNISTBlackboxF(unittest.TestCase):
    def test_mnist_blackbox(self):
        from tutorials import mnist_blackbox

        # Run the MNIST tutorial on a dataset of reduced size, reduced number
        # of data augmentations, increased substitute holdout for faster runtime.
        mnist_blackbox_args = {'train_start': 0,
                               'train_end': 5000,
                               'test_start': 0,
                               'test_end': 733,
                               'data_aug': 2,
                               'holdout': 400}
        report = mnist_blackbox.mnist_blackbox(**mnist_blackbox_args)

        # Check accuracy values contained in the AccuracyReport object
        self.assertTrue(report['bbox'] > 0.8, report['bbox'])
        self.assertTrue(report['sub'] > 0.47, report['sub'])
        self.assertTrue(report['bbox_on_sub_adv_ex'] < 0.7, report['bbox_on_sub_adv_ex'])

if __name__ == '__main__':
    unittest.main()

@@ -491,11 +491,12 @@
 d': 
-733
+2000
 ,%0A  
@@ -536,17 +536,17 @@
 a_aug': 
-2
+1
 ,%0A      
@@ -585,11 +585,108 @@
 t': 
-400
+1000,%0A                               'nb_epochs': 2,%0A                               'nb_epochs_s': 6
 %7D%0A  
@@ -872,9 +872,9 @@
 %3E 0.
-8
+7
 , re
@@ -933,10 +933,9 @@
 %3E 0.
-47
+3
 , re

make bbox tutorial test faster