<jupyter_start><jupyter_text>Simulate build_prediction and  _build_gaussian_tf by simulating tensors<jupyter_code>from IPython.core.display import display, HTML
display(HTML("<style>.container { width:90% !important; }</style>"))

%matplotlib inline
%load_ext autoreload
%autoreload 2


import sys
sys.path.append('../')
from mrcnn.utils import trim_zeros_graph
import numpy as np
import tensorflow as tf

np.set_printoptions(linewidth=100, precision=4)
sess = tf.InteractiveSession()

batch_size = 3
num_rois = 32
num_classes = 4
num_detections = 100
num_rois_per_image = 32
rois_per_image = 32<jupyter_output><empty_output><jupyter_text>Generate 3D Pred Array Tensors  Create class score tensor 1- Class_Score tensor:   batch_size x  num_dectections x num_classes                2- Output_ROIs :         batch_size x  num_detections  x 4     3- mrcnn_bboxes:         batch_size x  num_detections  x num_clases x 4   Generate pred_array: tensor [ seq_id,  max_score,  bbox coordinates, class_id]  1 -  Get max prediction score and  corresponding class for each ROI  2 -  Concat sequence id, max_score, bbox coordinates and class id     Generate meshgrid of image_id (batch grid) and bbox_ids  (roi_grid)  Generate scatter indices to scatter bbox information by image_id and class_id   The the highest score for each detection (highest score amongst classes) and the class corresponding to the highest score     Split Box/ Class info for each Image dim(0) by class and place into new tensor using tf.scatter_nd    Output is converted to NUMN_BATCHES x NUM_CLASSES x NUM_ROIS x columns(7)  Sort pred_scatter by score     Sort each slice based on column  1 (score) using top_k    Generate calss_grid, batch_grid and roi-grid based using mesh_grid     Stack results to generate gather indices     Generate final result - pred_tensor - using tf.gather_nd()<jupyter_code>mrcnn_class_init_0 = tf.Variable(tf.random_uniform([num_rois-10, num_classes], minval=0, maxval=1  , dtype=tf.float32, seed=1234), name = 'var1')
mrcnn_class_init_1 = tf.Variable(tf.random_uniform([num_rois-4 , num_classes], minval=0, maxval=1  , dtype=tf.float32, seed=5678), name = 'var2')
mrcnn_class_init_2 = tf.Variable(tf.random_uniform([num_rois-2 , num_classes], minval=0, maxval=1  , dtype=tf.float32, seed=9012), name = 'var3')

output_rois_init_0a = tf.Variable(tf.random_uniform([5, 4] , minval=0, maxval=128, dtype=tf.float32, seed=1969), name = 'var4')
output_rois_init_0b = tf.Variable(tf.random_uniform([15, 4], minval=0, maxval=128, dtype=tf.float32, seed=1192), name = 'var5')
output_rois_init_1  = tf.Variable(tf.random_uniform([num_rois-4 , 4], minval=0, maxval=128, dtype=tf.float32, seed=1999), name = 'var6')
output_rois_init_2  = tf.Variable(tf.random_uniform([num_rois-2 , 4], minval=0, maxval=128, dtype=tf.float32, seed=2012), name = 'var7')

mrcnn_bboxes       = tf.Variable(tf.random_uniform([batch_size, num_rois, num_classes, 4], minval=0, maxval=128, dtype=tf.float32, seed=8976),
                                 name = 'var8')

init = tf.global_variables_initializer().run()
np.set_printoptions(linewidth=100, precision=4)

mrcnn_class_filler_0 = tf.zeros([10, num_classes], dtype = tf.float32)
mrcnn_class_filler_1 = tf.zeros([4, num_classes], dtype = tf.float32)
mrcnn_class_filler_2 = tf.zeros([2, num_classes], dtype = tf.float32)
mrcnn_class_0 = tf.concat([mrcnn_class_init_0, mrcnn_class_filler_0],axis = 0)
mrcnn_class_1 = tf.concat([mrcnn_class_init_1, mrcnn_class_filler_1],axis = 0)
mrcnn_class_2 = tf.concat([mrcnn_class_init_2, mrcnn_class_filler_2],axis = 0)
print(' mrcnn_class 0 shape: ', mrcnn_class_0.shape)
print(' mrcnn_class 1 shape: ', mrcnn_class_1.shape)
print(' mrcnn_class 2 shape: ', mrcnn_class_2.shape)

mrcnn_class   = tf.stack([mrcnn_class_0, mrcnn_class_1, mrcnn_class_2], axis = 0 )
print(' mrcnn_class shape: ', mrcnn_class.shape)
# print(mrcnn_class.eval())

output_rois_filler_0a = tf.zeros([2, 4], dtype = tf.float32)
output_rois_filler_0b = tf.zeros([10, 4], dtype = tf.float32)
output_rois_filler_1  = tf.zeros([4 , 4], dtype = tf.float32)
output_rois_filler_2  = tf.zeros([2 , 4], dtype = tf.float32)
# output_rois_filler  = tf.zeros([batch_size, 2, 4], dtype = tf.float32)
output_rois_0         = tf.concat([output_rois_init_0a, output_rois_filler_0a, output_rois_init_0b, output_rois_filler_0b],axis = 0)
output_rois_1         = tf.concat([output_rois_init_1, output_rois_filler_1],axis = 0)
output_rois_2         = tf.concat([output_rois_init_2, output_rois_filler_2],axis = 0)
print(' output_rois 0 shape: ', output_rois_0.shape)
print(' output_rois 1 shape: ', output_rois_1.shape)
print(' output_rois 2 shape: ', output_rois_2.shape)

output_rois   = tf.stack([output_rois_0, output_rois_1, output_rois_2], axis = 0 )

print(' output_rois shape: ', output_rois.shape)
# print(output_rois[0].eval())


one   = tf.ones([num_rois, 4])
test_bboxes = tf.stack([one, one+1, one+2, one+3], axis = 0) 
print(output_rois.eval())
pred_classes     = tf.argmax(mrcnn_class,axis=-1, output_type = tf.int32)
pred_classes_exp = tf.to_float(tf.expand_dims(pred_classes,axis=-1))
print('pred_classes (classes with highest scores):', pred_classes.get_shape())
print(pred_classes.eval())
print('pred_classes expanded shape:', pred_classes_exp.get_shape())

pred_scores  = tf.reduce_max(mrcnn_class,axis=-1, keepdims=True)
print('pred_scores shape', pred_scores.shape)
# print(pred_scores.eval())

batch_grid, roi_grid = tf.meshgrid( tf.range(batch_size, dtype=tf.int32), tf.range(num_rois, dtype=tf.int32), indexing = 'ij' )
bbox_idx   = tf.to_float(tf.expand_dims(roi_grid, axis = -1))

print('roi_grid shape    :',tf.shape(roi_grid).eval())
print(roi_grid.eval())
print('batch_grid  shape :',tf.shape(batch_grid).eval())
print(batch_grid.eval())

# changed o make pred_array simnialr to detections array 
# pred_array = tf.concat([ bbox_idx, pred_scores , output_rois, pred_classes_exp], axis=-1)
pred_array = tf.concat([ bbox_idx, output_rois, pred_classes_exp, pred_scores ], axis=-1)
scatter_classes = tf.stack([batch_grid , pred_classes, roi_grid],axis = -1)
print(pred_array.get_shape())

# aa = tf.reduce_sum(pred_array[:,:, 2:6],2)
# print('aa is : ', type(aa), aa.shape)
# print( aa.eval())
# non_zero = tf.not_equal(aa, 0)
# print(' non_zero  ', non_zero.get_shape())
# print(non_zero.eval())
# non_zero_ind = tf.where(non_zero)
# print(' non_zero_ind  ', non_zero_ind.get_shape())
# print(non_zero_ind.eval())

# # non_zero_exp = tf.concat([tf.expand_dims(non_zero[:,0], axis = -1), non_zero], axis = -1 )
# # print(' non_zero  ', non_zero_exp.get_shape())
# # print(non_zero_exp.eval())

# pred_nz_array   = tf.scatter_nd(non_zero_ind, pred_array,[3, 32,7])
# print(' non_zero_array  ', pred_nz_array.shape)
# print(pred_nz_array.eval())
np.set_printoptions(linewidth=120, precision=6)
print('pred_array shape', pred_array.shape)
print('Image 0 ------')
print(pred_array[0].eval())
print('Image 1 ------')
print(pred_array[1].eval())
print('Image 2 ------')
print(pred_array[2].eval())

print('-- scatter to classes ----')
print('pred_array shape     ', pred_array.get_shape())
print('scatter_classes shape', scatter_classes.get_shape())
# scatter_classes gives indcies to scatter the pred_array by image / class 
# print(scatter_classes[0].eval())<jupyter_output><empty_output><jupyter_text>pred_scatt is the array with ROIs scattered out (separated)  by Image &  Class<jupyter_code>pred_scatt = tf.scatter_nd(scatter_classes, pred_array, [batch_size, num_classes, num_rois,7])
print('pred_scatter shape is ', pred_scatt.get_shape())
print('------------')
print(pred_scatt[0].eval())
print('------------')
print(pred_scatt[1].eval())
print('------------')
print(pred_scatt[2].eval())<jupyter_output><empty_output><jupyter_text>Sort `pred_scattered` by score (last column) and create `pred_tensor`<jupyter_code>'''
# sort_vals, sort_inds = tf.nn.top_k(pred_scatt[:,:,:,1], k=pred_scatt.shape[2])
'''
np.set_printoptions(linewidth=130, precision=6)
sort_vals, sort_inds = tf.nn.top_k(pred_scatt[:,:,:,-1], k=pred_scatt.shape[2])

# print(' Sort vals: ',tf.shape(sort_vals).eval())
# print(sort_vals.eval())
print()
print(' Sort inds: ',tf.shape(sort_inds).eval())
print(sort_inds.eval())

class_grid, batch_grid, roi_grid = tf.meshgrid(tf.range(num_classes),tf.range(batch_size), tf.range(num_rois))
roi_grid_exp = tf.to_float(tf.expand_dims(roi_grid, axis = -1))
gather_inds = tf.stack([batch_grid , class_grid, sort_inds],axis = -1)
pred_tensor = tf.gather_nd(pred_scatt, gather_inds)

# print(class_grid.eval())
# print(batch_grid.eval())
print('    class_grid  ', type(class_grid) , 'shape', class_grid.get_shape())
print('    batch_grid  ', type(batch_grid) , 'shape', batch_grid.get_shape())
print('    roi_grid shape', roi_grid.get_shape(), 'roi_grid_exp shape ', roi_grid_exp.get_shape())
print('    gather_inds ', type(gather_inds), 'shape', gather_inds.get_shape())
print('    pred_tensor (gathered)  : ', pred_tensor.get_shape())

print('-- stack results ----')
print('pred_scatt ', type(pred_scatt), 'shape',tf.shape(pred_scatt).eval())
print('gather_inds', type(gather_inds), 'shape',tf.shape(gather_inds).eval())
# print(gather_inds.eval())


# this line adds a sequence id to the end of each row
# pred_tensor  = tf.concat([pred_tensor, roi_grid_exp], axis = -1)


# print(pred_tensor.eval())
pred_cls_cnt = tf.count_nonzero(pred_tensor[:,:,:,0], axis = -1, name = 'pred_cls_count')
pred_cls_cnt1 = tf.count_nonzero(pred_tensor[:,:,:,-1], axis = -1, name = 'pred_cls_count')

print()
print('    -- pred_tensor results (bboxes sorted by score) ----')    
print('    final pred_tensor shape  : ', pred_tensor.get_shape())
print('    final pred_cls_cnt shape : ',pred_cls_cnt.get_shape())
print(pred_cls_cnt.eval())
print(pred_cls_cnt1.eval())
print('    complete')<jupyter_output><empty_output><jupyter_text>Display for visual check - `pred_tensor` is the final result which is passed on to  `build_gaussian_tf()`<jupyter_code>np.set_printoptions(linewidth=100, precision=4)
print('scatter shape is ', pred_scatt.get_shape())
print('pred_tensor shape is ', pred_tensor.get_shape() )

img = 1
print('Image ', img , '/ Class 0 ------------')
print(pred_tensor[img,0].eval())
print('Image ', img , '/ Class 1 ------------')
print(pred_tensor[img,1].eval())
print('Image ', img , '/ Class 2 ------------')
print(pred_tensor[img,2].eval())
print('Image ', img , '/ Class 3 ------------')
print(pred_tensor[img,3].eval())
 
cls = 3
print('Image 0 / Class ', cls ,' ------------')
print(pred_tensor[0, cls].eval())
print('Image 1 / Class ', cls ,' ------------')
print(pred_tensor[1, cls].eval())
print('Image 2 / Class ', cls ,' ------------')
print(pred_tensor[2, cls].eval())<jupyter_output><empty_output><jupyter_text>`build_gaussian_tf ()` -- Generate Multivariate Normal Distribution from Pred_Tensor<jupyter_code># def build_gaussian_tf(in_tensor) :   #, pred_cls_cnt, config):
## rois per image is determined by size of input tensor 
##   detection mode:   config.TRAIN_ROIS_PER_IMAGE 
##   ground_truth  :   config.DETECTION_MAX_INSTANCES    
'''
in_tensor :     [N, num_rois, 8 { index, ]
'''
in_tensor = pred_tensor

# num_detections  = config.DETECTION_MAX_INSTANCES
# batch_size      = config.BATCH_SIZE
# num_classes     = config.NUM_CLASSES  

num_detections  = 32          # config.DETECTION_MAX_INSTANCES
h, w            = [128, 128]  #   config.IMAGE_SHAPE[:2]
num_cols        = 7 
img_h, img_w    = [128, 128]  # config.IMAGE_SHAPE[:2]
batch_size      = 3           # config.BATCH_SIZE
num_classes     = 4           # config.NUM_CLASSES  

print('\n ')
print('  > BUILD_GAUSSIAN_TF() for ' )

# rois per image is determined by size of input tensor 
#   detection mode:   config.TRAIN_ROIS_PER_IMAGE 
#   ground_truth  :   config.DETECTION_MAX_INSTANCES

print('    orignal in_tensor shape : ', in_tensor.shape)   
# in_tensor = in_tensor[:,:,:,2:7]
in_tensor = in_tensor[:,:,:,1:]
print('    modified in_tensor shape : ', in_tensor.get_shape())

rois_per_image  = tf.to_int32(in_tensor.shape[2])
strt_cls        = 0 if rois_per_image == 32 else 1
print('    num of bboxes per class is : ', rois_per_image)
# print(in_tensor[0].eval())
print(in_tensor[0,0].eval())
print()
print(in_tensor[0,1].eval())<jupyter_output><empty_output><jupyter_text>Build Position meshgird<jupyter_code>##-----------------------------------------------------------------------------
## Build mesh-grid to hold pixel coordinates 
##-----------------------------------------------------------------------------
X = tf.range(img_w, dtype=tf.int32)
Y = tf.range(img_h, dtype=tf.int32)
X, Y = tf.meshgrid(X, Y)
# grid1 = tf.stack([X,Y], axis=-1)
# print('grid1 shape ', grid1.shape)
# print(grid1[0,:,:].eval())

print('   -- build meshgrid -----')
print('   X and Y meshgrid shapes ', X.get_shape(), Y.get_shape())
# print( ' X : \n',X.eval())
# print( ' Y : \n',Y.eval())

# ## hear we repeat X and Y  batch_size x rois_per_image times
ones = tf.ones([batch_size, rois_per_image,1, 1], dtype = tf.int32)
print('   ones: ',ones.shape)                
# # ones = tf.expand_dims(ones,-1)
# print(' ones with exp dims ',ones.shape)

rep_X = ones * X
rep_Y = ones * Y 
# print(' ones_exp * X', ones.shape, '*', X.shape, '= ',rep_X.shape)
# print(' ones_exp * Y', ones.shape, '*', Y.shape, '= ',rep_Y.shape)

# # stack the X and Y grids 
bef_pos = tf.to_float(tf.stack([rep_X,rep_Y], axis = -1))
print(' before transpse ', bef_pos.get_shape())
pos_grid = tf.transpose(bef_pos,[2,3,0,1,4])
print('   after transpose ', pos_grid.get_shape())    

# print(pos_grid_1[:,:,0,0,:].eval())<jupyter_output><empty_output><jupyter_text>`pred_tensor[:,:,:,1:7]`  == `[116.9736  21.8213  36.2715  45.6026   0.    0.9139   ]`Detections returned by `detect()` routine:`[[ 25.          18.          80.          72.           2.           0.99936014]  [ 51.           3.         106.          71.           3.           0.99924326]`  Build stacked_listList of stacked tensors (one per image) - each stacked tensor shape is `[? , 6]`<jupyter_code>##-----------------------------------------------------------------------------
## Stack non_zero bboxes from in_tensor into pt2_dense 
##-----------------------------------------------------------------------------
pt2_sum = tf.reduce_sum(tf.abs(in_tensor[:,:,:,:-1]) ,axis =-1)
print('   pt2_sum shape ',pt2_sum.shape)
print(pt2_sum[0].eval())

pt2_mask = tf.greater(pt2_sum , 0)
print('   pt2_mask shape ', pt2_mask.get_shape())
# print(pt2_mask.eval())

pt2_ind  = tf.where(pt2_mask)
print('   pt2_ind shape  ', pt2_ind.get_shape())
print(pt2_ind.eval())
# pt2_ind_float  =  tf.to_float(pt2_ind[:,0:1])

dense1 = tf.gather_nd( in_tensor, pt2_ind)
dense1 = tf.concat([tf.to_float(pt2_ind[:,0:1]), dense1],axis=1)
print('   dense1 shape ',dense1.get_shape())
print(dense1.eval())

stacked_list = tf.dynamic_partition(dense1, tf.to_int32(pt2_ind[:,0]),num_partitions = batch_size )
     
print(stacked_list[0].shape)
print(stacked_list[0].eval())<jupyter_output><empty_output><jupyter_text>Build stacked tensor<jupyter_code>import keras.backend as KB
print('   -- Build Stacked output from dynamically partitioned lists --------------')  

stacked_output=[]
for img, item  in enumerate(stacked_list) :
    # rois_in_image, cols  = tf.shape(stacked_list[img]).eval()
    rois_in_image  = tf.shape(item).eval()[0]

    print('\n   ===> list item #', img)       
    print('   img ', img, ' stacked_list[img] ', tf.shape(item).eval(), ' rois_in_image : ', rois_in_image)
    #     print(stacked_list[img].eval())            
    pad_item =  tf.pad(item,[[0, rois_per_image - rois_in_image ],[0,0]])
    stacked_output.append(pad_item)
    print('   tensor_list item pos padding :', tf.shape(pad_item).eval())
    #     print(stacked_list[img].eval())

print()    
print('   -- Stacked output contents --------------')    
print('    stacked_output shape : ', len(stacked_output))
for img, item  in enumerate(stacked_output) :
    print('   img ', img, ' stacked_list[img] ', tf.shape(item).eval() ) 
print('   stacked_tensor shape : ', tf.shape(stacked_tensor).eval())

print('   stacked tensor : ', tf.shape(stacked_tensor).eval(), stacked_tensor.shape, stacked_tensor.get_shape())
print(stacked_tensor[0].eval())
print()
print(stacked_tensor[1].eval())
print()
print(stacked_tensor[2].eval())
##-----------------------------------------------------------------------------
##  Build mean and convariance tensors for Multivariate Normal Distribution
##-----------------------------------------------------------------------------
width  = stacked_tensor[:,:,4] - stacked_tensor[:,:,2]
height = stacked_tensor[:,:,3] - stacked_tensor[:,:,1]
cx     = stacked_tensor[:,:,2] + ( width  / 2.0)
cy     = stacked_tensor[:,:,1] + ( height / 2.0)
means  = tf.stack((cx,cy),axis = -1)
covar_orig  = tf.stack((width * 0.5 , height * 0.5), axis = -1)
covar  = tf.sqrt(covar_orig)
# print(means.eval())
# print(covar.eval())

# print('width shape ',width.get_shape()) 
# print(mns.eval())
tfd = tf.contrib.distributions
mvn = tfd.MultivariateNormalDiag(
    loc  = means,
    scale_diag = covar)

print('   means shape ',means.get_shape(), '  ', means.get_shape())
print('   covar shape ',covar.get_shape(), '  ', covar.get_shape())
print('   from MVN :  \t mean shape     :', mvn.mean().shape, '\t stddev shape', mvn.stddev().shape )
print('   from MVN :  \t mean shape     :', mvn.mean().get_shape(), '\t stddev shape', mvn.stddev().get_shape())
print('   from MVN :  \t mvn.batch_shape:', mvn.batch_shape , '\t mvn.event_shape ',  mvn.event_shape)
print('   Linear OP shape      ', mvn.scale.shape, ' Linear Op batch shape ',mvn.scale.batch_shape)
print('   Linear op Range Dim  ', mvn.scale.range_dimension)
print('   Linear op Domain Dim ', mvn.scale.domain_dimension) 
 
prob_grid = mvn.prob(pos_grid)
# print(prob.eval())
#     eq = tf.equal(grid, pos)
#     print( ' pos and grid probabalitiy matricies equal  -->', tf.reduce_all(eq).eval())
trans_grid = tf.transpose(prob_grid,[2,3,0,1])

print('    >> input to MVN.PROB: pos_grid (meshgrid) shape: ', pos_grid.get_shape())
print('    << output probabilities shape:' , prob_grid.get_shape())
print('       transposed_grid shape: ', trans_grid.shape)

#--------------------------------------------------------------------------------
# kill distributions of NaN boxes (resulting from bboxes with height/width of zero
# which cause singular sigma cov matrices
#--------------------------------------------------------------------------------
# gauss_grid = tf.where(tf.is_nan(trans_grid),  tf.zeros_like(trans_grid), trans_grid)
#--------------------------------------------------------------------------------
## scatter out the probability distributions based on class
#--------------------------------------------------------------------------------
print('    Scatter out the probability distributions based on class --------------')     

'''
class_inds      = tf.to_int32(stacked_tensor[:,:,-1])
'''
class_inds      = tf.to_int32(stacked_tensor[:,:,-2])
batch_grid, roi_grid = tf.meshgrid( tf.range(batch_size, dtype=tf.int32), tf.range(rois_per_image, dtype=tf.int32),
                                    indexing = 'ij' )
scatter_classes = tf.stack([batch_grid, class_inds, roi_grid ],axis = -1)
gauss_scatt     = tf.scatter_nd(scatter_classes, gauss_grid, [batch_size, num_classes, rois_per_image, img_w, img_h])

print('    gaussian_grid      : ', gauss_grid.shape)    
print('    class shape        : ', class_inds.shape)
print(class_inds.eval())
print('    roi_grid shape     : ', roi_grid.get_shape() )
print('    batch_grid shape   : ', batch_grid.get_shape())
print('    scatter_classes    : ', scatter_classes.get_shape())
print('    gaussian scattered : ', gauss_scatt.shape)   

print('   means shape ',means.get_shape(), '\n', means.eval())
print('   covar orig shape ',covar_orig.get_shape(), '\n', covar_orig.eval())


#--------------------------------------------------------------------------------
## sum based on class
#--------------------------------------------------------------------------------
print('    Reduce sum based on class ---------------------------------------------')         
gauss_sum = tf.reduce_sum(gauss_scatt, axis=2, name='pred_gaussian')
print('    gaussian_sum shape : ', gauss_sum.get_shape())    

gauss_sum = tf.where(gauss_sum > 1e-6, gauss_sum,tf.zeros_like(gauss_sum))
gauss_sum = tf.transpose(gauss_sum,[0,2,3,1])
print('    gaussian_sum shape : ', gauss_sum.get_shape())    
print( gauss_sum[0,0].eval())

print('    complete')
gauss_scatt[0,0,0].eval()<jupyter_output><empty_output><jupyter_text>Softmax Sparse Cross Entropy Ignoring Last Label -- Used in Keras FCN<jupyter_code>import keras.backend as K 

y_pred = tf.placeholder(dtype=tf.float32, shape=(16,320,320,20))
y_true = tf.placeholder(dtype=tf.float32, shape=(16,320,320,1))
print(K.int_shape(y_pred), K.int_shape(y_true))
y_pred = K.reshape(y_pred, (-1, K.int_shape(y_pred)[-1]))
print(K.int_shape(y_pred))
log_softmax = tf.nn.log_softmax(y_pred)
print(K.int_shape(log_softmax))

y_true = K.flatten(y_true)
print(K.int_shape(y_true))

y_true = K.one_hot(tf.to_int32(y_true), K.int_shape(y_pred)[-1]+1)
print(K.int_shape(y_true))

unpacked = tf.unstack(y_true, axis=-1)
print(len(unpacked), unpacked[0].shape)

y_true = tf.stack(unpacked[:-1], axis=-1)
print(K.int_shape(y_true))


cross_entropy = -K.sum(y_true * log_softmax, axis=1)
print(K.int_shape(cross_entropy))

cross_entropy_mean = K.mean(cross_entropy)
print(K.int_shape(cross_entropy_mean))




# import keras.backend as K
# print(K.int_shape(bef_pos)[-1])
# unpacked  = K.flatten(test)
# unpacked.shape
# pos_grid_1[:,:,0,0,:].eval()<jupyter_output><empty_output><jupyter_text>Experimental code to Create mask for class bounding boxes<jupyter_code>print(gauss_scatt.shape)
print(gauss_grid.shape)
mask_arr = tf.zeros_like(gauss_grid)
mask = mask_arr[0]
mask.shape
boxes = tf.round(stacked_tensor[0,:,1:5]).eval()
boxes[0]
##-----------------------------------------------------------------------------
## Build mesh-grid to hold pixel coordinates 
##-----------------------------------------------------------------------------
X = tf.range(img_w, dtype=tf.int32)
Y = tf.range(img_h, dtype=tf.int32)
X, Y = tf.meshgrid(X, Y)
# grid1 = tf.stack([X,Y], axis=-1)
# print('grid1 shape ', grid1.shape)
# print(grid1[0,:,:].eval())<jupyter_output><empty_output><jupyter_text>Comparing Scipy / Tensorflow Multivar normal distribution<jupyter_code>tfd        = tf.contrib.distributions
grid       = pos_grid_1[:,:,0,0,:]
covar      = np.array([27.7818, 26.6678],dtype = np.float32)
covar_sqrt = np.sqrt(covar)
covar_sqrd = covar ** 2
full_covar = np.array([[27.7818, 0],[0, 26.6678]],dtype = np.float32)
mean       = np.array([48.8926, 36.101 ],dtype = np.float32)

print('   grid :', grid.dtype, grid.shape)
print('   Covar sqrt :', covar_sqrt)
print('   Covar sqrd :', covar_sqrd)

mvn1  = tfd.MultivariateNormalDiag(loc=mean,scale_diag=covar_sqrt)
prob1 = mvn1.prob(grid2)
print()
print('   mvn1 mean             ', mvn1.mean().eval())
print('   mvn1 std deviation    ', mvn1.stddev().eval())
print('   mvn1 covariance:      ', '\n', mvn1.covariance().eval())
print('   mvn1 location         ', mvn1.loc.eval())
print('   Linear OP shape       ', mvn1.scale.shape)
print('   Linear Op batch shape ', mvn1.scale.batch_shape)
print('   Linear op Range Dim   ', mvn1.scale.range_dimension)
print('   Linear op Domain Dim  ', mvn1.scale.domain_dimension) 
print('   Linear op Domain Dim  ', mvn1.scale.diag_part().eval()) 

mvn2  = tfd.MultivariateNormalDiag(loc=mean,scale_diag=covar)
prob2 = mvn2.prob(grid2)
print()
print('   mvn2 mean             ', mvn2.mean().eval())
print('   mvn2 std deviation    ', mvn2.stddev().eval())
print('   mvn2 covariance:      ', '\n', mvn2.covariance().eval())
print('   mvn2 location         ', mvn2.loc.eval())
print('   Linear OP shape       ', mvn2.scale.shape)
print('   Linear Op batch shape ', mvn2.scale.batch_shape)
print('   Linear op Range Dim   ', mvn2.scale.range_dimension)
print('   Linear op Domain Dim  ', mvn2.scale.domain_dimension) 
print('   Linear op Domain Dim  ', mvn2.scale.diag_part().eval()) 


mvn3  = tfd.MultivariateNormalFullCovariance( loc = mean, covariance_matrix = full_covar)
prob3 = mvn3.prob(grid2)
print()
print('   mvn3 mean             ', mvn3.mean().eval())
print('   mvn3 std deviation    ', mvn3.stddev().eval())
print('   mvn3 covariance:      ', '\n', mvn3.covariance().eval())
print('   mvn3 location         ', mvn3.loc.eval())
print('   Linear OP shape       ', mvn3.scale.shape)
print('   Linear Op batch shape ', mvn3.scale.batch_shape)
print('   Linear op Range Dim   ', mvn3.scale.range_dimension)
print('   Linear op Domain Dim  ', mvn3.scale.domain_dimension) 
print('   Linear op Domain Dim  ', mvn3.scale.diag_part().eval()) 

print('   << output probabilities shape:' )
print(' prob1 ', prob1.get_shape())
print(prob1.eval())
print(' prob2 ', prob2.get_shape())
print(prob2.eval())
print(' prob3 ', prob3.get_shape())
print(prob3.eval())
np.set_printoptions(linewidth=150, threshold=10000)
from scipy.stats import  multivariate_normal
# Build mesh-grid to hold pixel coordinates ----------------------------------
XX = np.arange(0, img_w, 1)
YY = np.arange(0, img_h, 1)
XX, YY = np.meshgrid(XX, YY)
pos  = np.empty(XX.shape + (2,))   # concatinate shape of x to make ( x.rows, x.cols, 2)
pos[:,:,0] = XX;
pos[:,:,1] = YY;
# print(XX)
# print(YY)
# print(pos[0,:,:])
# print(pos[0])
# print(grid[0].eval())
print(' pos type    ', type(pos), type(grid))
print(' grid shape ', pos.shape, grid.shape)
print(np.all(pos == grid.eval()))
print(' mean  ', mean)
print(' covar ', covar)
mvna    = multivariate_normal(mean, covar)
prob_a = mvna.pdf(pos)

mvnb = multivariate_normal(mean, covar_sqrd)
prob_b = mvnb.pdf(pos)

print(prob_a[35:50, 45:54])
max_a = np.max(prob_a)
print(np.unravel_index(np.argmax(prob_a) , prob_a.shape) )

print()

print(' covar ', covar_sqrd)
print(prob_b[35:50, 45:54])
max_b = np.max(prob_b)
print(np.unravel_index(np.argmax(prob_b) , prob_b.shape) )

print('max a , max_b ', max_a, max_b, max_a/max_b)<jupyter_output><empty_output><jupyter_text>Build indicies to gather bounding boxes from bboxes_4d corrsponding to predicted class Only used if we want to use mrcnn_bboxes (batch_size, num_rois, num_classes, 4)batch_size x nuum_detections x 4<jupyter_code>###  Build indicies to gather bounding boxes from bboxes_4d corrsponding to predicted class
###  Only used if we want to use mrcnn_bboxes (batch_size, num_rois, num_classes, 4)

# gather_boxes = tf.stack([batch_grid, roi_grid, pred_classes, ], axis = -1)

# print('-- gather_boxes  ----')
# print('gather_boxes inds', type(gather_boxes), 'shape',tf.shape(gather_boxes).eval())
# print(gather_boxes.eval())

# mrcnn_bboxes_selected = tf.gather_nd(mrcnn_bboxes, gather_boxes)
# print(' padding required for output_rois : ', mrcnn_bboxes_selected.get_shape())
# print(mrcnn_bboxes_selected[0].eval())

# print(' output_rois shape ', output_rois.get_shape())
# print(pred_classes[0].eval())
# print(output_rois[0].eval())<jupyter_output><empty_output><jupyter_text>Experimental Code   Experiment for sort ordering<jupyter_code>indices = tf.constant([
            [[0,2],[0,1],[0,0]],
            [[1,2],[1,1],[1,0]],
            [[2,0],[2,1],[2,1]],
          ])
params = tf.constant( [
          [['0-00', '0-01', '0-02', '0-03'], 
           ['0-10', '0-11', '0-12', '0-13'],
           ['0-20', '0-21', '0-22', '0-23'],
           ['0-30', '0-31', '0-32', '0-33']],
    
          [['1-00', '1-01', '1-02', '1-03'], 
           ['1-10', '1-11', '1-12', '1-13'],
           ['1-20', '1-21', '1-22', '1-23'],
           ['1-30', '1-31', '1-32', '1-33']],
    
          [['2-00', '2-01', '2-02', '2-03'], 
           ['2-10', '2-11', '2-12', '2-13'],
           ['2-20', '2-21', '2-22', '2-23'],
           ['2-30', '2-31', '2-32', '2-33']]    
         ])
print(params.shape, '   ', indices.shape)
res = tf.gather_nd(params, indices)
res.eval()<jupyter_output><empty_output><jupyter_text>Experiment with GATHER() for splitting into classes<jupyter_code># indices = tf.constant([
#             [0,2,2],[0,1,1],[0,0,0],
#             [1,2,2],[1,1,1],[1,0,0],
#             [2,0,0],[2,1,1],[2,1,1],
#            ])
# indices = tf.constant([ [0,2],[0,1],[0,0],[1,2],[1,1],[1,0],[1,2],[1,3], [2,3]] ) # 9 x 2
indices = tf.constant([
           [
            [[0,2],[0,1],[0,0],[1,2],[2,3]],
            [[1,1],[1,0],[1,2],[1,3],[1,1]]
           ]
           ])
params = tf.constant( [
          [['0-00', '0-01', '0-02', '0-03', '0-04', '0-05', '0-06'], 
           ['0-10', '0-11', '0-12', '0-13', '0-14', '0-15', '0-16'],
           ['0-20', '0-21', '0-22', '0-23', '0-24', '0-25', '0-26'],
           ['0-30', '0-31', '0-32', '0-33', '0-34', '0-35', '0-36'],
           ['0-40', '0-41', '0-42', '0-43', '0-44', '0-45', '0-46']],
                                                          
          [['1-00', '1-01', '1-02', '1-03', '1-04', '1-05', '1-06'], 
           ['1-10', '1-11', '1-12', '1-13', '1-14', '1-15', '1-16'],
           ['1-20', '1-21', '1-22', '1-23', '1-24', '1-25', '1-26'],
           ['1-30', '1-31', '1-32', '1-33', '1-34', '1-35', '1-36'],
           ['1-40', '1-41', '1-42', '1-43', '1-44', '1-45', '1-46']],
                                                          
          [['2-00', '2-01', '2-02', '2-03', '2-04', '2-05', '2-06'], 
           ['2-10', '2-11', '2-12', '2-13', '2-14', '2-15', '2-16'],
           ['2-20', '2-21', '2-22', '2-23', '2-24', '2-25', '2-26'],
           ['2-30', '2-31', '2-32', '2-33', '2-34', '2-35', '2-36'],
           ['2-40', '2-41', '2-42', '2-43', '2-44', '2-45', '2-46']]    
         ])
print(' params sahape: ', params.shape, ' parms[:axis]', params.shape[:0], params.shape[:1], params.shape[:2], params.shape[:3])
print(' params sahape: ', params.shape, ' parms[:axis]', params.shape[0:], params.shape[1:], params.shape[2:], params.shape[3:])
print(' indices.shape  ', indices.shape)
res = tf.gather_nd(params, indices)
print('result shape   ', res.get_shape())
res.eval()<jupyter_output><empty_output><jupyter_text>Experiment with SCATTER() for splitting into classes<jupyter_code># del indices, updates, shape

indices =  tf.constant([
                         #|
                         #| 
                        [[0,0,0], [0,3,0], [0,2,0], [0,1,0],[0,1,1],[0,1,2],[0,2,1],[0,3,2] ],
                        [[1,0,0], [1,3,0], [1,2,0], [1,1,0],[1,1,1],[1,1,2],[1,2,1],[1,3,2] ],
                        [[2,0,0], [2,3,0], [2,2,0], [2,1,0],[2,1,1],[2,1,2],[2,2,1],[2,3,2] ]
])
                            #<--- each one corrsponds to an entry in  the updates array [row , position in resulting array]
    
#                        [[1,1], [3,1], [2,1], [1,1],[1,1],[1,2],[2,1],[3,2] ],
#                        [[0,0], [3,0], [2,0], [1,0],[1,1],[1,2],[2,1],[3,2] ]
    
###----------------------------------------------------
### Our bounding box array
###---------------------------------------------------
updates = tf.constant( [
          [[1000, 1001, 1002, 1003, 1004, 1005, 1006], 
           [1010, 1011, 1012, 1013, 1014, 1015, 1016],
           [1020, 1021, 1022, 1023, 1024, 1025, 1026],
           [1030, 1031, 1032, 1033, 1034, 1035, 1036],
           [1040, 1041, 1042, 1043, 1044, 1045, 1046],
           [1050, 1051, 1052, 1053, 1054, 1055, 1056],
           [1060, 1061, 1062, 1063, 1064, 1065, 1066],
           [1070, 1071, 1072, 1073, 1074, 1075, 1076] ],
          
          [[2000, 2001, 2002, 2003, 2004, 2005, 2006], 
           [2010, 2011, 2012, 2013, 2014, 2015, 2016],
           [2020, 2021, 2022, 2023, 2024, 2025, 2026],
           [2030, 2031, 2032, 2033, 2034, 2035, 2036],
           [2040, 2041, 2042, 2043, 2044, 2045, 2046],
           [2050, 2051, 2052, 2053, 2054, 2055, 2056],
           [2060, 2061, 2062, 2063, 2064, 2065, 2066],
           [2070, 2071, 2072, 2073, 2074, 2075, 2076]] , 

          [[3000, 3001, 3002, 3003, 3004, 3005, 3006], 
           [3010, 3011, 3012, 3013, 3014, 3015, 3016],
           [3020, 3021, 3022, 3023, 3024, 3025, 3026],
           [3030, 3031, 3032, 3033, 3034, 3035, 3036],
           [3040, 3041, 3042, 3043, 3044, 3045, 3046],
           [3050, 3051, 3052, 3053, 3054, 3055, 3056],
           [3060, 3061, 3062, 3063, 3064, 3065, 3066],
           [3070, 3071, 3072, 3073, 3074, 3075, 3076] ]    
         ])                       

zeros = tf.zeros([3,4,8,7], dtype=tf.int32)
ref   = tf.Variable(zeros)
scatt1= tf.Variable(zeros)
shape = tf.constant([3,4, 8, 7])
 
# init = tf.global_variables_initializer().run()
# print('indices shape ', indices.shape,  'indices_shape [-1]', indices.shape[-1],  'indices_shape [:-1]', indices.shape[:-1])
# print('updates.shape ', updates.shape)
# print('ref     shape ', ref.get_shape(), ref)
# print('scatt1  shape ', scatt1.get_shape(), scatt)

scatt1 = tf.scatter_nd(indices, updates, shape)
# print('scatt1  shape ', scatt1.get_shape(), scatt)
# print(scatt1[0].eval())
# print('------------')
# print(scatt1[1].eval())
# print('------------')
# print(scatt1[2].eval())

# scatt = tf.scatter_nd_update(ref, indices, updates)
# print('scatter shape is ', scatt.get_shape(), scatt)
# print(scatt.eval())

print()

indices = tf.constant([
                        [[0,0], [3,0], [2,0], [1,0]]
#                     ,[[1,3], [1,3], [1,2], [1,0]]
                      ])
updates = tf.constant([
                       [[5, 5, 5, 5, 0], 
                        [6, 6, 6, 6, 3],
                        [7, 7, 7, 7, 2], 
                        [8, 8, 8, 8, 1]]
                       
#                      , [[5, 5, 5, 3], 
#                         [6, 6, 6, 3],
#                         [7, 7, 7, 2], 
#                         [8, 8, 8, 0]]
                       ])
shape = tf.constant([4, 4, 5])
print(indices.shape, 'inputs shape', updates.shape, shape.shape)
scatter = tf.scatter_nd(indices, updates, shape)
print('scatter shape is ', scatter.get_shape())
print(scatter.eval())
num_detections = 8
# batch_size is  3
gt_class_ids = tf.constant([[1,2,3,0,0,0,0,0],[3,3,3,0,0,0,0,0],[1,2,2,0,0,2,0,0]])
gt_bboxes    = tf.random_uniform([batch_size,3,4], maxval = 127, dtype=tf.int32)


gt_classes_exp = tf.to_float(tf.expand_dims(gt_class_ids ,axis=-1))
print('    gt_classes_exp: ' ,gt_classes_exp.get_shape())
print(gt_classes_exp.eval())


zeros        = tf.zeros([batch_size,5,4], dtype = tf.int32)
gt_bboxes    = tf.concat([gt_bboxes, zeros], axis = 1)
gt_bboxes    = tf.to_float(gt_bboxes)
print('\n    gt_bboxes: ' ,gt_bboxes.get_shape())
print(gt_bboxes.eval())

mask = tf.greater(gt_class_ids,0)
print(mask.eval())

gt_scores     = tf.where(mask, tf.ones_like(gt_class_ids), tf.zeros_like(gt_class_ids))
gt_scores_exp = tf.to_float(tf.expand_dims(gt_scores, axis=-1))
print('\n    gt_scores ', gt_scores_exp.get_shape())
print(gt_scores_exp.eval())

batch_grid, bbox_grid = tf.meshgrid( tf.range(batch_size    , dtype=tf.int32), 
                                    tf.range(num_detections , dtype=tf.int32), indexing = 'ij' )

print('\n    bbox_grid   ', type(bbox_grid)  , 'shape', bbox_grid.get_shape())
print(bbox_grid.eval())
print('\n    batch_grid ', type(batch_grid), 'shape', batch_grid.get_shape())
print(batch_grid.eval())
 
bbox_idx_zeros  = tf.zeros_like(bbox_grid)
bbox_idx        = tf.where(mask, bbox_grid , bbox_idx_zeros)
bbox_idx        = tf.to_float(tf.expand_dims(bbox_idx, axis = -1))    
print('    bbox_idx', type(bbox_idx), 'shape', bbox_idx.get_shape())
print(bbox_idx.eval())
gt_array        = tf.concat([bbox_idx, gt_scores_exp , gt_bboxes, gt_classes_exp], axis=2)
print('    gt_array  ',  type(gt_array), gt_array.shape)
print(gt_array.eval())

scatter_classes = tf.stack([batch_grid , gt_class_ids, bbox_grid],axis = -1)
print('\n    -- stack results ----')
print('\n    scatter_classes', type(scatter_classes), 'shape',tf.shape(scatter_classes).eval())
print(scatter_classes.eval())
gt_scatter = tf.scatter_nd(scatter_classes, gt_array, [batch_size, num_classes, num_detections,7])
print('    gt_tensor shape is ', gt_scatter.get_shape(), gt_scatter)
gt_scatter.eval()
sort_vals, sort_inds = tf.nn.top_k(gt_scatter[:,:,:,0], k=gt_scatter.shape[2])
print('    sort vals shape : ', sort_vals.get_shape())
print(sort_vals.eval())
print('    sort inds shape : ', sort_inds.get_shape())
print(sort_inds.eval())
# build gathering indexes to use in sorting 
class_grid, batch_grid, bbox_grid = tf.meshgrid(tf.range(num_classes),tf.range(batch_size), tf.range(num_detections))

print('    class_grid  ', type(class_grid) , 'shape', class_grid.get_shape())
print(class_grid.eval())
print('    batch_grid  ', type(batch_grid) , 'shape', batch_grid.get_shape())
print(class_grid.eval())
print('    bbox_grid    ', type(bbox_grid) , 'shape', bbox_grid.get_shape())
print(bbox_grid.eval())
gather_inds = tf.stack([batch_grid , class_grid, sort_inds],axis = -1)
print('    -- pred_tensor results (A-boxes sorted by score ----')
print('    gatehr_inds ', gather_inds.get_shape())
print(gather_inds.eval())

gt_tensor = tf.gather_nd(gt_scatter, gather_inds)
print('    -- pred_tensor results (A-boxes sorted by score ----')
print('    pred_tensor ', gt_tensor.get_shape())
print(gt_tensor.eval())<jupyter_output><empty_output><jupyter_text>Split Box/ Class info for each Image dim(0) by class and place into new tensor using tf.scatter_nd_update  obsolete -- use tf.scatter_nd instead    Output is converted to NUMN_BATCHES x NUM_CLASSES x num_rois x columns<jupyter_code># scatt1 = tf.scatter_nd(scatter_inds, pred_array, [batch_size, num_classes, num_rois])
# print('scatt1  shape ', scatt1.get_shape(), scatt)

# zeros = tf.zeros([batch_size,num_classes, num_rois,7], dtype=tf.float32)
# ref   = tf.Variable(zeros)
# init = tf.global_variables_initializer().run()

# pred_scatt_old = tf.scatter_nd_update(ref, scatter_classes, pred_array)


# print('scatter shape is ', pred_scatt.get_shape(), pred_scatt)
# print(pred_scatt[0].eval())
# print('------------')
# print(pred_scatt[1].eval())
# print('------------')
# print(pred_scatt[2].eval())<jupyter_output><empty_output><jupyter_text>Stack non-zero bbox infromaton from pred_array into stacked_array --- method 2 (obsolete)<jupyter_code># sep = tf.unstack(pred_tensor)
# print(len(sep))
# all_boxes = []
# for sub_sep in sep:
#     p_reduced = tf.reduce_sum(tf.abs(sub_sep), axis=-1)
#     print(' p_redcued shape ',p_reduced.shape)
# #     print(p_reduced.eval())
#     non_zeros = tf.cast(p_reduced, tf.bool)
#     print(' mask shape' ,non_zeros.shape) 
#     # print(non_zeros.eval())
#     boxes = tf.boolean_mask(sub_sep, non_zeros,axis = 0)
#     print(' boxes shape ',boxes.get_shape())
#     print(boxes.eval())

#     all_boxes.append(boxes)
# print(len(all_boxes))
# stacked_tensor = tf.stack(all_boxes)
# print('stacked shape',stacked_tensor.get_shape())

# p_reduced = tf.reduce_sum(tf.abs(pred_tensor), axis=-1)
# print(' p_redcued shape ',p_reduced.shape)
# print(p_reduced.eval())
# non_zeros = tf.cast(p_reduced, tf.bool)
# print(' mask shape' ,non_zeros.shape) 
# print(non_zeros.eval())
# boxes = tf.boolean_mask(pred_tensor, non_zeros,axis = 0)
# print(boxes.shape)
# print(boxes.eval())
# print(keepers[1].eval())
boxes1 = tf.Variable(tf.random_uniform([32, 4], minval=0, maxval=128, dtype=tf.float32, seed=1234), name = 'var')
boxes2 = tf.Variable(tf.random_uniform([100, 4], minval=0, maxval=128, dtype=tf.float32, seed=1234), name = 'var')

init = tf.global_variables_initializer().run()

# boxes1 = tf.placeholder(tf.float32,(32,4))
# boxes2 = tf.placeholder(tf.float32,(100,4))

print(boxes1.get_shape(), boxes2.get_shape())
aa = tf.expand_dims(boxes1, 1)
print(tf.shape(aa).eval())
aa = tf.tile(tf.expand_dims(boxes1, 1), [1, 1, tf.shape(boxes2)[0]])
print(tf.shape(aa).eval())                          

b1 = tf.reshape(tf.tile(tf.expand_dims(boxes1, 1), [1, 1, tf.shape(boxes2)[0]]), [-1, 4])
print('b1 : ', tf.shape(b1).eval())
b2 = tf.tile(boxes2, [tf.shape(boxes1)[0], 1])
print('b2 : ', tf.shape(b1).eval())
print('     overlaps_graph: shape of boxes1 after reshape: ',b1.shape)  # (?,4)
print('     overlaps_graph: shape of boxes2 after reshape: ',b2.shape)  # (?,4)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup