<jupyter_start><jupyter_text>SMC<jupyter_code>def get_nu_rho(n_satisfies, n_sampled_so_far, alpha, beta):
    ro = (n_satisfies + alpha) / (n_sampled_so_far + alpha + beta)
    nu = np.sqrt(((alpha + n_satisfies) * (n_sampled_so_far - n_satisfies + beta)) / (
            pow((alpha + n_sampled_so_far + beta), 2) * (alpha + n_sampled_so_far + beta + 1)))
    return ro, nu
# calculate time
elements_num = df_tokens.shape[0]
tmp = list(range(0, elements_num))
tokens_indx_rand = tmp.copy()
random.seed(42)
random.shuffle(tokens_indx_rand)
alpha = 1
beta = 1

n_satisfies_p5_list = []
n_sampled_so_far_list = []

smc_ro_estimates_p5 = []
smc_nu_estimates_p5 = []

n_satisfies_p5 = 0
n_sampled_so_far = 0

k = 0

for i in tokens_indx_rand:
    # TODO: vectorize this function using apply()
    # should be cumulative group by operation

    if p5_satisfies[i] == 1:
        n_satisfies_p5 = n_satisfies_p5 + 1

    n_sampled_so_far = n_sampled_so_far + 1
    rho_p5, nu_p5 = get_nu_rho(n_satisfies_p5, n_sampled_so_far, alpha, beta)

    n_satisfies_p5_list.append(n_satisfies_p5)
    n_sampled_so_far_list.append(n_sampled_so_far)

    smc_ro_estimates_p5.append(rho_p5)
    smc_nu_estimates_p5.append(nu_p5)
#     if k % 1000 == 0:
#         print(" k = {}.".format(k))
#     k = k + 1
    
_df_p5_smc = pd.DataFrame({'token_id': tokens_indx_rand,
                          'satisfies_p5': n_satisfies_p5_list,
                          'sampled': n_sampled_so_far_list,
                          'rho_p5': smc_ro_estimates_p5,
                          'nu_p5': smc_nu_estimates_p5
                          })
_df_p5_smc.shape
model_base_name = 'model_80_1_80_500'
verify_results_folder='E:/Dima/PhD/Papers/data/verify_results/'
_df_p5_smc.to_csv(verify_results_folder + "/" + model_base_name + "_p5_smc.csv", index=False)<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup