text:
using DataStructures, DataFrames, StatsFuns, GLM , JuMP,NLopt, HDF5, JLD, Distributions, MixedModels, RCall, StatsBase, xCommon

function loadDF()
    #cd("/media/u01/analytics/scoring/Healthy/modeling5/")
    #df_data = readtable("csv_final_healthychoice1_I3.csv",header=true); #lowercase(df_data)
    #df_h = readtable("Headers_healthy_choice3.csv",header=false); #lowercase(df_data)
    #names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
    cd("/media/u01/analytics/scoring/CDW5_792/")
    df_data = readtable("csv_final_cdw5_792.csv",header=false);
    df_h = readtable("Headers_cdw5.csv",header=false);
    names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
end
df_in=loadDF()

function isValid(df_data::DataFrame,cfg::OrderedDict)
    function checkValid(iarr::Array{Symbol})  length(setdiff(iarr, names(df_data))) > 0 ? false : true end
    !checkValid(cfg[:all_mandatory_vars]) ? error("ERROR: Not all mandatory_vars in dataset ") : println("VALID : mandatory_vars")
    !checkValid(cfg[:scoring_vars]) ? error("ERROR: Not all scoring_vars in dataset ") : println("VALID : scoring_vars")
end

"""
    cfg[:occ_y_var] = :Trps_POS_P1
    cfg[:occ_logvar] = :Trps_PRE_P1
    cfg[:dolocc_y_var] = :Dol_per_Trip_POS_P1
    cfg[:dolocc_logvar] = :Dol_per_Trip_PRE_P1
    cfg[:pen_y_var] = :Buyer_Pos_P1
    cfg[:pen_logvar] = :Buyer_Pre_P1
"""
const cfgDefaults=OrderedDict( :P2_Competitor => true
                        ,:pvalue_lvl => 0.20  #pvalue_lvl = 0.20
                        ,:excludedBreaks => String[]    #["estimated_hh_income","hh_age","number_of_children_in_living_un","person_1_gender"]
                        ,:excludedLevels => ["none"]
                        ,:excludedKeys => String[]
                        ,:exposed_flag_var => :exposed_flag
                        ,:sigLevel => "0.2"
                        ,:random_demos => [:estimated_hh_income,:hh_age,:number_of_children_in_living_Un,:person_1_gender]
                        ,:random_campaigns => []
                        ,:dropvars => [:exposed_flag]
                        ,:scoring_vars => [:Prd_1_Net_Pr_PRE,:Prd_1_Net_Pr_POS,:Buyer_Pos_P0,:Buyer_Pre_P0]
                        ,:occ_y_var => :Trps_POS_P1
                        ,:occ_logvar => :Trps_PRE_P1
                        ,:dolocc_y_var => :Dol_per_Trip_POS_P1
                        ,:dolocc_logvar => :Dol_per_Trip_PRE_P1
                        ,:pen_y_var => :Buyer_Pos_P1
                        ,:pen_logvar => :Buyer_Pre_P1
                       )

function getCFG(df_in::DataFrame)
    cfg=xCommon.loadCFG(cfgDefaults, pwd()*"/app.cfg")
    cfg[:exposed_flag_var] = :exposed_flag_new                  # Go In app.cfg
    cfg[:random_campaigns] = [:Publisher_Fct1,:Targeting_Fct1]  # Go In app.cfg

    cfg[:allrandoms] = vcat(cfg[:random_demos],cfg[:random_campaigns])
    #cfg[:ProScore] = grep1("MODEL",names(df_in))
    ps = filter(x->contains(string(x), "MODEL"), names(df_in)) # Set the ProScore variable in the dataset   #push!(cfg[:all_mandatory_vars], cfg[:ProScore])
    cfg[:ProScore] = length(ps) > 0 ? ps[1] : :MISSING_MODEL_VARIABLE_IN_DATA
    cfg[:num_products] = length( grep("Buyer_Pre_P",names(df_in)))-1  # get number of products in the data
    #rework vars after loading cfg
    cfg[:random_campaigns] = intersect(cfg[:random_campaigns],names(df_in))

    cfg[:all_mandatory_vars] = vcat( [:experian_id,
                                      cfg[:pen_y_var],cfg[:pen_logvar],
                                      cfg[:occ_y_var], cfg[:occ_logvar],
                                      cfg[:dolocc_y_var], cfg[:dolocc_logvar]
                                     ]
                                     , cfg[:random_demos]
                                   )
    return cfg
end
cfg=getCFG(df_in)

isValid(df_in,cfg)

function reworkCFG!(df_in::DataFrame,cfg::OrderedDict)
    xflags=[cfg[:exposed_flag_var],:exposed_flag]
    features = setdiff(names(df_in),xflags)

    if :state in names(df_in)
        cfg[:xVarsDemos]=vcat([:experian_id, :banner, :Prd_0_Qty_PRE],features[findfirst(features, :state):findfirst(features, :Mosaic)])   # exclude demos between....
    else
        cfg[:xVarsDemos]=Symbol[]
    end
    cfg[:xVarsDemos]=setdiff(cfg[:xVarsDemos],[:person_1_gender,:number_of_children_in_living_Un]) # exclude person, # from non-used demos
    cfg[:xVarsPost] = grep(["POS","Pos","Buyer_Pre_","Nonbuyer_Pre_"],features)   #exclude POST variables
    cfg[:iVarsPREPOS] = grep(["PRE_POS","Pre_Pos"],features)
    features=setdiff(features, setdiff(  vcat(cfg[:xVarsDemos],cfg[:xVarsPost]) ,cfg[:iVarsPREPOS])  )

    cfg[:xVarsP0] =  setdiff(grep("0", features ),[cfg[:ProScore]] ) #exclude category variables(exclude P0's)
    features=setdiff(features,cfg[:xVarsP0])

    cfg[:xVarsReports] = grep(["Perc_","Pr_per_"],features)  #exclude Reporting vars
    features=setdiff(features,cfg[:xVarsReports])

    cfg[:xvars] = vcat(xflags,cfg[:xVarsDemos], cfg[:xVarsPost],cfg[:xVarsP0], cfg[:xVarsReports],cfg[:dropvars])
    features=setdiff(features,cfg[:dropvars])

    cfg[:ivars] = vcat(cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars])

    cfg[:ALL_vars_to_exclude] = setdiff(cfg[:xvars], cfg[:ivars])

    df_in[:group] = df_in[cfg[:exposed_flag_var]]
    df_in[:panid] = df_in[:experian_id]
    features = setdiff(unique(vcat(features,cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars],[:group,:panid])  ) ,[:experian_id] )

    cfg[:negativevars] = grep(vec(hcat([["P"*string(i),string(i)*"_"] for i=3:cfg[:num_products]]...)),features)   # get variables that need to have negative sign
    cfg[:positivevars] = grep(["P1", "1_","MODEL"], features)    # get variables that need to have positive sign

    if cfg[:P2_Competitor] == true
        cfg[:negativevars] = unique(vcat(cfg[:negativevars],grep(["P2","2_"],features)))
    else
        cfg[:positivevars] = unique(vcat(cfg[:positivevars],grep(["P2","2_"],features)))
    end

    return df_in[features]
end

dfd = reworkCFG!(df_in,cfg)

#######################################
#--------- Data Manipulation ---------#
#######################################

function data_Prep(dfd::DataFrame, cfg::OrderedDict)
    dfd[:isO]=false
    dfd[:whyO]=""

    dfd[dfd[:number_of_children_in_living_Un].>=4,:number_of_children_in_living_Un] = 4  # aggregate #of children for 4+ L_114
    #vars = getvars()   # :GROUP
    #vars = DataFrame(names=names(dfd),eltypes=eltypes(dfd))
    if typeof(dfd[:group]) in [DataArray{String,1}]
        dfd[ findin(dfd[:group],["//N","\\N"]), :group] = "0"
        dfd[DataArrays.isna(dfd[:group]), :group]="0"
        dfd[:group] = [parse(Int64,s) for s = dfd[:group]]
    else
        dfd[ DataArrays.isnan(dfd[:group]), :group] = 0
    end

    vars = DataFrame(names=names(dfd),eltypes=eltypes(dfd))
    for c in setdiff(vars[findin(vars[:eltypes],[String]),:names],cfg[:allrandoms]) # set variables as.numeric and replace NA's with zero
        print("Convert String: String->Numeric: ",c)
        try dfd[c] = map( x -> DataArrays.isna(x) ?  NaN : convert(Float64, x)  , dfd[c]) catch e println("  (failed)") end  #NA to NaN
        try dfd[c] = convert(Array{Float64}, dfd[c]) catch e end   # To Float64
    end
    vars = DataFrame(names=names(dfd),eltypes=eltypes(dfd))
    #vars = getvars()
    for c in setdiff(vars[findin(vars[:eltypes],[Float64]),:names],cfg[:allrandoms])  # replace NaN's with zero for numeric variables
        println("Replace Float64 NaN (0.0): ",c)
        dfd[ DataArrays.isnan(dfd[c]), c] = 0.0
    end
    for c in  setdiff(vars[findin(vars[:eltypes],[Int64]),:names],cfg[:allrandoms])
        println("Replace Int64 NaN (0) : ",c)
        dfd[ DataArrays.isnan(dfd[c]), c] = 0
    end

    # create factor variables -  Need to understand how this impacts the data passed to R - might need to do this in R
    #pool!(dfd, cfg[:allrandoms])
    #pool!(dfd, [,:group])

    # NOTE : Do QCs here
    dfd[dfd[:person_1_gender].=="U",:isO] = true   # remove HHs with no gender info
    dfd[dfd[:person_1_gender].=="U",:whyO] = "person_1_gender=U; remove HHs with no gender info"
    dfd[findin(dfd[:estimated_hh_income],["U","L"]),:estimated_hh_income]="L" # aggregate U and L levels of hh income

    for r in cfg[:random_campaigns]    # check and drop exposed HHs with no publisher info or non-exposed HHs with publisher info
        dfd[findin(dfd[r],["\\N","NULL","0","NONE"])  ,r] ="none"
        dfd[ (dfd[:isO].==false) & (dfd[r].!="none") & (dfd[:group].==0) ,:whyO] = "non exposed HHs with publisher info"
        dfd[ (dfd[:isO].==false) & (dfd[r].!="none") & (dfd[:group].==0) ,:isO] = true
        println(r," non exposed HHs with publisher info : ",nrow(dfd[dfd[:whyO].=="non exposed HHs with publisher info",:] )   )
        dfd[ (dfd[:isO].==false) & (dfd[r].=="none") & (dfd[:group].!=0) ,:whyO] = "exposed HHs with no publisher info"
        dfd[ (dfd[:isO].==false) & (dfd[r].=="none") & (dfd[:group].!=0) ,:isO] = true
        println(r," exposed HHs with no publisher info : ",nrow(dfd[dfd[:whyO].=="exposed HHs with no publisher info",:] )   )
    end

    # segments for outliers detection
    dfd[:data_NB_NE_B] = false
    dfd[ (dfd[:Buyer_Pre_P1].==0 ) & (dfd[:group].==0 ) & (dfd[:Buyer_Pos_P1].==1 ) ,:data_NB_NE_B] = true
    dfd[:data_B_E_NB] = false
    dfd[ (dfd[:Buyer_Pre_P1].==1 ) & (dfd[:group].==0 ) & (dfd[:Buyer_Pos_P1].==0 ) ,:data_B_E_NB] = true
    dfd[:pen_reduction] = false
    dfd[ (dfd[:Buyer_Pre_P1].==1) & (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==0 )  ,:pen_reduction] = true
    dfd[:occ_reduction] = false
    dfd[ (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==1) & (dfd[:Trps_POS_P1].< dfd[:Trps_PRE_P1] ) ,:occ_reduction] = true
    dfd[:dolocc_reduction] = false
    dfd[  (dfd[:group].==0) & (dfd[:Buyer_Pos_P1].==1) & (dfd[:Dol_per_Trip_POS_P1].< dfd[:Dol_per_Trip_PRE_P1] )  , :dolocc_reduction] = true

    return dfd[dfd[:isO].==false, : ]   #[setdiff(names(dfd),[:isO,:whyO])]
end

dfd = data_Prep(dfd, cfg);

function MatchMe(dfd::DataFrame,cfg::OrderedDict)
    df=dfd[dfd[:isO].==false,:]
    df_exp     = df[df[:group].==1,:]
    df_unexp   = df[df[:group].==0,:]
    df_exp_dim   = nrow(df_exp)
    df_unexp_dim = nrow(df_unexp)
    new_unexp_dim = df_unexp_dim*(df_unexp_dim>2000000 ? 0.3 : df_unexp_dim>1000000 ? 0.4 : df_unexp_dim>750000 ? 0.6 : 0.7)
    #df_sample=DataFrame
    if length(string(cfg[:ProScore])) == 0
        df_unexp_1 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==1),:]
        df_unexp_0 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==0),:]

        df_exp_1_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==1),:])
        df_exp_0_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==0),:])
        df_unexp_1_dim = nrow(df_unexp_1)
        df_unexp_0_dim =  nrow(df_unexp_0)
        dim_sample0 = round(Int64, (new_unexp_dim-df_exp_dim  ) / (1+(df_exp_1_dim / df_exp_0_dim)) )
        dim_sample1 = round(Int64, new_unexp_dim - df_exp_dim - dim_sample0)

        new_df_unexp_1 = df_unexp_1[sample(1:size(df_unexp_1,1), dim_sample1 ),:]
        new_df_unexp_0 = df_unexp_0[sample(1:size(df_unexp_0,1), dim_sample0 ),:]
        dfd_sample  = vcat(df_exp,new_df_unexp_1,new_df_unexp_0)
    elseif length(string(cfg[:ProScore])) > 0
        sample_control_data=similar(df_unexp, 0)
        for (key, value) in countmap(df_exp[cfg[:ProScore]])
            sample_dim=round(Int64,new_unexp_dim*(value/df_exp_dim))
            temp_data = df_unexp[df_unexp[cfg[:ProScore]].==key,:]
            samp_data = temp_data[sample(1:size(temp_data,1), sample_dim, replace=false),:]
            sample_control_data = vcat(sample_control_data,    samp_data   )
        end

        sample_data = vcat(sample_control_data,df_exp)
        sample_df_unexp = sample_data[sample_data[:group].==0,:]
        sample_df_exp   = sample_data[sample_data[:group].==1,:]
        sample_df_unexp_1 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==1),:]
        sample_df_unexp_0 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==0),:]
        sample_df_exp_1 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==1) ,:]
        sample_df_exp_0 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==0) ,:]
        sample_df_unexp_1_dim = nrow(sample_df_unexp_1)
        sample_df_unexp_0_dim = nrow(sample_df_unexp_0)
        sample_df_exp_1_dim = nrow(sample_df_exp_1)
        sample_df_exp_0_dim = nrow(sample_df_exp_0)
        dim_sampleA = round(Int64,(sample_df_exp_1_dim/sample_df_exp_0_dim)*sample_df_unexp_0_dim)
        dim_sampleB = round(Int64,(sample_df_exp_0_dim/sample_df_exp_1_dim)*sample_df_unexp_1_dim)

        if sample_df_unexp_1_dim/sample_df_unexp_0_dim > sample_df_exp_1_dim/sample_df_exp_0_dim
            new_df_unexp_1 = sample_df_unexp_1[sample(1:sample_df_unexp_1_dim,dim_sampleA , replace=false),:]
            dfd_sample = vcat(sample_df_exp,new_df_unexp_1,sample_df_unexp_0)
        else
            new_df_unexp_0 = sample_df_unexp_0[sample(1:sample_df_unexp_0_dim, dim_sampleB, replace=false ),:]
            dfd_sample = vcat(sample_df_exp,sample_df_unexp_1,new_df_unexp_0)
        end
    end

"""
    #Generate Report!!!! ---   still need to remove :isO
    function dict2df(d::Dict,kc::String,c::String)
        dfo=DataFrame(k=Int64[],v=Int64[])
        for (key, value) in d
            push!(dfo,[key, value])
        end
        rename!(dfo,:k,Symbol(kc))
        rename!(dfo,:v,Symbol(c))
        return dfo
    end
    ds0=dict2df(countmap(dfd_sample[ dfd_sample[:group].==0,cfg[:ProScore]]),"ProScore","out_unexp")
    ds1=dict2df(countmap(dfd_sample[ dfd_sample[:group].==1,cfg[:ProScore]]),"ProScore","out_exp")
    do0=dict2df(countmap(df[ df[:group].==0,cfg[:ProScore]]),"ProScore","in_unexp")
    do1=dict2df(countmap(df[ df[:group].==1,cfg[:ProScore]]),"ProScore","in_exp")
    dfo=join(join(join(do0, do1, on = :ProScore,kind=:outer), ds0, on = :ProScore,kind=:outer),ds1, on = :ProScore,kind=:outer)
    for c in [:out_unexp, :out_exp, :in_unexp, :in_exp]
        dfo[DataArrays.isna(dfo[c]), c] = 0
    end
    b_ds0=dict2df(countmap(dfd_sample[ dfd_sample[:group].==0,:Buyer_Pre_P1]),"Buyer_Pre_P1","out_unexp")
    b_ds1=dict2df(countmap(dfd_sample[ dfd_sample[:group].==1,:Buyer_Pre_P1]),"Buyer_Pre_P1","out_exp")
    b_do0=dict2df(countmap(dfd[ dfd[:group].==0,:Buyer_Pre_P1]),"Buyer_Pre_P1","in_unexp")
    b_do1=dict2df(countmap(dfd[ dfd[:group].==1,:Buyer_Pre_P1]),"Buyer_Pre_P1","in_exp")
    dfx=DataFrame(Buyer_Pre_P1=[0,1],Buyer_Pre_Period=["No","Yes"])
    b_dfo1=join(b_do0, b_do1, on = :Buyer_Pre_P1,kind=:outer)
    b_dfo1[:in_proportion] = b_dfo1[:in_exp]./sum(b_dfo1[:in_exp])
    b_dfo2=join(b_ds0, b_ds1, on = :Buyer_Pre_P1,kind=:outer)
    b_dfo2[:out_proportion] =b_dfo2[:out_exp]./sum(b_dfo2[:out_exp])
    b_dfo=join(dfx,join(b_dfo1, b_dfo2, on = :Buyer_Pre_P1,kind=:outer), on = :Buyer_Pre_P1,kind=:outer)
    delete!(b_dfo, :Buyer_Pre_P1 )
    #still need to Write QC to file
"""
    rows2remove = setdiff(dfd[dfd[:isO].==false, :panid],dfd_sample[:panid])
    dfd[findin(dfd[:panid],rows2remove),:whyO]="NoMatch"
    dfd[findin(dfd[:panid],rows2remove),:isO]=true

    return dfd[dfd[:isO].==false, : ]  #[setdiff(names(dfd),[:isO,:whyO])]
end

dfd = MatchMe(dfd,cfg)

lowercase!(dfd)
cfg=lowercase(cfg)

#Not Used
dfd=dfd[setdiff(names(dfd),[:nonbuyer_pre_p1])]

######################################
#------------MODEL OBJECTS-----------#
######################################
abstract MModel

#!!! Multicollinearity, removing :[:fea_or_dis_trps_shr_dpp_p1,:fea_or_dis_trps_shr_dpp_p2,:fea_or_dis_trps_shr_dpp_p3,:fea_or_dis_trps_shr_dpp_p4]
type xGLM
    vfactors::Vector{Symbol}
    fmula::DataFrames.Formula
    xvars::Vector{Symbol}
    model::Any #DataFrameRegressionModel
    sdf::DataFrame
    wasSuccessful::Bool

    function xGLM(dfd::DataFrame, dist::Distribution, y_var::Symbol, logvar::Symbol , lnk::Link , vfactors::Array{Symbol} )
        this=new()
        this.wasSuccessful=false
        this.xvars=Symbol[]
        this.vfactors=setdiff(vfactors, [y_var,logvar])
        for l in 1:30
            #log_colname=Symbol("LOG_"*string(logvar))
            #this.vfactors=vcat(setdiff(this.vfactors,this.xvars),[log_colname])
            this.fmula = genFmula(y_var, this.vfactors, logvar  )
            try
                #this.model = glm(this.fmula,  dfd[vcat(this.vfactors,[y_var])], dist, LogLink(), offset=log(Array(dfd[logvar]  + 1)) )
                #dfd[log_colname]=log(Array(dfd[logvar]+1))
                f=this.fmula
                ##this.model = glm(this.fmula,  dfd[vcat(this.vfactors,[y_var,log_colname])], dist, m.lnk )   # LogLink()
                this.model = glm(f,  dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , dist, lnk )

                this.sdf = DataFrame(vars=vcat([:intercept],this.model.mf.terms.terms), coef=coef(this.model), se=stderr(this.model), zval=coef(this.model)./stderr(this.model) ,pval= ccdf(FDist(1, df_residual(this.model)), abs2(coef(this.model)./stderr(this.model))))
                this.wasSuccessful=true
                break
            catch e
                if isa(e, Base.LinAlg.PosDefException)
                    v=this.vfactors[e.info-1]
                    push!(this.xvars,v)
                    println("!!! Multicollinearity, removing :",v,"~~~",e.info-1, "\n~~~",e)
                else
                    println("....",e)
                    break
                end
            end
        end
        return this
    end
end

type MDolOcc <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    #mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MDolOcc(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="dolocc"; this.logvar=cfg[:dolocc_logvar]; this.y_var=cfg[:dolocc_y_var]; this.dist=Gamma()
        this.lnk=LogLink()
        #this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars= Symbol[ cfg[:occ_y_var],cfg[:occ_logvar],cfg[:pen_y_var],cfg[:pen_logvar], :buyer_pos_p1 ]
        this.Buyer_Pos_P1_is1=true
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mdolocc = MDolOcc(dfd,cfg)

type MOcc <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    #mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MOcc(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="occ"; this.logvar=cfg[:occ_logvar]; this.y_var=cfg[:occ_y_var]; this.dist=Poisson()
        this.lnk=LogLink()
        #this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars=Symbol[cfg[:dolocc_y_var],cfg[:dolocc_logvar],cfg[:pen_y_var],cfg[:pen_logvar], :buyer_pos_p1 ]
        this.Buyer_Pos_P1_is1=true
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mocc = MOcc(dfd,cfg)

type MPen <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    #mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MPen(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="pen"; this.logvar=cfg[:pen_logvar]; this.y_var=cfg[:pen_y_var]; this.dist=Bernoulli() #Binomial()
        this.lnk=LogitLink()
        #this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars=Symbol[cfg[:occ_y_var],cfg[:occ_logvar],cfg[:dolocc_y_var],cfg[:dolocc_logvar] ]
        this.Buyer_Pos_P1_is1=false
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mpen = MPen(dfd,cfg)

    include("/home/rmadmin/.julia/v0.4/RegTools/src/diagnostics.jl")
    include("/home/rmadmin/.julia/v0.4/RegTools/src/misc.jl")
    include("/home/rmadmin/.julia/v0.4/RegTools/src/modsel.jl")

#include("/media/u01/analytics/RegTools/diagnostics.jl")
#include("/media/u01/analytics/RegTools/misc.jl")
#include("/media/u01/analytics/RegTools/modsel.jl")

function vif!(g::xGLM)
    vdf=vif(g.model)
    vdf[:vars] = convert(Array{Symbol}, vdf[:variable])
    g.sdf = join(g.sdf,vdf[[:vars,:vif]], on = :vars, kind=:outer)
end

"""
    checksingularity(form::Formula, data::DataFrame, tolerance = 1.e-8)

Return a vector of terms in `form` that produce singularity in the model matrix
"""
function checksingularity(form::Formula, data::DataFrame, tolerance = 1.e-8)
    mf = ModelFrame(form, data)
    mm = ModelMatrix(mf)
    qrf = qrfact!(mm.m, Val{true})
    vals = abs.(diag(qrf[:R]))
    firstbad = findfirst(x -> x < min(tolerance, 0.5) * vals[1], vals)
    if firstbad == 0
        return Symbol[]
    end
    mf.terms.terms[view(mm.assign[qrf[:p]], firstbad:length(vals))]
end
# -- test --
#m=mocc
#custom_vars=[:dolocc_reduction,:occ_reduction,:pen_reduction,:data_b_e_nb,:data_nb_ne_b,:whyo,:iso]
#required_vars=vcat([m.y_var,:panid,m.logvar],cfg[:random_demos],cfg[:random_campaigns],cfg[:scoring_vars])
#vars=setdiff(m.vars,vcat(required_vars,custom_vars))
#m.removed_SingleLevelVars=FS_singleLevel(dfd,vars)
#vars = rmVars(m.removed_SingleLevelVars)
#f=genFmula(m.y_var,vars,m.logvar)

#checksingularity(f, dfd)
# -- test end ---

# =======================================================================================
# =======================================================================================

using IRImodels

factor_cols=vcat( [ cfg[:proscore], :group, :panid], cfg[:allrandoms] )

function featureSelection(dfd::DataFrame, m::MModel)
    function rmVars(v::Array{Symbol})
        v=setdiff(v,[:group])
        return setdiff(vars,v)
    end

    log_colname=Symbol("LOG_"*string(m.logvar))
    dfd[log_colname]=log(Array(dfd[m.logvar]+1))
    #this.vfactors=vcat(setdiff(this.vfactors,this.xvars),[log_colname])
    #this.fmula = genFmula(y_var, this.vfactors, logvar  )

    custom_vars=[:dolocc_reduction,:occ_reduction,:pen_reduction,:data_b_e_nb,:data_nb_ne_b,:whyo,:iso]
    required_vars=vcat([m.y_var,:panid,m.logvar],cfg[:random_demos],cfg[:random_campaigns],cfg[:scoring_vars])
    vars=setdiff(vcat(m.vars,[log_colname]),vcat(required_vars,custom_vars))

    println(uppercase(mocc.modelName)*" : SingleValue") #SingleValue
    m.removed_SingleLevelVars=FS_singleLevel(dfd,vars)
    vars = rmVars(m.removed_SingleLevelVars)

    println(uppercase(mocc.modelName)*" : Singularity : "*string(genFmula(m.y_var,vars,m.logvar))) # Singularity
    m.singularity_x = checksingularity(genFmula(m.y_var,vars,m.logvar), dfd)
    vars = rmVars(m.singularity_x)

    println(uppercase(mocc.modelName)*" : PVals") #PVals
    m.glm1_pvals = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , vars  )
    g1=m.glm1_pvals
    m.glm1_pvals_x=g1.sdf[g1.sdf[:pval].>0.7,:vars]
    vars = rmVars(vcat(m.glm1_pvals_x, g1.xvars ) )

    println(uppercase(mocc.modelName)*" : Z & Vif") #Z & Vif
    m.glm2_ZnVIF = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk ,vars  )
    g2=m.glm2_ZnVIF
    vif!(g2)
    z = g2.sdf[abs(g2.sdf[:zval]).<1.96,:vars]
    v = g2.sdf[ !DataArrays.isna(g2.sdf[:vif])&(g2.sdf[:vif].>15),:vars]
    m.glm2_ZnVIF_x =intersect(z,v)
    vars = rmVars(vcat(m.glm2_ZnVIF_x, g2.xvars) )

    function chkSigns(m::MModel, vars::Array{Symbol}, dfd::DataFrame, cfg::OrderedDict)  # Pvalue & Signs
        g = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , vars  )
        neutralvars = setdiff(vars,vcat(cfg[:negativevars],cfg[:positivevars]))
        neg=intersect(cfg[:negativevars],g.sdf[g.sdf[:coef].<0,:vars])
        pos=intersect(cfg[:positivevars],g.sdf[g.sdf[:coef].>0,:vars])
        varstokeep = intersect(vcat(neutralvars, pos,neg) ,  g.sdf[ g.sdf[:pval].<cfg[:pvalue_lvl] ,:vars] )
        return g, varstokeep
    end

    println(uppercase(mocc.modelName)*" : SIGN Check 1")
    (m.glm3_PnSigns, initialvars) = chkSigns(m, vars, dfd, cfg)
    println(uppercase(mocc.modelName)*" : SIGN Check 2")
    (m.glm4_PnSignsClean, vars_2) = chkSigns(m, convert(Array{Symbol},initialvars) , dfd, cfg)

    function getCorrVars(dfd::DataFrame, vars_2::Array{Symbol})
        rm_lst=Symbol[]
        if (length(vars_2) > 1) & (   length(getColswithType("num", dfd, convert(Array{Symbol},vars_2) ) ) > 1  )
            stackdf = corrDF(dfd,vars_2)
            stackdf[:variable_pval] = [ m.glm4_PnSignsClean.sdf[m.glm4_PnSignsClean.sdf[:vars].==c,:pval][1]   for c in stackdf[:variable]]
            stackdf[:vars_pval] = [ m.glm4_PnSignsClean.sdf[m.glm4_PnSignsClean.sdf[:vars].==c,:pval][1]   for c in stackdf[:vars]]
            stackdf[:most_Sig] = map((x,y) -> x < y ? "variable" : "vars" ,stackdf[:variable_pval],stackdf[:vars_pval])

            for row in eachrow(stackdf[(stackdf[:value].> 0.8) | (stackdf[:value].<-0.8),:])
                if row[:vars] == "group"
                    push!(rm_lst,row[:variable])
                elseif row[:variable] == "group"
                    push!(rm_lst,row[:vars])
                else
                    row[:most_Sig] == "variable" ? push!(rm_lst,row[:vars]) : push!(rm_lst,row[:variable])
                end
            end
        end
        return rm_lst
    end

    println(uppercase(mocc.modelName)*" : Correlation")
    m.corrvars_x = getCorrVars(dfd,convert(Array{Symbol},setdiff(vars_2,factor_cols)))
    vars_2 = setdiff(vars_2,m.corrvars_x)
    (m.glm5, m.finalvars) =  chkSigns(m, convert(Array{Symbol},vars_2), dfd, cfg)

    println(uppercase(mocc.modelName)*" : Final Review") # Final Review:
    m.glm6_final = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , convert(Array{Symbol},vcat(m.finalvars,[:group]))  )
end

poolit!(dfd,factor_cols)

featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], mocc)
featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], mdolocc)
featureSelection(dfd[(dfd[:iso].==false) ,:], mpen)

# ------------------------------------------------------------------------------------------------------------
# ------ Analysis of Dev -------------------------------------------------------------------------------------
# ------------------------------------------------------------------------------------------------------------

"""
    dropterm(f::Formula, trm::Symbol)

Return a copy of `f` without the term `trm`.

# Examples

julia> dropterm(foo ~ 1 + bar + baz, :bar)
foo ~ 1 + baz
"""
function dropterm(f::Formula, trm::Symbol)
    rhs = copy(f.rhs)
    args = rhs.args
    if !(Meta.isexpr(rhs, :call) && args[1] == :+ && (tpos = findlast(args, trm)) > 0)
        throw(ArgumentError("$trm is not a summand in `$rhs`"))
    end
    Formula(f.lhs, Expr(:call, :+, deleteat!(args, [1, tpos])...))
end

"""
    drop1(m::DataFrameRegressionModel, terms::Vector{Symbol}=Symbol[])

Perform a `drop1` analysis of deviance on `terms` in model `m`

Model `m` is modified and refit dropping the terms in `terms` one at a time.
An analysis of deviance is applied to the refit model.
"""
function drop1(m, terms::Vector{Symbol}=Symbol[])
    form = Formula(m.mf.terms)
    basedev = deviance(m)
    deviances, degsfreedom, Χ², pvalues, nms = [basedev], [0], [0.], [NaN], ["<none>"]
    ncoef = length(coef(m))
    dat = m.mf.df
    D = Distribution(m)
    dinst = D <: Poisson ? Poisson() :
        D == Bernoulli ? Bernoulli() :
        D == Binomial ? Binomial() :
        D <: Gamma ? Gamma(1.,1.) :
        throw(ArgumentError("Distribution $D not known in table"))
    L = Link(m)()
    if length(terms) == 0
        terms = filter(x -> isa(x, Symbol) || isa(x, Expr), form.rhs.args[2:end])
    end
    for trm in terms
        newform = dropterm(form, trm)
        newm = glm(newform, dat, dinst, L)
        newdev = deviance(newm)
        newdof = ncoef - length(coef(newm))
        newΧ² = newdev - basedev
        push!(deviances, newdev)
        push!(degsfreedom, newdof)
        push!(Χ², newΧ²)
        push!(pvalues, ccdf(Chisq(newdof), newΧ²))
        push!(nms, string(trm))
    end
    CoefTable([deviances, Χ², degsfreedom, pvalues], ["Deviance", "Χ²", "degfree", "P[> Χ²]"], nms)
end

Distributions.Distribution{T,D,L}(m::GlmResp{T,D,L}) = D
Distributions.Distribution(m::GeneralizedLinearModel) = Distribution(m.rr)
Distributions.Distribution{T<:GeneralizedLinearModel}(m::DataFrames.DataFrameRegressionModel{T}) = Distribution(m.model)

GLM.Link{T,D,L}(m::GlmResp{T,D,L}) = L
GLM.Link(m::GeneralizedLinearModel) = Link(m.rr)
GLM.Link{T<:GeneralizedLinearModel}(m::DataFrames.DataFrameRegressionModel{T}) = Link(m.model)

# -------
poolit!(dfd,cfg[:random_campaigns])
f=genFmula(mocc.y_var,vcat(mocc.finalvars,[:group],cfg[:random_campaigns]),mocc.logvar)
model_vars = convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))
glm_occ=glm(f, dfd[ (dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1) , model_vars ] , mocc.dist, mocc.lnk)
checksingularity(genFmula(m.y_var,vars,m.logvar), dfd)

glm(f,  dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , dist, lnk )
featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], mocc)

# ---- 2 -----
#dev
poolit!(dfd,cfg[:random_campaigns])
m=mocc
for c in vcat([:group],cfg[:random_campaigns])
    f=genFmula(m.y_var,vcat(m.finalvars,[c]),m.logvar)
    g=glm(f, dfd[ (dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1) , convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end])) ] , m.dist, m.lnk)
    d1=drop1(g, [c])
    d_g = d1.cols[find( x->(x == "P[> Χ²]"), d1.colnms)[1]][find( x->(x == string(c)), d1.rownms)[1]]
    println("----> ",d_g)
end

f=genFmula(mocc.y_var,vcat(mocc.finalvars,[:group]),mocc.logvar)
f=genFmula(mocc.y_var,vcat(mocc.finalvars,[:group],cfg[:random_campaigns]),mocc.logvar)

g=glm(f, dfd[ (dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1) , model_vars ] , mocc.dist, mocc.lnk)

drop1(g)

y.cols[end][end]

----------
find( x->(x == "P[> Χ²]"), y.colnms)
# time: 2016-09-26 21:05:07 UTC
# mode: julia
        find( x->(x == "P[> Χ²]"), y.colnms)[1]
# time: 2016-09-26 21:05:28 UTC
# mode: julia
        y.cols[find( x->(x == "P[> Χ²]"), y.colnms)[1]]   [find( x->(x == "group"), y.rownms)[1]]
# time: 2016-09-26 21:05:32 UTC
# mode: julia
        y.cols[find( x->(x == "P[> Χ²]"), y.colnms)[1]][find( x->(x == "group"), y.rownms)[1]]

-----------

-------------- Campels--------
using DataStructures, DataFrames, StatsFuns, GLM , JuMP,NLopt, HDF5, JLD, Distributions, MixedModels, RCall, StatsBase, xCommon
 R""" #load("/media/u01/analytics/Campbell_Soup_785/Victor/Glen_finaldata_dolocc.RDS")
dfd <- readRDS("/media/u01/analytics/Glen_finaldata_dolocc.RDS")
         ls()
     """
df_in = rcopy("dfd")

#Campels
    cfg[:all_mandatory_vars] = []

cfg[:all_mandatory_vars] = [:experian_id, cfg[:dolocc_y_var],cfg[:dolocc_logvar],
                            :hh_age,:estimated_hh_income,:number_of_children_in_living_Un,:person_1_gender]

------------------------------

function genRFmula(y::Symbol, logvar::Symbol, fvars::Array{Symbol}, rvars::Array{Symbol} )
    xvars=setdiff(fvars,[y,logvar]) # " ~ 1"
    vout = string(y)*" ~ 1"* reduce(*, [ " + "*  string(c) for c in xvars ] )
    f1 = replace(vout," ~ 1 + "," ~ ")
    f1 = f1 * " + offset(log("*string(logvar)*"+1)) + group" * reduce(*, [ " + (0+group|"*string(c)*")" for c in rvars ] )
end
f = genRFmula(mocc.y_var, mocc.logvar, mocc.finalvars, cfg[:random_campaigns] )

#include("/home/rmadmin/g/StatStack/src/RCode.jl")

function runRglmm(dfd::DataFrame,f::String, modelName::String)
    R"""
    rm(list=ls())
    Start.Time <- Sys.time()
    list.of.packages <- c("data.table","bit64","optimx","lme4",'languageR','lmerTest','glmnet','lsmeans','car','RDS','DMwR','stringr','nloptr','ggplot2','reshape2','plyr','viridis','gmodels')
    new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
    if(length(new.packages)) install.packages(new.packages)
    lapply(list.of.packages,require,character.only=TRUE)

    defaultControl <- list(algorithm="NLOPT_LN_BOBYQA",xtol_rel=1e-6,maxeval=1e5)
    nloptwrap2 <- function(fn,par,lower,upper,control=list(),...)
       {
         for (n in names(defaultControl))
             if (is.null(control[[n]])) control[[n]] <- defaultControl[[n]]
                 res <- nloptr(x0=par,eval_f=fn,lb=lower,ub=upper,opts=control,...)
             with(res,list(par=solution,
                    fval=objective,
                    feval=iterations,
                    conv=if (status>0) 0 else status,
                    message=message))
       }
    """

    dfname="/home/rmadmin/glmm_dataset_"*modelName*".dat"
    writetable(dfname, dfd, header = true)
    @rput dfname
    @rput f
    @rput modelName

    R2fact = vcat(cfg[:random_campaigns],[:group])
    R2fact = map(x-> string(x),R2fact)
    @rput R2fact
    R2num = vcat(m.finalvars,[cfg[:occ_y_var],cfg[:occ_logvar],cfg[:dolocc_y_var],cfg[:dolocc_logvar],cfg[:pen_y_var],cfg[:pen_logvar]] )
    R2num = map(x-> string(x),R2num)
    @rput R2num

    R""" dfd <- fread(   dfname  ,colClasses = "as.character",header=T)
    dfd$trps_pre_p1

    for(i in noquote(R2fact)){ data[i] <- lapply(data[i],function(x) as.factor(x))  }

    if (modelName=="occ")
       { cat("occ")
         a1$Trps_POS_P1 <- as.factor(a1$Trps_POS_P1)

    dfd$targeting_fct1 <- as.factor(dfd$targeting_fct1)
    dfd$publisher_fct1 <- as.factor(dfd$publisher_fct1)
         occ_model <- glmer(f,data=dfd,family=poisson(link="log"),control = glmerControl(calc.derivs = FALSE, optimizer="nloptwrap2"))

       }
    else if (modelName=="dolocc")
       { cat("dolocc")
         dolocc_model <- glmer(f,data=dfd,family=Gamma(link="log"),control = glmerControl(calc.derivs = FALSE, optimizer="nloptwrap2"))
       }
    else
       {  cat("pen")
          pen_model <- glmer(f,data=dfd,family=binomial(link="logit"),control = glmerControl(calc.derivs = FALSE, optimizer="nloptwrap2"))
       }

    """

end

m = mocc

runRglmm(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:] ,f, m.modelName)

for(i in R2fact)
    { eval(parse(text=dfd[i])) ]

# -----------------------------------------------------------------------------------------------------------------
#    OLD
# -----------------------------------------------------------------------------------------------------------------
"""

using DataStructures, DataFrames, StatsFuns, GLM, NLopt, HDF5, JLD, Distributions, MixedModels, RCall, StatsBase, xCommon

function loadDF()
    #cd("/media/u01/analytics/scoring/Healthy/modeling5/")
    #df_data = readtable("csv_final_healthychoice1_I3.csv",header=true); #lowercase(df_data)
    #df_h = readtable("Headers_healthy_choice3.csv",header=false); #lowercase(df_data)
    #names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
    cd("/media/u01/analytics/scoring/CDW5_792/")
    df_data = readtable("csv_final_cdw5_792.csv",header=false);
    df_h = readtable("Headers_cdw5.csv",header=false);
    names!(df_data, convert(Array{Symbol}, df_h[:x1]) )
end
df_in=loadDF()

function isValid(df_data::DataFrame,cfg::OrderedDict)
    function checkValid(iarr::Array{Symbol})  length(setdiff(iarr, names(df_data))) > 0 ? false : true end
    !checkValid(cfg[:all_mandatory_vars]) ? error("ERROR: Not all mandatory_vars in dataset ") : println("VALID : mandatory_vars")
    !checkValid(cfg[:scoring_vars]) ? error("ERROR: Not all scoring_vars in dataset ") : println("VALID : scoring_vars")
end

const cfgDefaults=OrderedDict( :P2_Competitor => true
                        ,:pvalue_lvl => 0.20  #pvalue_lvl = 0.20
                        ,:excludedBreaks => String[]    #["estimated_hh_income","hh_age","number_of_children_in_living_un","person_1_gender"]
                        ,:excludedLevels => ["none"]
                        ,:excludedKeys => String[]
                        ,:exposed_flag_var => :exposed_flag
                        ,:sigLevel => "0.2"
                        ,:random_demos => [:estimated_hh_income,:hh_age,:number_of_children_in_living_Un,:person_1_gender]
                        ,:random_campaigns => []
                        ,:dropvars => [:exposed_flag]
                        ,:scoring_vars => [:Prd_1_Net_Pr_PRE,:Prd_1_Net_Pr_POS,:Buyer_Pos_P0,:Buyer_Pre_P0]
                        ,:all_mandatory_vars => [:Buyer_Pos_P1,:Buyer_Pre_P1,:Trps_POS_P1,:Trps_PRE_P1,:Dol_per_Trip_POS_P1,:Dol_per_Trip_PRE_P1,
                                                 :Nonbuyer_Pre_P1,:hh_age,:estimated_hh_income,:number_of_children_in_living_Un,
                                                 :person_1_gender,:experian_id]
                       )

function getCFG(df_in::DataFrame)
    cfg=loadCFG(cfgDefaults, "app.cfg")
    cfg[:exposed_flag_var] = :exposed_flag_new                  # Go In app.cfg
    cfg[:random_campaigns] = [:Publisher_Fct1,:Targeting_Fct1]  # Go In app.cfg

    cfg[:allrandoms] = vcat(cfg[:random_demos],cfg[:random_campaigns])
    #cfg[:ProScore] = grep1("MODEL",names(df_in))
    ps = filter(x->contains(string(x), "MODEL"), names(df_in)) # Set the ProScore variable in the dataset   #push!(cfg[:all_mandatory_vars], cfg[:ProScore])
    cfg[:ProScore] = length(ps) > 0 ? ps[1] : :MISSING_MODEL_VARIABLE_IN_DATA
    cfg[:num_products] = length( grep("Buyer_Pre_P",names(df_in)))-1  # get number of products in the data
    #rework vars after loading cfg
    cfg[:random_campaigns] = intersect(cfg[:random_campaigns],names(df_in))

    cfg[:occ_y_var] = :trps_pos_p1
    cfg[:occ_logvar] = :trps_pre_p1
    cfg[:dolocc_y_var] = :dol_per_trip_pos_p1
    cfg[:dolocc_logvar] = :dol_per_trip_pre_p1
    cfg[:pen_y_var] = :buyer_pos_p1
    cfg[:pen_logvar] = :buyer_pre_p1

    return cfg
end
cfg=getCFG(df_in)

isValid(df_in,cfg)

function reworkCFG!(df_in::DataFrame,cfg::OrderedDict)
    xflags=[cfg[:exposed_flag_var],:exposed_flag]
    features = setdiff(names(df_in),xflags)

    cfg[:xVarsDemos]=vcat([:experian_id, :banner, :Prd_0_Qty_PRE],features[findfirst(features, :state):findfirst(features, :Mosaic)])   # exclude demos between....
    cfg[:xVarsDemos]=setdiff(cfg[:xVarsDemos],[:person_1_gender,:number_of_children_in_living_Un]) # exclude person, # from non-used demos
    cfg[:xVarsPost] = grep(["POS","Pos","Buyer_Pre_","Nonbuyer_Pre_"],features)   #exclude POST variables
    cfg[:iVarsPREPOS] = grep(["PRE_POS","Pre_Pos"],features)
    features=setdiff(features, setdiff(  vcat(cfg[:xVarsDemos],cfg[:xVarsPost]) ,cfg[:iVarsPREPOS])  )

    cfg[:xVarsP0] =  setdiff(grep("0", features ),[cfg[:ProScore]] ) #exclude category variables(exclude P0's)
    features=setdiff(features,cfg[:xVarsP0])

    cfg[:xVarsReports] = grep(["Perc_","Pr_per_"],features)  #exclude Reporting vars
    features=setdiff(features,cfg[:xVarsReports])

    cfg[:xvars] = vcat(xflags,cfg[:xVarsDemos], cfg[:xVarsPost],cfg[:xVarsP0], cfg[:xVarsReports],cfg[:dropvars])
    features=setdiff(features,cfg[:dropvars])

    cfg[:ivars] = vcat(cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars])

    cfg[:ALL_vars_to_exclude] = setdiff(cfg[:xvars], cfg[:ivars])

    df_in[:group] = df_in[cfg[:exposed_flag_var]]
    df_in[:panid] = df_in[:experian_id]
    features = setdiff(unique(vcat(features,cfg[:iVarsPREPOS],cfg[:all_mandatory_vars],cfg[:scoring_vars],[:group,:panid])  ) ,[:experian_id] )

    cfg[:negativevars] = grep(vec(hcat([["P"*string(i),string(i)*"_"] for i=3:cfg[:num_products]]...)),features)   # get variables that need to have negative sign
    cfg[:positivevars] = grep(["P1", "1_","MODEL"], features)    # get variables that need to have positive sign

    if cfg[:P2_Competitor] == true
        cfg[:negativevars] = unique(vcat(cfg[:negativevars],grep(["P2","2_"],features)))
    else
        cfg[:positivevars] = unique(vcat(cfg[:positivevars],grep(["P2","2_"],features)))
    end

    return df_in[features]
end

dfd = reworkCFG!(df_in,cfg)

#######################################
#--------- Data Manipulation ---------#
#######################################

function data_Prep(df_data::DataFrame, cfg::OrderedDict)
    df_data[:isO]=false
    df_data[:whyO]=""

    df_data[df_data[:number_of_children_in_living_Un].>=4,:number_of_children_in_living_Un] = 4  # aggregate #of children for 4+ L_114
    #vars = getvars()   # :GROUP
    #vars = DataFrame(names=names(df_data),eltypes=eltypes(df_data))
    if typeof(df_data[:group]) in [DataArray{String,1}]
        df_data[ findin(df_data[:group],["//N","\\N"]), :group] = "0"
        df_data[DataArrays.isna(df_data[:group]), :group]="0"
        df_data[:group] = [parse(Int64,s) for s = df_data[:group]]
    else
        df_data[ DataArrays.isnan(df_data[:group]), :group] = 0
    end

    vars = DataFrame(names=names(df_data),eltypes=eltypes(df_data))
    for c in setdiff(vars[findin(vars[:eltypes],[String]),:names],cfg[:allrandoms]) # set variables as.numeric and replace NA's with zero
        print("Convert String: String->Numeric: ",c)
        try df_data[c] = map( x -> DataArrays.isna(x) ?  NaN : convert(Float64, x)  , df_data[c]) catch e println("  (failed)") end  #NA to NaN
        try df_data[c] = convert(Array{Float64}, df_data[c]) catch e end   # To Float64
    end
    vars = DataFrame(names=names(df_data),eltypes=eltypes(df_data))
    #vars = getvars()
    for c in setdiff(vars[findin(vars[:eltypes],[Float64]),:names],cfg[:allrandoms])  # replace NaN's with zero for numeric variables
        println("Replace Float64 NaN (0.0): ",c)
        df_data[ DataArrays.isnan(df_data[c]), c] = 0.0
    end
    for c in  setdiff(vars[findin(vars[:eltypes],[Int64]),:names],cfg[:allrandoms])
        println("Replace Int64 NaN (0) : ",c)
        df_data[ DataArrays.isnan(df_data[c]), c] = 0
    end

    # create factor variables -  Need to understand how this impacts the data passed to R - might need to do this in R
    #pool!(df_data, cfg[:allrandoms])
    #pool!(df_data, [,:group])

    # NOTE : Do QCs here
    df_data[df_data[:person_1_gender].=="U",:isO] = true   # remove HHs with no gender info
    df_data[df_data[:person_1_gender].=="U",:whyO] = "person_1_gender=U; remove HHs with no gender info"
    df_data[findin(df_data[:estimated_hh_income],["U","L"]),:estimated_hh_income]="L" # aggregate U and L levels of hh income

    for r in cfg[:random_campaigns]    # check and drop exposed HHs with no publisher info or non-exposed HHs with publisher info
        df_data[findin(df_data[r],["\\N","NULL","0","NONE"])  ,r] ="none"
        df_data[ (df_data[:isO].==false) & (df_data[r].!="none") & (df_data[:group].==0) ,:whyO] = "non exposed HHs with publisher info"
        df_data[ (df_data[:isO].==false) & (df_data[r].!="none") & (df_data[:group].==0) ,:isO] = true
        println(r," non exposed HHs with publisher info : ",nrow(df_data[df_data[:whyO].=="non exposed HHs with publisher info",:] )   )
        df_data[ (df_data[:isO].==false) & (df_data[r].=="none") & (df_data[:group].!=0) ,:whyO] = "exposed HHs with no publisher info"
        df_data[ (df_data[:isO].==false) & (df_data[r].=="none") & (df_data[:group].!=0) ,:isO] = true
        println(r," exposed HHs with no publisher info : ",nrow(df_data[df_data[:whyO].=="exposed HHs with no publisher info",:] )   )
    end

    # segments for outliers detection
    df_data[:data_NB_NE_B] = false
    df_data[ (df_data[:Buyer_Pre_P1].==0 ) & (df_data[:group].==0 ) & (df_data[:Buyer_Pos_P1].==1 ) ,:data_NB_NE_B] = true
    df_data[:data_B_E_NB] = false
    df_data[ (df_data[:Buyer_Pre_P1].==1 ) & (df_data[:group].==0 ) & (df_data[:Buyer_Pos_P1].==0 ) ,:data_B_E_NB] = true
    df_data[:pen_reduction] = false
    df_data[ (df_data[:Buyer_Pre_P1].==1) & (df_data[:group].==0) & (df_data[:Buyer_Pos_P1].==0 )  ,:pen_reduction] = true
    df_data[:occ_reduction] = false
    df_data[ (df_data[:group].==0) & (df_data[:Buyer_Pos_P1].==1) & (df_data[:Trps_POS_P1].< df_data[:Trps_PRE_P1] ) ,:occ_reduction] = true
    df_data[:dolocc_reduction] = false
    df_data[  (df_data[:group].==0) & (df_data[:Buyer_Pos_P1].==1) & (df_data[:Dol_per_Trip_POS_P1].< df_data[:Dol_per_Trip_PRE_P1] )  , :dolocc_reduction] = true

    return df_data[df_data[:isO].==false, : ]   #[setdiff(names(df_data),[:isO,:whyO])]
end

dfd = data_Prep(dfd, cfg);

function MatchMe(df_data::DataFrame,cfg::OrderedDict)
    df=df_data[df_data[:isO].==false,:]
    df_exp     = df[df[:group].==1,:]
    df_unexp   = df[df[:group].==0,:]
    df_exp_dim   = nrow(df_exp)
    df_unexp_dim = nrow(df_unexp)
    new_unexp_dim = df_unexp_dim*(df_unexp_dim>2000000 ? 0.3 : df_unexp_dim>1000000 ? 0.4 : df_unexp_dim>750000 ? 0.6 : 0.7)
    #df_sample=DataFrame
    if length(string(cfg[:ProScore])) == 0
        df_unexp_1 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==1),:]
        df_unexp_0 =  df[(df[:group].==0)&(df[:Buyer_Pre_P1].==0),:]

        df_exp_1_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==1),:])
        df_exp_0_dim = nrow(df[(df[:group].==1)&(df[:Buyer_Pre_P1].==0),:])
        df_unexp_1_dim = nrow(df_unexp_1)
        df_unexp_0_dim =  nrow(df_unexp_0)
        dim_sample0 = round(Int64, (new_unexp_dim-df_exp_dim  ) / (1+(df_exp_1_dim / df_exp_0_dim)) )
        dim_sample1 = round(Int64, new_unexp_dim - df_exp_dim - dim_sample0)

        new_df_unexp_1 = df_unexp_1[sample(1:size(df_unexp_1,1), dim_sample1 ),:]
        new_df_unexp_0 = df_unexp_0[sample(1:size(df_unexp_0,1), dim_sample0 ),:]
        df_data_sample  = vcat(df_exp,new_df_unexp_1,new_df_unexp_0)
    elseif length(string(cfg[:ProScore])) > 0
        sample_control_data=similar(df_unexp, 0)
        for (key, value) in countmap(df_exp[cfg[:ProScore]])
            sample_dim=round(Int64,new_unexp_dim*(value/df_exp_dim))
            temp_data = df_unexp[df_unexp[cfg[:ProScore]].==key,:]
            samp_data = temp_data[sample(1:size(temp_data,1), sample_dim, replace=false),:]
            sample_control_data = vcat(sample_control_data,    samp_data   )
        end

        sample_data = vcat(sample_control_data,df_exp)
        sample_df_unexp = sample_data[sample_data[:group].==0,:]
        sample_df_exp   = sample_data[sample_data[:group].==1,:]
        sample_df_unexp_1 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==1),:]
        sample_df_unexp_0 = sample_data[(sample_data[:group].==0)&(sample_data[:Buyer_Pre_P1].==0),:]
        sample_df_exp_1 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==1) ,:]
        sample_df_exp_0 =  sample_data[(sample_data[:group].==1)&(sample_data[:Buyer_Pre_P1].==0) ,:]
        sample_df_unexp_1_dim = nrow(sample_df_unexp_1)
        sample_df_unexp_0_dim = nrow(sample_df_unexp_0)
        sample_df_exp_1_dim = nrow(sample_df_exp_1)
        sample_df_exp_0_dim = nrow(sample_df_exp_0)
        dim_sampleA = round(Int64,(sample_df_exp_1_dim/sample_df_exp_0_dim)*sample_df_unexp_0_dim)
        dim_sampleB = round(Int64,(sample_df_exp_0_dim/sample_df_exp_1_dim)*sample_df_unexp_1_dim)

        if sample_df_unexp_1_dim/sample_df_unexp_0_dim > sample_df_exp_1_dim/sample_df_exp_0_dim
            new_df_unexp_1 = sample_df_unexp_1[sample(1:sample_df_unexp_1_dim,dim_sampleA , replace=false),:]
            df_data_sample = vcat(sample_df_exp,new_df_unexp_1,sample_df_unexp_0)
        else
            new_df_unexp_0 = sample_df_unexp_0[sample(1:sample_df_unexp_0_dim, dim_sampleB, replace=false ),:]
            df_data_sample = vcat(sample_df_exp,sample_df_unexp_1,new_df_unexp_0)
        end
    end

    rows2remove = setdiff(df_data[df_data[:isO].==false, :panid],df_data_sample[:panid])
    df_data[findin(df_data[:panid],rows2remove),:whyO]="NoMatch"
    df_data[findin(df_data[:panid],rows2remove),:isO]=true

    return df_data[df_data[:isO].==false, : ]  #[setdiff(names(df_data),[:isO,:whyO])]
end

dfd = MatchMe(dfd,cfg)

lowercase!(dfd)
cfg=lowercase(cfg)

#Not Used
dfd=dfd[setdiff(names(dfd),[:nonbuyer_pre_p1])]

######################################
#------------MODEL OBJECTS-----------#
######################################
abstract MModel

#!!! Multicollinearity, removing :[:fea_or_dis_trps_shr_dpp_p1,:fea_or_dis_trps_shr_dpp_p2,:fea_or_dis_trps_shr_dpp_p3,:fea_or_dis_trps_shr_dpp_p4]
type xGLM
    vfactors::Vector{Symbol}
    fmula::DataFrames.Formula
    xvars::Vector{Symbol}
    model::Any #DataFrameRegressionModel
    sdf::DataFrame
    wasSuccessful::Bool

    function xGLM(dfd::DataFrame, dist::Distribution, y_var::Symbol, logvar::Symbol , lnk::Link , vfactors::Array{Symbol} )
        this=new()
        this.wasSuccessful=false
        this.xvars=Symbol[]
        this.vfactors=setdiff(vfactors, [y_var,logvar])
        for l in 1:30
            #log_colname=Symbol("LOG_"*string(logvar))
            #this.vfactors=vcat(setdiff(this.vfactors,this.xvars),[log_colname])
            this.fmula = genFmula(y_var, this.vfactors, logvar  )
            try
                #this.model = glm(this.fmula,  dfd[vcat(this.vfactors,[y_var])], dist, LogLink(), offset=log(Array(dfd[logvar]  + 1)) )
                #dfd[log_colname]=log(Array(dfd[logvar]+1))
                f=this.fmula
                ##this.model = glm(this.fmula,  dfd[vcat(this.vfactors,[y_var,log_colname])], dist, m.lnk )   # LogLink()
                this.model = glm(f,  dfd[convert(Array{Symbol},vcat(f.lhs,f.rhs.args[3:end]))]  , dist, lnk )

                this.sdf = DataFrame(vars=vcat([:intercept],this.model.mf.terms.terms), coef=coef(this.model), se=stderr(this.model), zval=coef(this.model)./stderr(this.model) ,pval= ccdf(FDist(1, df_residual(this.model)), abs2(coef(this.model)./stderr(this.model))))
                this.wasSuccessful=true
                break
            catch e
                if isa(e, Base.LinAlg.PosDefException)
                    v=this.vfactors[e.info-1]
                    push!(this.xvars,v)
                    println("!!! Multicollinearity, removing :",v,"~~~",e.info-1, "\n~~~",e)
                else
                    println("....",e)
                    break
                end
            end
        end
        return this
    end
end

type MDolOcc <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MDolOcc(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="dolocc"; this.logvar=cfg[:dolocc_logvar]; this.y_var=cfg[:dolocc_y_var]; this.dist=Gamma()
        this.lnk=LogLink()
        this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars= Symbol[ cfg[:occ_y_var],cfg[:occ_logvar],cfg[:pen_y_var],cfg[:pen_logvar] ]
        this.Buyer_Pos_P1_is1=true
        if this.Buyer_Pos_P1_is1  this.exclude_vars = vcat(this.exclude_vars, [:buyer_pos_p1] )  end
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mdolocc = MDolOcc(dfd,cfg)

type MOcc <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MOcc(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="occ"; this.logvar=cfg[:occ_logvar]; this.y_var=cfg[:occ_y_var]; this.dist=Poisson()
        this.lnk=LogLink()
        this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars=Symbol[cfg[:dolocc_y_var],cfg[:dolocc_logvar],cfg[:pen_y_var],cfg[:pen_logvar] ]
        this.Buyer_Pos_P1_is1=true
        if this.Buyer_Pos_P1_is1  this.exclude_vars = vcat(this.exclude_vars, [:buyer_pos_p1] )  end
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mocc = MOcc(dfd,cfg)

type MPen <: MModel
    vars::Vector{Symbol}
    finalvars::Vector{Symbol}
    y_var::Symbol
    dist::Distribution
    lnk::Link
    mandatory_vars::Vector{Symbol}
    exclude_vars::Vector{Symbol}
    singularity_x::Vector{Symbol}
    removed_SingleLevelVars::Vector{Symbol}
    glm1_pvals::xGLM
    glm1_pvals_x::Vector{Symbol}
    glm2_ZnVIF::xGLM
    glm2_ZnVIF_x::Vector{Symbol}
    glm3_PnSigns::xGLM
    glm3_PnSigns_x::Vector{Symbol}
    glm4_PnSignsClean::xGLM
    glm4_PnSignsClean_x::Vector{Symbol}
    glm5::xGLM
    glm5_Z_x::Vector{Symbol}
    corrvars_x::Vector{Symbol}
    glm6_final::xGLM
    Buyer_Pos_P1_is1::Bool
    modelName::String
    logvar::Symbol
    function MPen(dfd::DataFrame,cfg::OrderedDict=Dict())
        this=new(); this.modelName="pen"; this.logvar=cfg[:pen_logvar]; this.y_var=cfg[:pen_y_var]; this.dist=Bernoulli() #Binomial()
        this.lnk=LogitLink()
        this.mandatory_vars=Symbol[this.y_var]
        this.removed_SingleLevelVars=Symbol[]
        this.glm1_pvals_x=Symbol[]
        this.glm2_ZnVIF_x=Symbol[]
        this.glm3_PnSigns_x=Symbol[]
        this.glm4_PnSignsClean_x=Symbol[]
        this.corrvars_x=Symbol[]
        this.exclude_vars=Symbol[cfg[:occ_y_var],cfg[:occ_logvar],cfg[:dolocc_y_var],cfg[:dolocc_logvar] ]
        this.Buyer_Pos_P1_is1=false
        if this.Buyer_Pos_P1_is1  this.exclude_vars = vcat(this.exclude_vars, [:buyer_pos_p1] )  end
        this.vars=setdiff(names(dfd),vcat(this.exclude_vars,[:iso,:whyo,:data_nb_ne_b, :data_b_e_nb ,:pen_reduction,:occ_reduction,:dolocc_reduction]))
        return this
    end
end
mpen = MPen(dfd,cfg)

    include("/home/rmadmin/.julia/v0.4/RegTools/src/diagnostics.jl")
    include("/home/rmadmin/.julia/v0.4/RegTools/src/misc.jl")
    include("/home/rmadmin/.julia/v0.4/RegTools/src/modsel.jl")

#include("/media/u01/analytics/RegTools/diagnostics.jl")
#include("/media/u01/analytics/RegTools/misc.jl")
#include("/media/u01/analytics/RegTools/modsel.jl")

function vif!(g::xGLM)
    vdf=vif(g.model)
    vdf[:vars] = convert(Array{Symbol}, vdf[:variable])
    g.sdf = join(g.sdf,vdf[[:vars,:vif]], on = :vars, kind=:outer)
end

function checksingularity(form::Formula, data::DataFrame, tolerance = 1.e-8)
    mf = ModelFrame(form, data)
    mm = ModelMatrix(mf)
    qrf = qrfact!(mm.m, Val{true})
    vals = abs.(diag(qrf[:R]))
    firstbad = findfirst(x -> x < min(tolerance, 0.5) * vals[1], vals)
    if firstbad == 0
        return Symbol[]
    end
    mf.terms.terms[view(mm.assign[qrf[:p]], firstbad:length(vals))]
end
# -- test --
#m=mocc
#custom_vars=[:dolocc_reduction,:occ_reduction,:pen_reduction,:data_b_e_nb,:data_nb_ne_b,:whyo,:iso]
#required_vars=vcat([m.y_var,:panid,m.logvar],cfg[:random_demos],cfg[:random_campaigns],cfg[:scoring_vars])
#vars=setdiff(m.vars,vcat(required_vars,custom_vars))
#m.removed_SingleLevelVars=FS_singleLevel(dfd,vars)
#vars = rmVars(m.removed_SingleLevelVars)
#f=genFmula(m.y_var,vars,m.logvar)

#checksingularity(f, dfd)
# -- test end ---

# =======================================================================================
# =======================================================================================

using IRImodels

factor_cols=vcat( [ cfg[:proscore], :group, :panid], cfg[:allrandoms] )

function featureSelection(dfd::DataFrame, m::MModel)
    function rmVars(v::Array{Symbol})
        v=setdiff(v,[:group])
        return setdiff(vars,v)
    end

    log_colname=Symbol("LOG_"*string(m.logvar))
    dfd[log_colname]=log(Array(dfd[m.logvar]+1))
    #this.vfactors=vcat(setdiff(this.vfactors,this.xvars),[log_colname])
    #this.fmula = genFmula(y_var, this.vfactors, logvar  )

    custom_vars=[:dolocc_reduction,:occ_reduction,:pen_reduction,:data_b_e_nb,:data_nb_ne_b,:whyo,:iso]
    required_vars=vcat([m.y_var,:panid,m.logvar],cfg[:random_demos],cfg[:random_campaigns],cfg[:scoring_vars])
    vars=setdiff(vcat(m.vars,[log_colname]),vcat(required_vars,custom_vars))

    #SingleValue
    m.removed_SingleLevelVars=FS_singleLevel(dfd,vars)
    vars = rmVars(m.removed_SingleLevelVars)

    # Singularity
    m.singularity_x = checksingularity(genFmula(m.y_var,vars,m.logvar), dfd)
    vars = rmVars(m.singularity_x)

    #PVals
    m.glm1_pvals = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , vars  )
    g1=m.glm1_pvals
    m.glm1_pvals_x=g1.sdf[g1.sdf[:pval].>0.7,:vars]
    vars = rmVars(vcat(m.glm1_pvals_x, g1.xvars ) )

    #Z & Vif
    m.glm2_ZnVIF = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk ,vars  )
    g2=m.glm2_ZnVIF
    vif!(g2)
    z = g2.sdf[abs(g2.sdf[:zval]).<1.96,:vars]
    v = g2.sdf[ !DataArrays.isna(g2.sdf[:vif])&(g2.sdf[:vif].>15),:vars]
    m.glm2_ZnVIF_x =intersect(z,v)
    vars = rmVars(vcat(m.glm2_ZnVIF_x, g2.xvars) )

    function chkSigns(m::MModel, vars::Array{Symbol}, dfd::DataFrame, cfg::OrderedDict)  # Pvalue & Signs
        g = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , vars  )
        neutralvars = setdiff(vars,vcat(cfg[:negativevars],cfg[:positivevars]))
        neg=intersect(cfg[:negativevars],g.sdf[g.sdf[:coef].<0,:vars])
        pos=intersect(cfg[:positivevars],g.sdf[g.sdf[:coef].>0,:vars])
        varstokeep = intersect(vcat(neutralvars, pos,neg) ,  g.sdf[ g.sdf[:pval].<cfg[:pvalue_lvl] ,:vars] )
        return g, varstokeep
    end

    (m.glm3_PnSigns, initialvars) = chkSigns(m, vars, dfd, cfg)
    (m.glm4_PnSignsClean, vars_2) = chkSigns(m, convert(Array{Symbol},initialvars) , dfd, cfg)

    function getCorrVars(dfd::DataFrame, vars_2::Array{Symbol})
        rm_lst=Symbol[]
        if (length(vars_2) > 1) & (   length(getColswithType("num", dfd, convert(Array{Symbol},vars_2) ) ) > 1  )
            stackdf = corrDF(dfd,vars_2)
            stackdf[:variable_pval] = [ m.glm4_PnSignsClean.sdf[m.glm4_PnSignsClean.sdf[:vars].==c,:pval][1]   for c in stackdf[:variable]]
            stackdf[:vars_pval] = [ m.glm4_PnSignsClean.sdf[m.glm4_PnSignsClean.sdf[:vars].==c,:pval][1]   for c in stackdf[:vars]]
            stackdf[:most_Sig] = map((x,y) -> x < y ? "variable" : "vars" ,stackdf[:variable_pval],stackdf[:vars_pval])

            for row in eachrow(stackdf[(stackdf[:value].> 0.8) | (stackdf[:value].<-0.8),:])
                if row[:vars] == "group"
                    push!(rm_lst,row[:variable])
                elseif row[:variable] == "group"
                    push!(rm_lst,row[:vars])
                else
                    row[:most_Sig] == "variable" ? push!(rm_lst,row[:vars]) : push!(rm_lst,row[:variable])
                end
            end
        end
        return rm_lst
    end

    m.corrvars_x = getCorrVars(dfd,convert(Array{Symbol},setdiff(vars_2,factor_cols)))
    vars_2 = setdiff(vars_2,m.corrvars_x)
    (m.glm5, m.finalvars) =  chkSigns(m, convert(Array{Symbol},vars_2), dfd, cfg)

    # Final Review:
    m.glm6_final = xGLM(dfd, m.dist, m.y_var, m.logvar, m.lnk , convert(Array{Symbol},vcat(m.finalvars,[:group]))  )
end

featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], mocc)
featureSelection(dfd[(dfd[:iso].==false)&(dfd[:buyer_pos_p1].==1),:], mdolocc)
featureSelection(dfd[(dfd[:iso].==false) ,:], mpen)

"""

max_stars_repo_name: werbac/jlib

max_issues_repo_name: werbac/jlib

max_forks_repo_name: werbac/jlib


text:
const DISCRETE_INPLACE_DEFAULT = convert(DiscreteFunction{true},(du,u,p,t) -> du.=u)
const DISCRETE_OUTOFPLACE_DEFAULT = convert(DiscreteFunction{false},(u,p,t) -> u)

"""
$(TYPEDEF)

Defines a discrete problem.

# Fields

$(FIELDS)
"""
struct DiscreteProblem{uType,tType,isinplace,P,F,K} <: AbstractDiscreteProblem{uType,tType,isinplace}
  """The function in the map."""
  f::F
  """The initial condition."""
  u0::uType
  """The timespan for the problem."""
  tspan::tType
  """The parameter values of the function."""
  p::P
  """ A callback to be applied to every solver which uses the problem."""
  kwargs::K
  @add_kwonly function DiscreteProblem{iip}(f::AbstractDiscreteFunction{iip},
                                            u0,tspan::Tuple,p=NullParameters();
                                            kwargs...) where {iip}
    _tspan = promote_tspan(tspan)
    new{typeof(u0),typeof(_tspan),isinplace(f,4),
        typeof(p),
        typeof(f),typeof(kwargs)}(f,u0,_tspan,p,kwargs)
  end

  function DiscreteProblem{iip}(u0::Nothing,tspan::Nothing,p=NullParameters();
                                callback = nothing) where {iip}
    if iip
      f = DISCRETE_INPLACE_DEFAULT
    else
      f = DISCRETE_OUTOFPLACE_DEFAULT
    end
    new{Nothing,Nothing,iip,typeof(p),
        typeof(f),typeof(callback)}(f,nothing,nothing,p,callback)
  end

  function DiscreteProblem{iip}(f,u0,tspan,p=NullParameters();kwargs...) where {iip}
    DiscreteProblem(convert(DiscreteFunction{iip},f),u0,tspan,p;kwargs...)
  end
end

"""
    DiscreteProblem{isinplace}(f,u0,tspan,p=NullParameters(),callback=nothing)

Defines a discrete problem with the specified functions.
"""
function DiscreteProblem(f::AbstractDiscreteFunction,u0,tspan::Tuple,p=NullParameters();kwargs...)
  DiscreteProblem{isinplace(f)}(f,u0,tspan,p;kwargs...)
end

function DiscreteProblem(f::Base.Callable,u0,tspan::Tuple,p=NullParameters();kwargs...)
  iip = isinplace(f,4)
  DiscreteProblem(convert(DiscreteFunction{iip},f),u0,tspan,p;kwargs...)
end

"""
$(SIGNATURES)

Define a discrete problem with the identity map.
"""
function DiscreteProblem(u0::Union{AbstractArray,Number},tspan::Tuple,p=NullParameters();kwargs...)
  iip = typeof(u0) <: AbstractArray
  if iip
    f = DISCRETE_INPLACE_DEFAULT
  else
    f = DISCRETE_OUTOFPLACE_DEFAULT
  end
  DiscreteProblem(f,u0,tspan,p;kwargs...)
end

ext: jl


text:
function extract_thunk(f)
    if isexpr(f, :->, 2) && isexpr(f.args[1], :tuple, 1)
        arg, = f.args[1].args
        body = f.args[2]
        return (arg, body)
    else
        error("invalid argument: ", f)
    end
end

macro and_then(f, ex)
    arg, body = extract_thunk(f)
    quote
        result = $(esc(ex))
        if Try.isok(result)
            let $(esc(arg)) = Try.unwrap(result)
                $(esc(body))
            end
        else
            result
        end
    end
end

macro or_else(f, ex)
    arg, body = extract_thunk(f)
    quote
        result = $(esc(ex))
        if Try.iserr(result)
            let $(esc(arg)) = Try.unwrap_err(result)
                $(esc(body))
            end
        else
            result
        end
    end
end


text:
abstract type Weight end

#=
    Sum over the bins that are within `threshold`% of the minimum.
    A large value means the well is shallow.
=#
struct ThresholdSum <: Weight
    threshold::Float64
    function ThresholdSum(threshold::Real)
        0 < threshold && return new(1.0 + threshold)
        throw(ArgumentError("Threshold must be less than 1"))
    end
end

#=
  Sum over the bins that are within the rectangle inscribing the
  the bins which are `threshold`% of the minimum.
  A large value means the well is shallow. Punished asymmetric wells
  more than ThresholdSum
=#
struct ThresholdArea <: Weight
    threshold::Float64
    function ThresholdArea(threshold::Real)
        0 < threshold && return new(1.0 + threshold)
        throw(ArgumentError("Threshold must be less than 1"))
    end
end

struct ThresholdSumShift <: Weight
    threshold::Float64
    shift::Float64
    function ThresholdSumShift(threshold::Real)
        0 < threshold && return new(1.0 + threshold, 1.0)
        throw(ArgumentError("Threshold must be less than 1"))
    end
end

#=
   Sum over the bins within the `radius` distance of the
   minimum, minus the minimum. A high score means the well is sharp.
=#
struct WellSum <: Weight
    radius::Int
end

function (w::ThresholdSum)(x)
    min = minimum(x)
    sum(x .≤ w.threshold*min)
end

function (w::ThresholdArea)(x)
    min = minimum(x)
    thresholded = x .≤ w.threshold*min
    maxx = maxy = -1
    miny = minx = 1e5
    for x in 1:size(thresholded, 1), y in 1:size(thresholded, 2)
        if thresholded[x, y]
            if x < minx
                minx = x
            end
            if x > maxx
                maxx = x
            end
            if y < miny
                miny = y
            end
            if y > maxy
                maxy = y
            end
        end
    end
    sidex = maxx - minx
    sidey = maxy - miny
    (sidex > 0 ? sidex : 1) * (sidey > 0 ? sidey : 1)
end

function (w::ThresholdSumShift)(x)
    min = minimum(x) + w.shift
    sum(x .≤ w.threshold*min)
end

function (w::WellSum)(x)
    truth = argmin(x)
    r = w.radius
    xstart, ystart = truth.I .- r
    xstop, ystop = truth.I .+ r
    xstart = xstart > 0 ? xstart : 1
    ystart = ystart > 0 ? ystart : 1
    xstop = xstop < size(x, 1) ? xstop : size(x, 1)
    ystop = ystop < size(x, 2) ? ystop : size(x, 2)
    sum(@view x[xstart:xstop, ystart:ystop]) - minimum(x)
end

max_stars_repo_name: Caronthir/JSort

max_issues_repo_name: Caronthir/JSort

max_forks_repo_name: Caronthir/JSort


text:
@testset "Strings" begin

@testset "Strings: Is Palindrome" begin
    s = "AA"     # A palindrome
    @test is_palindrome(s) == true
    s = "ABC"     # Not a palindrome
    @test is_palindrome(s) == false
    s = "Statistics"    # Not a palindrome
    @test is_palindrome(s) == false
    s = "Stats"    # A palindrome
    @test is_palindrome(s) == true
    s = "Racecar"      # A palindrome
    @test is_palindrome(s) == true
    x = "Hello"      # Not a palindrome
    @test is_palindrome(s) == false
end

end

max_stars_repo_name: arubhardwaj/Julia

max_issues_repo_name: arubhardwaj/Julia

max_forks_repo_name: arubhardwaj/Julia


text:
using Sherlogs
using Test

include("sherlogs.jl")
include("logbook.jl")


text:
using VacBEM
using Test


text:
export IterationsPlot2D, RelaxationPlot2D

using Relaxation
using PyPlot

function IterationsPlot2D(un::Int64,deux::Int64,
                          trois::Int64,quatre::Int64,
                          orb::OutputRelaxationmod.OutputRelaxation;
                          i::Int64=-1,title::String="",
                          xlabel::String="x_1",
                          ylabel::String="x_2")

 if i<0 #on fait toutes les iterations
  i=1;niter=size(orb.inner_output_alas[i].xtab,1)
 else
  it=i;niter=1;
 end

 for j=i:niter
  figure(j) #on pourrait aussi faire un subplot(111)
  PyPlot.plot(orb.inner_output_alas[j].xtab[un,:], orb.inner_output_alas[j].xtab[deux,:], color="blue",marker="*",label="xk")

  PyPlot.plot(orb.inner_output_alas[j].xtab[trois,:], orb.inner_output_alas[j].xtab[quatre,:], color="green",marker="*",label="sk")

  ub=max(maximum(orb.inner_output_alas[j].xtab[un,:]),maximum(orb.inner_output_alas[j].xtab[deux,:]))
  RelaxationPlot2D(ub,orb.rtab[j],j,orb)

  if xlabel!=""
   PyPlot.xlabel(xlabel)
  end
  if xlabel!=""
   PyPlot.ylabel(ylabel)
  end
  if title!=""
   PyPlot.title(title)
  end
 end

 nothing
end

function RelaxationPlot2D(ub::Float64,
                          tb::Float64,
                          i::Int64,
                          orb::OutputRelaxationmod.OutputRelaxation;
                          legend::String="Butterfly",
                          pas::Float64=0.1)

 x1=orb.stab[i]:pas:ub
 x2=collect(Relaxation.psi(x1,orb.rtab[i],orb.stab[i],orb.ttab[i]))

 PyPlot.plot(x1, x2, color="red",label=legend)
 PyPlot.plot(x2, x1, color="red")
 PyPlot.plot(-tb:pas:ub,-tb*ones(-tb:pas:ub), color="red")
 PyPlot.plot(-tb*ones(-tb:pas:ub),-tb:pas:ub, color="red")
 PyPlot.plot(0:pas:ub,zeros(0:pas:ub), color="black")
 PyPlot.plot(zeros(0:pas:ub),0:pas:ub, color="black")
 PyPlot.axis("equal")
 PyPlot.legend(loc="upper left")

 nothing
end


text:
# result store keeps results of completed tasks.
# also publishes events when a result is published.
#
# creation:
# - the subspace is cleared
# - key for event created with atomic zero addition
#
# addition:
# - create a new versionstamped key with taskid and result set as value
# - atomic increment event
#
# deletion (on refcount becoming zero): (not implemented yet)
# - create a new versionstamped key with taskid and result set as nothing
# - atomic increment event
#
# on event (update in-memory state):
# - results are maintained in a dict
# - get all key values from last versionstamp till the last key
# - for every key-value
#   - if this is a new result
#     - add taskid and result to in-memory dict
#     - update current key
#   - if this is a deletion dut to refcount becoming zero
#     - delete taskid from in-memory dict
#     - update current key

mutable struct FdbResultStore
    db::FDBDatabase
    dictpath::Vector{UInt8}
    eventpath::Vector{UInt8}
    dict::Dict{TaskIdType,Vector{UInt8}}
    watchhandle::Union{Void,FDBFuture}
    versionstamp::Vector{UInt8}
    eventprocessor::Union{Void,Task}
    callback::Union{Void,Function}
    run::Bool

    function FdbResultStore(db::FDBDatabase, path::String; callback=nothing)
        dictpath = convert(Vector{UInt8}, joinpath(path, "r") * "/")
        eventpath = convert(Vector{UInt8}, joinpath(path, "re"))
        new(db, dictpath, eventpath, Dict{TaskIdType,Tuple}(), nothing, UInt8[], nothing, callback, true)
    end
end
new_key(rs::FdbResultStore) = prep_atomic_key!(copy(rs.dictpath), length(rs.dictpath)+1)

set_callback(rs::FdbResultStore, callback) = (rs.callback = callback)

function clear(rs::FdbResultStore)
    start_key = vcat(rs.dictpath, zeros(UInt8, 10))
    end_key = vcat(rs.dictpath, ones(UInt8, 10) * 0xff)
    open(FDBTransaction(rs.db)) do tran
        clearkeyrange(tran, start_key, end_key)
        clearkey(tran, rs.eventpath)
    end
    nothing
end

function init(rs::FdbResultStore)
    open(FDBTransaction(rs.db)) do tran
        atomic_add(tran, rs.eventpath, 0)
    end
    nothing
end

function publish_result(rs::FdbResultStore, tid::TaskIdType, result::Vector{UInt8})
    open(FDBTransaction(rs.db)) do tran
        val = vcat(reinterpret(UInt8, [tid]), result)
        atomic_setval(tran, new_key(rs), val, FDBMutationType.SET_VERSIONSTAMPED_KEY)
        atomic_add(tran, rs.eventpath, 1)
    end
    nothing
end

function range(rs::FdbResultStore)
    r1 = isempty(rs.versionstamp) ? vcat(rs.dictpath, UInt8[0]) : rs.versionstamp
    r2 = vcat(rs.dictpath, UInt8[0xff])
    r1, r2
end

function read_updates(rs::FdbResultStore, maxretries::Int=5)
    while maxretries > 0
        maxretries -= 1
        try
            return open(FDBTransaction(rs.db)) do tran
                startkey,endkey = range(rs)
                getrange(tran, keysel(FDBKeySel.first_greater_than, startkey), keysel(FDBKeySel.first_greater_than, endkey); limit=500)
            end
        catch ex
            ((maxretries > 0) && isa(ex, FDBError) && (ex.code == 1007)) || rethrow(ex)
        end
    end
end

function on_event(rs::FdbResultStore)
    more = true

    while more
        # process all updates
        kvs, more = read_updates(rs)
        if (kvs !== nothing) && !isempty(kvs)
            for kv in kvs
                key,_val = kv
                tid = reinterpret(TaskIdType, _val[1:sizeof(TaskIdType)])[1]
                result = _val[(sizeof(TaskIdType)+1):end]
                rs.dict[tid] = result
                if rs.callback !== nothing
                    rs.callback(tid, result)
                end
            end

            # remember the last versionstamp processed
            key,_val = kvs[end]
            rs.versionstamp = key
        end
    end
    nothing
end

function start_watch(rs::FdbResultStore)
    open(FDBTransaction(rs.db)) do tran
        watchhandle = FDBFuture()
        watchtask = watchkey(tran, rs.eventpath; handle=watchhandle)
        rs.watchhandle = watchhandle
        watchtask
    end
end

function process_events(rs::FdbResultStore)
    while rs.run
        watchtask = start_watch(rs)

        # process events
        # - either because an event triggered watchtask
        # - or those that we might have missed while not watching
        on_event(rs)

        try
            wait(watchtask)
        catch ex
            (isa(ex, FDBError) && (ex.code == 1101)) || rethrow(ex)
        end
    end
    nothing
end

function start_processing_events(rs::FdbResultStore)
    rs.run = true
    if rs.eventprocessor === nothing
        rs.eventprocessor = @schedule process_events(rs)
    end
    nothing
end

function stop_processing_events(rs::FdbResultStore)
    rs.run = false
    if rs.eventprocessor !== nothing
        if !istaskdone(rs.eventprocessor)
            open(FDBTransaction(rs.db)) do tran
                atomic_add(tran, rs.eventpath, 1)
            end
            wait(rs.eventprocessor)
        end
        rs.eventprocessor = nothing
    end
    nothing
end


text:
### DATA STRUCTURES ###
"""    Player(i, owned::Array, location, injail, money, getoutjailfree)"""
mutable struct Player
    i::Integer
    owned::Array{Integer}
    location::Integer
    injail::Bool
    money::Integer
    getoutjailfree::Integer
end
"""    Square(i, name, color, owner, actiontype, actionQTY, built, rents::Array)
Square.built = 0 for nothing; 1,2,3,4 for houses; 5 for hotel"""
mutable struct Square
    i::Integer
    name::String
    color::String
    owner::Integer
    actiontype::String
    actionQTY::Integer
    built::Integer
    rents::Array{Integer}
end
"""    Community(i, description, actiontype, actionQTY)"""
struct Community
    i::Integer
    description::String
    actiontype::String
    actionQTY::Integer
end
"""    Chance(i, description, actiontype, actionQTY)"""
struct Chance
    i::Integer
    description::String
    actiontype::String
    actionQTY::Integer
end
"""    Game(currentplayer, players, squares, chestN, chestcards, chanceN, chancecards)"""
mutable struct Game
    currentplayer::Integer
    players::Array{Player}
    squares::Array{Square}
    chestN::Integer
    chestcards::Array{Community}
    chanceN::Integer
    chancecards::Array{Chance}
end

### DATA TABLES ###
"""    Square(i, name, color, owner, actiontype, actionQTY, built, rents::Array)
Square.built = 0 for nothing; 1,2,3,4 for houses; 5 for hotel"""
const origsquares = [
    Square(1, "Go", "", 0, "none", 0, 0, []),
    Square(2, "Old Kent Road", "brown", 0, "rent", 0, 0, [2, 10, 30, 90, 160, 250]),
    Square(3, "Community Chest", "", 0, "community", 0, 0, []),
    Square(4, "Whitechapel Road", "brown", 0, "rent", 0, 0, [4, 20, 60, 180, 320, 450]),
    Square(5, "Income Tax", "", 0, "pay", 200, 0, []),
    Square(6, "Kings Cross Station", "", 0, "station", 0, 0, []),
    Square(7, "The Angel Islington", "cyan", 0, "rent", 0, 0, [6, 30, 90, 270, 400, 550]),
    Square(8, "Chance", "", 0, "chance", 0, 0, []),
    Square(9, "Euston Road", "cyan", 0, "rent", 0, 0, [6, 30, 90, 270, 400, 550]),
    Square(10, "Pentonville Road", "cyan", 0, "rent", 0, 0, [8, 40, 100, 300, 450, 600]),
    Square(11, "Jail", "", 0, "none", 0, 0, []),
    Square(12, "Pall Mall", "pink", 0, "rent", 0, 0, [10, 50, 150, 450, 625, 750]),
    Square(13, "Electric Company", "", 0, "utility", 0, 0, []),
    Square(14, "Whitehall", "pink", 0, "rent", 0, 0, [10, 50, 150, 450, 625, 750]),
    Square(15, "Northumberland Avenue", "pink", 0, "rent", 0, 0, [12, 60, 180, 500, 700, 900]),
    Square(16, "Marylebone Station", "", 0, "station", 0, 0, []),
    Square(17, "Bow Street", "orange", 0, "rent", 0, 0, [14, 70, 200, 550, 750, 950]),
    Square(18, "Community Chest", "", 0, "community", 0, 0, []),
    Square(19, "Marlborough Street", "orange", 0, "rent", 0, 0, [14, 70, 200, 550, 750, 950]),
    Square(20, "Vine Street", "orange", 0, "rent", 0, 0, [16, 80, 220, 600, 800, 1000]),
    Square(21, "Free Parking", "", 0, "none", 0, 0, []),
    Square(22, "Strand", "red", 0, "rent", 0, 0, [18, 90, 250, 700, 875, 1050]),
    Square(23, "Chance", "", 0, "chance", 0, 0, []),
    Square(24, "Fleet Street", "red", 0, "rent", 0, 0, [18, 90, 250, 700, 875, 1050]),
    Square(25, "Trafalgar Square", "red", 0, "rent", 0, 0, [20, 100, 300, 750, 925, 1100]),
    Square(26, "Fenchurch Street Station", "", 0, "station", 0, 0, []),
    Square(27, "Leicester Square", "yellow", 0, "rent", 0, 0, [22, 110, 330, 800, 975, 1150]),
    Square(28, "Coventry Street", "yellow", 0, "rent", 0, 0, [22, 110, 330, 800, 975, 1150]),
    Square(29, "Water Works", "", 0, "utility", 0, 0, []),
    Square(30, "Piccadilly", "yellow", 0, "rent", 0, 0, [24, 120, 360, 850, 1025, 1200]),
    Square(31, "Go To Jail", "", 0, "jail", 0, 0, []),
    Square(32, "Regent Street", "green", 0, "rent", 0, 0, [26, 130, 390, 900, 1100, 1275]),
    Square(33, "Oxford Street", "green", 0, "rent", 0, 0, [26, 130, 390, 900, 1100, 1275]),
    Square(34, "Community Chest", "", 0, "community", 0, 0, []),
    Square(35, "Bond Street", "green", 0, "rent", 0, 0, [28, 150, 450, 1000, 1200, 1400]),
    Square(36, "Liverpool Street Station", "", 0, "station", 0, 0, []),
    Square(37, "Chance", "", 0, "chance", 0, 0, []),
    Square(38, "Park Lane", "blue", 0, "rent", 0, 0, [35, 175, 500, 1100, 1300, 1500]),
    Square(39, "Super Tax", "", 0, "pay", 100, 0, []),
    Square(40, "Mayfair", "blue", 0, "rent", 0, 0, [50, 200, 600, 1400, 1700, 2000])
]
const origchestcards = [
    Community(1, "Advance to Go", "goto", 1),
    Community(2, "Go back to Old Kent Road", "goto", 2),
    Community(3, "Go to jail. Move directly to jail. Do not pass Go. Do not collect £200", "jail", 0),
    Community(4, "Pay hospital £100", "pay", 100),
    Community(5, "Doctor's fee. Pay £50", "pay", 50),
    Community(6, "Pay your insurance premium £50", "pay", 50),
    Community(7, "Bank error in your favour. Collect £200", "receive", 200),
    Community(8, "Annuity matures. Collect £100", "receive", 100),
    Community(9, "You inherit £100", "receive", 100),
    Community(10, "From sale of stock you get £50", "receive", 50),
    Community(11, "Receive interest on 7% preference shares: £25", "receive", 25),
    Community(12, "Income tax refund. Collect £20", "receive", 20),
    Community(13, "You have won second prize in a beauty contest. Collect £10", "receive", 10),
    Community(14, "It is your birthday. Collect £10 from each player", "10each", 0),
    Community(15, "Get out of jail free. This card may be kept until needed or sold", "getoutofjail", 1),
    Community(16, "Pay a £10 fine or take a Chance", "pay", 10)
]
const origchancecards = [
    Chance(1, "Advance to Go", "goto", 1),
    Chance(2, "Go to jail. Move directly to jail. Do not pass Go. Do not collect £200", "jail", 0),
    Chance(3, "Advance to Pall Mall. If you pass Go collection £200", "goto", 12),
    Chance(4, "Take a trip to Marylebone Station and if you pass Go collect £200", "goto", 16),
    Chance(5, "Advance to Trafalgar Square. If you pass Go collect £200", "goto", 25),
    Chance(6, "Advance to Mayfair", "goto", 40),
    Chance(7, "Go back three spaces", "goback", 3),
    Chance(8, "Make general repairs on all of your houses. For each house pay £25. For each hotel pay £100", "repairs", 0),
    Chance(9, "You are assessed for street repairs: £40 per house; £115 per hotel", "st_repairs", 0),
    Chance(10, "Pay school fees of £150", "pay", 150),
    Chance(11, "Drunk in charge fine £20", "pay", 20),
    Chance(12, "Speeding fine £15", "pay", 15),
    Chance(13, "Your building loan matures. Receive £150", "receive", 150),
    Chance(14, "You have won a crossword competition. Collect £100", "receive", 100),
    Chance(15, "Bank pays you dividend of £50", "receive", 50),
    Chance(16, "Get out of jail free. This card may be kept until needed or sold", "getoutofjail", 2)
]

# actiontypes
# "none"
# "pay" -- QTY gives amount
# "community"
# "chance"
# "jail"
# "goto" -- QTY gives location code to mode to
# "receive" -- QTY gives amount
# "getoutofjail" -- QTY 1 for chestcards. 2 for chancecards
# "goback" -- QTY gives n steps backward
# "10each" -- receive $10 from each player
# "repairs" -- For each house pay £25. For each hotel pay £100
# "st_repairs" -- Pay £40 per house, £115 per hotel
# "rent" -- pay rent to owner
# "station" -- pay rent to owner
# "utility" -- pay rent to owner


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

@doc raw"""
    ObjsSubteam(;
        auto_provision=nothing,
        auto_type=nothing,
        channel_count=nothing,
        created_by=nothing,
        date_create=nothing,
        date_delete=nothing,
        date_update=nothing,
        deleted_by=nothing,
        description=nothing,
        enterprise_subteam_id=nothing,
        handle=nothing,
        id=nothing,
        is_external=nothing,
        is_subteam=nothing,
        is_usergroup=nothing,
        name=nothing,
        prefs=nothing,
        team_id=nothing,
        updated_by=nothing,
        user_count=nothing,
        users=nothing,
    )

    - auto_provision::Bool
    - auto_type::Any
    - channel_count::Int32
    - created_by::DefsUserId
    - date_create::Int32
    - date_delete::Int32
    - date_update::Int32
    - deleted_by::Any
    - description::String
    - enterprise_subteam_id::String
    - handle::String
    - id::DefsSubteamId
    - is_external::Bool
    - is_subteam::Bool
    - is_usergroup::Bool
    - name::String
    - prefs::ObjsSubteamPrefs
    - team_id::DefsTeam
    - updated_by::DefsUserId
    - user_count::Int32
    - users::Vector{DefsUserId}
"""
mutable struct ObjsSubteam <: SwaggerModel
    auto_provision::Any # spec type: Union{ Nothing, Bool } # spec name: auto_provision
    auto_type::Any # spec type: Union{ Nothing, Any } # spec name: auto_type
    channel_count::Any # spec type: Union{ Nothing, Int32 } # spec name: channel_count
    created_by::Any # spec type: Union{ Nothing, DefsUserId } # spec name: created_by
    date_create::Any # spec type: Union{ Nothing, Int32 } # spec name: date_create
    date_delete::Any # spec type: Union{ Nothing, Int32 } # spec name: date_delete
    date_update::Any # spec type: Union{ Nothing, Int32 } # spec name: date_update
    deleted_by::Any # spec type: Union{ Nothing, Any } # spec name: deleted_by
    description::Any # spec type: Union{ Nothing, String } # spec name: description
    enterprise_subteam_id::Any # spec type: Union{ Nothing, String } # spec name: enterprise_subteam_id
    handle::Any # spec type: Union{ Nothing, String } # spec name: handle
    id::Any # spec type: Union{ Nothing, DefsSubteamId } # spec name: id
    is_external::Any # spec type: Union{ Nothing, Bool } # spec name: is_external
    is_subteam::Any # spec type: Union{ Nothing, Bool } # spec name: is_subteam
    is_usergroup::Any # spec type: Union{ Nothing, Bool } # spec name: is_usergroup
    name::Any # spec type: Union{ Nothing, String } # spec name: name
    prefs::Any # spec type: Union{ Nothing, ObjsSubteamPrefs } # spec name: prefs
    team_id::Any # spec type: Union{ Nothing, DefsTeam } # spec name: team_id
    updated_by::Any # spec type: Union{ Nothing, DefsUserId } # spec name: updated_by
    user_count::Any # spec type: Union{ Nothing, Int32 } # spec name: user_count
    users::Any # spec type: Union{ Nothing, Vector{DefsUserId} } # spec name: users

    function ObjsSubteam(;auto_provision=nothing, auto_type=nothing, channel_count=nothing, created_by=nothing, date_create=nothing, date_delete=nothing, date_update=nothing, deleted_by=nothing, description=nothing, enterprise_subteam_id=nothing, handle=nothing, id=nothing, is_external=nothing, is_subteam=nothing, is_usergroup=nothing, name=nothing, prefs=nothing, team_id=nothing, updated_by=nothing, user_count=nothing, users=nothing)
        o = new()
        validate_property(ObjsSubteam, Symbol("auto_provision"), auto_provision)
        setfield!(o, Symbol("auto_provision"), auto_provision)
        validate_property(ObjsSubteam, Symbol("auto_type"), auto_type)
        setfield!(o, Symbol("auto_type"), auto_type)
        validate_property(ObjsSubteam, Symbol("channel_count"), channel_count)
        setfield!(o, Symbol("channel_count"), channel_count)
        validate_property(ObjsSubteam, Symbol("created_by"), created_by)
        setfield!(o, Symbol("created_by"), created_by)
        validate_property(ObjsSubteam, Symbol("date_create"), date_create)
        setfield!(o, Symbol("date_create"), date_create)
        validate_property(ObjsSubteam, Symbol("date_delete"), date_delete)
        setfield!(o, Symbol("date_delete"), date_delete)
        validate_property(ObjsSubteam, Symbol("date_update"), date_update)
        setfield!(o, Symbol("date_update"), date_update)
        validate_property(ObjsSubteam, Symbol("deleted_by"), deleted_by)
        setfield!(o, Symbol("deleted_by"), deleted_by)
        validate_property(ObjsSubteam, Symbol("description"), description)
        setfield!(o, Symbol("description"), description)
        validate_property(ObjsSubteam, Symbol("enterprise_subteam_id"), enterprise_subteam_id)
        setfield!(o, Symbol("enterprise_subteam_id"), enterprise_subteam_id)
        validate_property(ObjsSubteam, Symbol("handle"), handle)
        setfield!(o, Symbol("handle"), handle)
        validate_property(ObjsSubteam, Symbol("id"), id)
        setfield!(o, Symbol("id"), id)
        validate_property(ObjsSubteam, Symbol("is_external"), is_external)
        setfield!(o, Symbol("is_external"), is_external)
        validate_property(ObjsSubteam, Symbol("is_subteam"), is_subteam)
        setfield!(o, Symbol("is_subteam"), is_subteam)
        validate_property(ObjsSubteam, Symbol("is_usergroup"), is_usergroup)
        setfield!(o, Symbol("is_usergroup"), is_usergroup)
        validate_property(ObjsSubteam, Symbol("name"), name)
        setfield!(o, Symbol("name"), name)
        validate_property(ObjsSubteam, Symbol("prefs"), prefs)
        setfield!(o, Symbol("prefs"), prefs)
        validate_property(ObjsSubteam, Symbol("team_id"), team_id)
        setfield!(o, Symbol("team_id"), team_id)
        validate_property(ObjsSubteam, Symbol("updated_by"), updated_by)
        setfield!(o, Symbol("updated_by"), updated_by)
        validate_property(ObjsSubteam, Symbol("user_count"), user_count)
        setfield!(o, Symbol("user_count"), user_count)
        validate_property(ObjsSubteam, Symbol("users"), users)
        setfield!(o, Symbol("users"), users)
        o
    end
end # type ObjsSubteam

const _property_map_ObjsSubteam = Dict{Symbol,Symbol}(Symbol("auto_provision")=>Symbol("auto_provision"), Symbol("auto_type")=>Symbol("auto_type"), Symbol("channel_count")=>Symbol("channel_count"), Symbol("created_by")=>Symbol("created_by"), Symbol("date_create")=>Symbol("date_create"), Symbol("date_delete")=>Symbol("date_delete"), Symbol("date_update")=>Symbol("date_update"), Symbol("deleted_by")=>Symbol("deleted_by"), Symbol("description")=>Symbol("description"), Symbol("enterprise_subteam_id")=>Symbol("enterprise_subteam_id"), Symbol("handle")=>Symbol("handle"), Symbol("id")=>Symbol("id"), Symbol("is_external")=>Symbol("is_external"), Symbol("is_subteam")=>Symbol("is_subteam"), Symbol("is_usergroup")=>Symbol("is_usergroup"), Symbol("name")=>Symbol("name"), Symbol("prefs")=>Symbol("prefs"), Symbol("team_id")=>Symbol("team_id"), Symbol("updated_by")=>Symbol("updated_by"), Symbol("user_count")=>Symbol("user_count"), Symbol("users")=>Symbol("users"))
const _property_types_ObjsSubteam = Dict{Symbol,String}(Symbol("auto_provision")=>"Bool", Symbol("auto_type")=>"Any", Symbol("channel_count")=>"Int32", Symbol("created_by")=>"DefsUserId", Symbol("date_create")=>"Int32", Symbol("date_delete")=>"Int32", Symbol("date_update")=>"Int32", Symbol("deleted_by")=>"Any", Symbol("description")=>"String", Symbol("enterprise_subteam_id")=>"String", Symbol("handle")=>"String", Symbol("id")=>"DefsSubteamId", Symbol("is_external")=>"Bool", Symbol("is_subteam")=>"Bool", Symbol("is_usergroup")=>"Bool", Symbol("name")=>"String", Symbol("prefs")=>"ObjsSubteamPrefs", Symbol("team_id")=>"DefsTeam", Symbol("updated_by")=>"DefsUserId", Symbol("user_count")=>"Int32", Symbol("users")=>"Vector{DefsUserId}")
Base.propertynames(::Type{ ObjsSubteam }) = collect(keys(_property_map_ObjsSubteam))
Swagger.property_type(::Type{ ObjsSubteam }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_ObjsSubteam[name]))}
Swagger.field_name(::Type{ ObjsSubteam }, property_name::Symbol) =  _property_map_ObjsSubteam[property_name]

function check_required(o::ObjsSubteam)
    (getproperty(o, Symbol("auto_provision")) === nothing) && (return false)
    (getproperty(o, Symbol("auto_type")) === nothing) && (return false)
    (getproperty(o, Symbol("created_by")) === nothing) && (return false)
    (getproperty(o, Symbol("date_create")) === nothing) && (return false)
    (getproperty(o, Symbol("date_delete")) === nothing) && (return false)
    (getproperty(o, Symbol("date_update")) === nothing) && (return false)
    (getproperty(o, Symbol("deleted_by")) === nothing) && (return false)
    (getproperty(o, Symbol("description")) === nothing) && (return false)
    (getproperty(o, Symbol("enterprise_subteam_id")) === nothing) && (return false)
    (getproperty(o, Symbol("handle")) === nothing) && (return false)
    (getproperty(o, Symbol("id")) === nothing) && (return false)
    (getproperty(o, Symbol("is_external")) === nothing) && (return false)
    (getproperty(o, Symbol("is_subteam")) === nothing) && (return false)
    (getproperty(o, Symbol("is_usergroup")) === nothing) && (return false)
    (getproperty(o, Symbol("name")) === nothing) && (return false)
    (getproperty(o, Symbol("prefs")) === nothing) && (return false)
    (getproperty(o, Symbol("team_id")) === nothing) && (return false)
    (getproperty(o, Symbol("updated_by")) === nothing) && (return false)
    true
end

function validate_property(::Type{ ObjsSubteam }, name::Symbol, val)
end


text:
# TODO: Document this.
macro select(src, exprs...)
    Expr(
        :call,
        :Projection,
        esc(src),
        Expr(
            :tuple,
            expression_expr.(exprs)...,
        ),
    )
end


text:
using Distributed

@everywhere function calc_pi(n::Integer)
   x = rand(n)
   y = rand(n)
   num_in_unit_circle = sum(x.*x.+y.*y .< 1)
   num_in_unit_circle * 4/n
end

n = 100_000;
mapreduce(calc_pi, +, Iterators.repeated(1, nworkers()) ) / nworkers()
@time pi_estimate = mapreduce(calc_pi, +, Iterators.repeated(n, nworkers()) ) / nworkers()
println("# After ", n, " itterations on each of ", nworkers(), " workers, estimated pi to be...")
println(pi_estimate)


text:
#=
    bp_ops: operators used in BP that apply to all sfuncs
=#

using Scruff.Utils
using Folds

export compute_lambda,
       compute_bel,
       send_pi,
       outgoing_pis,
       outgoing_lambdas

    #    @op_impl Scruff function Compute_lambda{SFunc}((range, lambda_msgs))
    #     return normalized_product(lambda_msgs, length(range))
    # end

@impl begin
    struct SFuncComputeLambda end

    function compute_lambda(sf::SFunc, range::__OptVec, lambda_msgs::Vector{<:Score})::Score

        if isempty(lambda_msgs)
            ps = zeros(Float64, length(range))
        else
            lams = [[get_log_score(l, o) for l in lambda_msgs] for o in range]
            ps = [sum(lams[i]) for i in 1:length(range)]
        end
        # avoid underflow
        m = maximum(ps)
        qs = ps .- m
        return LogScore(range, qs)
    end
end

@impl begin
    struct SFuncComputeBel end

    function compute_bel(sf::SFunc{I,O}, range::__OptVec{O},  pi::Dist{O}, lambda::Score)::Dist{<:O} where {I,O}
        ps = [cpdf(pi, (), x) * get_score(lambda, x) for x in range]
        return Cat(range, normalize(ps))
    end
end

@impl begin
    struct SFuncSendPi end

    function send_pi(sf::SFunc{I,O}, range::__OptVec{O}, bel::Dist{O}, lambda_msg::Score)::Dist{<:O} where {I,O}
        # pi_msg = [get_score(bel, x) / max.(1e-8, get_score(lambda_msg, x)) for x in range]
        f(x,y) = y == -Inf ? -Inf : x - y
        ps = [f(logcpdf(bel, (), x), get_log_score(lambda_msg, x)) for x in range]
        # delay exponentiation until after avoiding underflow
        m = maximum(ps)
        qs = ps .- m
        exped = exp.(qs)
        return Cat(range, normalize(exped))
    end
end

@impl begin
    struct SFuncOutgoingPis end

    function outgoing_pis(sf::SFunc, range::__OptVec, bel::Dist,
            incoming_lambdas::__OptVec{<:Score})::Vector{<:Dist}
        return [send_pi(sf, range, bel, l) for l in incoming_lambdas]
    end
end

@impl begin
    struct SFuncOutgoingLambdas end

    function outgoing_lambdas(sf::SFunc{I,O},
        lambda::Score,
        range::__OptVec,
        parranges::NTuple{N,Vector},
        incoming_pis::Tuple)::Vector{<:Score} where {N,I,O}

        lambdas = Score[]
        for i = 1:length(incoming_pis)
            msg = send_lambda(sf, lambda, range, parranges, incoming_pis, i)
            push!(lambdas, msg)
        end
        return lambdas
    end

end


text:
function gpdps_enep!(u,v,data,params)
    σ,τ,ω,maxit = params
    S,Sᵀ,z₁,z₂,α,a,b,ue₁,ue₂ = data

    # preallocate
    uold = similar(u)
    ξ = similar(u)
    u₁,u₂ = view(u,:,1), view(u,:,2)
    v₁,v₂ = view(v,:,1), view(v,:,2)
    ξ₁,ξ₂ = view(ξ,:,1), view(ξ,:,2)
    ρ₁,ρ₂ = similar(u₁), similar(v₁)
    Kᵤ,Kᵥ = similar(u), similar(v)
    for it = 1:maxit
        # Kx
        yᵥᵤ = S(v₁,u₂)
        yᵤᵥ = S(u₁,v₂)
        yᵤᵤ = S(u₁,u₂)
        @. ρ₁ = 2yᵤᵤ - yᵤᵥ - z₁
        @. ρ₂ = 2yᵤᵤ - yᵥᵤ - z₂
        Kᵤ .= Sᵀ(ρ₁,ρ₂) .+ α.*u

        # print output
        Psi = (norm(yᵤᵤ-z₁)^2/2 + α/2*norm(u₁) - norm(yᵥᵤ-z₁)^2/2 - α/2*norm(v₁) +
               norm(yᵤᵤ-z₂)^2/2 + α/2*norm(u₂) - norm(yᵤᵥ-z₂)^2/2 - α/2*norm(v₂)) /
              length(z₁)
        err = (norm(u₁-ue₁)^2 + norm(u₂-ue₂)^2)/length(z₁)
        @printf("it %3d: Psi = %1.3e\t Err = %1.3e\n", it, Psi, err)

        # x = prox_G(x-τKx)
        @. uold = u
        @. u = clamp(u-τ*Kᵤ,a,b)
        @. ξ = u + ω*(u-uold)

        # Ky
        yᵥᵤ = S(v₁,ξ₂)
        yᵤᵥ = S(ξ₁,v₂)
        @. ρ₁ = z₁ - yᵤᵥ
        @. ρ₂ = z₂ - yᵥᵤ
        Kᵥ .= Sᵀ(ρ₁,ρ₂) .- α.*v

        # y = prox_F*(y+σKy)
        @. v = clamp(v+σ*Kᵥ,a,b)
    end
end

function setup_enep(N,α,a,b)
    x₁,x₂,A = setup_fdm(N)

    B₁ = x₂.<0.5
    B₂ = x₂.>0.5

    y  = @. sin(π*x₁)*sin(π*x₂)
    p₁ = @. sin(2π*x₁)*sin(2π*x₂)
    p₂ = @. sin(3π*x₁)*sin(3π*x₂)

    z₁ = y .+ A*p₁
    z₂ = y .+ A*p₂

    u₁ = clamp.(B₁.*p₁./α,a,b)
    u₂ = clamp.(B₂.*p₂./α,a,b)

    f = A*y .- B₁.*u₁ .- B₂.*u₂

    C  = cholesky(A)
    S  = (u₁,u₂) -> C\(B₁.*u₁ .+ B₂.*u₂ .+ f)
    Sᵀ = (q₁,q₂) -> hcat(B₁.*(C\q₁),B₂.*(C\q₂))

    return S,Sᵀ,z₁,z₂,u₁,u₂
end

function setup_fdm(N)
    dx = 1/(N+1)
    xm = range(dx,stop=1-dx,length=N)
    xx,yy = reshape(xm, 1, N), reshape(xm, N, 1)
    x₁,x₂ = repeat(xx,outer=(N,1))[:], repeat(yy,outer=(1,N))[:]

    e  = fill(1/dx^2,N)
    D₂ = spdiagm(-1=>-e[2:end],0=>2e,1=>-e[2:end])
    Id = sparse(I,N,N)
    A  = kron(D₂,Id)+kron(Id,D₂)
    return x₁,x₂,A
end


text:
#https://www.sciencedirect.com/topics/mathematics/euclidean-distance#:~:text=The%20weighted%20Euclidean%20distance%20measure,element%20of%20the%20average%20profile.

dispersion = abs(a + b)
@propagate_inbounds function EuclidDist(a, b) #sumInterval subtracts 1 (complies with julia logic)
    cond = dispersion - 1
    @inbounds if cond >= 1
        return cond
    end
end

@benchmark sumInterval(1, 10)

max_stars_repo_name: adamwillisMastery/DeepLearner

max_issues_repo_name: adamwillisMastery/DeepLearner

max_forks_repo_name: adamwillisXanax/DeepLearner