text:
function _polygon_error_handler(request_body_dictionary::Dict{String, Any})::Tuple

    # initialize -
    error_response_dictionary = Dict{String,Any}()

    # what are my error keys?
    error_keys = keys(request_body_dictionary)
    for key ∈ error_keys
        error_response_dictionary[key] = request_body_dictionary[key]
    end

    # return -
    return (error_response_dictionary, nothing)
end

function _process_previous_close_polygon_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    status_flag = request_body_dictionary["status"]
    if (status_flag == "ERROR")
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame(

        ticker=String[],
        volume=Float64[],
        volume_weighted_average_price=Float64[],
        open=Float64[],
        close=Float64[],
        high=Float64[],
        low=Float64[],
        timestamp=Date[],
        number_of_transactions=Int[]
    )

    # fill in the header dictionary -
    header_keys = [
        "ticker", "queryCount", "adjusted", "status", "request_id", "count"
    ]

    # check - do we have a count (if not resturn zero)
    get!(request_body_dictionary, "count", 0)

    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # check - do we have a resultsCount field?
    results_count = get!(request_body_dictionary, "resultsCount", 0)
    if (results_count == 0) # we have no results ...
        # return the header and nothing -
        return (header_dictionary, nothing)
    end

    # populate the results DataFrame -
    results_array = request_body_dictionary["results"]
    for result_dictionary ∈ results_array

        # build a results tuple -
        result_tuple = (

            ticker = result_dictionary["T"],
            volume = result_dictionary["v"],
            volume_weighted_average_price = result_dictionary["vw"],
            open = result_dictionary["o"],
            close = result_dictionary["c"],
            high = result_dictionary["h"],
            low = result_dictionary["l"],
            timestamp = unix2datetime(result_dictionary["t"]*(1/1000)),
            number_of_transactions = result_dictionary["n"]
        )

        # push that tuple into the df -
        push!(df, result_tuple)
    end

    # return -
    return (header_dictionary, df)
end

function _process_aggregates_polygon_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    status_flag = request_body_dictionary["status"]
    if (status_flag == "ERROR")
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame(

        volume=Float64[],
        volume_weighted_average_price=Float64[],
        open=Float64[],
        close=Float64[],
        high=Float64[],
        low=Float64[],
        timestamp=Date[],
        number_of_transactions=Int[]
    )

    # fill in the header dictionary -
    header_keys = [
        "ticker", "queryCount", "adjusted", "status", "request_id", "count"
    ]

    # check - do we have a count (if not resturn zero)
    get!(request_body_dictionary, "count", 0)
    get!(request_body_dictionary, "ticker", "N/A")

    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # check - do we have a resultsCount field?
    results_count = get!(request_body_dictionary, "resultsCount", 0)
    if (results_count == 0) # we have no results ...
        # return the header and nothing -
        return (header_dictionary, nothing)
    end

    # populate the results DataFrame -
    results_array = request_body_dictionary["results"]
    for result_dictionary ∈ results_array

        # set some defaults in case missing fields -
        get!(result_dictionary, "vw", 0.0)
        get!(result_dictionary, "n", 0)

        # build a results tuple -
        result_tuple = (

            volume = result_dictionary["v"],
            volume_weighted_average_price = result_dictionary["vw"],
            open = result_dictionary["o"],
            close = result_dictionary["c"],
            high = result_dictionary["h"],
            low = result_dictionary["l"],
            timestamp = unix2datetime(result_dictionary["t"]*(1/1000)),
            number_of_transactions = result_dictionary["n"]
        )

        # push that tuple into the df -
        push!(df, result_tuple)
    end

    # return -
    return (header_dictionary, df)
end

function _process_daily_open_close_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    status_flag = request_body_dictionary["status"]
    if (status_flag == "ERROR" || status_flag == "NOT_FOUND")
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame(

        ticker=String[],
        volume=Float64[],
        open=Float64[],
        close=Float64[],
        high=Float64[],
        low=Float64[],
        from=Date[],
        afterHours=Float64[],
        preMarket=Float64[]
    )

    header_keys = [
        "status"
    ]
    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # build a results tuple -
    result_tuple = (

        volume = request_body_dictionary["volume"],
        open = request_body_dictionary["open"],
        close = request_body_dictionary["close"],
        high = request_body_dictionary["high"],
        low = request_body_dictionary["low"],
        from = Date(request_body_dictionary["from"]),
        ticker = request_body_dictionary["symbol"],
        afterHours = request_body_dictionary["afterHours"],
        preMarket = request_body_dictionary["preMarket"]
    )

    # push that tuple into the df -
    push!(df, result_tuple)

    # return -
    return (header_dictionary, df)
end

function _process_options_reference_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    status_flag = request_body_dictionary["status"]
    if (status_flag == "ERROR")
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame(

        cfi=String[],
        contract_type=String[],
        exercise_style=String[],
        expiration_date=Date[],
        primary_exchange=String[],
        shares_per_contract=Int64[],
        strike_price=Float64[],
        ticker = String[],
        underlying_ticker = String[]
    )

    # fill in the header dictionary -
    header_keys = [
        "status", "request_id", "count", "next_url"
    ]
    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # populate the results DataFrame -
    results_array = request_body_dictionary["results"]
    for result_dictionary ∈ results_array

        # build a results tuple -
        result_tuple = (

            cfi = result_dictionary["cfi"],
            contract_type = result_dictionary["contract_type"],
            exercise_style = result_dictionary["exercise_style"],
            expiration_date = Date(result_dictionary["expiration_date"]),
            primary_exchange = result_dictionary["primary_exchange"],
            shares_per_contract = result_dictionary["shares_per_contract"],
            strike_price = result_dictionary["strike_price"],
            ticker = result_dictionary["ticker"],
            underlying_ticker = result_dictionary["underlying_ticker"]
        )

        # push that tuple into the df -
        push!(df, result_tuple)
    end

    # return -
    return (header_dictionary, df)
end

function _process_ticker_news_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    status_flag = request_body_dictionary["status"]
    if (status_flag == "ERROR")
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame();

    # fill in the header dictionary -
    header_keys = [
        "status", "request_id", "count", "next_url"
    ]
    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # return -
    return (header_dictionary, df)
end

function _process_ticker_details_call_response(body::String)

    # convert to JSON -
    request_body_dictionary = JSON.parse(body)

    # before we do anything - check: do we have an error?
    if (haskey(request_body_dictionary,"error") == true)
        return _polygon_error_handler(request_body_dictionary)
    end

    # initialize -
    header_dictionary = Dict{String,Any}()
    df = DataFrame(

        logo = String[],
        listdate = Date[],
        cik = String[],
        bloomberg = String[],
        figi = Union{String,Nothing}[],
        lei = Union{String,Nothing}[],
        sic = Int[],
        country = String[],
        industry = String[],
        sector = String[],
        marketcap = Int[],
        employees = Int[],
        phone = String[],
        ceo = String[],
        url = String[],
        description = String[],
        exchange = String[],
        name = String[],
        symbol = String[],
        exchangeSymbol = String[],
        hq_address = String[],
        hq_state = String[],
        hq_country = String[],
        type = String[],
        updated = Date[],
        tags = Array{Array{String,1},1}(),
        similar = Array{Array{String,1},1}()
    );

    # fill in the header dictionary -
    header_keys = [
        "active"
    ];
    for key ∈ header_keys
        header_dictionary[key] = request_body_dictionary[key]
    end

    # updated comes back in "non-standard" mm/dd/yyyy date format -
    date_components = split(request_body_dictionary["updated"],"/")

    # build a results tuple -
    result_tuple = (

        logo = request_body_dictionary["logo"],
        listdate = Date(request_body_dictionary["listdate"]),
        figi = request_body_dictionary["figi"],
        cik = request_body_dictionary["cik"],
        bloomberg = request_body_dictionary["bloomberg"],
        lei = request_body_dictionary["lei"],
        sic = request_body_dictionary["sic"],
        country = request_body_dictionary["country"],
        industry = request_body_dictionary["industry"],
        sector = request_body_dictionary["sector"],
        marketcap = request_body_dictionary["marketcap"],
        employees = request_body_dictionary["employees"],
        phone = request_body_dictionary["phone"],
        ceo = request_body_dictionary["ceo"],
        url = request_body_dictionary["url"],
        description = request_body_dictionary["description"],
        exchange = request_body_dictionary["exchange"],
        name = request_body_dictionary["name"],
        symbol = request_body_dictionary["symbol"],
        exchangeSymbol = request_body_dictionary["exchangeSymbol"],
        hq_address = request_body_dictionary["hq_address"],
        hq_state = request_body_dictionary["hq_state"],
        hq_country = request_body_dictionary["hq_country"],
        type = request_body_dictionary["type"],
        updated = Date("$(date_components[3])-$(date_components[1])-$(date_components[2])"),
        tags = request_body_dictionary["tags"],
        similar = request_body_dictionary["similar"]
    )

    # push that tuple into the df -
    push!(df, result_tuple)

    # return -
    return (header_dictionary, df)
end


text:
export
    AbstractFeature,
    FeatureValue,
    FeatureState,

    is_feature_valid,
    is_symbol_a_feature,
    allfeatures,
    symbol2feature

"""
    Features can be extracted from SceneRecords.
They always return a FeatureValue, which allows the encoding of discrete / continuous / missing values,
which can also be forced to a Float64.
"""
abstract type AbstractFeature end

baremodule FeatureState
    # good
    const GOOD        = 0 # value is perfectly A-okay

    # caution
    const INSUF_HIST  = 1 # value best-guess was made due to insufficient history (ex, acceleration set to zero due to one timestamp)

    # bad (in these cases fval.v is typically set to NaN as well)
    const MISSING     = 2 # value is missing (no car in front, etc.)
    const CENSORED_HI = 3 # value is past an operating threshold
    const CENSORED_LO = 4 # value is below an operating threshold
end

immutable FeatureValue
    v::Float64 # feature value
    i::Int # used to encode

    FeatureValue(v::Float64, i::Int=FeatureState.GOOD) = new(v, i)
end

is_feature_valid(fval::FeatureValue) = fval.i == FeatureState.GOOD || fval.i == FeatureState.INSUF_HIST
Base.convert(::Type{Float64}, fval::FeatureValue) = fval.v

const SYMBOL_TO_FEATURE = Dict{Symbol, AbstractFeature}()

is_symbol_a_feature(sym::Symbol) = haskey(SYMBOL_TO_FEATURE, sym)
allfeatures() = values(SYMBOL_TO_FEATURE)
function symbol2feature(sym::Symbol)
    # if !haskey(SYMBOL_TO_FEATURE, sym)
    #     str = string(sym)
    #     if startswith(str, "isclean_")
    #         target = symbol(str[9:end])
    #         println("IsClean target: ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_IsClean{target}()
    #     elseif ismatch(r"^past_(\d)+_", str)
    #         history = parse(Int, match(r"(\d)+", str).match)
    #         len_of_header = length(match(r"^past_(\d)+_", str).match)
    #         target = symbol(str[len_of_header+1:end])
    #         println("Past history, target: ", history, "  ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_Past{target, history}()
    #     elseif ismatch(r"^mean_(\d)+_", str)
    #         history = parse(Int, match(r"(\d)+", str).match)
    #         len_of_header = length(match(r"^mean_(\d)+_", str).match)
    #         target = symbol(str[len_of_header+1:end])
    #         println("Mean over history, target: ", history, "  ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_Mean_Over_History{target, history}()
    #     elseif ismatch(r"^std_(\d)+_", str)
    #         history = parse(Int, match(r"(\d)+", str).match)
    #         len_of_header = length(match(r"^std_(\d)+_", str).match)
    #         target = symbol(str[len_of_header+1:end])
    #         println("Std over history, target: ", history, "  ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_Std_Over_History{target, history}()
    #     elseif ismatch(r"^max_(\d)+_", str)
    #         history = parse(Int, match(r"(\d)+", str).match)
    #         len_of_header = length(match(r"^max_(\d)+_", str).match)
    #         target = symbol(str[len_of_header+1:end])
    #         println("Max over history, target: ", history, "  ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_Max_Over_History{target, history}()
    #     elseif ismatch(r"^min_(\d)+_", str)
    #         history = parse(Int, match(r"(\d)+", str).match)
    #         len_of_header = length(match(r"^min_(\d)+_", str).match)
    #         target = symbol(str[len_of_header+1:end])
    #         println("Min over history, target: ", history, "  ", target)
    #         SYMBOL_TO_FEATURE[sym] = Feature_Min_Over_History{target, history}()
    #     end
    # end

    SYMBOL_TO_FEATURE[sym]
end

# ----------------------------------

function generate_feature_functions(
    name::AbstractString,
    sym::Symbol,
    inherent_type::DataType,
    units::AbstractString;
    lowerbound::Float64=-Inf,
    upperbound::Float64=Inf,
    can_be_missing::Bool=false,
    censor_lo::Float64=NaN,
    censor_hi::Float64=NaN,
    history::Int=1,
    )

    for feature in values(SYMBOL_TO_FEATURE)
        @assert(sym != Symbol(feature), "symb: $name -> $feature")
    end
    @assert(lowerbound ≤ upperbound)

    feature_name = Symbol("Feature_" * name)
    const_name   = Symbol(uppercase(name))
    sym_feature  = Meta.quot(sym)

    @eval begin
        export $const_name
        immutable $feature_name <: AbstractFeature end
        const       $const_name  = ($feature_name)()
        units(          ::$feature_name)  = $(units)
        inherent_type(  ::$feature_name)  = $(inherent_type)
        lowerbound(     ::$feature_name)  = $(lowerbound)
        upperbound(     ::$feature_name)  = $(upperbound)
        can_be_missing( ::$feature_name)  = $(can_be_missing)
        history(        ::$feature_name)  = $(history)
        Base.Symbol(    ::$feature_name)  = $sym_feature
        SYMBOL_TO_FEATURE[Symbol($const_name)] = $const_name
    end
end

# ----------------------------------

generate_feature_functions("PosFt", :posFt, Float64, "m")
function Base.get(::Feature_PosFt, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    FeatureValue(rec[vehicle_index, pastframe].state.posF.t)
end
generate_feature_functions("PosFyaw", :posFyaw, Float64, "rad")
function Base.get(::Feature_PosFyaw, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    FeatureValue(rec[vehicle_index, pastframe].state.posF.ϕ)
end
generate_feature_functions("Speed", :speed, Float64, "m/s")
function Base.get(::Feature_Speed, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    FeatureValue(rec[vehicle_index, pastframe].state.v)
end

generate_feature_functions("VelFs", :velFs, Float64, "m/s")
function Base.get(::Feature_VelFs, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    FeatureValue(veh.state.v*cos(veh.state.posF.ϕ))
end
generate_feature_functions("VelFt", :velFt, Float64, "m/s")
function Base.get(::Feature_VelFt, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    FeatureValue(veh.state.v*sin(veh.state.posF.ϕ))
end

function get_feature_derivative_backwards(
    f::AbstractFeature,
    rec::SceneRecord,
    roadway::Roadway,
    vehicle_index::Int,
    pastframe::Int=0,
    frames_back::Int=1,
    )

    id = rec[vehicle_index, pastframe].def.id

    retval = FeatureValue(0.0, FeatureState.INSUF_HIST)
    pastframe2 = pastframe - frames_back

    # println(f, "  ", pastframe, " ", pastframe2, " ", length(rec), "  ", record_length(rec), "  ", pastframe_inbounds(rec, pastframe2))

    if pastframe_inbounds(rec, pastframe) && pastframe_inbounds(rec, pastframe2)

        veh_index_curr = vehicle_index
        veh_index_prev = get_index_of_first_vehicle_with_id(rec, id, pastframe2)

        if veh_index_prev != 0
            curr = convert(Float64, get(f, rec, roadway, veh_index_curr, pastframe))
            past = convert(Float64, get(f, rec, roadway, veh_index_prev, pastframe2))
            Δt = get_elapsed_time(rec, pastframe2, pastframe)
            retval = FeatureValue((curr - past) / Δt)
        end
    end

    retval
end

generate_feature_functions("TurnRateG", :turnrateG, Float64, "rad/s")
function Base.get(::Feature_TurnRateG, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0; frames_back::Int=1)

    id = rec[vehicle_index, pastframe].def.id

    retval = FeatureValue(0.0, FeatureState.INSUF_HIST)
    pastframe2 = pastframe - frames_back
    if pastframe_inbounds(rec, pastframe2)

        veh_index_curr = vehicle_index
        veh_index_prev = get_index_of_first_vehicle_with_id(rec, id, pastframe2)

        if veh_index_prev != 0
            curr = rec[veh_index_curr, pastframe].state.posG.θ
            past = rec[veh_index_prev, pastframe2].state.posG.θ
            Δt = get_elapsed_time(rec, pastframe2, pastframe)
            retval = FeatureValue(deltaangle(past, curr) / Δt)
        end
    end

    retval
end
generate_feature_functions("TurnRateF", :turnrateF, Float64, "rad/s")
function Base.get(::Feature_TurnRateF, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(POSFYAW, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("AngularRateG", :angrateG, Float64, "rad/s²")
function Base.get(::Feature_AngularRateG, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(TURNRATEG, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("AngularRateF", :angrateF, Float64, "rad/s²")
function Base.get(::Feature_AngularRateF, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(TURNRATEF, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("Acc", :acc, Float64, "m/s^2")
function Base.get(::Feature_Acc, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(SPEED, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("AccFs", :accFs, Float64, "m/s²")
function Base.get(::Feature_AccFs, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(VELFS, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("AccFt", :accFt, Float64, "m/s²")
function Base.get(::Feature_AccFt, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(VELFT, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("Jerk", :jerk, Float64, "m/s³")
function Base.get(::Feature_Jerk, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(ACC, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("JerkFs", :jerkFs, Float64, "m/s³")
function Base.get(::Feature_JerkFs, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(ACCFS, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("JerkFt", :jerkFt, Float64, "m/s³")
function Base.get(::Feature_JerkFt, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    get_feature_derivative_backwards(ACCFT, rec, roadway, vehicle_index, pastframe)
end
generate_feature_functions("DesiredAngle", :desang, Float64, "rad")
function Base.get(::Feature_DesiredAngle, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    kp_desired_angle::Float64 = 1.0,
    )

    retval = FeatureValue(0.0, FeatureState.INSUF_HIST)
    if pastframe_inbounds(rec, pastframe) && pastframe_inbounds(rec, pastframe-1)

        id = rec[vehicle_index, pastframe].def.id

        pastϕ = get_vehiclestate(rec, id, pastframe-1).posF.ϕ
        currϕ = get_vehiclestate(rec, id, pastframe).posF.ϕ

        Δt = rec.timestep
        expconst = exp(-kp_desired_angle*Δt)
        retval = FeatureValue((currϕ - pastϕ*expconst) / (1.0 - expconst))
    end
    retval
end

generate_feature_functions("MarkerDist_Left", :d_ml, Float64, "m")
function Base.get(::Feature_MarkerDist_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh_ego = rec[vehicle_index, pastframe]
    t = veh_ego.state.posF.t
    lane = roadway[veh_ego.state.posF.roadind.tag]

    if n_lanes_left(lane, roadway) > 0
        footpoint = get_footpoint(veh_ego)
        lane_left = roadway[LaneTag(lane.tag.segment, lane.tag.lane + 1)]
        lane_width = -proj(footpoint, lane_left, roadway).curveproj.t
    else
        lane_width = lane.width
    end

    FeatureValue(lane_width/2 - t)
end
generate_feature_functions("MarkerDist_Right", :d_mr, Float64, "m")
function Base.get(::Feature_MarkerDist_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh_ego = rec[vehicle_index, pastframe]
    t = veh_ego.state.posF.t
    lane = roadway[veh_ego.state.posF.roadind.tag]

    if n_lanes_right(lane, roadway) > 0
        footpoint = get_footpoint(veh_ego)
        lane_left = roadway[LaneTag(lane.tag.segment, lane.tag.lane - 1)]
        lane_width = proj(footpoint, lane_left, roadway).curveproj.t
    else
        lane_width = lane.width
    end

    FeatureValue(lane_width/2 + t)
end
generate_feature_functions("MarkerDist_Left_Left", :d_mll, Float64, "m", can_be_missing=true)
function Base.get(::Feature_MarkerDist_Left_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    #=
    Distance to the left lane marker one lane to the left
    =#

    veh = rec[vehicle_index, pastframe]
    lane = roadway[veh.state.posF.roadind.tag]
    if n_lanes_left(lane, roadway) > 0
        offset = veh.state.posF.t
        lane_left = roadway[LaneTag(lane.tag.segment, lane.tag.lane + 1)]
        FeatureValue(lane.width/2 - offset + lane_left.width)
    else
        FeatureValue(NaN, FeatureState.MISSING) # there is no left lane
    end
end
generate_feature_functions("MarkerDist_Right_Right", :d_mrr, Float64, "m", can_be_missing=true)
function Base.get(::Feature_MarkerDist_Right_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    #=
    Distance to the right lane marker one lane to the right
    =#

    veh = rec[vehicle_index, pastframe]
    lane = roadway[veh.state.posF.roadind.tag]
    if n_lanes_right(lane, roadway) > 0
        offset = veh.state.posF.t
        lane_right = roadway[LaneTag(lane.tag.segment, lane.tag.lane - 1)]
        FeatureValue(lane.width/2 + offset + lane_right.width)
    else
        FeatureValue(NaN, FeatureState.MISSING) # there is no right lane
    end
end
generate_feature_functions("RoadEdgeDist_Left", :d_edgel, Float64, "m")
function Base.get(::Feature_RoadEdgeDist_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    offset = veh.state.posF.t
    footpoint = get_footpoint(veh)
    seg = roadway[veh.state.posF.roadind.tag.segment]
    lane = seg.lanes[end]
    roadproj = proj(footpoint, lane, roadway)
    curvept = roadway[RoadIndex(roadproj)]
    lane = roadway[roadproj.tag]
    FeatureValue(lane.width/2 + abs(curvept.pos - footpoint) - offset)
end
generate_feature_functions("RoadEdgeDist_Right", :d_edger, Float64, "m")
function Base.get(::Feature_RoadEdgeDist_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    offset = veh.state.posF.t
    footpoint = get_footpoint(veh)
    seg = roadway[veh.state.posF.roadind.tag.segment]
    lane = seg.lanes[1]
    roadproj = proj(footpoint, lane, roadway)
    curvept = roadway[RoadIndex(roadproj)]
    lane = roadway[roadproj.tag]
    FeatureValue(lane.width/2 + abs(curvept.pos - footpoint) + offset)
end
generate_feature_functions("LaneOffsetLeft", :posFtL, Float64, "m", can_be_missing=true)
function Base.get(::Feature_LaneOffsetLeft, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh_ego = rec[vehicle_index, pastframe]
    t = veh_ego.state.posF.t
    lane = roadway[veh_ego.state.posF.roadind.tag]
    if n_lanes_left(lane, roadway) > 0
        lane_left = roadway[LaneTag(lane.tag.segment, lane.tag.lane + 1)]
        lane_offset = t - lane.width/2 - lane_left.width/2
        FeatureValue(lane_offset)
    else
        FeatureValue(NaN, FeatureState.MISSING)
    end
end
generate_feature_functions("LaneOffsetRight", :posFtR, Float64, "m", can_be_missing=true)
function Base.get(::Feature_LaneOffsetRight, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh_ego = rec[vehicle_index, pastframe]
    t = veh_ego.state.posF.t
    lane = roadway[veh_ego.state.posF.roadind.tag]
    if n_lanes_right(lane, roadway) > 0
        lane_right = roadway[LaneTag(lane.tag.segment, lane.tag.lane - 1)]
        lane_offset = t + lane.width/2 + lane_right.width/2
        FeatureValue(lane_offset)
    else
        FeatureValue(NaN, FeatureState.MISSING)
    end
end
generate_feature_functions("N_Lane_Right", :n_lane_right, Int, "-", lowerbound=0.0)
function Base.get(::Feature_N_Lane_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    nlr = rec[vehicle_index, pastframe].state.posF.roadind.tag.lane - 1
    FeatureValue(convert(Float64, nlr))
end
generate_feature_functions("N_Lane_Left", :n_lane_left, Int, "-", lowerbound=0.0)
function Base.get(::Feature_N_Lane_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    seg = roadway[veh.state.posF.roadind.tag.segment]
    nll = length(seg.lanes) - veh.state.posF.roadind.tag.lane
    FeatureValue(convert(Float64, nll))
end
generate_feature_functions("Has_Lane_Right", :has_lane_right, Bool, "-", lowerbound=0.0, upperbound=1.0)
function Base.get(::Feature_Has_Lane_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    val = get(N_LANE_RIGHT, rec, roadway, vehicle_index, pastframe).v > 0.0
    FeatureValue(convert(Float64, val))
end
generate_feature_functions("Has_Lane_Left", :has_lane_left, Bool, "-", lowerbound=0.0, upperbound=1.0)
function Base.get(::Feature_Has_Lane_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    val = get(N_LANE_LEFT, rec, roadway, vehicle_index, pastframe).v > 0.0
    FeatureValue(convert(Float64, val))
end
generate_feature_functions("LaneCurvature", :curvature, Float64, "1/m", can_be_missing=true)
function Base.get(::Feature_LaneCurvature, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    veh = rec[vehicle_index, pastframe]
    curvept = roadway[veh.state.posF.roadind]
    val = curvept.k
    if isnan(val)
        FeatureValue(0.0, FeatureState.MISSING)
    else
        FeatureValue(val)
    end
end

# Dist_Merge
# Dist_Split

generate_feature_functions("TimeToCrossing_Right", :ttcr_mr, Float64, "s", lowerbound=0.0, censor_hi=10.0)
function Base.get(::Feature_TimeToCrossing_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    d_mr = get(MARKERDIST_RIGHT, rec, roadway, vehicle_index, pastframe).v
    velFt = get(VELFT, rec, roadway, vehicle_index, pastframe).v

    if d_mr > 0.0 && velFt < 0.0
        FeatureValue(-d_mr / velFt)
    else
        FeatureValue(Inf)
    end
end
generate_feature_functions("TimeToCrossing_Left", :ttcr_ml, Float64, "s", lowerbound=0.0, censor_hi=10.0)
function Base.get(::Feature_TimeToCrossing_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    d_ml = get(MARKERDIST_RIGHT, rec, roadway, vehicle_index, pastframe).v
    velFt = get(VELFT, rec, roadway, vehicle_index, pastframe).v

    if d_ml > 0.0 && velFt < 0.0
        FeatureValue(-d_ml / velFs)
    else
        FeatureValue(Inf)
    end
end
generate_feature_functions("EstimatedTimeToLaneCrossing", :est_ttcr, Float64, "s", lowerbound=0.0, censor_hi=10.0)
function Base.get(::Feature_EstimatedTimeToLaneCrossing, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    ttcr_left = get(TIMETOCROSSING_LEFT, rec, roadway, vehicle_index, pastframe).v
    ttcr_right = get(TIMETOCROSSING_RIGHT, rec, roadway, vehicle_index, pastframe).v
    FeatureValue(min(ttcr_left, ttcr_right))
end
generate_feature_functions("A_REQ_StayInLane", :a_req_stayinlane, Float64, "m/s²", can_be_missing=true)
function Base.get(::Feature_A_REQ_StayInLane, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    velFt = get(VELFT, rec, roadway, vehicle_index, pastframe).v

    if velFt > 0.0
        d_mr = get(MARKERDIST_RIGHT, rec, roadway, vehicle_index, pastframe).v
        if d_mr > 0.0
            return FeatureValue(0.5velFt*velFt / d_mr)
        else
            return FeatureValue(NaN, FeatureState.MISSING)
        end
    else
        d_ml = get(MARKERDIST_LEFT, rec, roadway, vehicle_index, pastframe)
        if d_ml < 0.0
            return FeatureValue(-0.5velFt*velFt / d_ml)
        else
            return FeatureValue(NaN, FeatureState.MISSING)
        end
    end
end

generate_feature_functions("Time_Consecutive_Brake", :time_consec_brake, Float64, "s", lowerbound=0.0)
function Base.get(::Feature_Time_Consecutive_Brake, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)

    prev_accel = convert(Float64, get(ACC, rec, roadway, vehicle_index, pastframe))
    if prev_accel ≥ 0.0
        FeatureValue(0.0)
    else
        pastframe_orig = pastframe
        id = rec[vehicle_index, pastframe].def.id
        while pastframe_inbounds(rec, pastframe-1) &&
              convert(Float64, get(ACC, rec, roadway,
                                   get_index_of_first_vehicle_with_id(rec, id,
                                                                      pastframe-1))) < 0.0

            pastframe -= 1
        end

        FeatureValue(get_elapsed_time(rec, pastframe, pastframe_orig))
    end
end
generate_feature_functions("Time_Consecutive_Accel", :time_consec_accel, Float64, "s", lowerbound=0.0)
function Base.get(::Feature_Time_Consecutive_Accel, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)

    prev_accel = convert(Float64, get(ACC, rec, roadway, vehicle_index, pastframe))
    if prev_accel ≤ 0.0
        FeatureValue(0.0)
    else

        pastframe_orig = pastframe
        id = rec[vehicle_index, pastframe].def.id
        while pastframe_inbounds(rec, pastframe-1) &&
              #get(ACC, rec, roadway, get_index_of_first_vehicle_with_id(rec, id, pastframe-1)) > 0.0
              convert(Float64,get(ACC, rec, roadway,
                                  get_index_of_first_vehicle_with_id(rec, id,
                                                                     pastframe-1))) > 0.0

            pastframe -= 1
        end

        FeatureValue(get_elapsed_time(rec, pastframe, pastframe_orig))
    end
end
generate_feature_functions("Time_Consecutive_Throttle", :time_consec_throttle, Float64, "s", lowerbound=0.0)
function Base.get(::Feature_Time_Consecutive_Throttle, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0)
    tc_accel = get(TIME_CONSECUTIVE_ACCEL, rec, roadway, vehicle_index, pastframe).v
    tc_brake = get(TIME_CONSECUTIVE_BRAKE, rec, roadway, vehicle_index, pastframe).v
    FeatureValue(tc_accel ≥ tc_brake ? tc_accel : -tc_brake)
end

#############################################
#
# FRONT
#
#############################################

generate_feature_functions("Dist_Front", :d_front, Float64, "m", lowerbound=0.0, can_be_missing=true)
function Base.get(::Feature_Dist_Front, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    censor_hi::Float64=100.0,
    )

    if neighborfore.ind == 0
        FeatureValue(100.0, FeatureState.CENSORED_HI)
    else
        len_ego = rec[vehicle_index, pastframe].def.length
        len_oth = rec[neighborfore.ind, pastframe].def.length
        FeatureValue(neighborfore.Δs - len_ego/2 - len_oth/2)
    end
end
generate_feature_functions("Speed_Front", :v_front, Float64, "m/s", can_be_missing=true)
function Base.get(::Feature_Speed_Front, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    )

    if neighborfore.ind == 0
        FeatureValue(0.0, FeatureState.MISSING)
    else
        FeatureValue(rec[neighborfore.ind, pastframe].state.v)
    end
end
generate_feature_functions("Timegap", :timegap, Float64, "s", can_be_missing=true)
function Base.get(::Feature_Timegap, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    censor_hi::Float64 = 100.0,
    )

    v = rec[vehicle_index, pastframe].state.v

    if v ≤ 0.0 || neighborfore.ind == 0
        FeatureValue(10.0, FeatureState.CENSORED_HI)
    else
        len_ego = rec[vehicle_index, pastframe].def.length
        len_oth = rec[neighborfore.ind, pastframe].def.length
        Δs = neighborfore.Δs - len_ego/2 - len_oth/2

        if Δs > 0.0
            FeatureValue(Δs / v)
        else
            FeatureValue(0.0) # collision!
        end
    end
end
generate_feature_functions("Inv_TTC", :inv_ttc, Float64, "1/s", can_be_missing=true)
function Base.get(::Feature_Inv_TTC, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    missing_surrogate::Float64 = 0.0,
    censor_hi::Float64 = 10.0,
    )

    if neighborfore.ind == 0
        FeatureValue(0.0, FeatureState.MISSING)
    else
        veh_fore = rec[neighborfore.ind, pastframe]
        veh_rear = rec[vehicle_index, pastframe]

        len_ego = veh_fore.def.length
        len_oth = veh_rear.def.length
        Δs = neighborfore.Δs - len_ego/2 - len_oth/2

        Δv = veh_fore.state.v - veh_rear.state.v

        if Δs < 0.0 # collision!
            FeatureValue(censor_hi, FeatureState.CENSORED_HI)
        elseif Δv > 0.0 # front car is pulling away
            FeatureValue(0.0)
        else
            f = -Δv/Δs
            if f > censor_hi
                FeatureValue(f, FeatureState.CENSORED_HI)
            else
                FeatureValue(f)
            end
        end
    end
end

#############################################
#
# FRONT LEFT
#
#############################################
generate_feature_functions("Dist_Front_Left", :d_front_left, Float64, "m", lowerbound=0.0, can_be_missing=true)
function Base.get(::Feature_Dist_Front_Left, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_left_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    )

    get(DIST_FRONT, rec, roadway, vehicle_index, pastframe, neighborfore=neighborfore)
end

#############################################
#
# FRONT RIGHT
#
#############################################

generate_feature_functions("Dist_Front_Right", :d_front_right, Float64, "m", lowerbound=0.0, can_be_missing=true)
function Base.get(::Feature_Dist_Front_Right, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    neighborfore::NeighborLongitudinalResult = get_neighbor_fore_along_right_lane(get_scene(rec, pastframe), vehicle_index, roadway),
    )

    get(DIST_FRONT, rec, roadway, vehicle_index, pastframe, neighborfore=neighborfore)
end

#############################################
#
# SCENE WISE
#
#############################################

generate_feature_functions("Is_Colliding", :is_colliding, Bool, "-", lowerbound=0.0, upperbound=1.0)
function Base.get(::Feature_Is_Colliding, rec::SceneRecord, roadway::Roadway, vehicle_index::Int, pastframe::Int=0;
    mem::CPAMemory=CPAMemory(),
    )

    scene = get_scene(rec, pastframe)
    is_colliding = convert(Float64, get_first_collision(scene, vehicle_index, mem).is_colliding)
    FeatureValue(is_colliding)
end

ext: jl


text:
# Spinnaker.jl: wrapper for FLIR/Point Grey Spinnaker SDK
# Copyright (C) 2019 Samuel Powell

# examples/format.jl: example to configure advanced image format controls
using Spinnaker

camlist = CameraList()
cam = camlist[0]
gammenset = gammaenable!(cam, false)
@assert gammenset == false
adcbits!(cam, "Bit12")
adcbits(cam)
pixelformat(cam)
pixelformat!(cam, "Mono12p")
@assert pixelformat(cam) == "Mono12p"
triggersource!(cam, "Software")
triggermode!(cam, "On")
acquisitionmode!(cam, "Continuous")
gain!(cam, 12)
exposure!(cam, 1e6)
start!(cam)
trigger!(cam)
saveimage(cam, joinpath(@__DIR__, "iformat.png"), spinImageFileFormat(6))
stop!(cam)


text:
struct Command
    name::Symbol
    help::String
    cmd::Function
end

const COMMANDS = Dict{Symbol, Command}()

function addCommand(name::Symbol, cmd::Function; help::String="")
    COMMANDS[name] = Command(name, help, cmd)
end


text:
# Autogenerated wrapper script for ghr_jll for armv6l-linux-gnueabihf
export ghr

JLLWrappers.@generate_wrapper_header("ghr")
JLLWrappers.@declare_executable_product(ghr)
function __init__()
    JLLWrappers.@generate_init_header()
    JLLWrappers.@init_executable_product(
        ghr,
        "bin/ghr",
    )

    JLLWrappers.@generate_init_footer()
end  # __init__()


text:
#######################
## MAXIMUM STEP
#######################

# while maintaining 0.1 * μ <= s' x

function maximum_step(vars::class_variables, direction::class_variables)
	try
		alpha = Inf;
		alpha = maximum_step(alpha, x(vars), x(direction));
		alpha = maximum_step(alpha, s(vars), s(direction));
		alpha = maximum_step(alpha, tau(vars), tau(direction));
		alpha = maximum_step(alpha, kappa(vars), kappa(direction));

		return alpha;
	catch e
		println("ERROR in maximum_step")
		throw(e)
	end
end

function maximum_step(alpha::Float64, array_point::Array{Float64}, array_direction::Array{Float64})
	try
		@assert(length(array_point) == length(array_direction))
		for i in 1:length(array_point)
			alpha = maximum_step(alpha, array_point[i], array_direction[i]);
		end

		return alpha
	catch e
		println("ERROR in maximum_step(::Float64,::Array,::Array)")
		throw(e)
	end
end

function maximum_step(alpha::Float64, var::Float64, dir::Float64)
	try
		candidate_alpha = -var/dir;
		if candidate_alpha >= 0
			alpha = min(alpha, candidate_alpha)
		end

		return alpha
	catch e
		println("ERROR in maximum_step(::Float64,::Float64,::Float64)")
		throw(e)
	end
end

max_forks_repo_name: ohinder/Everest


text:
"""
Memory-Map a set of 3 binary files created by convertDSVNet
Return Vector of nodes, Vector of the corresponding node's degree, Matrix of neighbours of each node
"""
function mmapNetU(file_dir::String, file_name::String, index::Int)
  file = open("$(file_dir)/$(file_name)/nodes$(index).bin")
  columns = read(file, Int)
  nodeArray = mmap_array(Int, (columns, 1), file)
  close(file)
  file = open("$(file_dir)/$(file_name)/degrees$(index).bin")
  degreeArray = mmap_array(Float64, (columns, 1), file)
  close(file)
  file = open("$(file_dir)/$(file_name)/neighbours$(index).bin")
  rows = read(file, Int)
  neighbourMatrix = mmap_array(Int, (rows, columns), file)
  close(file)
  return (nodeArray, degreeArray, neighbourMatrix)
end

"""
Memory-Map a set of 4 binary files created by convertDSVNet
Return Vector of nodes, Vector of the corresponding node's degree, Matrix of neighbours of each node, Matrix of corresponding neighbour's weight
"""
function mmapNetW(file_dir::String, file_name::String, index::Int)
  file = open("$(file_dir)/$(file_name)/nodes$(index).bin")
  columns = read(file, Int)
  nodeArray = mmap_array(Int, (columns, 1), file)
  close(file)
  file = open("$(file_dir)/$(file_name)/degrees$(index).bin")
  degreeArray = mmap_array(Float64, (columns, 1), file)
  close(file)
  file = open("$(file_dir)/$(file_name)/neighbours$(index).bin")
  rows = read(file, Int)
  neighbourMatrix = mmap_array(Int, (rows, columns), file)
  close(file)
  file = open("$(file_dir)/$(file_name)/weights$(index).bin")
  weightMatrix = mmap_array(Float64, (rows, columns), file)
  close(file)
  return (nodeArray, degreeArray, neighbourMatrix, weightMatrix)
end

"""
Converts the network file into sets of binary files, one set each time the maxMB threshold is reached
- *nodes.bin: vector to map internal to external node index
- *degrees.bin: vector of the node degee values
- *neighbours.bin: square array of the neighbours of each node
- *weights.bin: square array of the weight of the corresponding neighbour
Return number of file sets created
"""
function convertDSVNet(dsv_file::String, file_dir::String, file_name::String, delim::Char, maxMB::Int)
  fileCount = 0
  firstNode = -1
  lastNode = -1
  fileWeighted = 1
  # read in file as stream
  fstream = open(dsv_file)
  positionConvert = position(fstream)
  startConvert = false
  lineValues = 0
  while !startConvert || !eof(fstream)
    line = strip(readline(fstream), ['\n', '\r'])
    sline = split(line, delim)
    pline = parse(sline[1]; raise=false)
    if !isa(pline, Int)
      positionConvert = position(fstream)
      println("Skipped line: $(line)")
    else
      startConvert = true
      fstream = seek(fstream, positionConvert)
      if length(sline) == 2
        fileWeighted = 0
        fileCount, firstNode, lastNode = convertDSVunweighted(fstream, file_dir, file_name, delim, maxMB)
      elseif length(sline) == 3
        fileCount, firstNode, lastNode = convertDSVweighted(fstream, file_dir, file_name, delim, maxMB)
      else
        error("Wrong file format.")
      end
    end
  end
  file = open("$(file_dir)/$(file_name)/info.bin", "w")
  write(file, fileCount, firstNode, lastNode, fileWeighted)
  close(file)
  return fileCount, firstNode, lastNode, fileWeighted
end

function convertDSVweighted(fstream::IOStream, file_dir::String, file_name::String, delim::Char, maxMB::Int)
  fileCount = 0
  firstNode = typemax(Int)
  lastNode = 0
  # hashtable: key = node, value = (degree, neighbours, weights)
  data = Dict{Int, (Float64, Vector{Int}, Vector{Float64})}()
  totalNodes = 0
  maxNeighbours = 0
  while !eof(fstream)
    # parse stream line per line
    line = split(strip(readline(fstream), ['\n', '\r']), delim)
    node = int(line[1])
    neighbour = int(line[2])
    weight = float64(line[3])
    # update the data
    firstNode = min(firstNode, node, neighbour)
    lastNode = max(lastNode, node, neighbour)
    totalNodes, maxNeighbours = updateData!(data, node, neighbour, weight, totalNodes, maxNeighbours)
    totalNodes, maxNeighbours = updateData!(data, neighbour, node, weight, totalNodes, maxNeighbours)
    # check if no more data or size is greater than bound
    if eof(fstream) || checkMemSize(data, maxMB, totalNodes, maxNeighbours)
      # array to map internal to external node index
      nodeArray = collect(keys(data))
      degreeArray = Array(Float64, length(nodeArray))
      # pad neighbour vector with -1's and weight vector with 0's as empty neighbours
      neighbourMatrix = fill(-1, (maxNeighbours, length(nodeArray)))
      weightMatrix = fill(0.0, (maxNeighbours, length(nodeArray)))
      for i in 1:length(nodeArray)
        degreeArray[i] = data[nodeArray[i]][1]
        for j in 1:length(data[nodeArray[i]][2])
          neighbourMatrix[j, i] = data[nodeArray[i]][2][j]
          weightMatrix[j, i] = data[nodeArray[i]][3][j]
        end
      end
      fileCount += 1
      println("Writing components to file set $(fileCount)")
      file = open("$(file_dir)/$(file_name)/nodes$(fileCount).bin", "w")
      write(file, totalNodes, nodeArray)
      close(file)
      file = open("$(file_dir)/$(file_name)/degrees$(fileCount).bin", "w")
      write(file, degreeArray)
      close(file)
      file = open("$(file_dir)/$(file_name)/neighbours$(fileCount).bin", "w")
      write(file, maxNeighbours, neighbourMatrix)
      close(file)
      file = open("$(file_dir)/$(file_name)/weights$(fileCount).bin", "w")
      write(file, weightMatrix)
      close(file)
      empty!(data)
      gc()
      totalNodes = 0
      maxNeighbours = 0
    end
  end
  return fileCount, firstNode, lastNode
end

function convertDSVunweighted(fstream::IOStream, file_dir::String, file_name::String, delim::Char, maxMB::Int)
  fileCount = 0
  firstNode = typemax(Int)
  lastNode = 0
  # hashtable: key = node, value = neighbours
  data = Dict{Int, Vector{Int}}()
  totalNodes = 0
  maxNeighbours = 0
  while !eof(fstream)
    # parse stream line per line
    line = split(strip(readline(fstream), ['\n', '\r']), delim)
    node = int(line[1])
    neighbour = int(line[2])
    # update the data
    firstNode = min(firstNode, node, neighbour)
    lastNode = max(lastNode, node, neighbour)
    totalNodes, maxNeighbours = updateData!(data, node, neighbour, totalNodes, maxNeighbours)
    #totalNodes, maxNeighbours = updateData!(data, neighbour, node, totalNodes, maxNeighbours)
    # check if no more data or size is greater than bound
    if eof(fstream) || checkMemSize(data, maxMB, totalNodes, maxNeighbours)
      fileCount += 1
      println("Writing components to file set $(fileCount)")
      # array to map internal to external node index
      nodeArray = collect(keys(data))
      degreeArray = Array(Float64, length(nodeArray))
      # pad neighbour vector with -1's and weight vector with 0's as empty neighbours
      neighbourMatrix = fill(-1, (maxNeighbours, length(nodeArray)))
      for i in 1:length(nodeArray)
        degreeArray[i] = float(length(data[nodeArray[i]]))
        for j in 1:length(data[nodeArray[i]])
          neighbourMatrix[j, i] = data[nodeArray[i]][j]
        end
      end
      file = open("$(file_dir)/$(file_name)/nodes$(fileCount).bin", "w")
      write(file, totalNodes, nodeArray)
      close(file)
      file = open("$(file_dir)/$(file_name)/degrees$(fileCount).bin", "w")
      write(file, degreeArray)
      close(file)
      file = open("$(file_dir)/$(file_name)/neighbours$(fileCount).bin", "w")
      write(file, maxNeighbours, neighbourMatrix)
      close(file)
      empty!(data)
      totalNodes = 0
      maxNeighbours = 0
    end
  end
  return fileCount, firstNode, lastNode
end

"""
Update the data dictionary with the new node data
Return totalNodes and maxNeighbours
"""
function updateData!(data::Dict{Int, (Float64, Vector{Int}, Vector{Float64})}, node::Int, neighbour::Int, weight::Float64, totalNodes::Int, maxNeighbours::Int)
  # check if node already stored
  if haskey(data, node)
    # neighbour not yet stored
    if findfirst(data[node][2], neighbour) == 0
      # update the node degree and add the neighbour
      data[node] = (data[node][1]+weight, push!(data[node][2], neighbour), push!(data[node][3], weight))
      # check if maxNeighbours needs to be increased
      if length(data[node][2]) > maxNeighbours
        maxNeighbours = length(data[node][2])
      end
    end
  else
    data[node] = (weight, [neighbour], [weight])
    totalNodes += 1
  end
  return (totalNodes, maxNeighbours)
end

function updateData!(data::Dict{Int, Vector{Int}}, node::Int, neighbour::Int, totalNodes::Int, maxNeighbours::Int)
  # check if node already stored
  if haskey(data, node)
    # check if neighbour not yet stored
    if findfirst(data[node], neighbour) == 0
      # update the node degree and add the neighbour
      data[node] = push!(data[node], neighbour)
      # check if maxNeighbours needs to be increased
      if length(data[node]) > maxNeighbours
        maxNeighbours = length(data[node])
      end
    end
  else
    data[node] = [neighbour]
    totalNodes += 1
  end
  return (totalNodes, maxNeighbours)
end

"""
Check if the memory footprint is still smaller than the threshold
Return true if threshold reached
"""
function checkMemSize(data::Dict{Int, (Float64, Vector{Int}, Vector{Float64})}, maxMB::Int, totalNodes::Int, maxNeighbours::Int)
  # estimate current needed memory in Bytes: size of dictionary + arrays that need to be generated for storage
  size = sizeof(data) + (totalNodes + totalNodes*maxNeighbours)*sizeof(Int) + totalNodes*sizeof(Float64)
  # return true if size larger than maximum allowed MB threshold
  return (size < maxMB*1000000)?false:true
end

function checkMemSize(data::Dict{Int, Vector{Int}}, maxMB::Int, totalNodes::Int, maxNeighbours::Int)
  # estimate current needed memory in Bytes: size of dictionary + arrays that need to be generated for storage
  size = sizeof(data) + (totalNodes + totalNodes*maxNeighbours)*(sizeof(Int) + sizeof(Float64))
  # return true if size larger than maximum allowed MB threshold
  return (size < maxMB*1000000)?false:true
end

"""
Create an undirected network file with duplicate entries from the original network file
"""
function createMirrored(dsvFile::String, delim::Char, newFile::String)
  println("Creating mirrored network file")
  tic()
  # read in dsvfile as stream
  fstream = open(dsvFile)
  # write to newfile as stream
  nstream = open(newFile, "w")
  while !eof(fstream)
    # parse stream line per line
    line = readline(fstream)
    sline = split(strip(line, ['\n', '\r']), delim)
    pline = parse(sline[1]; raise=false)
    if !isa(pline, Int)
      println("Skipped line: $(line)")
    else
      if length(sline) == 2
        node = int(sline[1])
        neighbour = int(sline[2])
        write(nstream, "$(node)$(delim)$(neighbour)\n")
        write(nstream, "$(neighbour)$(delim)$(node)\n")
      elseif length(sline) == 3
        node = int(sline[1])
        neighbour = int(sline[2])
        weight = float(sline[3])
        write(nstream, "$(node)$(delim)$(neighbour)$(delim)$(weight)\n")
        write(nstream, "$(neighbour)$(delim)$(node)$(delim)$(weight)\n")
      end
    end
  end
  close(nstream)
  close(fstream)
  timeMirrored = toq()
  println("Done, time elapsed $(timeMirrored)s")
end


text:
using Documenter
using ExactDiagonalization

DocMeta.setdocmeta!(ExactDiagonalization, :DocTestSetup, :(using ExactDiagonalization); recursive=true)

makedocs(;
    modules=[ExactDiagonalization],
    authors="waltergu <waltergu1989@gmail.com> and contributors",
    repo="https://github.com/Quantum-Many-Body/ExactDiagonalization.jl/blob/{commit}{path}#{line}",
    sitename="ExactDiagonalization.jl",
    format=Documenter.HTML(;
        prettyurls=get(ENV, "CI", "false") == "true",
        canonical="https://Quantum-Many-Body.github.io/ExactDiagonalization.jl",
        assets=String[],
    ),
    pages=[
        "Home" => "index.md",
        "Examples" => [
            "examples/Introduction.md",
            ]
        ]
)

deploydocs(;
    repo="github.com/Quantum-Many-Body/ExactDiagonalization.jl.git"
)


text:
# ======================== sup(i) ==============================================

function _superscript(i::Int)

    c = i < 0 ? [Char(0x207B)] : []

    for d ∈ reverse(digits(abs(i)))
        d == 0 ? push!(c, Char(0x2070)) :
        d == 1 ? push!(c, Char(0x00B9)) :
        d == 2 ? push!(c, Char(0x00B2)) :
        d == 3 ? push!(c, Char(0x00B3)) : push!(c, Char(0x2070+d))
    end

    return join(c)

end

function _subscript(i::Int)

    c = i < 0 ? [Char(0x208B)] : []

    for d ∈ reverse(digits(abs(i)))
        push!(c, Char(0x2080+d))
    end

    return join(c)

end

# ======================== sup(i) ==============================================

@doc raw"""
    sup(i)

Superscript notation for integers and rational numbers
#### Examples:
```
sup(3) * 'P'
 "³P"
```
"""
function sup(i::T) where T<:Real

    sgn = i < 0 ? Char(0x207B) : ""

    num = _superscript(numerator(abs(i)))
    den = _superscript(denominator(abs(i)))

    return T == Rational{Int} ? (sgn * num * '\U141F' * den) : (sgn * num)

end

# ======================== sub(i) ==============================================

@doc raw"""
    sub(i)

Subscript notation for integers and rational numbers
#### Examples:
```
'D' * sub(5//2)
 "D₅⸝₂"
```
"""
function sub(i::T) where T<:Real

    sgn = i < 0 ? Char(0x208B) : ""

    num = _subscript(numerator(abs(i)))
    den = _subscript(denominator(abs(i)))

    return T == Rational{Int} ? (sgn * num * '\U2E1D' * den) : (sgn * num)

end

@doc raw"""
    frac(i)

Fraction notation for rational numbers
#### Examples:
```
frac(-5//2)
 "-⁵/₂"
```
"""
function frac(i::Rational{Int})

    sgn = i < 0 ? "-" : ""

    num = _superscript(numerator(abs(i)))
    den = _subscript(denominator(abs(i)))

    return sgn * num *  '/' * den

end


text:
module DifferentialDynamicsModels

using LinearAlgebra
using StaticArrays

export @maintain_type
export AbstractState, State, AbstractControl, Control, DifferentialDynamics, CostFunctional
export Time, TimePlusQuadraticControl
export ControlInterval, StepControl, RampControl, BVPControl
export SteeringBVP, SteeringConstraints, SteeringCache, EmptySteeringConstraints, EmptySteeringCache
export SingleIntegratorDynamics, BoundedControlNorm, SingleIntegratorSteering, GeometricSteering
export state_dim, control_dim, duration
export Propagate, InstantaneousControl, propagate, instantaneous_control, waypoints, waypoints_itr
export issymmetric

include("utils.jl")

# States, Controls, Dynamics, Cost Functionals, and Control Intervals
abstract type AbstractState end
const State = Union{AbstractState, AbstractVector{<:Number}}
abstract type AbstractControl end
const Control = Union{AbstractControl, AbstractVector{<:Number}}
abstract type DifferentialDynamics end
abstract type CostFunctional end
struct Time <: CostFunctional end
struct TimePlusQuadraticControl{Du,TR<:SMatrix{Du,Du}} <: CostFunctional
    R::TR
end
Base.zero(::Type{TimePlusQuadraticControl{Du,TR}}) where {Du,TR} = TimePlusQuadraticControl(zero(TR))
abstract type ControlInterval end
Base.zero(::CI) where {CI<:ControlInterval} = zero(CI)

(::Time)(c) = duration(c)
duration(cs) = sum(duration(c) for c in cs)
# (cost::CostFunctional)(cs) = sum(cost(c) for c in cs)    # JuliaLang/julia#29440
(cost::TimePlusQuadraticControl)(cs) = sum(cost(c) for c in cs)

# State/Control Sequences
include("iterators.jl")

## Propagation (state as a function of time)
propagate(f::DifferentialDynamics, x::State, cs) = foldl((x, c) -> propagate(f, x, c), cs, init=x)
propagate(f::DifferentialDynamics, x::State, cs, s::Number) = first(Propagate(f, x, cs, s))
propagate(f::DifferentialDynamics, x::State, cs, ss) = collect(Propagate(f, x, cs, ss))

## Instantaneous Controls (control as a function of time)
instantaneous_control(cs, s::Number) = first(InstantaneousControl(cs, s))
instantaneous_control(cs, ss) = collect(InstantaneousControl(cs, ss))

## Waypoints (convenience methods for state propagation)
function waypoints_itr(f::DifferentialDynamics, x::State, cs, dt::AbstractFloat)
    Propagate(f, x, cs, 0:dt:oftype(dt, duration(cs)))
end
function waypoints_itr(f::DifferentialDynamics, x::State, cs, N::Int)
    Propagate(f, x, cs, range(0, stop=duration(cs), length=N))
end
waypoints(f::DifferentialDynamics, x::State, cs, dt_or_N) = collect(waypoints_itr(f, x, cs, dt_or_N))

# Control Intervals
function propagate_ode(f::DifferentialDynamics, x::State, c::ControlInterval, s::Number=duration(c); N=10)
    s > 0 ? ode_rk4((y, t) ->  f(y, instantaneous_control(c,  t)), x,  s, zero(s), N) :
            ode_rk4((y, t) -> -f(y, instantaneous_control(c, -t)), x, -s, zero(s), N)
end
propagate(f::DifferentialDynamics, x::State, c::ControlInterval) = propagate_ode(f, x, c)    # general fallback

## Step Control
struct StepControl{N,T,S<:StaticVector{N}} <: ControlInterval
    t::T
    u::S
    function (::Type{SC})(t::T, u::S) where {N,T,S<:StaticVector{N},SC<:StepControl}
        new{N,T,S}(t, u)
    end
end
const ZeroOrderHoldControl{N,T,S} = StepControl{N,T,S}
duration(c::StepControl) = c.t
Base.zero(::Type{StepControl{N,T,S}}) where {N,T,S} = StepControl(zero(T), zero(S))
function propagate(f::DifferentialDynamics, x::State, c::StepControl, s::Number)
    s <= 0           ? x :
    s >= duration(c) ? propagate(f, x, c) :
                       propagate(f, x, StepControl(s, c.u))
end
instantaneous_control(c::StepControl, s::Number) = c.u
(cost::TimePlusQuadraticControl)(c::StepControl) = c.t*(1 + c.u'*cost.R*c.u)

## Ramp Control
struct RampControl{N,T,S0<:StaticVector{N},Sf<:StaticVector{N}} <: ControlInterval
    t::T
    u0::S0
    uf::Sf
    function (::Type{RC})(t::T, u0::S0, uf::Sf) where {N,T,S0<:StaticVector{N},Sf<:StaticVector{N},RC<:RampControl}
        new{N,T,S0,Sf}(t, u0, uf)
    end
end
const FirstOrderHoldControl{N,T,S0,Sf} = RampControl{N,T,S0,Sf}
RampControl(c::StepControl) = RampControl(c.t, c.u, c.u)
duration(c::RampControl) = c.t
Base.zero(::Type{RampControl{N,T,S0,Sf}}) where {N,T,S0,Sf} = RampControl(zero(T), zero(S0), zero(Sf))
function propagate(f::DifferentialDynamics, x::State, c::RampControl, s::Number)
    s <= 0           ? x :
    s >= duration(c) ? propagate(f, x, c) :
                       propagate(f, x, RampControl(s, c.u0, instantaneous_control(c, s)))
end
instantaneous_control(c::RampControl, s::Number) = c.u0 + (s/c.t)*(c.uf - c.u0)
(cost::TimePlusQuadraticControl)(c::RampControl) = (Δu = c.uf - c.u0; c.t*(1 + c.u0'*cost.R*c.uf + Δu'*cost.R*Δu/3))

## BVP Control
struct BVPControl{T,S0<:State,Sf<:State,Fx<:Function,Fu<:Function} <: ControlInterval
    t::T
    x0::S0
    xf::Sf
    x::Fx
    u::Fu
end
duration(c::BVPControl) = c.t
function Base.zero(::Type{BVPControl{T,S0,Sf,Fx,Fu}}) where {T,S0,Sf,Fx,Fu}
    BVPControl(zero(T), zero(S0), zero(Sf), Fx.instance, Fu.instance)
end
propagate(f::DifferentialDynamics, x::State, c::BVPControl) = (x - c.x0) + c.xf
propagate(f::DifferentialDynamics, x::State, c::BVPControl, s::Number) = (x - c.x0) + c.x(c.x0, c.xf, c.t, s)
instantaneous_control(c::BVPControl, s::Number) = c.u(c.x0, c.xf, c.t, s)
function (cost::TimePlusQuadraticControl)(c::BVPControl{T}; N=10) where {T}
    c.t + ode_rk4((y, t) -> (u = instantaneous_control(c, t); u'*cost.R*u), zero(T), c.t, zero(T), N)
end

# Steering Two-Point Boundary Value Problems (BVPs)
abstract type SteeringConstraints end
abstract type SteeringCache end
struct EmptySteeringConstraints <: SteeringConstraints end
struct BoundedControlNorm{P,T} <: SteeringConstraints
    b::T
end
BoundedControlNorm(b::T=1) where {T} = BoundedControlNorm{2,T}(b)
BoundedControlNorm{P}(b::T) where {P,T} = BoundedControlNorm{P,T}(b)
struct EmptySteeringCache <: SteeringCache end
struct SteeringBVP{D<:DifferentialDynamics,C<:CostFunctional,SC<:SteeringConstraints,SD<:SteeringCache}
    dynamics::D
    cost::C
    constraints::SC
    cache::SD
end
function SteeringBVP(dynamics::DifferentialDynamics, cost::CostFunctional;
                     constraints::SteeringConstraints=EmptySteeringConstraints(),
                     cache::SteeringCache=EmptySteeringCache())
    SteeringBVP(dynamics, cost, constraints, cache)
end
LinearAlgebra.issymmetric(bvp::SteeringBVP) = false                           # general fallback
(bvp::SteeringBVP)(x0::State, xf::State, cost_bound::Number) = bvp(x0, xf)    # general fallback

# Single Integrator
struct SingleIntegratorDynamics{N} <: DifferentialDynamics end

state_dim(::SingleIntegratorDynamics{N}) where {N} = N
control_dim(::SingleIntegratorDynamics{N}) where {N} = N

(::SingleIntegratorDynamics{N})(x::StaticVector{N}, u::StaticVector{N}) where {N} = u
propagate(f::SingleIntegratorDynamics{N}, x::StaticVector{N}, c::StepControl{N}) where {N} = x + c.t*c.u
propagate(f::SingleIntegratorDynamics{N}, x::StaticVector{N}, c::RampControl{N}) where {N} = x + c.t*(c.u0 + c.uf)/2

LinearAlgebra.issymmetric(bvp::SteeringBVP{<:SingleIntegratorDynamics,<:CostFunctional,<:BoundedControlNorm}) = true
const GeometricSteering{N,T} = SteeringBVP{SingleIntegratorDynamics{N},Time,BoundedControlNorm{2,T}}
const SingleIntegratorSteering{N,T} = GeometricSteering{N,T}
GeometricSteering{N}(b=1) where {N} = SteeringBVP(SingleIntegratorDynamics{N}(), Time(), constraints=BoundedControlNorm(b))
GeometricSteering(N, b=1) = GeometricSteering{N}(b)
function (bvp::GeometricSteering{N})(x0::StaticVector{N}, xf::StaticVector{N}) where {N}
    c = norm(xf - x0)/bvp.constraints.b
    ctrl = StepControl(c, SVector((xf - x0)*(c > 0 ? inv(c) : 0)))    # @benchmark appears faster than ifelse
    (cost=c, controls=ctrl)
end

end # module


text:
getcompweights(comp::Array{Float64,1}, crosscors::Array{Float64,3}) =
reshape(cor(comp, crosscors[:,:]), size(crosscors,2,3))

function getweights(outICA, allcrosscors, nan2zeros=false, normalise=true)
    num_components = size(outICA.W,2)
    compweights = [getcompweights(outICA.W[:,x],allcrosscors) for x in 1:num_components]
    nan2zeros && nan2zero!.(compweights)
    normalise && zscore!.(compweights, mean.(compweights),std.(compweights))
    return compweights
end

function getfit(stat, crosscors, num_components)
    @show myid(), num_components
    statfit = fit(stat, convert(Array{Float64,2}, crosscors), num_components, verbose=false, maxiter=10000);
end

function getfitsummary(stat, crosscors, num_components)
    fit = getfit(stat, crosscors, num_components)
    getfitsummary(fit, crosscors)
end

function getfitsummary(fit::MultivariateStats.ICA, crosscors)
    sum(abs(getweights(fit, crosscors)), (2,3))
end

function getbestexamples(weights, n, num_components)
    map(com->selectperm(weights[com,:,:][:], 1:n, rev=false),1:num_components)
end

significantcellpairs(args...) = significantweights(args...)[2]

function significantweights(outICA, allcrosscors, threshold)
    weights = EPhys.getweights(outICA, allcrosscors, true)
    [(diagind(x) = 0) for x in weights]
    cellpairs = (x->abs.(x).>threshold).(weights)

    threshweights = deepcopy(weights)
    [x[!c]=0. for (x,c) in zip(threshweights,cellpairs)]
    return threshweights, cellpairs
end

function getstrength!(out,pattern,crosscors)
    @inbounds for ind in 1:size(crosscors,2)
        out[ind] += dot(view(crosscors,:,ind),pattern)
    end
end

function weightcors!{T<:Real}(cors, weight::T)
    (weight == 0) && (fill!(cors,0);return)
    @inbounds for i in eachindex(cors)
        cors[i] *= weight
    end
end

getstrengthpoint(args) = getstrengthpoint(args...)

function getstrengthpoint(ICw, spikes::SpikeTimes, lags, cellpairs2use::BitArray)
    #pre-allocate
    activation = spzeros(spikes.spiketimes[1].maxtime, sum(cellpairs2use))
    pointscor = zeros(Float64, length(lags),
                      maximum(map(x->length(x),spikes.spiketimes)))
    cpind=0
    # for each cell pair
    # for (ind,spiketimes) in enumerate(spikes.spiketimes)
    #    for (ind2,spiketimes2) in enumerate(spikes.spiketimes)
    for (ind,ind2) in zip(findn(cellpairs2use)...)
        cpind += 1
        # get view of temp array
        pc = view(pointscor,:, eachindex(spikes.spiketimes[ind].timestamps))
        # get instantaneous cross-correletion
        slpxcorr!(pc,spikes.spiketimes[ind2].timestamps,
                  spikes.spiketimes[ind].timestamps,lags)
        # dot product of component pattern with instant xcorr
        getstrength!(
                     view(activation,floor(Int,spikes.spiketimes[ind].timestamps),cpind),
        ICw, pc)
        # reset pre-allocated xcorr array
        view(pointscor, :, 1:length(spikes.spiketimes[ind].timestamps)) .= 0
    end
    activation::SparseMatrixCSC{Float64,Int64}
end

"""
getassemblyactivationpoints(assemblyactivations, thresh)
Get discrete times of activation from continuous activation strength.
"""
function getassemblyactivationpoints(assemblyactivations, thresh)
    ispeak = map(falses, assemblyactivations)
    [(y[getindex.(findlocalmaxima(x),1)]=true) for (x,y) in zip(assemblyactivations,ispeak)]

    isoverthresh =  map(falses,assemblyactivations)
    [(y[zscore(x).>thresh]=true) for (x,y) in zip(assemblyactivations,isoverthresh)]

    isactivationpeak = [x & y for (x,y) in zip(ispeak, isoverthresh)]
    return isactivationpeak
end

"""
function parallel_fiteachcomp(crosscorsIN, min_comp, max_comp, stat=ICA)
Applies ICA with N components where N is a unit range
"""
function parallel_fiteachcomp(processors, crosscorsIN, min_comp, max_comp, stat=ICA)
    fits = pmap(processors, x->getfit(stat, crosscorsIN,x), min_comp:max_comp)
    return fits
end

"""
fitNICAComponentsCrosscors([processors,] spiketimes, bins; normalise_crosscors=true,
usediag=false, just_lower=true, min_comp=3, max_comp=20, verbose=false)
- Cacluates Cross Correlegrams
- Selects which ones to use
- Applies ICA with N components where N is a unit range
"""
function calc_fit_crosscors(spiketimes::SpikeTimes; kwargs...)
    allcrosscors, corselected = calc_select_crosscors(spiketimes::SpikeTimes; kwargs...)
    outICA, outPCA, num_components = pcaica(allcrosscors[:,corselected])
    return allcrosscors, corselected, outICA, outPCA, num_components
end

function calc_select_crosscors(spiketimes::SpikeTimes;
                               bins=-250:5:250, normalise_crosscors=true, usediag=false, just_lower=true,
                               verbose=false, cells2use=:, intrapairstoignore=[], _ignore...)

    verbose && println("Getting Crosscorrelations...")
    allcrosscors = getallcrosscors(spiketimes, bins, normalise_crosscors,
                                   cells2use)

    verbose && println("Selecting Crosscorrelations...")
    excludecors = falses(allcrosscors[1,:,:])
    excludecors[intrapairstoignore,intrapairstoignore] = true
    corselected = selectcrosscors!(allcrosscors,
                                   usediag, just_lower,
                                   excludecors[:])

    return allcrosscors, corselected
end

function pcaica(matrix)
    outPCA,λmax,num_components = EPhys.pca_npatterns(matrix)
    outICA = EPhys.getfit(ICA, matrix, num_components)
    return outICA, outPCA, num_components
end

if false
    function fitNICAComponents2ShufCrosscors(processors::Base.AbstractWorkerPool, orig_spikematrix,
                                             nshuf::Int; kwargs...)
        shufICAComponentfits = Array{Tuple{Array{Float64,3},Array{Bool,1},Array{Any,1}},1}(nshuf)
        for i in 1:nshuf
            println("Shuffle $i of $nshuf")
            shufspikematrix = mapslices(shuffle, orig_spikematrix, 1)
            shufspiketimes = getspiketimesarray(shufspikematrix)
            shufICAComponentfits[i] = fitNICAComponents2Crosscors(default_worker_pool(),
                                                                  shufspiketimes; kwargs...)
        end
        return shufICAComponentfits
    end
end

"""
function getstrength(pattern,crosscors[,weights])
Get the strength of a particular pattern for a set of crosscorrelations
Use `weights` to weight output by a matrix with the same dims as crosscors
"""
function getstrength(pattern,crosscors)
    tweights = mapslices(x->dot(x,pattern),crosscors,1)
end

function getstrength(pattern,crosscors, weights)
    tweights = mapslices(x->dot(x,pattern),crosscors,1).*weights
    sum(tweights)::Float64
end
"""
function ptrack_ensemble{T}(processors, spikematrix, windows, ,bins, ICw::Array{T,2},
Cweights::Array{T,3}; use_proj=true, batch_size=1, temp="none")
Track assembly strength over time using multiple processors.
NB This function is still under development
"""
function ptrack_ensemble{T}(processors, spikematrix, ICw::Array{T,2}, ICweights::Array{T,3};
                            use_proj=true, bins=-200:5:200, batch_size=10, windows=0,
                            verbose=false, metadata=Dict(), ignore_...)

    options = Dict(:use_proj=>use_proj, :batch_size=>batch_size, :metadata=>metadata, :bins=>bins,
                   :windows=>windows)
    try
        num_components = size(ICw,2)
        trackedensembles = zeros(length(windows), num_components)
        verbose && println("Windowing Spikes")

        sliding_spikes = pmap(processors, win->getspikes(spikematrix, win), windows,
                              batch_size=100);

        verbose && println("Computing Sliding Crosscorrelations")
        #chunksize = length(processors.workers)*batch_size
        chunksize = 100
        @show length(processors.workers),batch_size,chunksize
        (length(windows) < chunksize) && (chunksize = length(windows))
        time_created = string(Dates.format(now(),"yymmdd_HHMMSS"))
        verbose && println("Using groupname var_$time_created")

        sliding_cor_chunk = [zeros(Int, length(bins), size(spikematrix,2), size(spikematrix,2))
                             for x in 1:chunksize]
        scor = zeros(Int, length(bins), size(spikematrix,2), size(spikematrix,2), chunksize)

        @show length(sliding_spikes)
        for index in 1:chunksize:length(sliding_spikes)
            ## Chunking
            if index < (length(sliding_spikes)-chunksize)
                chunkrange = index:(index+chunksize-1)
            else
                chunkrange = index:length(sliding_spikes)
            end
            println("Processing chunk $chunkrange of $(length(sliding_spikes))")
            ## Sliding Cross Cor
            sliding_cor_chunk[1:length(chunkrange)] = pmap(processors, spks->allcross_mat(spks, bins), sliding_spikes[chunkrange],
                                                           batch_size=batch_size)
            for (ind, val) in enumerate(sliding_cor_chunk[1:length(chunkrange)])
                scor[:,:,:,ind] = val
            end

            ## Track Portion
            println("Projecting Chunk Onto Components...")
            for (ind,chunkbit) in enumerate(chunkrange)
                for component in 1:num_components
                    trackedensembles[chunkbit, component] = getstrength(ICw[:,component],scor[:,:,:,ind],
                                                                        ICweights[component,:,:])
                end
            end
        end

        println("Saving Tracked Ensembles...")
        savebigfile(metadata["bsnm"], "ensemble_tracked", [trackedensembles],
                    ["trackedensembles"], options)
        return trackedensembles
    catch error
        @everywhere gc(true)
        sleep(0.5)
        @everywhere gc(true)
        throw(error)
    end
end

#function projensemblecor{T}(processors, bsnm, loadoptions; use_proj=true)
#loadbigfile(bsnm, "assembly", ["outICA", "weights"], loadoptions)
#loadbigfile(bsnm, "slidingcor", "sliding_cor", loadoptions, memorymap=true)
#i
#end

function projensemblecor{T}(processors, sliding_cor, ICw::Array{T,2},
                            ICweights::Array{T,3}; use_proj=true, batch_size=10, metadata=0, bins=0, windows=0, _ignore...)

    options = Dict(:use_proj=>use_proj, :batch_size=>batch_size, :metadata=>metadata, :bins=>bins,
                   :windows=>windows)

    num_components = size(ICw,2)
    @show processors
    println("Projecting Component Weights back onto Sliding CrossCorrelatons")
    trackedensembles = projensemblecor_main(sliding_cor,
                                            ICw, ICweights, num_components, batch_size, use_proj)

    savebigfile(metadata["bsnm"], "ensemble_tracked", [trackedensembles],
                ["trackedensembles"], options)
    return trackedensembles
end

function projensemblecor_main(sliding_cor, ICw, ICweights, num_components,
                              batch_size,use_proj)
    if use_proj
        out = zeros(num_components, size(sliding_cor,4))
        for index in 1:batch_size:size(sliding_cor,4)
            if index < (size(sliding_cor,4)-batch_size)
                chunkrange = index:(index+batch_size-1)
            else
                chunkrange = index:size(sliding_cor,4)
            end
            println("Processing $chunkrange...")
            for chunkbit in chunkrange
                scor = sliding_cor[:,:,:,chunkbit]
                for component in 1:num_components
                    out[component, chunkbit] = getstrength(ICw[:,component],scor,
                                                           ICweights[component,:,:])
                end
            end

            #out[chunkrange] = pmap(processors, y->getstrength(ICw,y,ICweights),
            #sliding_cor[:,:,:,chunkrange])
        end
        return out
    else
        return pmap(processors, y->getstrength(ICw,y), sliding_cor)
    end
end

#=
function ptrack_ensemble{T}(spikematrix, windows, bins, ICw::Array{T,2},
    ICweights::Array{T,3}; use_proj=true)
    num_components = size(ICw,2)
    trackedensembles = zeros(length(windows), num_components)

    for component in 1:num_components
        @show component
        trackedensembles[:,component] = projensemblecor(spikematrix, windows,
        bins, ICw[:,component], ICweights[component,:])
        sleep(0.5)
    end
    trackedensembles
end=#

"""
function ptrack_ensemble{T}(processors, spikematrix, windows, bins, ICw::Array{T,2},
Cweights::Array{T,3}; use_proj=true, batch_size=1, temp="none")
Track assembly strength over time using multiple processors.
NB This function is still under development
"""
function ptrack_ensemble_old{T}(processors, spikematrix, ICw::Array{T,2}, ICweights::Array{T,3};
                                use_proj=true, bins=-200:5:200, batch_size=10, windows=0,
                                verbose=false, metadata=Dict(), ignore_...)

    options = Dict(:use_proj=>use_proj, :batch_size=>batch_size, :metadata=>metadata, :bins=>bins,
                   :windows=>windows)
    try
        num_components = size(ICw,2)
        trackedensembles = zeros(length(windows), num_components)
        verbose && println("Windowing Spikes")

        sliding_spikes = pmap(processors, win->getspikes(spikematrix, win), windows,
                              batch_size=100);

        verbose && println("Computing Sliding Crosscorrelations")
        #chunksize = length(processors.workers)*batch_size
        chunksize = 100
        @show length(processors.workers),batch_size,chunksize
        (length(windows) < chunksize) && (chunksize = length(windows))
        time_created = string(Dates.format(now(),"yymmdd_HHMMSS"))
        verbose && println("Using groupname var_$time_created")

        sliding_cor_chunk = [zeros(Int, length(bins), size(spikematrix,2), size(spikematrix,2))
                             for x in 1:chunksize]
        sliding_cor_chunk_mat = zeros(Int, length(bins), size(spikematrix,2), size(spikematrix,2), chunksize)

        @show length(sliding_spikes)
        for index in 1:chunksize:length(sliding_spikes)
            if index < (length(sliding_spikes)-chunksize)
                chunkrange = index:(index+chunksize-1)
            else
                chunkrange = index:length(sliding_spikes)
            end
            println("Processing chunk $chunkrange of $(length(sliding_spikes))")
            sliding_cor_chunk[1:length(chunkrange)] = pmap(processors, spks->allcross_mat(spks, bins), sliding_spikes[chunkrange],
                                                           batch_size=batch_size)
            println("Writing chunk ...")
            for (ind, val) in enumerate(sliding_cor_chunk[1:length(chunkrange)])
                sliding_cor_chunk_mat[:,:,:,ind] = val
            end
            savebigfile(metadata["bsnm"], "slidingcor", [sliding_cor_chunk_mat[:,:,:,1:length(chunkrange)]], ["sliding_cor"],
                        time_created, (:,:,:,chunkrange,), (size(sliding_cor_chunk_mat)[1:3]...,length(sliding_spikes)),
                        options)
        end

        println("sliding cors saved")
        return nothing
    catch error
        @everywhere gc(true)
        sleep(0.5)
        @everywhere gc(true)
        throw(error)
    end
end


text:
#= Instructions:

- Pkg.add("PkgBenchmark.jl")
- using PkgBenchmark

- results = benchmarkpkg("IntervalArithmetic")
- show(results)

To compare the current version with a given release use `judge`:

- results = judge("IntervalArithmetic", tag)
- show(results)

To export the benchmark results to a markdown file, do:

- export_markdown("results.md", results)
=#

using BenchmarkTools, IntervalArithmetic

const SUITE = BenchmarkGroup() # parent BenchmarkGroup to contain our suite

S = SUITE["Constructors"] = BenchmarkGroup()
S["Interval"] = @benchmarkable Interval(1., 2.)

S = SUITE["Intervals"] = BenchmarkGroup()
S["interval"] = @benchmarkable interval(0.1)
S["pi"] = @benchmarkable interval(pi)
S["expression"] = @benchmarkable sin(interval(0.1)) + cos(interval(0.2))

fma_mul(x, y) = fma(x, y, zero(x))

S = SUITE["Arithmetic"] = BenchmarkGroup()
begin
    intervals = AbstractInterval{Float64}[
        Interval(1., 2.),
        Interval(0., 4.),
      #  emptyinterval(Float64),
        Interval(-1., 2.),
        Interval(-2., -1.),
        Interval(0., 0.),
        Interval(-1., 0.),
    ]

    for op in (+, -, *, /, fma_mul)
        S[string(op)] = @benchmarkable foreach(x -> foldr($(op), $(rand(intervals, 10)), init = $(interval(5) + interval(-abs(randn()), abs(randn())))), $(1:100))
    end
end

S = SUITE["Elementary functions"] = BenchmarkGroup()
begin
    for op in (exp, log ∘ abs, sin, tan, abs)
        S[string(op)] = @benchmarkable foreach(q -> foldl($((res, x) -> op(res)), $(1:10), init = $(rand(intervals))), $(1:100))
    end
end

S = SUITE["Sum"] = BenchmarkGroup()
begin
    sum1(N) = sum(interval(i, i+1) for i in 1:N)
    sum2(N) = (one = interval(1); sum(one / interval(i^2) for i in 1:N))
    sum3(N) = foldl((x, i) -> x + x^-i, 1:N, init = interval(1))
    sum4(N) = foldl((x, i) -> x + x^-i, 1:N, init = emptyinterval())
    S["Sum1"] = @benchmarkable sum1(1000)
    S["Sum2"] = @benchmarkable sum2(1000)
    S["Sum3"] = @benchmarkable sum3(1000)
    S["Sum4"] = @benchmarkable sum4(1000)
end

display(run(SUITE))
println()


text:
# Here be all conversions from Laszip C Types to LasIO types

"Conversion to LasIO header"
LasHeader(h::LazHeader) =
    LasIO.LasHeader(
        h.file_source_ID,
        h.global_encoding,
        h.project_ID_GUID_data_1,
        h.project_ID_GUID_data_2,
        h.project_ID_GUID_data_3,
        join(convert(NTuple{8, Char}, h.project_ID_GUID_data_4)),
        h.version_major,
        h.version_minor,
        join(convert(NTuple{32, Char}, h.system_identifier)),
        join(convert(NTuple{32, Char}, h.generating_software)),
        h.file_creation_day,
        h.file_creation_year,
        h.header_size,
        h.offset_to_point_data,
        h.number_of_variable_length_records,
        h.point_data_format,
        h.point_data_record_length,
        h.number_of_point_records,
        [n for n in h.number_of_points_by_return],
        h.x_scale_factor,
        h.y_scale_factor,
        h.z_scale_factor,
        h.x_offset,
        h.y_offset,
        h.z_offset,
        h.max_x,
        h.min_x,
        h.max_y,
        h.min_y,
        h.max_z,
        h.min_z,
        Vector{LasVariableLengthRecord}(),
        Vector{UInt8}()
    )

"ASPRS LAS point data record format 0"
Base.convert(::Type{LasPoint0}, p::LazPoint) =
    LasPoint0(
        p.X,
        p.Y,
        p.Z,
        p.intensity,
        p.return_number,
        p.classification,
        p.scan_angle_rank,
        p.user_data,
        p.point_source_ID
    )

"ASPRS LAS point data record format 1"
Base.convert(::Type{LasPoint1}, p::LazPoint) =
    LasPoint1(
        p.X,
        p.Y,
        p.Z,
        p.intensity,
        p.return_number,
        p.classification,
        p.scan_angle_rank,
        p.user_data,
        p.point_source_ID,
        p.gps_time
    )

"ASPRS LAS point data record format 2"
Base.convert(::Type{LasPoint2}, p::LazPoint) =
    LasPoint2(
        p.X,
        p.Y,
        p.Z,
        p.intensity,
        p.return_number,
        p.classification,
        p.scan_angle_rank,
        p.user_data,
        p.point_source_ID,
        reinterpret(N0f16, p.rgb[1]),
        reinterpret(N0f16, p.rgb[2]),
        reinterpret(N0f16, p.rgb[3])
    )

"ASPRS LAS point data record format 3"
Base.convert(::Type{LasPoint3}, p::LazPoint) =
    LasPoint3(
        p.X,
        p.Y,
        p.Z,
        p.intensity,
        p.return_number,
        p.classification,
        p.scan_angle_rank,
        p.user_data,
        p.point_source_ID,
        p.gps_time,
        reinterpret(N0f16, p.rgb[1]),
        reinterpret(N0f16, p.rgb[2]),
        reinterpret(N0f16, p.rgb[3])
    )


text:
#!/usr/bin/env julia
#=
Package Gnar
del2z <delta.z@aliyun.com>
=#
module Gnar

include("struct.jl")
export Polar, Model

include("data/Data.jl")
include("wordvec/WordVec.jl")

end # module


text:
println("Running tests ...")
include("test_hb.jl")


text:
const QuiqboxSubModules = ["Molden"]

# Initialization function.
function __init__()
    tryIncluding.(QuiqboxSubModules)

    # Force JIT compilation
    # GTBasis(genBasisFunc.([[0,0,0], [0,0,1]], "STO-3G") |> flatten)
end


text:
export BrickletTemperatureIRAmbientTemperatureCallbackThreshold
struct BrickletTemperatureIRAmbientTemperatureCallbackThreshold
    option::Char
    min::Integer
    max::Integer
end

export BrickletTemperatureIRObjectTemperatureCallbackThreshold
struct BrickletTemperatureIRObjectTemperatureCallbackThreshold
    option::Char
    min::Integer
    max::Integer
end

export BrickletTemperatureIRIdentity
struct BrickletTemperatureIRIdentity
    uid::String
    connected_uid::String
    position::Char
    hardware_version::Vector{Integer}
    firmware_version::Vector{Integer}
    device_identifier::Integer
end

export BrickletTemperatureIR
"""
Measures contactless object temperature between -70°C and +380°C
"""
mutable struct BrickletTemperatureIR <: TinkerforgeDevice
    ipcon::IPConnection
    deviceInternal::PyObject

    """
    Creates an object with the unique device ID *uid* and adds it to
    the IP Connection *ipcon*.
    """
    function BrickletTemperatureIR(uid::String, ipcon::IPConnection)
        package = pyimport("tinkerforge.bricklet_temperature_ir")
        deviceInternal = package.BrickletTemperatureIR(uid, ipcon.ipconInternal)

        return new(ipcon, deviceInternal)
    end
end

export get_ambient_temperature
"""
    $(SIGNATURES)

Returns the ambient temperature of the sensor.

If you want to get the ambient temperature periodically, it is recommended
to use the :cb:`Ambient Temperature` callback and set the period with
:func:`Set Ambient Temperature Callback Period`.
"""
function get_ambient_temperature(device::BrickletTemperatureIR)
    return device.deviceInternal.get_ambient_temperature()
end

export get_object_temperature
"""
    $(SIGNATURES)

Returns the object temperature of the sensor, i.e. the temperature
of the surface of the object the sensor is aimed at.

The temperature of different materials is dependent on their `emissivity
<https://en.wikipedia.org/wiki/Emissivity>`__. The emissivity of the material
can be set with :func:`Set Emissivity`.

If you want to get the object temperature periodically, it is recommended
to use the :cb:`Object Temperature` callback and set the period with
:func:`Set Object Temperature Callback Period`.
"""
function get_object_temperature(device::BrickletTemperatureIR)
    return device.deviceInternal.get_object_temperature()
end

export set_emissivity
"""
    $(SIGNATURES)

Sets the `emissivity <https://en.wikipedia.org/wiki/Emissivity>`__ that is
used to calculate the surface temperature as returned by
:func:`Get Object Temperature`.

The emissivity is usually given as a value between 0.0 and 1.0. A list of
emissivities of different materials can be found
`here <https://www.infrared-thermography.com/material.htm>`__.

The parameter of :func:`Set Emissivity` has to be given with a factor of
65535 (16-bit). For example: An emissivity of 0.1 can be set with the
value 6553, an emissivity of 0.5 with the value 32767 and so on.

.. note::
 If you need a precise measurement for the object temperature, it is
 absolutely crucial that you also provide a precise emissivity.

The emissivity is stored in non-volatile memory and will still be used after a restart or power cycle of the Bricklet.
"""
function set_emissivity(device::BrickletTemperatureIR, emissivity)
    device.deviceInternal.set_emissivity(emissivity)
end

export get_emissivity
"""
    $(SIGNATURES)

Returns the emissivity as set by :func:`Set Emissivity`.
"""
function get_emissivity(device::BrickletTemperatureIR)
    return device.deviceInternal.get_emissivity()
end

export set_ambient_temperature_callback_period
"""
    $(SIGNATURES)

Sets the period with which the :cb:`Ambient Temperature` callback is
triggered periodically. A value of 0 turns the callback off.

The :cb:`Ambient Temperature` callback is only triggered if the temperature has
changed since the last triggering.
"""
function set_ambient_temperature_callback_period(device::BrickletTemperatureIR, period)
    device.deviceInternal.set_ambient_temperature_callback_period(period)
end

export get_ambient_temperature_callback_period
"""
    $(SIGNATURES)

Returns the period as set by :func:`Set Ambient Temperature Callback Period`.
"""
function get_ambient_temperature_callback_period(device::BrickletTemperatureIR)
    return device.deviceInternal.get_ambient_temperature_callback_period()
end

export set_object_temperature_callback_period
"""
    $(SIGNATURES)

Sets the period with which the :cb:`Object Temperature` callback is
triggered periodically. A value of 0 turns the callback off.

The :cb:`Object Temperature` callback is only triggered if the temperature
has changed since the last triggering.
"""
function set_object_temperature_callback_period(device::BrickletTemperatureIR, period)
    device.deviceInternal.set_object_temperature_callback_period(period)
end

export get_object_temperature_callback_period
"""
    $(SIGNATURES)

Returns the period as set by :func:`Set Object Temperature Callback Period`.
"""
function get_object_temperature_callback_period(device::BrickletTemperatureIR)
    return device.deviceInternal.get_object_temperature_callback_period()
end

export set_ambient_temperature_callback_threshold
"""
    $(SIGNATURES)

Sets the thresholds for the :cb:`Ambient Temperature Reached` callback.

The following options are possible:

.. csv-table::
 :header: "Option", "Description"
 :widths: 10, 100

 "'x'",    "Callback is turned off"
 "'o'",    "Callback is triggered when the ambient temperature is *outside* the min and max values"
 "'i'",    "Callback is triggered when the ambient temperature is *inside* the min and max values"
 "'<'",    "Callback is triggered when the ambient temperature is smaller than the min value (max is ignored)"
 "'>'",    "Callback is triggered when the ambient temperature is greater than the min value (max is ignored)"
"""
function set_ambient_temperature_callback_threshold(device::BrickletTemperatureIR, option, min, max)
    device.deviceInternal.set_ambient_temperature_callback_threshold(option, min, max)
end

export get_ambient_temperature_callback_threshold
"""
    $(SIGNATURES)

Returns the threshold as set by :func:`Set Ambient Temperature Callback Threshold`.
"""
function get_ambient_temperature_callback_threshold(device::BrickletTemperatureIR)
    return device.deviceInternal.get_ambient_temperature_callback_threshold()
end

export set_object_temperature_callback_threshold
"""
    $(SIGNATURES)

Sets the thresholds for the :cb:`Object Temperature Reached` callback.

The following options are possible:

.. csv-table::
 :header: "Option", "Description"
 :widths: 10, 100

 "'x'",    "Callback is turned off"
 "'o'",    "Callback is triggered when the object temperature is *outside* the min and max values"
 "'i'",    "Callback is triggered when the object temperature is *inside* the min and max values"
 "'<'",    "Callback is triggered when the object temperature is smaller than the min value (max is ignored)"
 "'>'",    "Callback is triggered when the object temperature is greater than the min value (max is ignored)"
"""
function set_object_temperature_callback_threshold(device::BrickletTemperatureIR, option, min, max)
    device.deviceInternal.set_object_temperature_callback_threshold(option, min, max)
end

export get_object_temperature_callback_threshold
"""
    $(SIGNATURES)

Returns the threshold as set by :func:`Set Object Temperature Callback Threshold`.
"""
function get_object_temperature_callback_threshold(device::BrickletTemperatureIR)
    return device.deviceInternal.get_object_temperature_callback_threshold()
end

export set_debounce_period
"""
    $(SIGNATURES)

Sets the period with which the threshold callbacks

* :cb:`Ambient Temperature Reached`,
* :cb:`Object Temperature Reached`

are triggered, if the thresholds

* :func:`Set Ambient Temperature Callback Threshold`,
* :func:`Set Object Temperature Callback Threshold`

keep being reached.
"""
function set_debounce_period(device::BrickletTemperatureIR, debounce)
    device.deviceInternal.set_debounce_period(debounce)
end

export get_debounce_period
"""
    $(SIGNATURES)

Returns the debounce period as set by :func:`Set Debounce Period`.
"""
function get_debounce_period(device::BrickletTemperatureIR)
    return device.deviceInternal.get_debounce_period()
end

export get_identity
"""
    $(SIGNATURES)

Returns the UID, the UID where the Bricklet is connected to,
the position, the hardware and firmware version as well as the
device identifier.

The position can be 'a', 'b', 'c', 'd', 'e', 'f', 'g' or 'h' (Bricklet Port).
A Bricklet connected to an :ref:`Isolator Bricklet <isolator_bricklet>` is always at
position 'z'.

The device identifier numbers can be found :ref:`here <device_identifier>`.
|device_identifier_constant|
"""
function get_identity(device::BrickletTemperatureIR)
    return device.deviceInternal.get_identity()
end

export register_callback
"""
Registers the given *function* with the given *callback_id*.
"""
function register_callback(device::BrickletTemperatureIR, callback_id, function_)
    if isnothing(function_)
        device.registered_callbacks.pop(callback_id, None)
    else
        device.registered_callbacks[callback_id] = function_
    end
end


text:
#=Copyright (c) 2017 Gabriel Goh

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.=#

gen_rand2(n,x) = [(0.1*randn(n,1) + x[1]) (0.1*randn(n,1) - x[2])]

X = [gen_rand2(12, [1,0.7]);
     gen_rand2(12, [-1,0.7]);
     gen_rand2(12, [0,-1])];

scatter(X[:,1], X[:,2])

function Φ(x)
  o = Float64[]
  for i = 0:5, j = 0:5
    push!(o, (x[1]^i)*x[2]^j )
  end
  return o
end

res = 500
Z = [Φ([i,j])[10] for i = linspace(-1,1,res), j = linspace(-1,1,res)]

D = cat(2,[Φ(X[i,:]) for i = 1:size(X,1)]...)
S,U = eig(D*D')

n = 3
Z = [vecdot(Φ([i,j]),U[:,n]) for i = linspace(-2,2,res), j = linspace(-2,2,res)]
c = 0.1*sqrt(S[n])
#heatmap(linspace(-2,2,res),linspace(-2,2,res),min(max(Z,-c),c))
heatmap(linspace(-2,2,res),linspace(-2,2,res),log(abs(Z)))
scatter!(X[:,2], X[:,1])

max_stars_repo_name: sbeleidy/post--momentum

max_issues_repo_name: sbeleidy/post--momentum

max_forks_repo_name: sbeleidy/post--momentum


text:
# ---------------------------------------------------------------------------- #
#
#   juliaTikz.jl
#
#   Julia Library to write to Tikz
#
#   juliaTikz
#   Spring 2018
#
#   Max Opgenoord
#
# ---------------------------------------------------------------------------- #

# __precompile__()

"""
    Tikz.jl

Julia Library to write to Tikz

Max Opgenoord

Spring 2018
"""
module Tikz

    using Printf
    import Dates

    export tikzStyle
    include("tikzStyle.jl")

    export tikzObj, closeObj
    include("tikzObj.jl")

    export axisObj
    include("axisObj.jl")

    export plot
    include("plot.jl")

    export node
    include("node.jl")

end

max_stars_repo_name: mopg/julia-tikz

max_issues_repo_name: mopg/julia-tikz

max_forks_repo_name: mopg/julia-tikz


text:
"""
    GradField
A structure that contains all the necessary items for automatic differentiation.
All function types in JITrench have this type as a field named "grad_field".
"""
mutable struct GradField
    inputs :: Union{Nothing, Vector{Variable}}
    outputs :: Union{Nothing, Vector{Variable}}
    generation :: Union{Nothing, Int}
    function GradField(inputs=nothing, outputs=nothing, generation=nothing)
        new(inputs, outputs, generation)
    end
end

function Base.show(io::IO, f::DiffableFunction)
    print(io, typeof(f))
end

function Base.show(io::IO, ::MIME"text/plain", f::DiffableFunction)
    print(io,"""
    Type: $(typeof(f))
      GradField:
        inputs: $(f.grad_field.inputs)
        outputs: $(f.grad_field.outputs)
        generation: $(f.grad_field.generation)""")
end


text:
using RedBlackGraph
using LinearAlgebra
using Test

@testset "Avus Linear Algebra Tests" begin
    r = red_one(1)
    o = one(r)

    # test dot product of myself and father
    u = [r, 2, 3, 0, 0]
    v = [2, r, 0, 0, 0]
    @test u ⋅ v == 2

    # test dot product of myself and mother
    u = [r, 2, 3, 0, 0]
    v = [3, 0, o, 0, 0]
    @test u ⋅ v == 3

    # sample from docs
    u = [2, 0, 0, 0, r]
    v = [3, 0, r, 0, 0]
    @test u ⋅ v == 5

    # test dot product of my daughter and my father
    u = [2, 0, 0, 0, o]
    v = [2, r, 0, 0, 0]
    @test u ⋅ v == 4

    A = [r 2 3 0 0;
         0 r 0 2 0;
         0 0 1 0 0;
         0 0 0 r 0;
         2 0 0 0 1]
    expected_result =  [r 2 3 4 0;
                        0 r 0 2 0;
                        0 0 1 0 0;
                        0 0 0 r 0;
                        2 4 5 0 1]
    result = A * A
    @test result == expected_result

    expected_result =  [r 2 3 4 0;
                        0 r 0 2 0;
                        0 0 1 0 0;
                        0 0 0 r 0;
                        2 4 5 8 1]
    result = A * result
    @test result == expected_result

    A =  [r 2 3 4 0;
          0 r 0 2 0;
          0 0 1 0 0;
          0 0 0 r 0;
          2 4 5 0 1]
    u = [2 0 0 0 0]
    v = [0, 3, 0, 0, 0]

    result = u * A
    u_lambda = [2 4 5 8 0]
    @test result == u_lambda

    result = A * v
    v_lambda = [5; 3; 0; 0; 9]
    @test result == v_lambda

    A = [r 2 3 0 0;
         0 r 0 2 0;
         0 0 1 0 0;
         0 0 0 r 0;
         2 0 0 0 1]
    @test A == A * I

end