text:
using Flux
using Flux.Tracker
using Statistics
import LinearAlgebra.Diagonal
using Random

module CycleWorld
using Random

"""
 CycleWorld

   1 -> 0 -> 0 -> ... -> 0 -|
   ^------------------------|

chain_length: size of cycle
actions: Progress

"""

mutable struct CycleWorldEnv
    chain_length::Int64
    agent_state::Int64
    actions::AbstractSet
    partially_observable::Bool
    CycleWorldEnv(chain_length::Int64; rng=Random.GLOBAL_RNG, partially_observable=true) =
        new(chain_length,
            0,
            Set(1:1),
            partially_observable)
end

function reset!(env::CycleWorldEnv; rng = Random.GLOBAL_RNG, kwargs...)
    env.agent_state = 0
end

get_actions(env::CycleWorldEnv) = env.actions

function environment_step!(env::CycleWorldEnv, action::Int64; rng = Random.GLOBAL_RNG, kwargs...)
    # actions 1 == Turn Left
    # actions 2 == Turn Right
    # actions 3 == Up
    # JuliaRL.step()
    env.agent_state = (env.agent_state + 1) % env.chain_length
    # JuliaRL
end

function get_reward(env::CycleWorldEnv) # -> get the reward of the environment
    return 0
end

function get_state(env::CycleWorldEnv) # -> get state of agent
    if env.partially_observable
        return partially_observable_state(env)
    else
        return fully_observable_state(env)
    end
end

function fully_observable_state(env::CycleWorldEnv)
    return [env.agent_state]
end

function partially_observable_state(env::CycleWorldEnv)
    state = zeros(1)
    if env.agent_state == 0
        state[1] = 1
    end
    return state
end

function is_terminal(env::CycleWorldEnv) # -> determines if the agent_state is terminal
    return false
end

function Base.show(io::IO, env::CycleWorldEnv)
    println(env.agent_state)
end

function start!(env::CycleWorldEnv; rng=Random.GLOBAL_RNG, kwargs...)
    reset!(env; rng=rng, kwargs...)
    return env, get_state(env)
end

function step!(env::CycleWorldEnv, action; rng = Random.GLOBAL_RNG, kwargs...) # -> env, state, reward, terminal
    environment_step!(env, action; rng=rng, kwargs...)
    return env, get_state(env), get_reward(env), is_terminal(env)
end

end #end CycleWorld

build_features(s) = [1.0, s[1], 1-s[1]]

glorot_uniform(rng::Random.AbstractRNG, dims...) = (rand(rng, Float32, dims...) .- 0.5f0) .* sqrt(24.0f0/sum(dims))
glorot_normal(rng::Random.AbstractRNG, dims...) = randn(rng, Float32, dims...) .* sqrt(2.0f0/sum(dims))

abstract type AbstractGVFLayer end

function get_question_parameters(gvfn::AbstractGVFLayer, preds_tilde, stp1) end

mutable struct GVFLayer{F, A, V} <: AbstractGVFLayer
    σ::F
    Wx::A
    Wh::A
    cumulants::AbstractArray
    discounts::AbstractArray
    h::V
end

GVFLayer(num_gvfs, num_ext_features, cumulants, discounts; init=Flux.glorot_uniform, σ_int=σ) =
    GVFLayer(
        σ_int,
        param(0.1.*init(num_gvfs, num_ext_features)),
        param(0.1.*init(num_gvfs, num_gvfs)),
        cumulants,
        discounts,
        param(Flux.zeros(num_gvfs)))

function get_question_parameters(gvfn::GVFLayer{F,A,V}, preds_tilde, state_tp1) where {F, A, V}
    cumulants = [gvfn.cumulants[i](state_tp1, preds_tilde) for i in 1:length(gvfn.cumulants)]
    discounts = [gvfn.discounts[i](state_tp1) for i in 1:length(gvfn.cumulants)]
    return cumulants, discounts, ones(size(gvfn.discounts))
end

function (m::GVFLayer)(h, x)
    new_h = m.σ.(m.Wx*x + m.Wh*h)
    return new_h, new_h
end

Flux.hidden(m::GVFLayer) = m.h
Flux.@treelike GVFLayer
GVFN(args...; kwargs...) = Flux.Recur(GVFLayer(args...; kwargs...))

function reset!(m, h_init)
    Flux.reset!(m)
    m.state.data .= h_init
end

function jacobian(δ, pms)
    k  = length(δ)
    J = IdDict()
    for id in pms
        v = get!(J, id, zeros(k, size(id)...))
        for i = 1:k
            Flux.back!(δ[i], once = false) # Populate gradient accumulator
            v[i, :,:] .= id.grad
            id.grad .= 0 # Reset gradient accumulator
        end
    end
    J
end

function jacobian!(J::IdDict, δ::TrackedArray, pms::Params)
    k  = length(δ)
    for i = 1:k
        Flux.back!(δ[i], once = false) # Populate gradient accumulator
        for id in pms
            v = get!(J, id, zeros(typeof(id[1].data), k, size(id)...))::Array{typeof(id[1].data), 3}
            v[i, :, :] .= id.grad
            id.grad .= 0 # Reset gradient accumulator
        end
    end
end

abstract type Optimizer end

function train!(gvfn::Flux.Recur{T}, opt::Optimizer, h_init, state_seq, env_state_tp1) where {T <: AbstractGVFLayer} end
function train!(gvfn::AbstractGVFLayer, opt::Optimizer, h_init, state_seq, env_state_tp1)
    throw("$(typeof(opt)) not implemented for $(typeof(gvfn)). Try Flux.Recur{$(typeof(gvfn))} ")
end

mutable struct RTD_jacobian <: Optimizer
    α::Float64
    J::IdDict
    RTD_jacobian(α) = new(α, IdDict{Any, Array{Float64, 3}}())
end

function train!(gvfn::Flux.Recur{T}, opt::RTD_jacobian, h_init, states, env_state_tp1) where {T <: AbstractGVFLayer}

    α = opt.α
    reset!(gvfn, h_init)
    preds = gvfn.(states)
    preds_t = preds[end-1]
    preds_tilde = Flux.data(preds[end])

    cumulants, discounts, ρ = get_question_parameters(gvfn.cell, preds_tilde, env_state_tp1)

    targets = cumulants .+ discounts.*preds_tilde
    δ = targets .- preds_t

    jacobian!(opt.J, δ, Params([gvfn.cell.Wx, gvfn.cell.Wh]))

    for weights in Params([gvfn.cell.Wx, gvfn.cell.Wh])
        Flux.Tracker.update!(weights, -α.*sum(δ.*opt.J[weights]; dims=1)[1,:,:])
    end
end

mutable struct RTD <: Optimizer
    α::Float64
    RTD(α) = new(α)
end

function train!(gvfn::Flux.Recur{T}, opt::RTD, h_init, states, env_state_tp1) where {T <: AbstractGVFLayer}

    α = opt.α

    reset!(gvfn, h_init)
    preds = gvfn.(states)
    preds_t = preds[end-1]
    preds_tilde = Flux.data(preds[end])

    cumulants, discounts, ρ = get_question_parameters(gvfn.cell, preds_tilde, env_state_tp1)
    targets = cumulants .+ discounts.*preds_tilde
    δ = targets .- preds_t

    prms = params(gvfn)
    # println(prms)
    grads = Tracker.gradient(() ->mean(δ.^2), prms)
    for weights in prms
        Flux.Tracker.update!(weights, -α.*grads[weights])
    end
end

mutable struct RTDC <: Optimizer
    α::Float64
    β::Float64
    h::IdDict
    RTD(α) = new(α)
end

function train!(gvfn::Flux.Recur{T}, opt::RTDC, h_init, states, env_state_tp1) where {T <: AbstractGVFLayer}

    α = opt.α

    reset!(gvfn, h_init)
    preds = gvfn.(states)
    preds_t = preds[end-1]
    preds_tilde = Flux.data(preds[end])

    cumulants, discounts, ρ = get_question_parameters(gvfn.cell, preds_tilde, env_state_tp1)
    targets = cumulants .+ discounts.*preds_tilde
    δ = targets .- preds_t

    prms = params(gvfn)
    # println(prms)
    grads = Tracker.gradient(() ->mean(δ.^2), prms)
    for weights in prms
        Flux.Tracker.update!(weights, -α.*grads[weights])
    end
end

function test_GVFN_bptt()

    env = CycleWorld.CycleWorldEnv(6)
    num_gvfs = 6
    num_steps = 50000
    τ=7

    rng = Random.MersenneTwister(10)

    opt = RTD(0.6)
    discount = (args...)->0.0
    cumulant(i) = (s_tp1, p_tp1)-> i==1 ? s_tp1[1] : p_tp1[i-1]
    # cumulants = [[cumulant(i) for i in 1:6]; [cumulant(1)]]
    # discounts = [[discount for i in 1:6]; [(env_state)-> env_state[1] == 1 ? 0.0 : 0.9]]

    cumulants = [cumulant(i) for i in 1:6]
    discounts = [discount for i in 1:6]

    pred_strg = zeros(num_steps, num_gvfs)

    gvfn = GVFN(num_gvfs, 3, cumulants, discounts; init=(dims...)->glorot_uniform(rng, dims...))
    _, s_t = CycleWorld.start!(env)
    h_t = zeros(num_gvfs)
    h_tm1 = zeros(num_gvfs)

    state_list = [zeros(3) for t in 1:τ]
    popfirst!(state_list)
    push!(state_list, build_features(s_t))
    hidden_state_init = zeros(num_gvfs)

    for step in 1:num_steps
        _, s_tp1, _, _ = CycleWorld.step!(env, 1)

        if length(state_list) == (τ+1)
            # println(state_list)
            popfirst!(state_list)
        end
        push!(state_list, build_features(s_tp1))

        print(step, "\r")

        train!(gvfn, opt, hidden_state_init, state_list, s_tp1)

        reset!(gvfn, hidden_state_init)
        preds = gvfn.(state_list)
        pred_strg[step,:] .= preds[end].data
        # println(env.agent_state)
        # println(preds[end])
        s_t .= s_tp1
        hidden_state_init .= Flux.data(preds[1])
        # println(preds)
        # println(hidden_state_init)

    end
    return pred_strg

end


text:
@testset "Verify hesslogpdf for a Normal distribution" begin

N = Normal(1.5, 2.3)

Nx = 10
x = randn(Nx)

    for i =1:Nx
        @test abs(hesslog_pdf(x[i]) - (-1.0))<1e-10
    end
end

ext: jl


text:
using .ArcadeLearningEnvironment

"""
    AtariEnv(;kwargs...)

This implementation follows the guidelines in [Revisiting the Arcade Learning Environment: Evaluation Protocols and Open Problems for General Agents](https://arxiv.org/abs/1709.06009)

# Keywords

- `name::String="pong"`: name of the Atari environments. Use `ReinforcementLearningEnvironments.list_atari_rom_names()` to show all supported environments.
- `grayscale_obs::Bool=true`:if `true`, then gray scale observation is returned, otherwise, RGB observation is returned.
- `noop_max::Int=30`: max number of no-ops.
- `frame_skip::Int=4`: the frequency at which the agent experiences the game.
- `terminal_on_life_loss::Bool=false`: if `true`, then game is over whenever a life is lost.
- `repeat_action_probability::Float64=0.`
- `color_averaging::Bool=false`: whether to perform phosphor averaging or not.
- `max_num_frames_per_episode::Int=0`
- `full_action_space::Bool=false`: by default, only use minimal action set. If `true`, one need to call `legal_actions` to get the valid action set. TODO
- `seed::Int` is used to set the initial seed of the underlying C environment and the rng used by the this wrapper environment to initialize the number of no-op steps at the beginning of each episode.
- `log_level::Symbol`, `:info`, `:warning` or `:error`. Default value is `:error`.

See also the [python implementation](https://github.com/openai/gym/blob/c072172d64bdcd74313d97395436c592dc836d5c/gym/wrappers/atari_preprocessing.py#L8-L36)
"""
AtariEnv(name; kwargs...) = AtariEnv(; name = name, kwargs...)
function AtariEnv(;
    name = "pong",
    grayscale_obs = true,
    noop_max = 30,
    frame_skip = 4,
    terminal_on_life_loss = false,
    repeat_action_probability = 0.0,
    color_averaging = false,
    max_num_frames_per_episode = 0,
    full_action_space = false,
    seed = nothing,
    log_level = :error,
)
    frame_skip > 0 || throw(ArgumentError("frame_skip must be greater than 0!"))
    name in getROMList() || throw(
        ArgumentError(
            "unknown ROM name.\n\nRun `ReinforcementLearningEnvironments.list_atari_rom_names()` to see all the game names.",
        ),
    )

    ale = ALE_new()
    if isnothing(seed)
        rng = Random.GLOBAL_RNG
    else
        setInt(ale, "random_seed", Int32(seed % typemax(Int32)))
        rng = MersenneTwister(hash(seed + 1))
    end
    setInt(ale, "frame_skip", Int32(1))  # !!! do not use internal frame_skip here, we need to apply max-pooling for the latest two frames, so we need to manually implement the mechanism.
    setInt(ale, "max_num_frames_per_episode", max_num_frames_per_episode)
    setFloat(ale, "repeat_action_probability", Float32(repeat_action_probability))
    setBool(ale, "color_averaging", color_averaging)
    setLoggerMode!(log_level)
    loadROM(ale, name)

    observation_size =
        grayscale_obs ? (getScreenWidth(ale), getScreenHeight(ale)) :
        (3, getScreenWidth(ale), getScreenHeight(ale))  # !!! note the order
    observation_space = Space(
        ClosedInterval{
            Cuchar,
        }.(
            fill(typemin(Cuchar), observation_size),
            fill(typemax(Cuchar), observation_size),
        ),
    )

    actions = full_action_space ? getLegalActionSet(ale) : getMinimalActionSet(ale)
    action_space = Base.OneTo(length(actions))
    screens =
        (fill(typemin(Cuchar), observation_size), fill(typemin(Cuchar), observation_size))

    env = AtariEnv{grayscale_obs,terminal_on_life_loss,grayscale_obs ? 2 : 3,typeof(rng)}(
        ale,
        name,
        screens,
        actions,
        action_space,
        observation_space,
        noop_max,
        frame_skip,
        0.0f0,
        lives(ale),
        rng,
    )
    finalizer(env) do x
        ALE_del(x.ale)
    end
    env
end

update_screen!(env::AtariEnv{true}, screen) =
    ArcadeLearningEnvironment.getScreenGrayscale!(env.ale, vec(screen))
update_screen!(env::AtariEnv{false}, screen) =
    ArcadeLearningEnvironment.getScreenRGB!(env.ale, vec(screen))

function (env::AtariEnv{is_gray_scale,is_terminal_on_life_loss})(
    a,
) where {is_gray_scale,is_terminal_on_life_loss}
    r = 0.0f0

    for i in 1:env.frame_skip
        r += act(env.ale, env.actions[a])
        if i == env.frame_skip
            update_screen!(env, env.screens[1])
        elseif i == env.frame_skip - 1
            update_screen!(env, env.screens[2])
        end
    end

    # max-pooling
    if env.frame_skip > 1
        env.screens[1] .= max.(env.screens[1], env.screens[2])
    end

    env.reward = r
    nothing
end

is_terminal(env::AtariEnv{<:Any,true}) = game_over(env.ale) || (lives(env.ale) < env.lives)
is_terminal(env::AtariEnv{<:Any,false}) = game_over(env.ale)

RLBase.nameof(env::AtariEnv) = "AtariEnv($(env.name))"
RLBase.action_space(env::AtariEnv) = env.action_space
RLBase.reward(env::AtariEnv) = env.reward
RLBase.is_terminated(env::AtariEnv) = is_terminal(env)
RLBase.state(env::AtariEnv) = env.screens[1]
RLBase.state_space(env::AtariEnv) = env.observation_space

function Random.seed!(env::AtariEnv, s)
    @warn "seeding is not exposed in the atari env. so we just do nothing here"
end

function Base.copy(env::AtariEnv{G,T,N,S}) where {G,T,N,S}
    @warn "currently cloneSystemState/cloneState has a bug"
    env
end

function RLBase.reset!(env::AtariEnv)
    reset_game(env.ale)
    for _ in 1:rand(env.rng, 0:env.noopmax)
        act(env.ale, Int32(0))
    end
    update_screen!(env, env.screens[1])  # no need to update env.screens[2]
    env.reward = 0.0f0  # dummy
    env.lives = lives(env.ale)
    nothing
end

imshowgrey(x::AbstractArray{UInt8,2}) = imshowgrey(reshape(x, :), size(x))
imshowgrey(x::AbstractArray{UInt8,1}, dims) = imshow(reshape(x, dims), colormap = 2)
imshowcolor(x::AbstractArray{UInt8,3}) = imshowcolor(reshape(x, :), size(x))
function imshowcolor(x::AbstractArray{UInt8,1}, dims)
    clearws()
    setviewport(0, dims[1] / dims[2], 0, 1)
    setwindow(0, 1, 0, 1)
    y = (zeros(UInt32, dims...) .+ 0xff) .<< 24
    img = UInt32.(x)
    @simd for i in 1:length(y)
        @inbounds y[i] += img[3*(i-1)+1] + img[3*(i-1)+2] << 8 + img[3*i] << 16
    end
    drawimage(0, 1, 0, 1, dims..., y)
    updatews()
end

function Base.show(io::IO, m::MIME"image/png", env::AtariEnv)
    x = getScreenRGB(env.ale)
    p = imshowcolor(x, (Int(getScreenWidth(env.ale)), Int(getScreenHeight(env.ale))))
    show(io, m, p)
end

Base.show(io::IO, t::MIME"text/plain", env::AbstractEnv) = show(
    IOContext(io, :is_show_state => false, :is_show_state_space => false),
    MIME"text/markdown"(),
    env,
)

list_atari_rom_names() = getROMList()


text: include("geometrytypes.jl")


text:
# This file is part of Kpax3. License is MIT.

# TODO: Test A and B

function test_prior_col_exceptions()
  data = UInt8[0 0 0 0 0 1;
               1 1 1 1 1 0;
               0 0 1 0 1 1;
               1 1 0 1 0 0;
               1 1 0 0 0 0;
               0 0 0 1 1 0;
               1 1 1 0 0 0;
               0 0 0 1 1 1;
               0 0 1 0 0 0;
               1 0 0 1 0 1;
               0 1 0 0 1 0;
               0 0 0 0 0 1;
               1 1 1 0 0 0;
               0 0 0 1 1 0;
               1 1 0 0 1 1;
               0 0 1 1 0 0;
               1 1 0 1 0 0;
               0 0 1 0 1 1]

  γ = [0.6; 0.35; 0.05]
  r = log(0.001) / log(0.95)

  @test_throws Kpax3.KInputError Kpax3.AminoAcidPriorCol(data, [1.0; 1.0; 1.0; 1.0], r)

  @test_throws Kpax3.KDomainError Kpax3.AminoAcidPriorCol(data, [-1.0; 1.0; 1.0], r)
  @test_throws Kpax3.KDomainError Kpax3.AminoAcidPriorCol(data, [1.0; -1.0; 1.0], r)
  @test_throws Kpax3.KDomainError Kpax3.AminoAcidPriorCol(data, [1.0; 1.0; -1.0], r)
  @test_throws Kpax3.KDomainError Kpax3.AminoAcidPriorCol(data, γ, 0.0)
  @test_throws Kpax3.KDomainError Kpax3.AminoAcidPriorCol(data, γ, -1.0)

  nothing
end

test_prior_col_exceptions()

function test_prior_col_constructor()
  data = UInt8[0 0 0 0 0 1;
               1 1 1 1 1 0;
               0 0 1 0 1 1;
               1 1 0 1 0 0;
               1 1 0 0 0 0;
               0 0 0 1 1 0;
               1 1 1 0 0 0;
               0 0 0 1 1 1;
               0 0 1 0 0 0;
               1 0 0 1 0 1;
               0 1 0 0 1 0;
               0 0 0 0 0 1;
               1 1 1 0 0 0;
               0 0 0 1 1 0;
               1 1 0 0 1 1;
               0 0 1 1 0 0;
               1 1 0 1 0 0;
               0 0 1 0 1 1]

  (m, n) = size(data)

  γ = [0.6; 0.35; 0.05]
  r = log(0.001) / log(0.95)

  priorC = Kpax3.AminoAcidPriorCol(data, γ, r)

  logγ = [log(γ[1]); log(γ[2]); log(γ[3])]
  logω = Vector{Float64}[[log(k - 1) - log(k); -log(k)] for k in 1:n]

  @test priorC.logγ == logγ
  @test priorC.logω == logω

  # TODO: Test A and B
  nothing
end

test_prior_col_constructor()


text:
function equals(obj::Object, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function get_class(obj::Object)
    return jcall(obj, "getClass", Class, ())
end

function hash_code(obj::Object)
    return jcall(obj, "hashCode", jint, ())
end

function next_int(obj::IntRandomGenerator)
    return jcall(obj, "nextInt", jint, ())
end

function notify(obj::Object)
    return jcall(obj, "notify", void, ())
end

function notify_all(obj::Object)
    return jcall(obj, "notifyAll", void, ())
end

function set_seed(obj::AbstractWell, arg0::Vector{jint})
    return jcall(obj, "setSeed", void, (Vector{jint},), arg0)
end

function wait(obj::Object)
    return jcall(obj, "wait", void, ())
end

function wait(obj::Object, arg0::jlong)
    return jcall(obj, "wait", void, (jlong,), arg0)
end

function wait(obj::Object, arg0::jlong, arg1::jint)
    return jcall(obj, "wait", void, (jlong, jint), arg0, arg1)
end


text:
__precompile__()

module FastStyleTransfer

using Flux, Metalhead, Images, CuArrays, NNlib, BSON
using BSON: @save, @load
using Flux: @epochs, sub2, expand, initn
using Base.Iterators: partition
using Flux.Tracker: track, data, @grad, nobacksies

export train, stylize

include("utils.jl")
include("layers.jl")
include("vgg.jl")
include("transformer_net.jl")
include("neural_style.jl")

end # module


text:
# AUTO GENERATED FILE - DO NOT EDIT

export modal

"""
    modal(;kwargs...)
    modal(children::Any;kwargs...)
    modal(children_maker::Function;kwargs...)

A Modal component.
Centers content vertically and horizontally. For more information, see: https://mantine.dev/core/modal/
Keyword arguments:
- `children` (a list of or a singular dash component, string or number; optional): Content that should be centered vertically and horizontally
- `id` (String; optional): The ID of this component, used to identify dash components in callbacks
- `centered` (Bool; optional): Controls if modal should be centered
- `className` (String; optional): Often used with CSS to style elements with common properties
- `closeOnClickOutside` (Bool; optional): Should modal be closed when outside click was registered?
- `hideCloseButton` (Bool; optional): Hides close button, modal still can be closed with escape key and by clicking outside
- `opened` (Bool; optional): Mounts modal if true
- `overflow` (a value equal to: "inside", "outside"; optional): Control vertical overflow behavior
- `overlayColor` (String; optional): Overlay below modal color, defaults to theme.black in light theme and to theme.colors.dark[9] in dark theme
- `overlayOpacity` (Real; optional): Overlay below modal opacity, defaults to 0.75 in light theme and to 0.85 in dark theme
- `padding` (a value equal to: "xs", "sm", "md", "lg", "xl" | Real; optional): Modal padding from theme or number value for padding in px
- `radius` (a value equal to: "xs", "sm", "md", "lg", "xl" | Real; optional): Modal padding from theme or number value for padding in px
- `size` (a value equal to: "xs", "sm", "md", "lg", "xl" | Real | String; optional): Modal body width
- `style` (Dict; optional): Inline style override
- `title` (String; optional): Modal title, displayed in header before close button
- `zIndex` (Real; optional): Popper zIndex
"""
function modal(; kwargs...)
        available_props = Symbol[:children, :id, :centered, :className, :closeOnClickOutside, :hideCloseButton, :opened, :overflow, :overlayColor, :overlayOpacity, :padding, :radius, :size, :style, :title, :zIndex]
        wild_props = Symbol[]
        return Component("modal", "Modal", "dash_mantine_components", available_props, wild_props; kwargs...)
end

modal(children::Any; kwargs...) = modal(;kwargs..., children = children)
modal(children_maker::Function; kwargs...) = modal(children_maker(); kwargs...)


text:
using Compat
using ProtoBuf
import ProtoBuf.meta
import Base: hash, isequal, ==

# service methods for ApplicationMasterProtocolService
const _ApplicationMasterProtocolService_methods = MethodDescriptor[
        MethodDescriptor("registerApplicationMaster", 1, RegisterApplicationMasterRequestProto, RegisterApplicationMasterResponseProto),
        MethodDescriptor("finishApplicationMaster", 2, FinishApplicationMasterRequestProto, FinishApplicationMasterResponseProto),
        MethodDescriptor("allocate", 3, AllocateRequestProto, AllocateResponseProto)
    ] # const _ApplicationMasterProtocolService_methods
const _ApplicationMasterProtocolService_desc = ServiceDescriptor("ApplicationMasterProtocolService", 1, _ApplicationMasterProtocolService_methods)

ApplicationMasterProtocolService(impl::Module) = ProtoService(_ApplicationMasterProtocolService_desc, impl)

type ApplicationMasterProtocolServiceStub <: AbstractProtoServiceStub{false}
    impl::ProtoServiceStub
    ApplicationMasterProtocolServiceStub(channel::ProtoRpcChannel) = new(ProtoServiceStub(_ApplicationMasterProtocolService_desc, channel))
end # type ApplicationMasterProtocolServiceStub

type ApplicationMasterProtocolServiceBlockingStub <: AbstractProtoServiceStub{true}
    impl::ProtoServiceBlockingStub
    ApplicationMasterProtocolServiceBlockingStub(channel::ProtoRpcChannel) = new(ProtoServiceBlockingStub(_ApplicationMasterProtocolService_desc, channel))
end # type ApplicationMasterProtocolServiceBlockingStub

registerApplicationMaster(stub::ApplicationMasterProtocolServiceStub, controller::ProtoRpcController, inp::RegisterApplicationMasterRequestProto, done::Function) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[1], controller, inp, done)
registerApplicationMaster(stub::ApplicationMasterProtocolServiceBlockingStub, controller::ProtoRpcController, inp::RegisterApplicationMasterRequestProto) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[1], controller, inp)

finishApplicationMaster(stub::ApplicationMasterProtocolServiceStub, controller::ProtoRpcController, inp::FinishApplicationMasterRequestProto, done::Function) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[2], controller, inp, done)
finishApplicationMaster(stub::ApplicationMasterProtocolServiceBlockingStub, controller::ProtoRpcController, inp::FinishApplicationMasterRequestProto) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[2], controller, inp)

allocate(stub::ApplicationMasterProtocolServiceStub, controller::ProtoRpcController, inp::AllocateRequestProto, done::Function) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[3], controller, inp, done)
allocate(stub::ApplicationMasterProtocolServiceBlockingStub, controller::ProtoRpcController, inp::AllocateRequestProto) = call_method(stub.impl, _ApplicationMasterProtocolService_methods[3], controller, inp)

export ApplicationMasterProtocolService, ApplicationMasterProtocolServiceStub, ApplicationMasterProtocolServiceBlockingStub, registerApplicationMaster, finishApplicationMaster, allocate


text:
# I think this file is done 01/03/2020 WRS

"""
N-dimensional Point containers
"""
module Points

export Point
export Cartesian
export Polar
export Spherical

using Formatting
using LinearAlgebra
using RecipesBase
using StaticArrays

abstract type AbstractCoordinateType end
struct Cartesian <: AbstractCoordinateType end
struct Polar <: AbstractCoordinateType end
struct Spherical <: AbstractCoordinateType end

"""
	Point{N,C}

`N`-Dimensional point of coordinate type `C`
Can be accessed with fields `:x`, `:y`, `:z`, `:s`, `:r`, `:ϕ`, `:θ` and their variants.
"""
struct Point{N,C}
	vector::SVector{N,Float64}

	function Point{N,C}(vec::SVector{N,T}) where {N,C<:AbstractCoordinateType,T}
		!any(isnan.(vec)) || throw(ArgumentError("argument of `Point` $(sum(isnan.(vec))) contains NaNs"))
		all(isfinite.(vec)) || throw(ArgumentError("argument of `Point` contains $(sum(isfinite.(vec))) Infs"))
		new{N,C}(vec)
	end

	Point{Cartesian}(vec::SVector{N,T}) where {N,T<:Real} = Point{N,Cartesian}(vec)

	function Point{Polar}(vec::SVector{N,T}) where {N,T<:Real}
		1≤N≤3 || throw(ArgumentError("`Point{Polar}` requires 1≤N≤3, but N=$N"))
		0≤vec[1] || throw(ArgumentError("`Point{Polar}` requires 0≤r, but r=$(vec[1])"))
		return Point{N,Polar}(vec)
	end
	function Point{Spherical}(vec::SVector{N,T}) where {N,T<:Real}
		1≤N≤3 || throw(ArgumentError("`Point{Spherical}` requires 1≤N≤3, but N=$N"))
		0≤vec[1] || throw(ArgumentError("Point{Spherical}` requires 0≤r, but r=$(vec[1])"))
		if N==3 0≤vec[3]≤π || throw(ArgumentError("`Point{Spherical}` requires 0≤θ≤π, but θ=$(vec[3])")) end
		return Point{N,Spherical}(vec)
	end
end

# default Point constructor makes N-d Cartesian point
"""
	Point(x₁,x₂...,xⱼ) -> point::Point{j,Cartesian}
"""
Point(arg,args...) = Point{Cartesian}(arg,args...)

# Point{C} constructor takes an ntuple or a list of numbers
"""
	Point{C}(x₁,x₂...,xⱼ) -> point::Point{j,C}

`C` is an `AbstractCoordinateType`, one of `Cartesian`, `Polar`, `Spherical`
"""
Point{C}(vec::Vararg{Number,N}) where {N,C} = Point{C}(float.(vec))
Point{C}(vec::NTuple{N}) where {N,C} = Point{C}(SVector{N,Float64}(vec))

# make a new point from `p` with data in `vec`
Point(p::Point) = p
Point(p::Point{N,C},vec::SVector{N}) where {N,C} = Point{C}(vec)
Point{C}(p::Point{N,C},vec::SVector{N}) where {N,C} = Point{C}(vec)

# conversion to Cartesian point
function Cartesian(p::Point{N,C}) where {N,C}
	if C<:Cartesian
		return p
	elseif C<:Polar
		if N==1
			return Point{Cartesian}(p.x)
		elseif N==2
			return Point{Cartesian}(p.x,p.y)
		elseif N==3
			return Point{Cartesian}(p.x,p.y,p.z)
		else
			throw(ArgumentError("conversion from `Polar` to `Cartesian` in dimension N=$N not implemented"))
		end
	elseif C<:Spherical
		if N==1
			return Point{Cartesian}(p.x)
		elseif N==2
			return Point{Cartesian}(p.x,p.z)
		elseif N==3
			return Point{Cartesian}(p.x,p.y,p.z)
		else
			throw(ArgumentError("conversion from `Spherical` to `Cartesian` in dimension N=$N not implemented"))
		end
	else
		throw(ArgumentError("cannot convert `Point{$C}` to `Point{Cartesian}`"))
	end
end
# conversion to Polar point
function Polar(p::Point{N,C}) where {N,C}
	if C<:Polar
		return p
	elseif C<:Cartesian
		if N==1
			return Point{Polar}(p.s)
		elseif N==2
			return Point{Polar}(p.s,p.ϕ)
		elseif N==3
			return Point{Polar}(p.s,p.ϕ,p.z)
		else
			throw(ArgumentError("conversion from `Cartesian` to `Polar` in dimension N=$N not implemented"))
		end
	elseif C<:Spherical
		if N==1
			return Point{Polar}(p.s)
		elseif N==2
			throw(ArgumentError("conversion from `Spherical` to `Polar` in dimension N=2 not implemented"))
		elseif N==3
			return Point{Polar}(p.s,p.ϕ,p.z)
		else
			throw(ArgumentError("conversion from `Spherical` to `Polar` in dimension N=$N not implemented"))
		end
	else
		throw(ArgumentError("cannot convert `Point{$C}` to `Point{Polar}`"))
	end
end
# conversion to Spherical point
function Spherical(p::Point{N,C}) where {N,C}
	if C<:Spherical
		return p
	elseif C<:Cartesian
		if N==1
			return Point{Spherical}(p.r)
		elseif N==2
			return Point{Spherical}(p.r,π/2-p.ϕ)
		elseif N==3
			return Point{Spherical}(p.r,p.θ,p.ϕ)
		else
			throw(ArgumentError("conversion from `Cartesian` to `Spherical` in dimension N=$N not implemented"))
		end
	elseif C<:Polar
		if N==1
			return Point{Spherical}(p.r)
		elseif N==2
			return Point{Spherical}(p.r,p.θ)
		elseif N==3
			return Point{Spherical}(p.r,p.θ,p.ϕ)
		else
			throw(ArgumentError("conversion from `Polar` to `Spherical` in dimension N=$N not implemented"))
		end
	else
		throw(ArgumentError("cannot convert `Point{$C}` to `Point{Polar}`"))
	end
end

# define action of `convert` by using `Cartesian`, `Polar`, `Spherical`
Base.convert(::Type{Point{N,C}}, p::Point) where {N,C} = C(p)
# 1-dim conversion of point to single float
Base.convert(::Type{Float64}, p::Point{1}) = convert(Float64, p.x)

# for use in tests (see defs of norm later)
function Base.isapprox(x::Point, y::Point; atol::Real=0, rtol::Real=sqrt(eps(Float64)), nans::Bool=false)
    x == y || norm(x-y) ≤ max(atol, rtol*max(norm(x), norm(y)))
end

Base.:*(p::Point,a::Number) = a*p
Base.:*(a::Number,p::Point{N,Cartesian}) where N = Point{Cartesian}(a*p.vector)
Base.:*(a::Number,p::Point{1,Polar}) = Point{Polar}(a*p.vec[1])
Base.:*(a::Number,p::Point{2,Polar}) = Point{Polar}(a*p.vec[1],p.vec[2])
Base.:*(a::Number,p::Point{3,Polar}) = Point{Polar}(a*p.vec[1],p.vec[2],a*p.vec[3])
Base.:*(a::Number,p::Point{1,Spherical}) = Point{Spherical}(a*p.vec[1])
Base.:*(a::Number,p::Point{2,Spherical}) = Point{Spherical}(a*p.vec[1],p.vec[2])
Base.:*(a::Number,p::Point{3,Spherical}) = Point{Spherical}(a*p.vec[1],p.vec[2],p.vec[3])

Base.:\(a::Number,p::Point{N}) where N = p/a
Base.:/(p::Point,a::Number) where N = p*(1/a)

Base.:+(p1::Point{N,Cartesian},p2::Point{N,Cartesian}) where N = Point{Cartesian}(p1.vector+p2.vector)
Base.:-(p1::Point{N,Cartesian},p2::Point{N,Cartesian}) where N = Point{Cartesian}(p1.vector-p2.vector)
# differing types defaults to Cartesian
Base.:+(p1::Point{N,C1},p2::Point{N,C2}) where {N,C1,C2} = Cartesian(p1) + Cartesian(p2)
Base.:-(p1::Point{N,C1},p2::Point{N,C2}) where {N,C1,C2} = Cartesian(p1) - Cartesian(p2)
# same types preserves coordinate type
Base.:+(p1::Point{N,C},p2::Point{N,C}) where {N,C} = C(Cartesian(p1) + Cartesian(p2))
Base.:-(p1::Point{N,C},p2::Point{N,C}) where {N,C} = C(Cartesian(p1) - Cartesian(p2))

Base.:-(p::Point) = -1*p
Base.:-(p::Point{1,Polar}) = p
Base.:-(p::Point{2,Polar}) = Point{Polar}(p.s,mod2pi(p.ϕ+π))
Base.:-(p::Point{3,Polar}) = Point{Polar}(p.s,mod2pi(p.ϕ+π),-p.z)
Base.:-(p::Point{1,Spherical}) = p
Base.:-(p::Point{2,Spherical}) = Point{Spherical}(p.r, mod(p.θ+π,π))
Base.:-(p::Point{3,Spherical}) = Point{Spherical}(p.r, mod(p.ϕ+π,π), mod2pi(p.ϕ+π))

Base.:+(p1::Point{1},p2::Number) = Point{Cartesian}(p1.vec .+ p2)
Base.:+(p2::Number, p1::Point{1}) = p1 + p2
Base.:-(p1::Point{1},p2::Number) = +(p1,-p2)
Base.:-(p2::Number,p1::Point{1}) = +(p2,-p1)

LinearAlgebra.norm(p::Point{N}) where N = p.r

Base.ndims(::Point{N}) where N = N
Base.getindex(p::Point,index::Integer) = getfield(p,:vector)[index]
fnames = (:firstindex, :lastindex, :length, :size, :eltype, :iterate)
for fname ∈ fnames @eval Base.$fname(p::Point) = $fname(p.vec) end
Base.iterate(p::Point,state) = iterate(p.vec,state)
Base.IndexStyle(::Point) = IndexStyle(p.vec)

Base.isless(p1::Point{1},p2::Point{1}) = isless(p1.x,p2.x)
Base.isless(p::Point{1},x::Real) = isless(p.x,x)
Base.isless(x::Real,p::Point{1}) = isless(x,p.x)

################################################################################
# Properties

# Cartesian
function Base.getproperty(p::Point{N,Cartesian},sym::Symbol) where N
	if sym == :vec
		return getfield(p,:vector)
	elseif Base.sym_in(sym,(:x,:X))
		return p[1]
	elseif Base.sym_in(sym,(:y,:Y))
		if N≤1
			return 0.0
		else
			return p[2]
		end
	elseif Base.sym_in(sym,(:z,:Z))
		if N≤2
			return 0.0
		else
			return p[3]
		end
	elseif Base.sym_in(sym,(:s,))
		if N==1
			return abs(p[1])
		elseif N≥2
			return hypot(p[1],p[2])
		end
	elseif Base.sym_in(sym,(:r,))
		return norm(getfield(p,:vector))
	elseif Base.sym_in(sym,(:ϕ,:φ,:𝝋,:phi))
		if N==1
			return p[1]>0 ? 0.0 : π
		elseif 2≤N≤3
			return atan(p[2],p[1])
		else
			throw(ErrorException("$sym only defined for 1≤N≤3, but N=$N"))
		end
	elseif Base.sym_in(sym,(:θ,:theta,:ϑ))
		if N≤2
			return π/2
		elseif N==3
			θ = acos(p[3]/norm(getfield(p,:vector)))
			return isnan(θ) ? 0.0 : θ
		else
			throw(ErrorException("$sym only defined for 1≤N≤3, but N=$N"))
		end
	else
		return getfield(p,sym)
	end
end

# Polar
function Base.getproperty(p::Point{N,Polar},sym::Symbol) where N
	if sym == :vec
		return getfield(p,:vector)
	elseif Base.sym_in(sym,(:x,:X))
		if N==1
			return p[1]
		else
			return p[1]*cos(p[2])
		end
	elseif Base.sym_in(sym,(:y,:Y))
		if N==1
			return 0.0
		else
			return p[1]*sin(p[2])
		end
	elseif Base.sym_in(sym,(:z,:Z))
		if N≤2
			return 0.0
		else
			return p[3]
		end
	elseif Base.sym_in(sym,(:s,))
		return p[1]
	elseif Base.sym_in(sym,(:r,))
		if N≤2
			return p[1]
		else
			return hypot(p[1],p[3])
		end
	elseif Base.sym_in(sym,(:ϕ,:φ,:𝝋,:phi))
		if N==1
			return 0.0
		else
			return p[2]
		end
	elseif Base.sym_in(sym,(:θ,:theta,:ϑ))
		if N≤2
			return π/2
		else
			return acos(p[3]/hypot(p[1],p[3]))
		end
	else
		return getfield(p,sym)
	end
end

# Spherical
function Base.getproperty(p::Point{N,Spherical},sym::Symbol) where N
	if sym == :vec
		return getfield(p,:vector)
	elseif Base.sym_in(sym,(:x,:X))
		if N==1
			return p[1]
		elseif N==2
			return p[1]*sin(p[2])
		elseif N==3
			return p[1]*sin(p[2])*cos(p[3])
		else
			throw(ErrorException("N=$N and field :x not consistent with Point{Spherical}"))
		end
	elseif Base.sym_in(sym,(:y,:Y))
		if N≤2
			return 0.0
		else
			return p[1]*sin(p[2])*sin(p[3])
		end
	elseif Base.sym_in(sym,(:z,:Z))
		if N==1
			return 0.0
		else
			return p[1]*cos(p[2])
		end
	elseif Base.sym_in(sym,(:s,))
		if N==1
			return p[1]
		else
			return p[1]*sin(p[2])
		end
	elseif Base.sym_in(sym,(:r,))
		return p[1]
	elseif Base.sym_in(sym,(:θ,:ϑ,:𝜃,:theta))
		if N==1
			return 0.0
		else
			return p[2]
		end
	elseif Base.sym_in(sym,(:ϕ,:φ,:𝝋,:phi))
		if N≤2
			return 0.0
		else
			return p[3]
		end
	else
		return getfield(p,sym)
	end
end

#1D
function Base.propertynames(::Point{1,C},private=false) where C
	if private
		return fieldnames(Point)
	elseif C<:Cartesian
		return (:x, :vector)
	elseif C<:Polar
		return (:s, :vector)
	elseif C<:Spherical
		return (:r, :vector)
	end
end

#2D
function Base.propertynames(::Point{2,C},private=false) where C
	if private
		return fieldnames(Point)
	elseif C<:Cartesian
		return (:x, :y, :vector, :s, :ϕ)
	elseif C<:Polar
		return (:s, :ϕ, :vector, :x, :y)
	elseif C<:Spherical
		return (:r, :θ, :vector, :x, :z)
	end
end

#3D
function Base.propertynames(::Point{3,C},private=false) where C
	if private
		return fieldnames(Point)
	elseif C<:Cartesian
		return (:x, :y, :z, :vector, :r, :ϕ, :θ, :s)
	elseif C<:Polar
		return (:s, :ϕ, :z, :vector, :x, :y, :r, :θ)
	elseif C<:Spherical
		return (:r, :ϕ, :θ, :vector, :x, :y, :z, :s)
	end
end

################################################################################
# Pretty Printing

import ..PRINTED_COLOR_NUMBER
import ..PRINTED_COLOR_DARK

# 1d (Cartesian)
function Base.show(io::IO,p::Point{1,Cartesian})
	print(io,"1D Cartesian ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": ")
	printstyled(io, fmt("3.3f",p.x),color=PRINTED_COLOR_NUMBER)
end
# 1d (Polar)
function Base.show(io::IO,p::Point{1,Polar})
	print(io,"1D Polar ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": s=")
	printstyled(io, fmt("3.3f",p.s),color=PRINTED_COLOR_NUMBER)
end
# 1d (Spherical)
function Base.show(io::IO,p::Point{1,Spherical})
	print(io,"1D Spherical ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": r=")
	printstyled(io, fmt("3.3f",p.r),color=PRINTED_COLOR_NUMBER)
end

# 2d (Cartesian)
function Base.show(io::IO,p::Point{2,Cartesian})
	print(io,"2D Cartesian ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": x=")
	printstyled(io, fmt("3.3f",p.x),color=PRINTED_COLOR_NUMBER)
	print(io, ", y=")
	printstyled(io, fmt("3.3f",p.y),color=PRINTED_COLOR_NUMBER)
end
# 2d (Polar)
function Base.show(io::IO,p::Point{2,Polar})
	print(io,"2D Polar ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": s=")
	printstyled(io, fmt("3.3f",p.s),color=PRINTED_COLOR_NUMBER)
	print(io, ", ϕ=")
	printstyled(io, fmt("3.3f",180*p.ϕ/π),"°",color=PRINTED_COLOR_NUMBER)
end
# 2d (Spherical)
function Base.show(io::IO,p::Point{2,Spherical})
	print(io,"2D Spherical ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": r=")
	printstyled(io, fmt("3.3f",p.r),color=PRINTED_COLOR_NUMBER)
	print(io, ", θ=")
	printstyled(io, fmt("3.3f",180*p.θ/π),"°",color=PRINTED_COLOR_NUMBER)
end

# 3d (Cartesian)
function Base.show(io::IO,p::Point{3,Cartesian})
	print(io,"3D Cartesian ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": x=")
	printstyled(io, fmt("3.3f",p.x),color=PRINTED_COLOR_NUMBER)
	print(io, ", y=")
	printstyled(io, fmt("3.3f",p.y),color=PRINTED_COLOR_NUMBER)
	print(io, ", z=")
	printstyled(io, fmt("3.3f",p.z),color=PRINTED_COLOR_NUMBER)
end
# 3d (Polar)
function Base.show(io::IO,p::Point{3,Polar})
	print(io,"3D Polar ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": s=")
	printstyled(io, fmt("3.3f",p.s),color=PRINTED_COLOR_NUMBER)
	print(io, ", ϕ=")
	printstyled(io, fmt("3.3f",180*p.ϕ/π),"°",color=PRINTED_COLOR_NUMBER)
	print(io, ", z=")
	printstyled(io, fmt("3.3f",p.z),color=PRINTED_COLOR_NUMBER)
end
# 3d (Spherical)
function Base.show(io::IO,p::Point{3,Spherical})
	print(io,"3D Spherical ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
	print(io, ": r=")
	printstyled(io, fmt("3.3f",p.r),color=PRINTED_COLOR_NUMBER)
	print(io, ", θ=")
	printstyled(io, fmt("3.3f",180*p.θ/π),"°",color=PRINTED_COLOR_NUMBER)
	print(io, ", ϕ=")
	printstyled(io, fmt("3.3f",180*p.ϕ/π),"°",color=PRINTED_COLOR_NUMBER)
end

function Base.show(io::IO,p::Point{N,C}) where {N,C}
	print(io, "$(N)D $C ")
	printstyled(io,"Point",color=PRINTED_COLOR_DARK)
end

################################################################################
# Plotting

@recipe function f(p::Point{1})
	seriestype --> :scatter
	legend --> false
	([p.x],[0])
end

@recipe function f(p::Point{2})
	seriestype --> :scatter
	legend --> false
	([p.x],[p.y])
end

@recipe function f(p::Point{3})
	seriestype --> :scatter
	legend --> false
	([p.x],[p.y],[p.z])
end

@recipe function f(p::Array{T}) where T<:Point{1}
	seriestype --> :scatter
	legend --> false
	x = map(p -> p.x, p)
	y = map(p -> 0, p)
	(x,y)
end

@recipe function f(p::Array{T}) where T<:Point{2}
	seriestype --> :scatter
	legend --> false
	x = map(p -> p.x, p)
	y = map(p -> p.y, p)
	(x,y)
end

@recipe function f(p::Array{T}) where T<:Point{3}
	seriestype --> :scatter
	legend --> false
	x = map(p -> p.x, p)
	y = map(p -> p.y, p)
	z = map(p -> p.z, p)
	(x,y,z)
end

end # module


text:
# function ScheduleExtraction(; verboseOutput::Bool=false)

verboseOutput=true ## TEMP

## MAIN LOOP, THROUGH EACH DATA RECORD
# for n in 1:length(proj.dataIndices)

n=5 ## TEMP

## INIT LOCAL VARS
t = proj.smartMeters[n].tVec
dataSample = proj.smartMeters[n].dat
nDays = Dates.value(proj.smartMeters[n].nDays)

## SET UP VERBOSE OUTPUT IF REQUESTED
if verboseOutput == true
    proj.verbose[n] = DataFrame(
    date=[string(proj.smartMeters[1].tVec[i])[1:10] for i in 1:48:size(proj.smartMeters[1].tVec,1)],
    seed_L = fill!(Array{Float64}(undef,nDays),NaN),
    seed_M = fill!(Array{Float64}(undef,nDays),NaN),
    seed_H = fill!(Array{Float64}(undef,nDays),NaN),
    clusterAssignment = fill!(Array{Array{Int8}}(undef,nDays),fill!(Array{Int8}(undef,48),-99)),
    clusterAssignmentValue = fill!(Array{Array{Float64}}(undef,nDays),fill!(Array{Float64}(undef,48),NaN)))
    # clusterAssignment = fill!(Array{Array{Int8}}(undef,5),Array{Int8}(undef,48))
end

## INIT
nSamplesPlus1 = Int16(Second(24*3600)/proj.smartMeters[1].tRes)+1 ## PLUS ONE TO PICK UP EVENTS THAT TRANSCEND DAYS (OCCURING AT 00:00)

## SET UP DAY RANGE
days = 1:nDays

dataSample = FixJanFirst(t,dataSample) ## LINEARLY INTERPOLATE AT 00:00 1ST JAN IF ZERO VALUE
dataSample = FixMissedKwhReading(dataSample) ## INTERPOLATE FOR MISSED KWH READINGS

## PREPEND FINAL ROW FROM PREVIOUS DAY TO ALLOW FOR EVENTS OCCURING 00:00HRS
dataSample = vcat([dataSample[end,1] dataSample[end,1:end-1]'],dataSample)

## INITILALISE ARRAY
global binaryHeatingResponse
binaryHeatingResponse = fill!(Array{Float64}(undef,nSamplesPlus1,nDays),NaN)

## CREATE REGISTER FOR DAILY controlRegime
global controlRegime
controlRegime = Array{Union{Missing,String}}(missing,nDays)

## SCAN DAILY DATA
for dy in days
    ds = dataSample[:,dy]
    iNan = [i for i in 1:length(ds) if isnan(ds[i])]
    if length(iNan)==0

        ## SET SEEDS (FORCE 1D CLUSTER USING ZERO SECOND DIMENSION)
        dsCnt=[0 ((maximum(ds)-minimum(ds))*0.1+minimum(ds)) maximum(ds);zeros(1,3)]

        ## MAKE SUITABLE ARRAY FOR CLUSTERING (FORCE 1D CLUSTER USING ZERO SECOND DIMENSION)
        ds=[ds';zeros(1,nSamplesPlus1)]

        ## K-MEANS: 3 CLUSTERS FOR HIGH/MED/LOW
        clsr = Clustering.kmeans!(ds,dsCnt)

        ## RECORD CENTERS
        if verboseOutput == true
            clsrCenters = sort(clsr.centers[1,:])
            proj.verbose[n].seed_L[dy] = clsrCenters[1]
            proj.verbose[n].seed_M[dy] = clsrCenters[2]
            proj.verbose[n].seed_H[dy] = clsrCenters[3]
        end

        ## RECORD CLUSTER ASSIGNMENT VALUES
        pos = sortperm(clsr.centers[1,:])
        clsrAssgn = pos[clsr.assignments]
        if verboseOutput == true
            proj.verbose[n].clusterAssignmentValue[dy] = fill!(Array{Float32}(undef,48),NaN)
            proj.verbose[n].clusterAssignmentValue[dy][clsrAssgn[1:end-1].==1].=proj.verbose[n].seed_L[dy]
            proj.verbose[n].clusterAssignmentValue[dy][clsrAssgn[1:end-1].==2].=proj.verbose[n].seed_M[dy]
            proj.verbose[n].clusterAssignmentValue[dy][clsrAssgn[1:end-1].==3].=proj.verbose[n].seed_H[dy]
        end

        # REMOVE GENTLE RAMP AT START-UP
        rampBool = [join(clsrAssgn[i:(i+2)]).=="012" for i in 1:(nSamplesPlus1-2)]
        iRamp = [i for i in 1:length(rampBool) if rampBool[i]] .+ 1
        # TEMPORARY FIX
        [clsrAssgn[i] = 0 for i in iRamp]

        if verboseOutput == true
            proj.verbose[n].clusterAssignment[dy] = clsrAssgn[1:end-1] .-2
        end

        ## MAP HIGH/MED/LOW DATA TO 1/1/0
        clsrAssgn = Int8.((clsrAssgn).>1)

        binaryHeatingResponse[:,dy] = clsrAssgn
        # elseif maximum(ds)>0.2 && dataKrt<8 ## CATEGORISE AS "HtgCont" (CONTINUOUS)
        # controlRegime[dy] = 5
        # controlRegime[dy] = "c"
        # binaryHeatingResponse[:,dy] = ones(nSamplesPlus1,1)
        # else ## IF MAX LOAD IS <0.2kW AND KURTOSIS <8, CATEGORISE AS "HtgIdle"
        # controlRegime[dy] = 2
        # controlRegime[dy] = "i"
        # end
    else ## IF NaNS ARE IDENTIFIED, CATEGORISE AS "1_RejectedData"
        # controlRegime[dy] = 1
        controlRegime[dy] = "v"
    end
end

## BUILD ARRAY OF SCHEDULES FOR EACH DAY
sched = [[i/2 for i in 1:(size(binaryHeatingResponse,1)-1) if abs.(diff(binaryHeatingResponse[:,dy]))[i]==1] for dy in days]
## DETERMINE DAYS WHERE SCHEDULES ARE POPULATED
# println(sched)

global schedContInd, schedIdleInd, schedProgRandInd, schedRandInd
schedContInd = []
schedIdleInd = []
schedRandInd = []
schedProgRandInd = []
for dy in [dy for dy in days if !isequal(controlRegime[dy],"v")]
    if !isempty(sched[dy]) && iseven(length(sched[dy]))
        push!(schedProgRandInd,dy)
    elseif minimum(dataSample[:,dy]).>0.2
        push!(schedContInd,dy)
    elseif maximum(dataSample[:,dy]).<0.2
        push!(schedIdleInd,dy)
    else
        push!(schedRandInd,dy)
    end
end

# schedContInd = [dy for dy in days if isempty(sched[dy]) && maximum(dataSample[:,dy]).>0.2]
# schedIdleInd = [dy for dy in days if isempty(sched[dy]) && maximum(dataSample[:,dy]).<0.2]
# schedProgRandInd = [dy for dy in days if ~isempty(sched[dy])]

# println(schedRandInd)
# println(controlRegime)
for sR in schedRandInd
    controlRegime[sR] = "r"
end
## IF MAX LOAD IS >0.2kW, CATEGORISE AS "HtgCont" REGIME (CONTINUOUS)
for sC in schedContInd
    controlRegime[sC] = "c"
    binaryHeatingResponse[:,sC] = ones(nSamplesPlus1,1)
end
## IF MAX LOAD IS <0.2kW, CATEGORISE AS "HtgIdle" REGIME
for sI in schedIdleInd
    controlRegime[sI] = "i"
    binaryHeatingResponse[:,sI] = zeros(nSamplesPlus1,1)
end

## CONTINUOUS ("c"), IDLE ("i") AND VOID ("v") ARE NOW DEFINED "controlRegime" ⇒ PREPEND "H/L" DEPENDING ON SETPOINT STATUS AT 00:00 (APPLIES TO "p" AND REMAINING "r" REGIMES)
htgInitHigh = [dy for dy in days if binaryHeatingResponse[1,dy].==1]
randomOrProgram = [dy for dy in days if ismissing(controlRegime[dy])]
[sched[dy]=vcat(-1,sched[dy]) for dy in intersect(htgInitHigh,randomOrProgram)]

schedUnique = unique(sched[schedProgRandInd])
# [println("$(dy)\t→\t$(length(sched[dy]))\t→\t$(isempty(sched[dy]))\t→\t$(sched[dy])") for dy in schedProgRandInd]
# println(" ")
# [println(s) for s in schedUnique]

## CYCLE THROUGH DAYS YET TO BE CATEGORISED INTO "controlRegime" (MIXED HIGH/LOW SETPOINTS)
## ASSIGN EACH DAILY SCHEDULE TO A UNIQUE SCHEDULE ID
global schedId
schedId = fill!(Array{Int32}(undef,nDays),0)
# for d in [dy for dy in days if controlRegime[dy]==0]
# println([dy for dy in days if isequal(controlRegime[dy],missing)])
# println(schedId)

# println(controlRegime)
for d in [dy for dy in days if isequal(controlRegime[dy],missing)]
    schedId[d] = Int([sU for sU in 1:length(schedUnique) if sched[d]==schedUnique[sU]][1])
end
# println(schedId)

## ESTABLISH ARRAY OF LOCAL OFFSET FOR PATTERN RECOGNITION (-1WEEK, -1DAY, +1DAY, +1WEEK)
DayOffsets(x,xMax) = [x+i for i in [-7 -1 1 7] if (x+i)>0 && (x+i)<(xMax+1)]
# dayOffsets = [DayOffsets(dy,days[end]) for dy in days]

# for d in [dy for dy in days if controlRegime[dy]==0]
for d in [dy for dy in days if isequal(controlRegime[dy],missing)]
    dayOffsets = DayOffsets(d,days[end])
    # if isempty(findall(dayOffsets.==schedId[d]))
    if !isempty(findall(schedId[dayOffsets].==schedId[d]))
        # println("$(schedId[d]) \t→\t $(schedId[dayOffsets]) \t→\t $(d)")
        # controlRegime[d] = 3
        controlRegime[d] = "p"
    else
        # println("$(schedId[d]) \t→\t $(schedId[dayOffsets]) \t→\t $(d) x")
        # controlRegime[d] = 4
        controlRegime[d] = "r"
    end
end

## IDENTIFY AND REASSIGN PROFILES APPEARING IN HtgRand ("r") REGIME, WHICH ALSO APPEAR IN HtgProg ("p")
schedIdProg = unique(schedId[findall(controlRegime[:].=="p")])
pos = [i for i in 1:length(schedId) if !isempty(findall(schedIdProg.==schedId[i]))]
controlRegime[pos].="p"

## COUNT INSTANCES OF HEATING PROGRAMS
htgPrograms = [length(findall(schedId.==i)) for i in unique(schedId[findall(controlRegime[:].=="p")])]

## CHECK RANDOM DAYS FOR REPEATING BEHAVIOUR
schedIdRand = unique(schedId[findall(controlRegime[:].=="r")])
htgRandoms = [length(findall(schedId.==i)) for i in unique(schedId[findall(controlRegime[:].=="r")]) if !(i.==0)]

## IDENTIFY AND REASSIGN PROFILES THAT OCCUR MORE THAT 5 TIMES FROM "r" TO "p"
# println(schedIdRand[17])
# println(htgRandoms[17])

controlRegimes = [length(findall(controlRegime.==i)) for i in ["c","p","r","i","v"]] ## 'c'→CONTINUOUS,'p'→PROGRAM,'r'→RANDOM,'i'→IDLE,'v'→VOID
proj.summaryData.HtgCont[n] = controlRegimes[1]
proj.summaryData.HtgProg[n] = controlRegimes[2]
proj.summaryData.HtgRand[n] = controlRegimes[3]
proj.summaryData.HtgIdle[n] = controlRegimes[4]
proj.summaryData.HtgVoid[n] = controlRegimes[5]
proj.summaryData.NumProgs[n] = length(htgPrograms)
proj.summaryData.HtgPrograms[n] = "$(join(sort(htgPrograms,rev=true),"|"))"
proj.summaryData.RandomReoccur[n] = "$(join(sort(htgRandoms,rev=true)[1:6],"|"))"

## CREATE BOOL ARRAY FOR HIGH HEATING STATE AT MIDNIGHT
schedProg = schedUnique[schedIdProg]
htgSPHigh =  [schedProg[i][1].==-1 for i in 1:length(schedProg)]
[schedProg[i]=schedProg[i][2:end] for i in 1:length(schedProg) if htgSPHigh[i].==true]

## WRITE HTEAGIN SCEDULE DATAFRAM (GIVING SHAPE("|" DELIM); REF DWELLING; ID WRT THAT DWELLING; H/L BOOL AT MIDNIGHT)
for i in 1:length(schedIdProg)
    push!(proj.htgSchedules,[join(schedProg[i],"|") parse(Int32,split(proj.smartMeters[n].id,"_")[4]) schedIdProg[i] htgSPHigh[i]])
end
dwellingProfiles = DataFrame(ControlRegime=controlRegime, SchedID=schedId)
fullPath = join([proj.dataPath "GasProfileData" "GasProfileDataTest_$(split(proj.smartMeters[n].id,"_")[4]).csv"],"/")
CSV.write(fullPath,dwellingProfiles)

println("okay         →  $(n)")

# catch
#     println("unsuccessful →  $(n)")
# end
# end
# return proj
# end


text:
using ProgressLogging

using Term.progress
import Term.Consoles: clear, cursor_position
import Term.Progress: SPINNERS
import Term: tprintln, Panel, install_term_logger

install_term_logger()

function simple(; kwargs...)
    pbar = ProgressBar(; refresh_rate = 60, expand = false, width = 150, kwargs...)
    j1 = addjob!(pbar; N = 100, description = "First")

    with(pbar) do
        for i in 1:100
            update!(j1)
            sleep(0.01)
        end
    end
end

function simple_slow(; kwargs...)
    pbar = ProgressBar(;
        refresh_rate = 60,
        expand = false,
        columns = :detailed,
        width = 150,
        kwargs...,
    )
    j1 = addjob!(pbar; N = 5, description = "First")

    with(pbar) do
        for i in 1:5
            for k in 1:10000
                x = rand(100, 100)
                y = x .^ 2 .- rand(100, 100)
                # yield()
            end

            update!(j1)
            # println(i)
            # sleep(.001)
        end
    end
end

function multi_nested()
    pbar = ProgressBar(; refresh_rate = 60, expand = false, width = 150)
    with(pbar) do
        outer = addjob!(pbar; N = 3, description = "outer")
        for i in 1:3
            println("iii", i)
            inner = addjob!(pbar; N = 100, description = "inner $i")
            for j in 1:100
                update!(inner)
                sleep(0.005)
            end

            update!(outer)
            removejob!(pbar, inner)
            sleep(1)
        end
    end
end

function multi_nested_double()
    pbar = ProgressBar(; refresh_rate = 60, expand = false, width = 150)
    with(pbar) do
        outer = addjob!(pbar; N = 3, description = "outer")
        for i in 1:3
            println(i)
            inner = addjob!(pbar; N = 100, description = "inner $i")
            inner2 = addjob!(pbar; N = 100, description = "inner $i v2")
            for j in 1:100
                update!(inner)
                update!(inner2)
                sleep(0.005)
            end

            update!(outer)
            removejob!(pbar, inner)
        end
    end
end

function multi(; kwargs...)
    pbar = ProgressBar(; refresh_rate = 60, expand = false, width = 150, kwargs...)
    j1 = addjob!(pbar; N = 50, description = "First")
    j2 = addjob!(pbar; N = 75, description = "Second")
    j3 = addjob!(pbar; N = 100, description = "Third")

    text = [
        "   [italic white]Let",
        "   [green]the",
        "   [bright_blue]texts",
        "   [white]be",
        Panel("[bold red underline]shown!"; fit = true),
    ]
    idx = 1
    with(pbar) do
        for i in 1:100
            i % 20 == 0 && begin
                tprintln(text[idx])
                idx += 1
            end

            update!(j1)
            update!(j2)
            update!(j3)
            sleep(0.025)
        end
    end
end

function spinner()
    for spinner in keys(SPINNERS)
        columns_kwargs = Dict(
            :SpinnerColumn => Dict(:spinnertype => spinner, :style => "bold green"),
            :CompletedColumn => Dict(:style => "dim"),
        )

        pbar = ProgressBar(; columns = :spinner, columns_kwargs = columns_kwargs)
        with(pbar) do
            job = addjob!(pbar; description = "[orange1]$spinner...")
            for i in 1:500
                update!(job)
                sleep(0.0025)
            end
        end
    end
end

function mixed()
    columns_kwargs = Dict(
        :SpinnerColumn => Dict(:style => "bold green"),
        :CompletedColumn => Dict(:style => "dim"),
    )

    pbar = ProgressBar(; columns_kwargs = columns_kwargs)
    with(pbar) do
        j1 = addjob!(pbar; N = length(keys(SPINNERS)))

        for spinner in keys(SPINNERS)
            update!(j1)
            job = addjob!(pbar; description = "...")
            for i in 1:500
                update!(job)
                sleep(0.001)
            end
        end
    end
end

function transientjobs()
    columns_kwargs = Dict(
        :SpinnerColumn => Dict(:style => "bold green"),
        :CompletedColumn => Dict(:style => "dim"),
    )

    pbar = ProgressBar(; columns_kwargs = columns_kwargs)
    with(pbar) do
        j1 = addjob!(pbar; N = length(keys(SPINNERS)))

        for spinner in keys(SPINNERS)
            println(spinner)
            update!(j1)
            job = addjob!(pbar; N = 500, description = "...", transient = true)
            for i in 1:500
                update!(job)
                sleep(0.001)
            end
        end
    end
end

function progresslogging()
    @progress "outer...." for i in 1:6
        @progress "inner... $i" for j in 1:100
            sleep(0.01)
        end
    end
end

function _track()
    @track for i in 1:10
        sleep(0.1)
    end
end

function _track_slow()
    @track for i in 1:10
        for k in 1:10000
            x = rand(100, 100)
            y = x .^ 2 .- rand(100, 100)
        end
    end
end

clear()
println("starting")
print("_"^150)

# simple(; transient=false, columns=:detailed)
# multi(; transient=false)
# multi_nested()
# multi_nested_double()
# spinner()
# mixed()
# transientjobs()
# progresslogging()
# _track()

simple_slow()
# _track_slow()

println("done")


text:
abstract type AbstractTestBlock
end

struct TestBlockUnitTests <: AbstractTestBlock
end
struct TestBlockIntegration1 <: AbstractTestBlock
end
struct TestBlockIntegration2 <: AbstractTestBlock
end
struct TestBlockIntegration3 <: AbstractTestBlock
end
struct TestBlockIntegration4 <: AbstractTestBlock
end
struct TestBlockIntegration5 <: AbstractTestBlock
end
struct TestBlockIntegration6 <: AbstractTestBlock
end
struct TestBlockIntegration7 <: AbstractTestBlock
end

group_includes_block(::AbstractTestGroup, ::AbstractTestBlock) = true

group_includes_block(::TestGroupDefault, ::AbstractTestBlock) = true
group_includes_block(::TestGroupAll, ::AbstractTestBlock) = true
group_includes_block(::TestGroupTestPlots, ::AbstractTestBlock) = true

group_includes_block(::TestGroupImportOnly, ::AbstractTestBlock) = false

group_includes_block(::TestGroupTravis1, ::TestBlockUnitTests) = true
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration1) = true
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis1, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis2, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration2) = true
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis2, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis3, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration3) = true
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis3, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis4, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration4) = true
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis4, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis5, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration5) = true
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis5, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis6, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration6) = true
group_includes_block(::TestGroupTravis6, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupTravis7, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupTravis7, ::TestBlockIntegration7) = true

group_includes_block(::TestGroupDocker1, ::TestBlockUnitTests) = true
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupDocker1, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupDocker2, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration1) = true
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration2) = true
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration3) = true
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupDocker2, ::TestBlockIntegration7) = false

group_includes_block(::TestGroupDocker3, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration4) = true
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration5) = true
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration6) = true
group_includes_block(::TestGroupDocker3, ::TestBlockIntegration7) = true

group_includes_block(::TestGroupDocker4, ::TestBlockUnitTests) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration1) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration2) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration3) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration4) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration5) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration6) = false
group_includes_block(::TestGroupDocker4, ::TestBlockIntegration7) = false


text:
### For reading a string

"""
    readlengthandstring(io::IO)

Read  `n::Int64` from `io`. Then read `n` bytes from `io`
and return as a string.
"""
function readlengthandstring(io::IO)
    stringlength = read(io, Int64)
    return readstring(io, stringlength)
end

function readstring(io::IO, stringlength::Int32)
    buf = Array{UInt8}(undef, convert(Int64, stringlength))
    readbytes!(io, buf)
    return unsafe_string(pointer(buf))
end

"""
    readstring(io::IO, stringlength::Int64)

Read  `stringlength` bytes from `io` and return as a string.
"""
function readstring(io::IO, stringlength::Int64)
    buf = Array{UInt8}(undef, stringlength)
    readbytes!(io, buf)
    return unsafe_string(pointer(buf))
end

function peektypeold(io::IO, T::DataType)
    p = position(io)
    res = read(io, T)
    seek(io, p)
    return res
end

function peektype(io::IO, T::DataType)
    mark(io)
    res = read(io, T)
    reset(io)
    res
end


text:
using DynamicalSystems, Base.Test, OrdinaryDiffEq

println("\nTesting continuous system lyapunov exponents...")

@testset "Lorenz system" begin
  ds = Systems.lorenz()
  ds2 = ContinuousDS(ds.state, ds.eom)
  @testset "lyapunovs" begin
    λ = lyapunovs(ds, 2e4)
    @test 0.89 < λ[1] < 0.91
    @test -0.001 < λ[2] < 0.01
    @test -14.6 < λ[3] < -14.5

    λ = lyapunovs(ds, 2e4; dt = 0.1, Ttr = 10.0,
    diff_eq_kwargs = Dict(:abstol=>1e-9, :solver => DP5()))
    @test 0.89 < λ[1] < 0.91
    @test -0.001 < λ[2] < 0.01
    @test -14.6 < λ[3] < -14.5
  end

  @testset "lyapunovs ForwardDiff" begin
    λ = lyapunovs(ds2, 2e4)
    @test 0.89 < λ[1] < 0.91
    @test -0.001 < λ[2] < 0.01
    @test -14.6 < λ[3] < -14.5
  end

  @testset "lyapunov" begin
    λ1 = lyapunov(ds, 2000, dt =  0.1)
    λ2 = lyapunov(ds2, 2000, dt = 0.1,
    diff_eq_kwargs = Dict(:solver => DP5(), :abstol => 1e-9))
    @test 0.89 < λ1[1] < 0.91
    @test 0.89 < λ2[1] < 0.91
  end
end
#=
@testset "Roessler system" begin
  ds = Systems.roessler()
  ds2 = ContinuousDS(ds.state, ds.eom)
  @testset "lyapunovs" begin
    λ = lyapunovs(ds, 5e4)
    @test 0.06 < λ[1] < 0.08
    @test -0.01 < λ[2] < 0.01
    @test -5.4 < λ[3] < -5.39

    λ = lyapunovs(ds, 5e4; dt = 0.1, Ttr = 10.0,
    diff_eq_kwargs = Dict(:abstol=>1e-9, :solver => DP5()))
    @test 0.06 < λ[1] < 0.08
    @test -0.01 < λ[2] < 0.01
    @test -5.4 < λ[3] < -5.39
  end

  @testset "lyapunovs ForwardDiff" begin
    λ = lyapunovs(ds2, 5e4)
    @test 0.06 < λ[1] < 0.08
    @test -0.01 < λ[2] < 0.01
    @test -5.4 < λ[3] < -5.39
  end

  @testset "lyapunov" begin
    λ1 = lyapunov(ds, 5000, dt =  0.1)
    λ2 = lyapunov(ds2, 5000, dt = 0.1,
    diff_eq_kwargs = Dict(:solver => DP5(), :abstol => 1e-9))
    @test 0.065 < λ1[1] < 0.075
    @test 0.065 < λ2[1] < 0.075
  end
end
=#


text:
@testset "GaussianLikelihood" begin
    lik = GaussianLikelihood(1e-5)
    test_interface(lik, SqExponentialKernel(), rand(10); functor_args=(:σ²,))
end

@testset "HeteroscedasticGaussianLikelihood" begin
    lik = HeteroscedasticGaussianLikelihood()
    IN_DIM = 3
    OUT_DIM = 2 # one for the mean the other for the log-standard deviation
    N = 10
    X = MOInput([rand(IN_DIM) for _ in 1:N], OUT_DIM)
    test_interface(lik, IndependentMOKernel(SqExponentialKernel()), X)
end


text:
#####
##### `LPCM_SAMPLE_TYPE_UNION`
#####

const LPCM_SAMPLE_TYPE_UNION = Union{Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,Float32,Float64}

function julia_type_from_onda_sample_type(t::AbstractString)
    t == "int8" && return Int8
    t == "int16" && return Int16
    t == "int32" && return Int32
    t == "int64" && return Int64
    t == "uint8" && return UInt8
    t == "uint16" && return UInt16
    t == "uint32" && return UInt32
    t == "uint64" && return UInt64
    t == "float32" && return Float32
    t == "float64" && return Float64
    throw(ArgumentError("sample type $t is not supported by Onda"))
end

julia_type_from_onda_sample_type(T::Type{<:LPCM_SAMPLE_TYPE_UNION}) = T

function onda_sample_type_from_julia_type(T::Type)
    T === Int8 && return "int8"
    T === Int16 && return "int16"
    T === Int32 && return "int32"
    T === Int64 && return "int64"
    T === UInt8 && return "uint8"
    T === UInt16 && return "uint16"
    T === UInt32 && return "uint32"
    T === UInt64 && return "uint64"
    T === Float32 && return "float32"
    T === Float64 && return "float64"
    throw(ArgumentError("sample type $T is not supported by Onda"))
end

onda_sample_type_from_julia_type(t::AbstractString) = onda_sample_type_from_julia_type(julia_type_from_onda_sample_type(t))

convert_number_to_lpcm_sample_type(x::LPCM_SAMPLE_TYPE_UNION) = x
convert_number_to_lpcm_sample_type(x) = Float64(x)

#####
##### `SamplesInfo`
#####

"""
    const SamplesInfo = @row("onda.samples-info@1",
                             kind::String,
                             channels::Vector{String},
                             sample_unit::String,
                             sample_resolution_in_unit::LPCM_SAMPLE_TYPE_UNION,
                             sample_offset_in_unit::LPCM_SAMPLE_TYPE_UNION,
                             sample_type::String = Onda.onda_sample_type_from_julia_type(sample_type),
                             sample_rate::LPCM_SAMPLE_TYPE_UNION)

A type alias for [`Legolas.Row{typeof(Legolas.Schema("onda.samples-info@1"))}`](https://beacon-biosignals.github.io/Legolas.jl/stable/#Legolas.@row)
representing the bundle of `onda.signal` fields that are intrinsic to a signal's sample data,
leaving out extrinsic file or recording information. This is useful when the latter information
is irrelevant or does not yet exist (e.g. if sample data is being constructed/manipulated in-memory
without yet having been serialized).
"""
const SamplesInfo = @row("onda.samples-info@1",
                         kind::AbstractString = convert(String, kind),
                         channels::AbstractVector{<:AbstractString} = convert(Vector{String}, channels),
                         sample_unit::AbstractString = convert(String, sample_unit),
                         sample_resolution_in_unit::LPCM_SAMPLE_TYPE_UNION = convert_number_to_lpcm_sample_type(sample_resolution_in_unit),
                         sample_offset_in_unit::LPCM_SAMPLE_TYPE_UNION = convert_number_to_lpcm_sample_type(sample_offset_in_unit),
                         sample_type::AbstractString = onda_sample_type_from_julia_type(sample_type),
                         sample_rate::LPCM_SAMPLE_TYPE_UNION = convert_number_to_lpcm_sample_type(sample_rate))

#####
##### `Signal`
#####

# Note that the real field type restrictions here are more lax than the documented
# ones for improved compatibility with data produced by older Onda.jl versions and/or
# non-Julia producers.
"""
    const Signal = @row("onda.signal@1" > "onda.samples-info@1",
                        recording::UUID,
                        file_path::Any,
                        file_format::String = (file_format isa AbstractLPCMFormat ?
                                               Onda.file_format_string(file_format) :
                                               file_format),
                        span::TimeSpan)

A type alias for [`Legolas.Row{typeof(Legolas.Schema("onda.signal@1"))}`](https://beacon-biosignals.github.io/Legolas.jl/stable/#Legolas.@row)
representing an [`onda.signal` as described by the Onda Format Specification](https://github.com/beacon-biosignals/Onda.jl##ondasignal1).

Note that the `Signal` constructor will perform additional validation on underlying `onda.samples-info@1` fields to
ensure that these fields are compliant with the Onda specification; an `ArgumentError` will be thrown if any fields
are invalid.

This type primarily exists to aid in the validated row construction, and is not intended to be used as a type constraint
in function or struct definitions. Instead, you should generally duck-type any "signal-like" arguments/fields so that
other generic row types will compose with your code.
"""
const Signal = @row("onda.signal@1" > "onda.samples-info@1",
                    recording::Union{UInt128,UUID} = UUID(recording),
                    file_path::Any,
                    file_format::AbstractString = file_format isa AbstractLPCMFormat ? file_format_string(file_format) : file_format,
                    span::Union{NamedTupleTimeSpan,TimeSpan} = TimeSpan(span),
                    kind::AbstractString = _validate_signal_kind(kind),
                    channels::AbstractVector{<:AbstractString} = _validate_signal_channels(channels),
                    sample_unit::AbstractString = _validate_signal_sample_unit(sample_unit))

function _validate_signal_kind(x)
    is_lower_snake_case_alphanumeric(x) || throw(ArgumentError("invalid signal kind (must be lowercase/snakecase/alphanumeric): $x"))
    return x
end

function _validate_signal_sample_unit(x)
    is_lower_snake_case_alphanumeric(x) || throw(ArgumentError("invalid signal sample unit (must be lowercase/snakecase/alphanumeric): $x"))
    return x
end

function _validate_signal_channels(x)
    allunique(x) || throw(ArgumentError("invalid signal channels (duplicate channel names are disallowed): $x"))
    foreach(_validate_signal_channel, x)
    return x
end

function _validate_signal_channel(x)
    is_lower_snake_case_alphanumeric(x, ('-', '.')) || throw(ArgumentError("invalid channel name (must be lowercase/snakecase/alphanumeric): $x"))
    return x
end

extract_samples_info(signal) = @compat SamplesInfo(; signal.kind, signal.channels, signal.sample_unit,
                                                   signal.sample_resolution_in_unit, signal.sample_offset_in_unit,
                                                   signal.sample_type, signal.sample_rate)

"""
    write_signals(io_or_path, table; kwargs...)

Invoke/return `Legolas.write(path_or_io, signals, Schema("onda.signal@1"); kwargs...)`.
"""
write_signals(path_or_io, signals; kwargs...) = Legolas.write(path_or_io, signals, Legolas.Schema("onda.signal@1"); kwargs...)

"""
    validate_signals(signals)

Perform both table-level and row-level validation checks on the content of `signals`,
a presumed `onda.signal` table. Returns `signals`.

This function will throw an error in any of the following cases:

- `Legolas.validate(signals, Legolas.Schema("onda.signal@1"))` throws an error
- `Signal(row)` errors for any `row` in `Tables.rows(signals)`
- `signals` contains rows with duplicate `file_path`s
"""
validate_signals(signals) = _fully_validate_legolas_table(signals, Legolas.Schema("onda.signal@1"), :file_path)

#####
##### duck-typed utilities
#####

"""
    channel(x, name)

Return `i` where `x.channels[i] == name`.
"""
channel(x, name) = findfirst(isequal(name), x.channels)

"""
    channel(x, i::Integer)

Return `x.channels[i]`.
"""
channel(x, i::Integer) = x.channels[i]

"""
    channel_count(x)

Return `length(x.channels)`.
"""
channel_count(x) = length(x.channels)

"""
    sample_count(x, duration::Period)

Return the number of multichannel samples that fit within `duration` given `x.sample_rate`.
"""
sample_count(x, duration::Period) = TimeSpans.index_from_time(x.sample_rate, duration) - 1

"""
    sample_type(x)

Return `x.sample_type` as an `Onda.LPCM_SAMPLE_TYPE_UNION` subtype. If `x.sample_type` is an Onda-specified `sample_type` string (e.g. `"int16"`), it will be converted to the corresponding Julia type. If `x.sample_type <: Onda.LPCM_SAMPLE_TYPE_UNION`, this function simply returns `x.sample_type` as-is.
"""
sample_type(x) = julia_type_from_onda_sample_type(x.sample_type)

"""
    sizeof_samples(x, duration::Period)

Returns the expected size (in bytes) of an encoded `Samples` object corresponding to `x` and `duration`:

    sample_count(x, duration) * channel_count(x) * sizeof(x.sample_type)

"""
sizeof_samples(x, duration::Period) = sample_count(x, duration) * channel_count(x) * sizeof(sample_type(x))