text:
import Base: convert, haskey, setindex!, getindex
export slider, togglebutton, button,
       checkbox, textbox, textarea,
       radiobuttons, dropdown, selection,
       togglebuttons, html, latex, hbox, vbox,
       progress, widget, selection_slider,
       set!

const Empty = VERSION < v"0.4.0-dev" ? Nothing : Void

### Layout Widgets
type Layout <: Widget
    box::Any
end

type Box <: Widget
    layout::Any
    vert::Bool
    children::Vector{Widget}
    Box(layout, vert, children) = begin
        b = new(layout, vert, children)
        layout.box = b
        b
    end
end

hbox(children::Vararg{Widget}) = Box(Layout(nothing), false, collect(children))
vbox(children::Vararg{Widget}) = Box(Layout(nothing), true, collect(children))

### Input widgets
"""Helps init widget's value and signal depending on which ones were set"""
function init_wsigval(signal, value; default=value, typ=typeof(default))
    if signal == nothing
        if value == nothing
            value = default
        end
        _typ = typ === Void ? typeof(value) : typ
        signal = Signal(_typ, value)
    else
        #signal set
        if value == nothing
            value = signal.value
        else
            #signal set and value set
            push!(signal, value)
        end
    end
    signal, value
end

########################## Slider ############################

type Slider{T<:Number} <: InputWidget{T}
    signal::Signal{T}
    label::AbstractString
    value::T
    range::Range{T}
    orientation::String
    readout::Bool
    readout_format::AbstractString
    continuous_update::Bool
end

# differs from median(r) in that it always returns an element of the range
medianelement(r::Range) = r[(1+length(r))>>1]

slider(args...) = Slider(args...)
"""
    slider(range; value, signal, label="", readout=true, continuous_update=true)

Create a slider widget with the specified `range`. Optionally specify
the starting `value` (defaults to the median of `range`), provide the
(Reactive.jl) `signal` coupled to this slider, and/or specify a string
`label` for the widget.
"""
slider{T}(range::Range{T};
          value=nothing,
          signal=nothing,
          label="",
          orientation="horizontal",
          readout=true,
          readout_format=T <: Integer ? "d" : ".3f",
          continuous_update=true,
          syncsig=true) = begin
    signal, value = init_wsigval(signal, value; typ=T, default=medianelement(range))
    s = Slider(signal, label, value, range, orientation,
                readout, readout_format, continuous_update)
    if syncsig
        #keep the slider updated if the signal changes
        keep_updated(val) = begin
            if val != s.value
                s.value = val
                update_view(s)
            end
            nothing
        end
        preserve(map(keep_updated, signal; typ=Void))
    end
    s
end

######################### Checkbox ###########################

type Checkbox <: InputWidget{Bool}
    signal::Signal{Bool}
    label::AbstractString
    value::Bool
end

checkbox(args...) = Checkbox(args...)

"""
    checkbox(value=false; label="", signal)

Provide a checkbox with the specified starting (boolean)
`value`. Optional provide a `label` for this widget and/or the
(Reactive.jl) `signal` coupled to this widget.
"""
checkbox(value::Bool; signal=nothing, label="") = begin
    signal, value = init_wsigval(signal, value)
    Checkbox(signal, label, value)
end
checkbox(; label="", value=nothing, signal=nothing) = begin
    signal, value = init_wsigval(signal, value; default=false)
    Checkbox(signal, label, value)
end
###################### ToggleButton ########################

type ToggleButton <: InputWidget{Bool}
    signal::Signal{Bool}
    label::AbstractString
    value::Bool
end

togglebutton(args...) = ToggleButton(args...)

togglebutton(; label="", value=nothing, signal=nothing) = begin
    signal, value = init_wsigval(signal, value; default=false)
    ToggleButton(signal, label, value)
end

"""
    togglebutton(label=""; value=false, signal)

Create a toggle button. Optionally specify the `label`, the initial
state (`value=false` is off, `value=true` is on), and/or provide the
(Reactive.jl) `signal` coupled to this button.
"""
togglebutton(label; kwargs...) =
    togglebutton(label=label; kwargs...)

######################### Button ###########################

type Button{T} <: InputWidget{T}
    signal::Signal{T}
    label::AbstractString
    value::T
end

button(; value=nothing, label="", signal=Signal(value)) =
    Button(signal, label, value)

"""
    button(label; value=nothing, signal)

Create a push button. Optionally specify the `label`, the `value`
emitted when then button is clicked, and/or the (Reactive.jl) `signal`
coupled to this button.
"""
button(label; kwargs...) =
    button(label=label; kwargs...)

######################## Textbox ###########################

type Textbox{T} <: InputWidget{T}
    signal::Signal{T}
    label::String
    @compat range::Union{Empty, Range}
    value::T
end

function empty(t::Type)
    if is(t, Number) zero(t)
    elseif is(t, AbstractString) ""
    end
end

function Textbox(; label="",
                 value=nothing,
                 # Allow unicode characters even if initiated with ASCII
                 typ=Void,
                 range=nothing,
                 signal=nothing,
                 syncsig=true)
    if isa(value, AbstractString) && range != nothing
        throw(ArgumentError(
               "You cannot set a range on a string textbox"
             ))
    end
    signal, value = init_wsigval(signal, value; typ=typ, default="")
    t = Textbox(signal, label, range, value)
    if syncsig
        #keep the Textbox updated if the signal changes
        keep_updated(val) = begin
            if val != t.value
                t.value = val
                update_view(t)
            end
            nothing
        end
        preserve(map(keep_updated, signal; typ=Void))
    end
    t

end

textbox(;kwargs...) = Textbox(;kwargs...)

"""
    textbox(value=""; label="", typ=typeof(value), range=nothing, signal)

Create a box for entering text. `value` is the starting value; if you
don't want to provide an initial value, you can constrain the type
with `typ`. Optionally provide a `label`, specify the allowed range
(e.g., `-10.0:10.0`) for numeric entries, and/or provide the
(Reactive.jl) `signal` coupled to this text box.
"""
textbox(val; kwargs...) =
    Textbox(value=val; kwargs...)
textbox(val::String; kwargs...) =
    Textbox(value=val; kwargs...)

parse_msg{T<:Number}(w::Textbox{T}, val::AbstractString) = parse_msg(w, parse(T, val))
function parse_msg{T<:Number}(w::Textbox{T}, val::Number)
    v = convert(T, val)
    if isa(w.range, Range)
        # force value to stay in range
        v = max(first(w.range),
                min(last(w.range), v))
    end
    v
end

######################### Textarea ###########################

type Textarea{AbstractString} <: InputWidget{AbstractString}
    signal::Signal{AbstractString}
    label::AbstractString
    value::AbstractString
end

textarea(args...) = Textarea(args...)

textarea(; label="",
         value=nothing,
         signal=nothing) = begin
    signal, value = init_wsigval(signal, value; default="")
    Textarea(signal, label, value)
end

"""
    textarea(value=""; label="", signal)

Creates an extended text-entry area. Optionally provide a `label`
and/or the (Reactive.jl) `signal` associated with this widget. The
`signal` updates when you type.
"""
textarea(val; kwargs...) =
    textarea(value=val; kwargs...)

##################### SelectionWidgets ######################

immutable OptionDict
    dict::OrderedDict
    invdict::Dict
end
OptionDict(d::OrderedDict) = begin
    T1 = eltype([keys(d)...])
    T2 = eltype([values(d)...])
    OptionDict(OrderedDict{T1,T2}(d), Dict{T2,T1}(zip(values(d), keys(d))))
end

Base.getindex(x::OptionDict, y) = getindex(x.dict, y)
Base.haskey(x::OptionDict, y) = haskey(x.dict, y)
Base.keys(x::OptionDict) = keys(x.dict)
Base.values(x::OptionDict) = values(x.dict)
Base.length(x::OptionDict) = length(keys(x))

type Options{view, T} <: InputWidget{T}
    signal::Signal
    label::AbstractString
    value::T
    value_label::AbstractString
    options::OptionDict
    icons::AbstractArray
    tooltips::AbstractArray
    readout::Bool
    orientation::AbstractString
end

Options(view::Symbol, options::OptionDict;
        label = "",
        value_label=first(keys(options)),
        value=nothing,
        icons=[],
        tooltips=[],
        typ=valtype(options.dict),
        signal=nothing,
        readout=true,
        orientation="horizontal",
        syncsig=true,
        syncnearest=true,
        sel_mid_idx=0) = begin
    #sel_mid_idx set in selection_slider(...) so default value_label is middle of range
    sel_mid_idx != 0 && (value_label = collect(keys(options.dict))[sel_mid_idx])
    signal, value = init_wsigval(signal, value; typ=typ, default=options[value_label])
    typ = eltype(signal)
    ow = Options{view, typ}(signal, label, value, value_label,
                    options, icons, tooltips, readout, orientation)
    if syncsig
        syncselnearest = view == :SelectionSlider && typ <: Real && syncnearest
        if view != :SelectMultiple
            #set up map that keeps the value_label in sync with the value
            #TODO handle SelectMultiple. Need something similar to handle_msg
            keep_label_updated(val) = begin
                if syncselnearest
                    val = nearest_val(keys(ow.options.invdict), val)
                end
                if haskey(ow.options.invdict, val) &&
                  ow.value_label != ow.options.invdict[val]
                    ow.value_label = ow.options.invdict[val]
                    update_view(ow)
                end
                nothing
            end
            preserve(map(keep_label_updated, signal; typ=Void))
        end
        push!(signal, value)
    end
    ow
end

function Options(view::Symbol,
                    options::Union{Associative, AbstractArray};
                    kwargs...)
    Options(view, getoptions(options); kwargs...)
end

function getoptions(options)
    opts = OrderedDict()
    for el in options
        addoption!(opts, el)
    end
    optdict = OptionDict(opts)
end

addoption!(opts, v::NTuple{2}) = opts[string(v[1])] = v[2]
addoption!(opts, v) = opts[string(v)] = v

"""
    dropdown(choices; label="", value, typ, icons, tooltips, signal)

Create a "dropdown" widget. `choices` can be a vector of
options. Optionally specify the starting `value` (defaults to the
first choice), the `typ` of elements in `choices`, supply custom
`icons`, provide `tooltips`, and/or specify the (Reactive.jl) `signal`
coupled to this widget.

# Examples

    a = dropdown(["one", "two", "three"])

To link a callback to the dropdown, use

    f = dropdown(["turn red"=>colorize_red, "turn green"=>colorize_green])
    map(g->g(image), signal(f))
"""
dropdown(opts; kwargs...) =
    Options(:Dropdown, opts; kwargs...)

"""
radiobuttons: see the help for `dropdown`
"""
radiobuttons(opts; kwargs...) =
    Options(:RadioButtons, opts; kwargs...)

"""
selection: see the help for `dropdown`
"""
function selection(opts; multi=false, kwargs...)
    if multi
        signal = Signal(collect(opts)[1:1])
        Options(:SelectMultiple, opts; signal=signal, kwargs...)
    else
        Options(:Select, opts; kwargs...)
    end
end

Base.@deprecate select(opts; kwargs...) selection(opts, kwargs...)

"""
togglebuttons: see the help for `dropdown`
"""
togglebuttons(opts; kwargs...) =
    Options(:ToggleButtons, opts; kwargs...)

"""
selection_slider: see the help for `dropdown`
If the slider has numeric (<:Real) values, and its signal is updated, it will
update to the nearest value from the range/choices provided. To disable this
behaviour, so that the widget state will only update if an exact match for
signal value is found in the range/choice, use `syncnearest=false`.
"""
selection_slider(opts; kwargs...) = begin
    if !haskey(Dict(kwargs), :value_label)
        #default to middle of slider
        mid_idx = length(opts)÷2 + 1 # +1 to round up.
        push!(kwargs, (:sel_mid_idx, mid_idx))
    end
    Options(:SelectionSlider, opts; kwargs...)
end

function nearest_val(x, val)
    local valbest
    local dxbest = typemax(Float64)
    for v in x
        dx = abs(v-val)
        if dx < dxbest
            dxbest = dx
            valbest = v
        end
    end
    valbest
end

### Output Widgets

export Latex, Progress

Base.@deprecate html(value; label="")  HTML(value)

type Latex <: Widget
    label::AbstractString
    value::AbstractString
end
latex(label, value::AbstractString) = Latex(label, value)
latex(value::AbstractString; label="") = Latex(label, value)
latex(value; label="") = Latex(label, mimewritable("application/x-latex", value) ? stringmime("application/x-latex", value) : stringmime("text/latex", value))

## # assume we already have Latex
## writemime(io::IO, m::MIME{symbol("application/x-latex")}, l::Latex) =
##     write(io, l.value)

type Progress <: Widget
    label::AbstractString
    value::Int
    range::Range
    orientation::String
    readout::Bool
    readout_format::String
    continuous_update::Bool
end

progress(args...) = Progress(args...)
progress(;label="", value=0, range=0:100, orientation="horizontal",
            readout=true, readout_format="d", continuous_update=true) =
    Progress(label, value, range, orientation, readout, readout_format, continuous_update)

# Make a widget out of a domain
widget(x::Signal, label="") = x
widget(x::Widget, label="") = x
widget(x::Range, label="") = selection_slider(x, label=label)
widget(x::AbstractVector, label="") = togglebuttons(x, label=label)
widget(x::Associative, label="") = togglebuttons(x, label=label)
widget(x::Bool, label="") = checkbox(x, label=label)
widget(x::AbstractString, label="") = textbox(x, label=label, typ=AbstractString)
widget{T <: Number}(x::T, label="") = textbox(typ=T, value=x, label=label)

### Set!

"""
`set!(w::Widget, fld::Symbol, val)`

Set the value of a widget property and update all displayed instances of the
widget. If `val` is a `Signal`, then updates to that signal will be reflected in
widget instances/views.

If `fld` is `:value`, `val` is also `push!`ed to `signal(w)`
"""
function set!(w::Widget, fld::Symbol, val)
    fld == :value && push!(signal(w), val)
    setfield!(w, fld, val)
    update_view(w)
    w
end

set!(w::Widget, fld::Symbol, valsig::Signal) = begin
    map(val -> set!(w, fld, val), valsig) |> preserve
end

set!{T<:Options}(w::T, fld::Symbol, val::Union{Signal,Any}) = begin
    fld == :options && (val = getoptions(val))
    invoke(set!, (Widget, Symbol, typeof(val)), w, fld, val)
end


text:
# Code to be loaded into eFEMpart

abstract type AbstractMesh end
abstract type AbstractDictList end

# allows symbol -> list access, i.e. bdry[:wall]
struct Index
  lookup::Dict{Symbol, Int}      # name => names array position
  names::Vector{Symbol}
end

struct DictList{T} <: AbstractDictList
  id::Vector{Vector{T}}
  idnames::Index
end

mutable struct Node
  x::Float64
  y::Float64
end

mutable struct Element
  order::Int
  QuadRule::Symbol
  NodeList::Vector{Int}
  SiblingList::Vector{Int}      # list of sibling indices -- possibly changes size
  Lc::Int                       # current refinement
  Lt::Int                       # target refinement
  Ls::Int                       # max target refinment of siblings
  La::Int                       # max target refinement of edge-connected neighbors
  regtype::Symbol               # :regular or :irregular, to help with refinement
  refineEdge::Vector{Bool}      # 4-array of booleans for whether or not to refine face
end

struct ScalarMesh <: AbstractMesh
  xy::Vector{Node}
  cm::Vector{Element}
  bdry::DictList{Int}      # boundary indices
  order::Symbol
end

struct FluidMesh <: AbstractMesh
  xy::Vector{Node}
  cm::Vector{Element}
  bdry::DictList{Int}      # boundary indices
  order::Symbol
  xyp::Vector{Node}                    # List of Node values
  cmp::Vector{Element}                 # connectivity matrix for pressure
end

# constructor for Index
function Index(names::Vector{Symbol})
  lookup = Dict{Symbol, Int}(zip(names, 1:length(names)))
  Index(lookup, names)
end

# constructor for Element
function Element(nl::Vector{Int})
  length(nl)==4 ? order=1 : order=2
  order == 1 ? QuadRule = :GaussOrder2 : QuadRule = :GaussOrder3

  # ONLY FOR TESTING
  falseArr = [false,false,false,false]
  return Element(order,QuadRule,nl,[0],0,0,0,0,:parent,falseArr)
end

# df[SingleColumnIndex] => AbstractDataVector
function Base.getindex(dl::AbstractDictList, col_ind::Symbol)
    selected_column = index(dl)[col_ind]
    return dl.id[selected_column]
end

function Base.getindex(mesh::AbstractMesh, col_ind::Symbol)
    return getindex(mesh.bdry,col_ind)
end

index(dl::AbstractDictList) = dl.idnames.lookup
indexlist(dl::AbstractDictList) = dl.idnames.names

##### Constructors for Meshes #####

# Assigns to appropriate mesh constructor based on mesh file extension
function Mesh(file_name::String)
  fileExtension = file_name[end-3:end]

  fn::GMSHfile = GMSHfile(file_name)
  mesh::ScalarMesh = Mesh(fn)
  return mesh
end

function FluidMesh(file_name::String)
  fileExtension = file_name[end-3:end]

  fn::GMSHfile = GMSHfile(file_name)
  mesh::FluidMesh = FluidMesh(fn)
  return mesh
end

getBoundaries(mesh) = mesh.bdry.idnames.names

ext: jl


text:
using AssemblyCalculus
using Graphs
using Plots

function time_sim(assem_size::Int; mean_degree=100)
    assembly_sizes = [assem_size, assem_size]
    area_sizes = assembly_sizes .^ 2
    plasticities = [0.01, 0.01]
    n = sum(area_sizes)
    p = mean_degree / n
    g = erdos_renyi(n, p, is_directed=true)
    ba = Brain(g, area_sizes, assembly_sizes, plasticities)
    g = nothing
    stims = [rand_stim(ba[i]) for i in 1:2]
    assems = Assembly{Float64}[]
    timesteps = 50
    td = @timed as, sp = simulate!(stims, assems, timesteps)
    return td.time
end

# Precompile
time_sim(5)

assem_sizes = collect(100:50:350)
times = time_sim.(assem_sizes)
neurons = 2 .* assem_sizes .^ 2
β = [neurons ones(size(neurons))] \ times
m = round(β[1], sigdigits=1)
b = round(β[2], sigdigits=2)

plot(neurons, times, linewidth=3, color=:teal, legend=:bottomright)
plot!(neurons, times, smooth=true, label="Best Fit: $m x + $b")
xticks!(neurons, string.(neurons))
xlabel!("Number of neurons")
ylabel!("Seconds to run 50 iterations")
title!("Time Complexity of AssemblyCalculus.jl")


text:
export Atom, Bond, Molecule
export complete, removeHs, natoms, nbonds
export readsdf, writesdf
using Printf

struct Atom
    symbol::String
    x::Float64
    y::Float64
    z::Float64
end

Atom(symbol) = Atom(symbol, 0.0, 0.0, 0.0)

struct Bond
    i::Int
    j::Int
    type::Int
    stereo::Int
end

Bond(i, j) = Bond(i, j, 0, 0)

struct Molecule
    name::String
    atoms::Vector{Atom}
    bonds::Vector{Bond}
    props::Dict
end

Molecule(name::String, atoms, bonds) = Molecule(name, atoms, bonds, Dict())
# Molecule(atoms) = Molecule(atoms, Bond[])

Base.getindex(mol::Molecule, key) = mol.props[key]
Base.setindex!(mol::Molecule, value, key) = mol.props[key] = value

natoms(m::Molecule) = length(m.atoms)
nbonds(m::Molecule) = length(m.bonds)

function complete(mol::Molecule)
    dict = Dict((b.i,b.j) => b for b in mol.bonds)
    bonds = [mol.bonds...]
    for i = 1:natoms(mol)
        for j = i+1:natoms(mol)
            if !haskey(dict, (i,j))
                push!(bonds, Bond(i,j))
            end
        end
    end
    Molecule(mol.name, mol.atoms, bonds, copy(mol.props))
end

function removeHs(mol::Molecule)
    dict = Dict()
    atoms = Atom[]
    for i = 1:natoms(mol)
        a = mol.atoms[i]
        a.symbol == "H" && continue
        dict[i] = length(dict) + 1
        push!(atoms, a)
    end
    bonds = Bond[]
    for b in mol.bonds
        haskey(dict,b.i) || continue
        i = dict[b.i]
        haskey(dict,b.j) || continue
        j = dict[b.j]
        push!(bonds, Bond(i,j,b.type,b.stereo))
    end
    @assert !isempty(atoms)
    Molecule(atoms, bonds)
end

function secondorder(mol::Molecule)
    neighbors = [Int[] for _=1:natoms(mol)]
    for b in mol.bonds
        @assert b.i != b.j
        push!(neighbors[b.i], b.j)
        push!(neighbors[b.j], b.i)
    end
    bonds = Bond[]
    for i = 1:natoms(mol)
        ns = neighbors[i]
        set = Set(ns)
        for j in ns
            push!(set, neighbors[j]...)
            for k in neighbors[j]
                push!(set, neighbors[k]...)
                #for l in neighbors[k]
                #    push!(set, neighbors[l]...)
                #    for m in neighbors[l]
                #        push!(set, neighbors[m]...)
                #    end
                #end
            end
        end
        ks = filter(k -> i < k, collect(set))
        for k in ks
            push!(bonds, Bond(i,k,0,0))
        end
    end
    Molecule(mol.atoms, bonds)
end

function readsdf(filename::String)
    lines = open(readlines, filename)
    buffer = String[]
    mols = Molecule[]
    for line in lines
        if line == raw"$$$$"
            mol = parsemol(buffer)
            push!(mols, mol)
            buffer = String[]
        else
            push!(buffer, line)
        end
    end
    mols
end

function parsemol(lines::Vector{String})
    name = lines[1]
    line = lines[4]
    natoms = parse(Int, line[1:3])
    nbonds = parse(Int, line[4:6])
    atoms = Atom[]
    for i = 5:5+natoms-1
        line = lines[i]
        x = parse(Float64, line[1:10])
        y = parse(Float64, line[11:20])
        z = parse(Float64, line[21:30])
        symbol = strip(line[32:34])
        push!(atoms, Atom(symbol,x,y,z))
    end
    bonds = Bond[]
    k = 5 + natoms
    for i = k:k+nbonds-1
        line = lines[i]
        id1 = parse(Int, line[1:3])
        id2 = parse(Int, line[4:6])
        type = parse(Int, line[7:9])
        stereo = parse(Int, line[10:12])
        push!(bonds, Bond(id1,id2,type,stereo))
    end
    props = Dict{String,Any}()
    i = findnext(x -> x == "M  END", lines, k+nbonds) + 1
    while i < length(lines)
        line = lines[i]
        @assert line[1] == '>'
        s = first(findnext("<",line,3))
        e = first(findnext(">",line,s+1))
        key = line[s+1:e-1]
        j = findnext(isempty, lines, i+1)
        props[key] = join(lines[i+1:j-1])
        i = j + 1
    end
    Molecule(name, atoms, bonds, props)
end

function writesdf(filename::String, mols::Vector{Molecule})
    open(filename, "w") do io
        for m in mols
            println(io, m.name)
            println(io, "  LightChem")
            println(io, "")
            @printf(io, "%03s%03s%03s\n", natoms(m), nbonds(m), 0)
            for a in m.atoms
                @printf(io, "%10s%10s%10s %-03s\n", a.x, a.y, a.z, a.symbol)
            end
            for b in m.bonds
                @printf(io, "%03s%03s%03s%03s%03s%03s%03s\n", b.i, b.j, b.type, b.stereo, 0, 0, 0)
            end
            println(io, "M  END")
            for (k,v) in m.props
                println(io, "> <$k>")
                println(io, v)
                println(io, "")
            end
            println(io, raw"$$$$")
        end
    end
end

export firstorder
function firstorder(mol::Molecule)
    neighbors = [String[] for _=1:natoms(mol)]
    for b in mol.bonds
        s1 = mol.atoms[b.i].symbol
        s2 = mol.atoms[b.j].symbol
        push!(neighbors[b.i], s2)
        push!(neighbors[b.j], s1)
    end
    atoms = Atom[]
    for i = 1:natoms(mol)
        ns = sort!(neighbors[i])
        a = mol.atoms[i]
        s = "$(a.symbol) $(join(ns))"
        push!(atoms, Atom(s,a.x,a.y,a.z))
    end
    Molecule(mol.name, atoms, mol.bonds, copy(mol.props))
end


text:
#ModifiedHuberLoss
using LaTeXStrings, LossFunctions, Measures, Latexify

Plots.plot(ModifiedHuberLoss(), leg=:topright, c=:olive, linewidth=2.5)

max_stars_repo_name: Ellon-M/visualizations

max_issues_repo_name: Ellon-M/visualizations

max_forks_repo_name: Ellon-M/visualizations


text:
@testset "Testing Simple Simulation" begin
    num_cores = 200
    num_records = 2 * num_cores
    records = [TestPrograms.makerecord(i, zerofirst = true) for i in 1:num_records]

    # Expected results are these records in order - strip the learding zeros.
    expected_results = vcat((r[2:end] for r in records)...)

    # Iterate through several permutations.
    failing_permutations = []
    snakesort = TestPrograms.snakesort

    for _ in 1:2

        # Generate a random permutatation for the records.
        perm = randperm(num_records)

        # Create 4 sort cores.
        #
        # NOTE: These could be put in a vector and a lot of this done
        # automatically. I'm doing it by hand just for clarity sake.
        println("Creating Cores")
        cores = [test_core(snakesort(i == 1, i == num_cores, false)) for i in 1:num_cores]

        # Initialize cores
        println("Initializing Cores")
        for c in cores
            TestPrograms.initialize_snakesort(c)
        end

        # Connect fifos
        println("Connecting Cores")
        for i in 1:num_cores - 1, j in 1:2
            cores[i].outputs[j] = cores[i+1].fifos[j]
        end

        # Create a fifo for the output handler
        output_fifo = AsapSim.DualClockFifo(Int16, 32)
        output_handler = AsapSim.OutputHandler{Int16, typeof(output_fifo)}(
            input = output_fifo,
            stoplength = 50 * num_records
        )
        cores[num_cores].outputs[1] = output_fifo

        # Create an input handler and connect it to the ibuf of core 1
        input_handler = AsapSim.InputHandler{Int16,typeof(output_fifo)}(
            # End data with -1 to tell first core that we're done.
            data = vcat(records[perm]..., [-1, num_records]),
            output = cores[1].fifos[1],
        )

        core_vector = vcat(input_handler, cores, output_handler)

        # Randomize clock periods
        println("Randomizing clock periods")
        for c in core_vector
            c.clockperiod = rand(70:100)
        end

        # Create a vector of SimWrappers for all of these cores.
        println("Wrapping Simulation")
        simtime = 500000 * num_records
        sim = LightDES.Simulation(simtime)

        wrappers = [AsapSim.wrap!(sim, i) for i in core_vector]
        println("Starting Simulation")

        for wrapper in wrappers
            AsapSim.schedule!(sim, wrapper)
        end

        # Clear allocation data for better memory analysis.
        Profile.clear_malloc_data()
        Profile.clear()
        @time run(sim)
        println(now(sim))

        open("profile.txt", "w") do f
            Profile.print(f)
        end

        # for w in wrappers
        #     println(w.visits)
        #     core = w.core

        #     isa(core, AsapSim.AsapCore) || continue

        #     println("Testing input fifos")
        #     for (i, fifo) in enumerate(core.fifos)
        #         println(i, " ", AsapSim.stall_check_ibuf(core, i, false))
        #         println("Num Reads: ", fifo.num_reads)
        #         println("Num Writes: ", fifo.num_writes)
        #     end

        #     println("Testing output fifos")
        #     for (k,v) in core.outputs
        #         println(k, " ", AsapSim.stall_check_obuf(core, k, false))
        #         println("Num Reads: ", v.num_reads)
        #         println("Num Writes: ", v.num_writes)
        #     end
        #     println()
        # end

        passed = output_handler.data == expected_results
        @test passed
        if !passed
            push!(failing_permutations, perm)
        end

        @test length(output_handler.data) == length(expected_results)
    end

    # Print out failing permutations if any were encountered.
    if length(failing_permutations) > 0
        display(failing_permutations)
    end
end


text:
"""
    Noise(data, level; scale = :percent_max, type = :gaussian )

Adds noise (uniform, or gaussian) to an array of data.
The scale can be made relative to the percent_max, percent_average, or the absolute level values.

Note: for Gaussian noise it is likely convenient to scale by an inverse quantile(say 0.999)
"""
function Noise(data, level; scale = :percent_max, type = :gaussian )
    (maxes, minis) = ( [], [] )
    noise = []
    if type == :gaussian
        noise = randn( size( data ) )
    elseif type == :uniform
        noise = rand( Float64, size( data ) ) .- 0.5
    else #ToDo: Throw exception
        return
    end
    if scale == :percent_max
        if length( size( data ) ) > 1
            (maxes, minis) = ( reduce( max, data, dims = 2 ), reduce( min, data, dims = 2 ) )
            return ( noise .* (maxes .- minis) .* level ) .+ data
        else
            (maxes, minis) = ( reduce( max, data ), reduce( min, data ) )
            return ( noise .* (maxes .- minis) .* level ) .+ data
        end
    elseif scale == :percent_average
        if length(size(data)) > 1
            (maxes, minis) = ( reduce( max, data, dims = 2 ), reduce( min, data, dims = 2 ) )
            avg = mean( (data .- minis) ./ (maxes .- minis), dims = 2 )
            return ( noise .* avg .* level ) .+ data
        else
            (maxes, minis) = ( reduce( max, data ), reduce( min, data ) )
            avg = mean( (data .- minis) ./ (maxes .- minis) )
            return ( noise .* avg .* level ) .+ data
        end
    elseif scale == :absolute
        return ( noise .* level ) .+ data
    end
end

"""
    StandardNormalVariate(X)

Scales the columns of `X` by the mean and standard deviation of each row. Returns the scaled array.
"""
StandardNormalVariate(X) = ( X .- Statistics.mean(X, dims = 2) ) ./ Statistics.std(X, dims = 2)

"""
    Scale1Norm(X)

Scales the columns of `X` by the 1-Norm of each row. Returns the scaled array.
"""
Scale1Norm(X) = X ./ sum(abs.(X), dims = 2)

"""
    Scale2Norm(X)

Scales the columns of `X` by the 2-Norm of each row. Returns the scaled array.
"""
Scale2Norm(X) = X ./ sqrt.(sum(X .^ 2, dims = 2))

"""
    ScaleInfNorm(X)

Scales the columns of `X` by the Inf-Norm of each row. Returns the scaled array.
"""
ScaleInfNorm(X) = X ./ reduce(max, X, dims = 2)

"""
    ScaleFNorm(X)

Scales EACH entry of `X` by the Frobenius Norm of. Returns the scaled array.
"""
ScaleFNorm(X) = X ./ FNorm( X )

"""
    ScaleMinMax(X)

Scales the columns of `X` by the Min and Max of each row such that no observation is greater than 1 or less than zero.
Returns the scaled array.
"""
function ScaleMinMax(X)
    mini = reduce(min, X, dims = 2)
    maxi = reduce(max, X, dims = 2)
    return (X .- mini) ./ (maxi .- mini)
end

"""
    ScaleByIntensity(X, index)

Scales the columns of `X` by the value of a peak at a specified `index` in each row.
Returns the scaled array.
"""
ScaleByIntensity(X, index) = X ./ X[:,index]

"""
    offsetToZero(X)

Ensures that no observation(row) of Array `X` is less than zero, by ensuring the minimum value of each row is zero.
"""
offsetToZero(X) = X .- reduce(min, X, dims = 2)

"""
    boxcar(X; windowsize = 3, fn = mean)

Applies a boxcar function (`fn`) to each window of size `windowsize` to every row in `X`.
*Note: the function provided must support a dims argument/parameter.*
"""
function boxcar(X; windowsize = 3, fn = mean)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    @assert windowsize <= vars
    result = zeros(obs, vars - windowsize + 1)
    for v in 1 : (vars - windowsize + 1 )
        result[:, v] = fn(X[:, v : (v + windowsize - 1) ], dims = 2)
    end
    return result
end

"""
    ALSSmoother(X; lambda = 100, p = 0.001, maxiters = 10)

Applies an assymetric least squares smoothing function to a 2-Array `X`. The `lambda`, `p`, and `maxiters`
parameters control the smoothness. See the reference below for more information.

Paul H. C. Eilers, Hans F.M. Boelens. Baseline Correction with Asymmetric Least Squares Smoothing.  2005
"""
function ALSSmoother(X; lambda = 100, p = 0.001, maxiters::Int = 10)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    output = zeros(obs,vars)
    for r in 1:obs
        D = SecondDerivative( sparse( I, vars, vars )' )';
        w = ones(vars);
        z = zeros(vars)
        for it in 1 : maxiters
            W = spdiagm(0 => w);
            C = cholesky(W + lambda * D' * D).U;
            z = C \ (C' \ (w .* X[r,:]));
            w[ X[r,:] .> z ] .= p;
            w[ X[r,:] .< z ] .+= 1.0 - p;
        end
        output[r,:] = z
    end
    return output
end

"""
    PerfectSmoother(X; lambda = 100)

Applies an assymetric least squares smoothing function to a a 2-Array `X`. The `lambda`
parameter controls the smoothness. See the reference below for more information.

Paul H. C. Eilers. "A Perfect Smoother". Analytical Chemistry, 2003, 75 (14), pp 3631–3636.
"""
function PerfectSmoother(X; lambda = 100)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    output = zeros(obs,vars)
    for r in 1:obs
        D = SecondDerivative( sparse( I, vars, vars )' )';
        w = spdiagm(0 => ones(vars));
        C = cholesky(w + lambda * D' * D).U
        output[r,:] = C \ (C' \ (w * X[r,:]))
    end
    return output
end

#Pretty sure this is reversible like a transform, but don't have time to solve the inverse yet...
struct MultiplicativeScatterCorrection
    BiasedMeans
    Bias
    Coefficients
end

"""
    MultiplicativeScatterCorrection(Z::Array)

Creates a MultiplicativeScatterCorrection object from the data in Z

Martens, H. Multivariate calibration. Wiley
"""
function MultiplicativeScatterCorrection(Z::Array)
    BiasedMeans = hcat( ones( ( size(Z)[2], 1) ) , StatsBase.mean( Z, dims = 1 )[1,:] )
    Coeffs = ( BiasedMeans' * BiasedMeans ) \ ( Z * BiasedMeans )'
    MultiplicativeScatterCorrection( BiasedMeans, Coeffs[1,:], Coeffs[2,:] )
end

"""
    (T::MultiplicativeScatterCorrection)(Z)

Applies MultiplicativeScatterCorrection from a stored object `T` to Array `Z`.
"""
function (T::MultiplicativeScatterCorrection)(Z)
    Coeffs = ( T.BiasedMeans' * T.BiasedMeans ) \ ( Z * T.BiasedMeans )'
    return (Z .- Coeffs[1,:]) ./ Coeffs[2,:]
end

"""
    FirstDerivative(X)

Uses the finite difference method to compute the first derivative for every row in `X`.
*Note: This operation results in the loss of a column dimension.*
"""
function FirstDerivative(X)
    X = forceMatrixT(X)
    Xsize = size(X)
    XNew = zeros( Xsize[ 1 ] , Xsize[ 2 ] - 1)
    for c in 1 : ( Xsize[ 2 ] - 1 )
        XNew[ :, c ] = X[ :, c + 1 ] .- X[ :, c ]
    end
    return XNew
end

"""
    FirstDerivative(X)

Uses the finite difference method to compute the second derivative for every row in `X`.
*Note: This operation results in the loss of two columns.*
"""
function SecondDerivative(X)
    X = forceMatrixT(X)
    Xsize = size(X)
    XNew = zeros( Xsize[ 1 ], Xsize[ 2 ] - 2 )
    for c in 1 : ( Xsize[ 2 ] - 2 )
        XNew[ :, c ] = (X[ :, c + 2 ] .- X[ :, c + 1 ]) - (X[ :, c + 1 ] .- X[ : , c ] )
    end
    return XNew
end

"""
    FractionalDerivative(Y, X = 1 : length(Y); Order = 0.5)

Calculates the Grunwald-Leitnikov fractional order derivative on every row of Array Y.
Array `X` is a vector that has the spacing between column-wise entries in `Y`. `X` can be a scalar if that is constant (common in spectroscopy).
`Order` is the fractional order of the derivative.
*Note: This operation results in the loss of a column dimension.*

The Fractional Calculus, by Oldham, K.; and Spanier, J. Hardcover: 234 pages. Publisher: Academic Press, 1974. ISBN 0-12-525550-0
"""
function FractionalDerivative(Y, X = 1 : length(Y); Order = 0.5)
    Y = forceMatrixT(Y)
    (Obs,Vars) = size(Y)
    ddy = zeros(Obs,Vars-1)
    w = zeros(Vars)
    w[1] = 1.0
    for var in 2:Vars
        w[ var ] = w[ var - 1 ] * ( 1.0 - ( Order + 1.0 ) / ( var - 1.0 ) )
    end
    for var in 2:Vars
        h = (length(X) > 1) ? X[ var ] - X[ var - 1 ] : X
        for obs in 1 : Obs#This could definitely be broadcasted
            ddy[ obs, var-1 ] = w[ 1 : var ]' * Y[obs, var : -1: 1 ] ./ (h^Order)
        end
    end
    return ddy
end

"""
    ConvFilter1DFFT(a, filter)

Performs a 1D convolution of vector `filter` onto `a` via the FFT definition of a
 convolution. This method implicitly zeropads and truncates the result.

Note: This isn't highly optimized for performance. It was made when DSP.jl broke,
and broke the ChemometricsTools.jl package.
"""
function ConvFilter1DFFT(a, filter)
    @assert(length(filter) <= length(a), "Length of filter should be less than the length of the vector." )
    NewSize = length(a) + length(filter) - 1;
    ADiff = Int(round((NewSize - length(a)) ))
    ADiffhalf = Int(floor(ADiff / 2))
    filterDiff = Int(round((NewSize - length(filter)) ))
    X = fft( vcat( a, zeros( ADiff ) ) )
    H = fft( vcat( filter, zeros( filterDiff ) ) )
    return real.(ifft( X .* H ))[ (ADiffhalf + 1):(end-ADiffhalf) ]
end

"""
    SavitzkyGolay(X, Delta, PolyOrder, windowsize)

Performs SavitskyGolay smoothing across every row in an Array `X`.
The `window size` is the size of the convolution filter, `PolyOrder` is the order of the polynomial,
and `Delta` is the order of the derivative.

Savitzky, A.; Golay, M.J.E. (1964). "Smoothing and Differentiation of Data by Simplified Least Squares Procedures". Analytical Chemistry. 36 (8): 1627–39. doi:10.1021/ac60214a047.
"""
function SavitzkyGolay(X, DerivOrder, PolyOrder, windowsize::Int)
    @assert( (windowsize % 2) == 1, "Window size must be an odd number" )
    @assert( PolyOrder < (windowsize-1), "Polynomial order must be less than the window size." )
    X = forceMatrixT(X)
    (Obs,Vars) = size(X)
    windowspan = (windowsize - 1) / 2
    basis = ones( windowsize, PolyOrder+1 )
    basis[:,2:end] = ( (-windowspan:windowspan)' .^ ( 1 : PolyOrder ))'
    DerivFac = factorial( DerivOrder )
    FIR = inv( basis' * basis) * basis'
    output = DerivFac * ConvFilter1DFFT(X[1,:], FIR[DerivOrder+1,: ])'
    for r in 2:Obs
        sg = DerivFac * ConvFilter1DFFT(X[r,:], FIR[DerivOrder+1,: ])'
        output = vcat( output, sg )
    end
    offset = Int( windowspan )
    return output[:, (offset + 1) : (end - offset)]
end

"""
    LinearResample(X, newsize)

Resamples a vector `X` to be of size `newsize` via linear interpolation.
"""
function LinearResample(X, newsize)
   cursize = length(X)
   if cursize == newsize
      return X
   end
   newspacing = ( (cursize-1) / (newsize-1) )
   newsampling = 1 : newspacing : cursize
   interpolated = zeros(newsize)
   for i in 1 : ( cursize - 1 )
      #For large arrays this is costly - could change to track the indices
      #assuming uniform spacing.
      inds = findall( i .<= newsampling .<= ( i + 1 ) )
      slope = ( X[i+1] - X[i] )
      offset = X[i] - (slope*i)
      interpolated[inds] = (newsampling[inds] .* slope) .+ offset
   end
   return interpolated
end

"""
    SincInterpolation(Y, S, Up)
    Y - vector of a line shape
    S - Sampled domain of Y
    Up - Upsampled X vector

Refactored from: https://gist.github.com/endolith/1297227#file-sinc_interp-m
"""
SincInterpolation(Y, S, Up) = sinc.( (Up .- S') ./ (S[2] - S[1]) ) * Y

"""
    FourierUpsample(X, newlength)

zero pads a vector `X` in the frequency domain then converts it back to the time domain.

"""
function FourierUpsample(X, newlength)
    len = length(x)
    @assert(newlength > len, "New length must be greater than the vector length to upsample!")
    xfft = fft(x)
    left = Int( floor( (len+1) / 2 ) )
    return real.(ifft( vcat(xfft[1:left], zeros(newlength - len), xfft[(left+1):end] ) )) .* (newlength/len)
end

struct DirectStandardizationXform
    pca::PCA
    TransferMatrix::Array
end

"""
    DirectStandardization(InstrumentX1, InstrumentX2; Factors = minimum(collect(size(InstrumentX1))) - 1)

Makes a DirectStandardization object to facilitate the transfer from Instrument #2 to Instrument #1 .
The returned object can be used to transfer unseen data to the approximated space of instrument 1.
The number of `Factors` used are those from the internal orthogonal basis.

Yongdong Wang and Bruce R. Kowalski, "Calibration Transfer and Measurement Stability of Near-Infrared Spectrometers," Appl. Spectrosc. 46, 764-771 (1992)
"""
function DirectStandardization(InstrumentX1, InstrumentX2; Factors = minimum(collect(size(InstrumentX1))) - 1)
    pcamodel = PCA(InstrumentX1; Factors = Factors)
    InstrumentX2_DS = pcamodel(InstrumentX2; Factors = Factors, inverse = false)
    TransferMatrix = LinearAlgebra.pinv(InstrumentX2_DS) * pcamodel.Scores[:, 1:Factors]
    return DirectStandardizationXform(pcamodel, TransferMatrix)
end

"""
    (DSX::DirectStandardizationXform)(X; Factors = length(DSX.pca.Values))

Applies a the transform from a learned direct standardization object `DSX` to new data `X`.
"""
function (DSX::DirectStandardizationXform)(X; Factors = length(DSX.pca.Values))
    Into = DSX.pca(X; Factors = Factors)
    Bridge = Into * DSX.TransferMatrix[1:Factors,1:Factors]
    return DSX.pca(Bridge; Factors = Factors, inverse = true)
end

struct OrthogonalSignalCorrection
    Weights::Array
    Loadings::Array
    Residuals::Array
end

"""
    OrthogonalSignalCorrection(X, Y; Factors = 1)

Performs Thomas Fearn's Orthogonal Signal Correction to an endogenous `X` and exogenous `Y`.
The number of `Factors` are the number of orthogonal components to be removed from `X`.
This function returns an OSC object.

Tom Fearn. On orthogonal signal correction. Chemometrics and Intelligent Laboratory Systems. Volume 50, Issue 1, 2000, Pages 47-52.
"""
function OrthogonalSignalCorrection(X, Y; Factors = 1)
    (Obs,Vars) = size(X)
    W = zeros(Vars, Factors)
    Loadings = zeros( Vars, Factors)
    Residuals = copy(X)
    M = LinearAlgebra.Diagonal(ones(Vars)) .- ( X' * Y * Base.inv(Y' * X * X' * Y) * Y' * X )
    P = svd( X * M )
    for factor in 1 : Factors
        Projection = M * Residuals' * ( P.U[ : , factor ] ./ P.S[ factor ] )
        W[ : , factor ] = Projection / sqrt(Projection' * Projection)
        t = Residuals * Projection
        p = Residuals' * (t ./ ( t' * t ))
        Loadings[ : , factor ] = p
        Residuals = Residuals .- (t * p')
    end
    return OrthogonalSignalCorrection(W, Loadings, Residuals)
end

"""
    (OSC::OrthogonalSignalCorrection)(Z; Factors = 2)

Applies a the transform from a learned orthogonal signal correction object `OSC` to new data `Z`.
"""
function (OSC::OrthogonalSignalCorrection)(Z; Factors = 2)
    X = copy(Z)
    for factor in 1 : Factors
        X .-= (X * OSC.Weights[ : , factor ] * OSC.Loadings[:, factor ]')
    end
    return X
end

struct TransferByOrthogonalProjection
    Factors::Int
    vars::Int
    pca::PCA
end

"""
    TransferByOrthogonalProjection(X1, X2; Factors = 1)

Performs Thomas Fearns Transfer By Orthogonal Projection to facilitate transfer from `X1` to `X2`. Returns a TransferByOrthogonalProjection object.

Anne Andrew, Tom Fearn. Transfer by orthogonal projection: making near-infrared calibrations robust to between-instrument variation. Chemometrics and Intelligent Laboratory Systems. Volume 72, Issue 1, 2004, Pages 51-56,
"""
function TransferByOrthogonalProjection(X1, X2; Factors = 1)
    (Obs,Vars) = size(X1)
    OneToTwo = PCA(X1 .- X2; Factors = Factors)
    return TransferByOrthogonalProjection(Factors, Vars, OneToTwo)
end

"""
    (TbOP::TransferByOrthogonalProjection)(X1; Factors = TbOP.Factors)

Applies a the transform from a learned transfer by orthogonal projection object `TbOP` to new data `X1`.
"""
function (TbOP::TransferByOrthogonalProjection)(X1; Factors = TbOP.Factors)
    return X1 * (LinearAlgebra.Diagonal( ones( TbOP.vars ) ) .- (TbOP.pca.Loadings[1:Factors,:]' * TbOP.pca.Loadings[1:Factors,:]))
end

struct CORAL
    coralmat::Array
end
"""
    CORAL(X1, X2; lambda = 1.0)

Performs CORAL to facilitate covariance based transfer from `X1` to `X2` with regularization parameter `lambda`. Returns a CORAL object.

Correlation Alignment for Unsupervised Domain Adaptation. Baochen Sun, Jiashi Feng, Kate Saenko. https://arxiv.org/abs/1612.01939
"""
function CORAL(X1, X2; lambda = 1.0)
    (Obs1, vars) = size(X1)
    (Obs2, vars) = size(X2)
    d = lambda .* LinearAlgebra.Diagonal(repeat([1], vars))
    c1 = (1.0 / Obs1) * (X1' * X1) .+ d
    c2 = (1.0 / Obs2) * (X2' * X2) .+ d
    CORALXfer = c1 ^ (-1/2) * c2 ^ (1/2)
    return CORAL(CORALXfer)
end

"""
    (C::CORAL)(Z)

Applies a the transform from a learned `CORAL` object to new data `Z`.
"""
(C::CORAL)(Z) = Z * C.coralmat


text:
"""
    from_dict(::Type{T}, d::AbstractDict{String}; kw...) where T

Convert dictionary `d` to an option type `T`, the valud of valid fields of `T`
in this dictionary `d` can be override by keyword arguments.

# Example

```julia-repl
julia> @option struct OptionA
           name::String = "Sam"
           age::Int = 25
       end

julia> d = Dict{String, Any}(
           "name" => "Roger",
           "age" => 10,
       );

julia> from_dict(OptionA, d; age=25)
OptionA(;
    name = "Roger",
    age = 25,
)
```
"""
function from_dict(::Type{T}, d::AbstractDict{String}; kw...) where T
    # override dict values
    from_underscore_kwargs!(deepcopy(d), T; kw...)
    return from_dict_validate(T, d)
end

"""
    from_toml(::Type{T}, filename::String; kw...) where T

Convert a given TOML file `filename` to an option type `T`. Valid fields
can be override by keyword arguments. See also [`from_dict`](@ref).
"""
function from_toml(::Type{T}, filename::String; kw...) where T
    is_option(T) || error("not an option type")
    d = TOML.parsefile(filename)
    d["#filename#"] = filename
    return from_dict(T, d; kw...)
end

"""
    from_toml_if_exists(::Type{T}, filename::String; kw...) where T

Similar to [`from_toml`](@ref) but will create the option instance
via `from_kwargs(T;kw...)` instead of error if the file does not exist.
"""
function from_toml_if_exists(::Type{T}, filename::String; kw...) where T
    if isfile(filename)
        return from_toml(T, filename; kw...)
    else
        return from_kwargs(T; kw...)
    end
end

"""
    from_kwargs(convention!, ::Type{T}; kw...) where T

Convert keyword arguments to given option type `T` using `convention!`.
See also [`from_dict`](@ref).

# Convention

- `from_underscore_kwargs!`: use `_` to disambiguate subfields of the same name, this is the default behaviour.
- `from_field_kwargs!`: do not disambiguate subfields, errors if there are disambiguity
"""
function from_kwargs(convention!, ::Type{T}; kw...) where T
    d = OrderedDict{String, Any}()
    convention!(d, T; kw...)
    return from_dict_validate(T, d)
end

"""
    from_kwargs(::Type{T}; kw...) where T

Convert keyword arguments to given option type `T` using the underscore convention.
"""
from_kwargs(::Type{T}; kw...) where T = from_underscore_kwargs(T; kw...)

"""
    from_underscore_kwargs(::Type{T}; kw...) where T

Convert keyword arguments to given option type `T` using the underscore convention.
"""
from_underscore_kwargs(::Type{T}; kw...) where T = from_kwargs(from_underscore_kwargs!, T; kw...)

"""
    from_field_kwargs(::Type{T}; kw...) where T

Convert keyword arguments to given option type `T` using the field keyword convention.
"""
from_field_kwargs(::Type{T}; kw...) where T = from_kwargs(from_field_kwargs!, T; kw...)

function from_dict_validate(::Type{T}, d::AbstractDict{String}) where T
    assert_option(T)

    for k in keys(d)
        k == "#filename#" && continue
        Symbol(k) in fieldnames(T) || error("invalid key: $k")
    end

    for (name, default) in zip(fieldnames(T), field_defaults(T))
        if default === no_default
            haskey(d, string(name)) && continue
            error("$name is required")
        end
    end

    return from_dict_inner(T, d)
end

function assert_union_alias(::Type{T}, name=nothing) where T
    T isa Union || return
    name === nothing && return
    name == type_alias(T.a) && error("duplicated alias name: $name")
    return assert_union_alias(T.b, type_alias(T.a))
end

# we don't process other kind of value
"""
    pick_union(::Type, x) -> type, value

Pick a type `T` and its corresponding value from a `Union`. For option
types it should be a dictionary type. The value can be furthur converted
to this type `T` via [`convert_to_option`](@ref) or `Base.convert`.
"""
pick_union(::Type{T}, x) where T = T, x

function pick_union(::Type{T}, d::AbstractDict{String}) where T
    if !(T isa Union)
        T === Nothing && return
        is_option(T) || return T, d
        if haskey(d, type_alias(T))
            return T, d[type_alias(T)]
        else
            return
        end
    end

    assert_union_alias(T)

    ret_a = pick_union(T.a, d)
    ret_b = pick_union(T.b, d)

    if ret_a === nothing
        return ret_b
    else
        return ret_a
    end
end

"""
    from_dict_inner(::Type{T}, d::AbstractDict{String}) where T

Internal method to convert a dictionary (subtype of `AbstractDict`)
to type `T`, this method will not check if `T` is an option type
via `is_option`, and will not validate if all the required fields
are available in the dict object.
"""
function from_dict_inner(::Type{T}, @nospecialize(d)) where T
    d isa AbstractDict{String} || error("cannot convert $d to $T, expect $T <: AbstractDict{String}")
    args = Any[]
    for each in fieldnames(T)
        key = string(each)
        type = fieldtype(T, each)
        default = field_default(T, each)

        if default === no_default
            if type isa Union
                pick = pick_union(type, d[key])
                pick === nothing && error("alias name for multi-option $type is required")
                type, value = pick
            else
                value = d[key]
            end
        else
            if type isa Union && haskey(d, key)
                pick = pick_union(type, d[key])
            else
                pick = nothing
            end

            if pick === nothing
                value = get(d, key, default)
            else
                type, value = pick
            end
        end

        if is_option(type) && value isa AbstractDict{String}
            # need some assertions so we call from_dict_validate
            push!(args, from_dict_validate(type, value))
        elseif value isa AbstractDict && isempty(value) && Nothing <: type
            # empty collection
            push!(args, nothing)
        else
            v = convert_union_to_option(T, type, value)
            if v === nothing
                push!(args, convert(type, value))
            else
                push!(args, v)
            end
        end
    end

    return T(args...)
end

# NOTE: this is for compatibilty
"""
    from_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T

Internal method for inserting keyword arguments to given dictionary object `d`. It will overwrite
existing keys in `d` if it is specified by keyword argument.
"""
function from_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T
    return from_underscore_kwargs!(d, T, prefix; kw...)
end

function from_underscore_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T
    validate_keywords(T, underscore_keywords(T); kw...)
    unsafe_from_underscore_kwargs!(d, T, prefix; kw...)
end

function unsafe_from_underscore_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T
    return foreach_keywords!(d, T) do name, type
        key = underscore(prefix, name)

        from_kwargs_option_key!(d, type, name, key, kw) do field_d, field_type
            unsafe_from_underscore_kwargs!(field_d, field_type, key; kw...)
        end
    end
end

function from_field_kwargs!(d::AbstractDict{String}, ::Type{T}; kw...) where T
    validate_keywords(T, field_keywords(T); kw...)
    unsafe_from_field_kwargs!(d, T; kw...)
end

function unsafe_from_field_kwargs!(d::AbstractDict{String}, ::Type{T}; kw...) where T
    return foreach_keywords!(d, T) do name, type
        from_kwargs_option_key!(d, type, name, name, kw) do field_d, field_type
            unsafe_from_field_kwargs!(field_d, field_type; kw...)
        end
    end
end

function from_kwargs_option_key!(f, d::AbstractDict, ::Type{T}, name::Symbol, key::Symbol, kw) where T
    key_str = string(key)
    name_str = string(name)
    # shortcut
    if haskey(kw, key)
        d[name_str] = kw[key]
        return d
    end

    if is_option(T)
        field_d = OrderedDict{String, Any}()
        if haskey(d, name_str)
            d[name_str] isa AbstractDict || error("option $key_str must be specified using an AbstractDict")
            field_d = merge!(field_d, d[name_str])
        end

        f(field_d, T) # recurse into subfields
        if !isempty(field_d)
            d[name_str] = field_d
        end
    elseif T isa Union
        from_kwargs_option_key!(f, d, T.a, name, key, kw)
        from_kwargs_option_key!(f, d, T.b, name, key, kw)
    end
    return d
end

function validate_keywords(::Type{T}, keys = underscore_keywords(T); kw...) where T
    if length(keys) > 8
        hint = join(map(x->LIGHT_BLUE_FG(string(x)), keys[1:8]), ", ")
        hint *= "... please check documentation for other valid keys"
    else
        hint = join(map(x->LIGHT_BLUE_FG(string(x)), keys), ", ")
    end

    for (k, v) in kw
        k in keys || throw(ArgumentError("invalid key $(LIGHT_BLUE_FG(string(k))), possible keys are: $hint"))
    end
    return
end

"""
    field_keywords(::Type{T}) where T

Return all the option type field names given `T`, error if there are duplicated sub-fields.
"""
function field_keywords(::Type{T}) where T
    return collect_field_keywords!(Symbol[], T)
end

"""
    underscore_keywords(::Type{T}) where T

Return keywords given `T` using the underscore convention.
"""
function underscore_keywords(::Type{T}) where T
    return collect_underscore_keywords!(Symbol[], T)
end

function foreach_keywords!(f, list, ::Type{T}) where T
    is_option(T) || return list

    for name in fieldnames(T)
        type = fieldtype(T, name)
        f(name, type)
    end
    return list
end

function underscore(prefix::Maybe{Symbol}, name)
    if prefix === nothing
        return name
    else
        return Symbol(prefix, :_, name)
    end
end

function collect_field_keywords!(list::Vector{Symbol}, ::Type{T}) where T
    return foreach_keywords!(list, T) do name, type
        msg = "duplicated field $(LIGHT_BLUE_FG(string(name))) " *
            "in type $(GREEN_FG(string(T))) and its sub-fields"
        if is_option(type)
            collect_field_keywords!(list, type)
        elseif type isa Union
            name in list && error(msg)
            push!(list, name)
            # recurse into Union
            collect_field_keywords!(list, type.a)
            collect_field_keywords!(list, type.b)
        else
            name in list && error(msg)
            push!(list, name)
        end
    end
end

function collect_underscore_keywords!(list::Vector{Symbol}, ::Type{T}, prefix::Maybe{Symbol} = nothing) where T
    return foreach_keywords!(list, T) do name, type
        key = underscore(prefix, name)

        if is_option(type)
            collect_underscore_keywords!(list, type, key)
        elseif type isa Union
            push!(list, key)
            collect_underscore_keywords!(list, type.a, key)
            collect_underscore_keywords!(list, type.b, key)
        else
            push!(list, key)
        end
    end
end


text:
Coord = CartesianIndex{3}

const movements = Dict{String,Coord}(
    "e"  => Coord(1, -1, 0),
    "w"  => Coord(-1, 1, 0),
    "ne" => Coord(1, 0, -1),
    "nw" => Coord(0, 1, -1),
    "se" => Coord(0, -1, 1),
    "sw" => Coord(-1, 0, 1),
)

function tilecoord(line::AbstractString, movements::Dict{String,Coord}=movements)
    dirs = r"se|sw|ne|nw|e|w"
    directions = [line[m] for m in findall(dirs, line)]
    return sum(movements[d] for d in directions)
end

function flip!(tiles::Dict{Coord,Bool}, line::AbstractString)
    coord = tilecoord(line)
    tiles[coord] = !get(tiles, coord, false)
    return tiles
end

function init(lines::Vector{<:AbstractString})
    tiles = Dict{Coord,Bool}()
    for line in lines
        flip!(tiles, line)
    end
    return tiles
end

const tiles = init(readlines("day24.txt"))

println("Part 1: $(sum(values(tiles)))")

const neighbors = collect(values(movements))

function with_black_neighbors(blacktiles::Set{Coord}, neighbors::Vector{Coord}=neighbors)
    blackneighbors = Dict{Coord,Int}()
    for t in blacktiles, n in neighbors
        blackneighbors[t + n] =  get(blackneighbors, t + n, 0) + 1
    end
    return blackneighbors
end

function flipall(blacktiles::Set{Coord}, blackneighbors::Dict{Coord,Int})
    futuretiles = copy(blacktiles)
    for a in blacktiles
        n = get(blackneighbors, a, 0)
        if n == 0 || n > 2
            pop!(futuretiles, a)
        end
    end
    for (a, n) in blackneighbors
        if n == 2 && a ∉ blacktiles
            push!(futuretiles, a)
        end
    end
    return futuretiles
end

function part2(tiles::Dict{Coord,Bool})
    blacktiles = Set{Coord}(k for (k, v) in tiles if v)
    for i in 1:100
        blackneighbors = with_black_neighbors(blacktiles)
        blacktiles = flipall(blacktiles, blackneighbors)
    end
    return length(blacktiles)
end

println("Part 2: $(part2(tiles))")


text:
## https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-artist

"""
    artist_get(artist_id)

**Summary**: Get Spotify catalog information for a single artist identified by their unique Spotify ID.

# Arguments
- `artist_id` : The Spotify artist ID.

# Example
```julia-repl
julia> Spotify.artist_get("0YC192cP3KPCRWx8zr8MfZ")[1]
JSON3.Object{Base.CodeUnits{UInt8, String}, Vector{UInt64}} with 10 entries:
  :external_urls => {…
  :followers     => {…
  :genres        => ["german soundtrack", "soundtrack"]
  :href          => "https://api.spotify.com/v1/artists/0YC192cP3KPCRWx8zr8MfZ"
```
"""
function artist_get(artist_id)
    return spotify_request("artists/$artist_id")
end

## https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-artists-albums

"""
    artist_get_albums(artist_id; include_groups="None", country="US", limit=20, offset=0)

**Summary**: Get Spotify catalog information about an artist's albums.

# Arguments
- `artist_id` : The Spotify ID of the artist

# Optional keywords
- `include_groups` : A comma-separated list of keywords that will be used to filter the response.
                     If not supplied, all album types will be returned.
                     Valid values are:
                     * `album`
                     * `single`
                     * `compilation`
                     * `appears_on`
- `country` : An ISO 3166-1 alpha-2 country code string. Use this to limit the response to one particular
              geographical market. Default is set to "US".
- `limit` : The maximum number of tracks to return. Default is set to 20.
- `offset` : The index of the first track to return. Default is 0.

# Example
```julia-repl
julia> Spotify.artist_get_albums("0YC192cP3KPCRWx8zr8MfZ")[1]
JSON3.Object{Base.CodeUnits{UInt8, String}, Vector{UInt64}} with 10 entries:
  :external_urls => {…
  :followers     => {…
  :genres        => ["german soundtrack", "soundtrack"]
  :href          => "https://api.spotify.com/v1/artists/0YC192cP3KPCRWx8zr8MfZ"
```
"""
function artist_get_albums(artist_id; include_groups="album", country="US", limit=20, offset=0)

    url1 = "artists/$artist_id?include_groups=$include_groups"
    url2 = "&country=$country&limit=$limit&offset=$offset"

    return spotify_request(url1 * url2)

end

## https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-artists-top-tracks

"""
    artist_top_tracks(artist_id; country="US")

**Summary**: Get Spotify catalog information about an artist's top tracks by country.

# Arguments
- `artist_id` : The Spotify ID of the artist

# Optional keywords
- `country` : An ISO 3166-1 alpha-2 country code string. Use this to limit the response to one particular
              geographical market. Default is set to "US".

# Example
```julia-repl
julia> Spotify.artist_top_tracks("0YC192cP3KPCRWx8zr8MfZ")[1]
JSON3.Object{Base.CodeUnits{UInt8, String}, Vector{UInt64}} with 1 entry:
  :tracks => JSON3.Object[{…
```
"""
function artist_top_tracks(artist_id; country="US")
    return spotify_request("artists/$artist_id/top-tracks?country=$country")
end

## https://developer.spotify.com/documentation/web-api/reference/#/operations/get-an-artists-related-artists
"""
    artist_get_related_artists(artist_id)

**Summary**: Get spotify catalog information about artists similar to a given artist.
             Similarity is based on analysis of the Spotify community's listening history.

# Arguments
- `artist_id` : The Spotify ID of the artist

# Example
```julia-repl
julia> Spotify.artist_get_related_artists("0YC192cP3KPCRWx8zr8MfZ")[1]
JSON3.Object{Base.CodeUnits{UInt8, String}, Vector{UInt64}} with 1 entry:
  :artists => JSON3.Object[{…
```
"""
function artist_get_related_artists(artist_id)
    return spotify_request("artists/$artist_id/related-artists")
end


text:
##############################################################################
##
## group transform multiple PooledDataVector into one
## Output is a PooledArray where pool is type Int64, equal to ranking of group
## NA in some row mean result has NA on this row
##
##############################################################################

function reftype(sz)
	sz <= typemax(UInt8)  ? UInt8 :
	sz <= typemax(UInt16) ? UInt16 :
	sz <= typemax(UInt32) ? UInt32 :
	UInt64
end

#  similar todropunusedlevels! but (i) may be NA (ii) change pool to integer
function factorize!(refs::Array)
	uu = unique(refs)
	sort!(uu)
	has_na = uu[1] == 0
	T = reftype(length(uu)-has_na)
	dict = Dict{eltype(refs), T}(zip(uu, (1-has_na):convert(T, length(uu)-has_na)))
	@inbounds @simd for i in 1:length(refs)
		 refs[i] = dict[refs[i]]
	end
	PooledDataArray(RefArray(refs), collect(1:(length(uu)-has_na)))
end

function pool_combine!{T}(x::Array{UInt64, T}, dv::PooledDataVector, ngroups::Integer)
	@inbounds for i in 1:length(x)
	    # if previous one is NA or this one is NA, set to NA
	    x[i] = (dv.refs[i] == 0 || x[i] == zero(UInt64)) ? zero(UInt64) : x[i] + (dv.refs[i] - 1) * ngroups
	end
	return(x, ngroups * length(dv.pool))
end

"""
Group multiple variables into one DataArray

### Arguments
* `df` : AbstractDataFrame
* `cols` : A vector of symbols

### Returns
* `::PooledDataArray` where each value corresponds to a unique combination of values in `cols`

### Details
A typical formula is composed of one dependent variable, exogeneous variables, endogeneous variables, instruments, and high dimensional fixed effects

### Examples
```julia
using DataFrames, RDatasets, FixedEffectModels
df = dataset("plm", "Cigar")
df[:StateYearPooled] = group(df[:State], df[:Year])
df[:StateYearPooled] = group(df, [:State, :Year])
```
"""

function group(x::AbstractVector)
	v = PooledDataArray(x)
	PooledDataArray(RefArray(v.refs), collect(1:length(v.pool)))
end

# faster specialization
function group(x::PooledDataVector)
	PooledDataArray(RefArray(copy(x.refs)), collect(1:length(x.pool)))
end

function group(df::AbstractDataFrame)
	isempty(df) && throw("df is empty")
	ncols = size(df, 2)
	v = df[1]
	ncols = size(df, 2)
	ncols == 1 && return(group(v))
	if typeof(v) <: PooledDataVector
		x = convert(Array{UInt64}, v.refs)
	else
		v = PooledDataArray(v, v.na, UInt64)
		x = v.refs
	end
	ngroups = length(v.pool)
	for j = 2:ncols
		v = PooledDataArray(df[j])
		(x, ngroups) = pool_combine!(x, v, ngroups)
	end
	return(factorize!(x))
end
group(df::AbstractDataFrame, cols::Vector) =  group(df[cols])
group(df::AbstractDataFrame, args...) =  group(df, [a for a in args])
function group(args...)
	df = DataFrame(Any[a for a in args],
		Symbol[convert(Symbol, "v$i") for i in 1:length(args)])
	return group(df)
end


text:
"""
`getext(pathname)` split the input name into the [name,".",extension] array.
# Example:
```
	pathsplit = getext("inputfpath\\file.ext")
    pathsplit[1] = pathsplit[1]*"_copy";
	outputpath = join(pathsplit); # which returns "inputfpath\\file_copy.ext"
```
"""
function getext(pathname)
	dotpos = findall(isequal('.'),pathname);
	if isempty(dotpos)
		str_ext = "";
		str_beforedot = pathname;
	else
		str_ext = pathname[(dotpos[end]+1):length(pathname)];
		str_beforedot = pathname[1:(dotpos[end]-1)];
	end

	return [str_beforedot,".",str_ext]
end


text:
# Figures 5.5a, 5.6a and 5.6b in "Infinite GMRES for parameterized linear systems"
using SparseArrays
using LinearAlgebra
using PyPlot
using Statistics
pygui(true)
include("infgmres.jl");
include("create_helmholtz.jl");

n=5000;
a=0;
b=1;
c=1.5;

function envelope_k(x)
    L0=b;
    alpha=10;
    if (x<L0/2)
        return 5+10*(x/L0)*sin((1/L0)*alpha*pi*x);
    else
        return 5+10*(1-x/L0)*sin((1/L0)*alpha*pi*x);
    end
end

function k_eval2(x)
    k0 = envelope_k(b);
    if (x<b)
        return envelope_k(x) # k_interior
    else
        return k0;
    end

end

function beta_eval2(x)
    if (x<b)
        return sin((x-a)*2*pi/(b-a));
    else
        return 0;
    end

end

function h_eval2(x)
    if x<b
        return ((x-b)^2)/(a-b)^2;
    else
        return 0;
    end
end

# Number of derivatives
N=34;
myproblem=create_helmholtz(n,a,b,c,N,k_eval2,beta_eval2,h_eval2);

kk=N-1;
how_many = 5;

timestamps_mat1=zeros(kk,how_many); timestamps1=zeros(kk);
timestamps_mat2=zeros(kk,how_many); timestamps2=zeros(kk);
timestamps_mat3=zeros(kk,how_many); timestamps3=zeros(kk);
for i=1:how_many
    global ff1, hist1
    global ff2, hist2
    global ff3, hist3
    (ff1,hist1)=infgmres(myproblem,kk,method=:tiar);
    (ff2,hist2)=infgmres(myproblem,kk,method=:iar);
    (ff3,hist3)=infgmres(myproblem,kk,method=:iar_lr);
    timestamps1a=hist1[:time_count][1:kk] .+ hist1[:precomp_time]
    timestamps_mat1[:,i]=timestamps1a;
    timestamps2a=hist2[:time_count][1:kk] .+ hist2[:precomp_time]
    timestamps_mat2[:,i]=timestamps2a;
    timestamps3a=hist3[:time_count][1:kk] .+ hist3[:precomp_time]
    timestamps_mat3[:,i]=timestamps3a;
end
for i=1:kk
    timestamps1[i]=float(median(timestamps_mat1[i,:]))
    timestamps2[i]=float(median(timestamps_mat2[i,:]))
    timestamps3[i]=float(median(timestamps_mat3[i,:]))
end

mmu=1.6;

errv1=zeros(kk); errv2=zeros(kk); errv3=zeros(kk);
for k=1:kk
    global xx1, xx2, xx3
    xx1=ff1(mmu,k)
    xx2=ff2(mmu,k)
    xx3=ff3(mmu,k)
    errv1[k]=norm(myproblem.Afun(mmu)*xx1-myproblem.b)/norm(myproblem.b)
    errv2[k]=norm(myproblem.Afun(mmu)*xx2-myproblem.b)/norm(myproblem.b)
    errv3[k]=norm(myproblem.Afun(mmu)*xx3-myproblem.b)/norm(myproblem.b)
end
its=range(1,stop=kk,length=kk);

figure(1)
ax = PyPlot.gca()
#plot(myproblem.xv,real(xx1))
plot(myproblem.xv,real(xx2))
xlabel("xs")
ylabel("u_n(x)")
#plot(myproblem.xv,real(xx3))

figure(2)
ax = PyPlot.gca()
ax.set_yscale("log")
semilogy(its,errv1,"o")
semilogy(its,errv2,"+")
semilogy(its,errv3,"*")
xlabel("its")
ylabel("res")
ax.legend(["Tensor","Alg 1","Low-rank"])

figure(3)
ax = PyPlot.gca()
ax.set_yscale("log")
semilogy(timestamps1,errv1,"o")
semilogy(timestamps2,errv2,"+")
semilogy(timestamps3,errv3,"*")
xlabel("CPU-time")
ylabel("res")
ax.legend(["Tensor","Alg 1","Low-rank"])

max_stars_repo_name: siobhanie/InfGMRES

max_issues_repo_name: siobhanie/InfGMRES

max_forks_repo_name: siobhanie/InfGMRES


text:
using WaveletMatrices: WaveletMatrix, getindex, rank, select, freq
using IndexableBitVectors
using Test
using Random

Random.seed!(12345)

function seq_select(a, vec, j)
    if j ≤ 0
        return 0
    end
    for i in 1:length(vec)
        if vec[i] == a
            j -= 1
        end
        if j == 0
            return i
        end
    end
    return 0
end

@testset "WaveletMatrix" begin
    @testset "ordered" begin
        wm = WaveletMatrix([0x00, 0x01, 0x02, 0x03])
        @test length(wm) == 4
        # getindex
        @test wm[1] == 0x00
        @test wm[2] == 0x01
        @test wm[3] == 0x02
        @test wm[4] == 0x03
        @test typeof(wm[4]) == UInt8
        # 0x00
        @test rank(0x00, wm, 0) == 0
        @test rank(0x00, wm, 1) == 1
        @test rank(0x00, wm, 2) == 1
        @test rank(0x00, wm, 3) == 1
        @test rank(0x00, wm, 4) == 1
        @test rank(0x00, wm, 5) == 1
        # 0x01
        @test rank(0x01, wm, 0) == 0
        @test rank(0x01, wm, 1) == 0
        @test rank(0x01, wm, 2) == 1
        @test rank(0x01, wm, 3) == 1
        @test rank(0x01, wm, 4) == 1
        @test rank(0x01, wm, 5) == 1
        # 0x02
        @test rank(0x02, wm, 0) == 0
        @test rank(0x02, wm, 1) == 0
        @test rank(0x02, wm, 2) == 0
        @test rank(0x02, wm, 3) == 1
        @test rank(0x02, wm, 4) == 1
        @test rank(0x02, wm, 5) == 1
        # 0x03
        @test rank(0x03, wm, 0) == 0
        @test rank(0x03, wm, 1) == 0
        @test rank(0x03, wm, 2) == 0
        @test rank(0x03, wm, 3) == 0
        @test rank(0x03, wm, 4) == 1
        @test rank(0x03, wm, 5) == 1

        @test select(0x00, wm, 0) == 0
        @test select(0x01, wm, 0) == 0
        @test select(0x02, wm, 0) == 0
        @test select(0x03, wm, 0) == 0

        @test select(0x00, wm, 1) == 1
        @test select(0x01, wm, 1) == 2
        @test select(0x02, wm, 1) == 3
        @test select(0x03, wm, 1) == 4

        @test select(0x00, wm, 2) == 0
        @test select(0x01, wm, 2) == 0
        @test select(0x02, wm, 2) == 0
        @test select(0x03, wm, 2) == 0

        @test freq(0x00, wm, 1, 4) == 1
        @test freq(0x00, wm, 2, 4) == 0
        @test freq(0x01, wm, 1, 2) == 1
        @test freq(0x01, wm, 1, 1) == 0
        @test freq(0x02, wm, 4, 2) == 0
        @test freq(0x02, wm, 3, 3) == 1
        @test freq(0x03, wm, 4, 1) == 0
        @test freq(0x03, wm, 4, 4) == 1
    end

    @testset "unordered" begin
        wm = WaveletMatrix([0x01, 0x03, 0x02, 0x00])
        @test length(wm) == 4
        # getindex
        @test wm[1] == 0x01
        @test wm[2] == 0x03
        @test wm[3] == 0x02
        @test wm[4] == 0x00
        # 0x00
        @test rank(0x00, wm, 0) == 0
        @test rank(0x00, wm, 1) == 0
        @test rank(0x00, wm, 2) == 0
        @test rank(0x00, wm, 3) == 0
        @test rank(0x00, wm, 4) == 1
        @test rank(0x00, wm, 5) == 1
        # 0x01
        @test rank(0x01, wm, 0) == 0
        @test rank(0x01, wm, 1) == 1
        @test rank(0x01, wm, 2) == 1
        @test rank(0x01, wm, 3) == 1
        @test rank(0x01, wm, 4) == 1
        @test rank(0x01, wm, 5) == 1
        # 0x02
        @test rank(0x02, wm, 0) == 0
        @test rank(0x02, wm, 1) == 0
        @test rank(0x02, wm, 2) == 0
        @test rank(0x02, wm, 3) == 1
        @test rank(0x02, wm, 4) == 1
        @test rank(0x02, wm, 5) == 1
        # 0x03
        @test rank(0x03, wm, 0) == 0
        @test rank(0x03, wm, 1) == 0
        @test rank(0x03, wm, 2) == 1
        @test rank(0x03, wm, 3) == 1
        @test rank(0x03, wm, 4) == 1
        @test rank(0x03, wm, 5) == 1

        @test select(0x00, wm, 1) == 4
        @test select(0x01, wm, 1) == 1
        @test select(0x02, wm, 1) == 3
        @test select(0x03, wm, 1) == 2

        @test select(0x00, wm, 2) == 0
        @test select(0x01, wm, 2) == 0
        @test select(0x02, wm, 2) == 0
        @test select(0x03, wm, 2) == 0
    end

    @testset "homogeneous" begin
        wm = WaveletMatrix([0x01, 0x01, 0x01, 0x01])
        @test length(wm) == 4
        # getindex
        for i in 1:4
            @test wm[i] == 0x01
        end
        @test rank(0x01, wm, 0) == 0
        @test rank(0x01, wm, 1) == 1
        @test rank(0x01, wm, 2) == 2
        @test rank(0x01, wm, 3) == 3
        @test rank(0x01, wm, 4) == 4
        @test rank(0x01, wm, 5) == 4

        @test select(0x01, wm, 1) == 1
        @test select(0x01, wm, 2) == 2
        @test select(0x01, wm, 3) == 3
        @test select(0x01, wm, 4) == 4
        @test select(0x01, wm, 5) == 0

        @test select(0x00, wm, 1) == 0
        @test select(0x02, wm, 1) == 0

        for i in 1:4, j in 1:4
            @test freq(0x00, wm, i, j) == 0
            @test freq(0x01, wm, i, j) == max(0, j - i + 1)
        end
    end

    @testset "Vector{Uint64}" begin
        x = [
            0x0000000000000001,
            0xffffffffffffffff,
            0x0000000000000000,
            0x0000000000000001,
            0xffffffffffffffff,
            0xffffff171f410d54,
            0x0000000000000009,
            0x0000000000000000,
            0x1fffff171f410d54,
            0x0000000000000008,
        ]
        wm = WaveletMatrix(x)
        @test length(wm) == 10
        for i in 1:lastindex(x)
            @test wm[i] == x[i]
        end
        @test typeof(wm[1]) == UInt64
        for v in x, i in 1:length(x)
            @test rank(v, wm, i) == count(v′ -> v′ == v, x[1:i])
        end
        for v in x, j in 1:length(x)+1
            @test select(v, wm, j) == seq_select(v, x, j)
        end
    end

    @testset "2-bit encoding" begin
        x = rand(0x00:0x03, 100)
        wm = WaveletMatrix{2}(x)
        @test length(wm) == 100
        for i in 1:lastindex(x)
            @test wm[i] == x[i]
        end
        for a in 0x00:0x03, i in 1:100
            @test rank(a, wm, i) == count(a′ -> a′ == a, x[1:i])
        end
        for a in 0x00:0x03, j in 1:100+1
            @test select(a, wm, j) == seq_select(a, x, j)
        end
    end

    @testset "17-bit encoding" begin
        x = rand(0x00000000:0x00000011, 500)
        wm = WaveletMatrix{17}(x)
        @test length(wm) == 500
        for i in 1:lastindex(x)
            @test wm[i] == x[i]
        end
        for a in 0x00000000:0x00000011, i in 1:500
            @test rank(a, wm, i) == count(a′ -> a′ == a, x[1:i])
        end
        for a in 0x00000000:0x00000011, j in 1:500+1
            @test select(a, wm, j) == seq_select(a, x, j)
        end
    end

    @testset "destructive" begin
        x = rand(0x00:0x03, 1000)
        wm = WaveletMatrix{2}(x, destructive=false)
        @test length(x) == 1000
        wm = WaveletMatrix{2}(x, destructive=true)
        @test length(x) == 0
    end

    @testset "random" begin
        # there is at least one duplicated byte (pigeonhole principle)
        len = typemax(UInt8) + 1
        bytes = rand(UInt8, len)
        wm = WaveletMatrix(copy(bytes))
        for i in 1:len
            @test wm[i] == bytes[i]
        end
        for byte in 0x00:0xff, i in 1:len
            @test rank(byte, wm, i) == count(b -> b == byte, bytes[1:i])
        end
        for byte in 0x00:0xff, j in 0:len+1
            @test select(byte, wm, j) == seq_select(byte, bytes, j)
        end
        for byte in 0x00:0xff
            @test freq(byte, wm, len, 1) == 0
            @test freq(byte, wm, 1, len) == count(b -> b == byte, bytes)
            i, j = rand(1:len), rand(1:len)
            @test freq(byte, wm, i, j) == count(b -> b == byte, bytes[i:j])
        end
    end
end


text:
function read_graph(filename)
    ver_count = 1
    open(filename,"r") do f
        for line in readlines(f)
            str = split(chomp(line))
            str = map(x -> parse(Int64,x) + 1, str)
            temp = max(str[1], str[2])
            if(temp > ver_count)
                ver_count = temp
            end
        end
    end
    g = Graph(ver_count)
    open(filename,"r") do f
        for line in readlines(f)
            str = split(chomp(line))
            str = map(x -> parse(Int64,x) + 1, str)
            try
                add_edge!(g,str[1],str[2])
            catch
                println("something wrong!")
            end
        end
    end
    g
end

function output_format(res)
    temp1 = res[1]
    temp2 = res[2]
    for t = 1:length(temp2)
        println("----------------------")
        println("Iteration: $t")
        println("There are $(temp2[t]) groups")
    end
    println
    println("----------------------")
    println("There are $(maximum(temp1)) groups finally")
    for i = 1:length(temp1)
        println("node($i) -> group($(temp1[i]))")
    end
end

using LightGraphs
data_path = "test.txt"
g = read_graph(data_path)
res = label_propagation(g)
output_format(res)


text:
#=
Copyright (c) 2015, Intel Corporation

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Intel Corporation nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=#

using Latte
using FactCheck
using HDF5
facts("Testing HDF5 Layer") do
    _file = "temp"

    w = 224
    h = 224
    c = 3
    n = 16

    data_value = rand(Float32, w, h, c, n) * 256
    label_value = map(floor, rand(Float32, 1, n) * 10)

    h5open("$_file.hdf5", "w") do h5
        dset_data = d_create(h5, "data", datatype(Float32), dataspace(w, h, c, n))
        dset_label = d_create(h5, "label", datatype(Float32), dataspace(1, n))
        dset_data[:,:,:,:] = data_value
        dset_label[:,:] = label_value
    end

    open("$_file.txt", "w") do f
        write(f, "$_file.hdf5")
    end

    net = Net(8)
    data, label  = HDF5DataLayer(net, "$_file.txt", "$_file.txt"; shuffle=false)

    init(net)
    forward(net)
    @fact get_buffer(net, :datavalue) --> data_value[:,:,:,1:8]
    @fact get_buffer(net, :labelvalue) --> label_value[:,1:8]
    forward(net)
    @fact get_buffer(net, :datavalue) --> data_value[:,:,:,9:16]
    @fact get_buffer(net, :labelvalue) --> label_value[:,9:16]
    # Test wrap around
    forward(net)
    @fact get_buffer(net, :datavalue) --> data_value[:,:,:,1:8]
    @fact get_buffer(net, :labelvalue) --> label_value[:,1:8]
    rm("$_file.txt")
    rm("$_file.hdf5")
end

FactCheck.exitstatus()


text:
using Blosc2
import Blosc2.Lib as lb
@testset "init destroy" begin
    sc = lb.blosc2_schunk_new()
    lb.blosc2_schunk_free(sc)
    @test 1 == 1

end

@testset "blosc2_schunk_append_buffer / blosc2_schunk_decompress_chunk" begin

    sc = lb.blosc2_schunk_new()
    n = 100000
    data = rand(1:1000, n)

    sz = lb.blosc2_schunk_append_buffer(sc, data, sizeof(data))
    @test sz > 0
    @test unsafe_load(sc, 1).nchunks == 1
    res = Vector{Int64}(undef, n + 500)
    sz = lb.blosc2_schunk_decompress_chunk(sc, 0, res, sizeof(res))
    @test sz == sizeof(data)
    @test data == res[1:div(sz, sizeof(Int64))]

    sz = lb.blosc2_schunk_append_buffer(sc, data, sizeof(data))
    @test sz > 0

    lb.blosc2_schunk_free(sc)
end

@testset "dealing with chunks" begin

    sc = lb.blosc2_schunk_new()
    n = 10000
    chunks_n = 5
    ch_comp_sizes = Vector{Cint}(undef, chunks_n)
    datas = Vector{Vector{Int}}(undef, chunks_n)
    for i in 1:chunks_n
        datas[i] = rand(1:1000, n)
        sz = lb.blosc2_schunk_append_buffer(sc, datas[i], sizeof(datas[i]))
        @test sz > 0
        ch_comp_sizes[i] = sz
    end
    @test unsafe_load(sc, 1).nchunks == chunks_n

    buff = lb.blosc2_schunk_get_chunk(sc, 1)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[2]

    chn = lb.blosc2_schunk_append_chunk(sc, buff, true)
    @test chn == chunks_n + 1

    buff = lb.blosc2_schunk_get_chunk(sc, chunks_n)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[2]

    chn = lb.blosc2_schunk_insert_chunk(sc, 2, buff, true)
    @test chn == chunks_n + 2

    buff = lb.blosc2_schunk_get_chunk(sc, 2)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[2]

    chn = lb.blosc2_schunk_update_chunk(sc, 1, buff, true)
    @test chn == chunks_n + 2

    buff = lb.blosc2_schunk_get_chunk(sc, 1)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[2]

    chn = lb.blosc2_schunk_delete_chunk(sc, 0)
    @test chn == chunks_n + 1

    buff = lb.blosc2_schunk_get_chunk(sc, 0)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[2]

    lb.blosc2_schunk_free(sc)
end

@testset "blosc2_schunk_reorder_offsets" begin
    sc = lb.blosc2_schunk_new()
    n = 10000
    chunks_n = 5
    ch_comp_sizes = Vector{Int32}(undef, chunks_n)
    datas = Vector{Vector{Int}}(undef, chunks_n)
    for i in 1:chunks_n
        datas[i] = rand(1:1000, n)
        sz = lb.blosc2_schunk_append_buffer(sc, datas[i], sizeof(datas[i]))
        @test sz > 0
        ch_comp_sizes[i] = sz
    end
    @test unsafe_load(sc, 1).nchunks == chunks_n

    offsets = Int32.(collect(0:chunks_n-1))
    offsets[4] = 0
    offsets[1] = 3

    r = lb.blosc2_schunk_reorder_offsets(sc, offsets)
    @test r == 0

    buff = lb.blosc2_schunk_get_chunk(sc, 0)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[4]

    buff = lb.blosc2_schunk_get_chunk(sc, 3)
    res = Vector{Int64}(undef, n + 500)
    dctx = lb.blosc2_create_dctx()
    dsz = div(lb.blosc2_decompress_ctx(dctx, buff, sizeof(buff), res, sizeof(res)), sizeof(Int64))
    lb.blosc2_free_ctx(dctx)
    @test dsz == n
    resize!(res, dsz)
    @test res == datas[1]

    lb.blosc2_schunk_free(sc)
end

@testset "blosc2_schunk_get_cparams / dparams" begin
    sc = lb.blosc2_schunk_new()
    params = lb.blosc2_schunk_get_cparams(sc)
    @test params.compcode == 0
    @test params.clevel == 5

    dparams = lb.blosc2_schunk_get_dparams(sc)
    @test dparams.nthreads == 1
    lb.blosc2_schunk_free(sc)
end


text:
Base.@kwdef struct LinearAnalysis{I}
    npoints::Int=200
    dropcollinear::Bool=false
    interval::I=automatic
end

add_intercept_column(x::AbstractVector{T}) where {T} = [ones(T, length(x)) x]

# TODO: add multidimensional version
function (l::LinearAnalysis)(input::ProcessedLayer)
    output = map(input) do p, n
        x, y = p
        weights = get(n, :weights, similar(x, 0))
        default_interval = length(weights) > 0 ? nothing : :confidence
        interval = l.interval === automatic ? default_interval : l.interval
        # FIXME: handle collinear case gracefully
        lin_model = GLM.lm(add_intercept_column(x), collect(y); wts=weights, l.dropcollinear)
        x̂ = range(extrema(x)..., length=l.npoints)
        pred = GLM.predict(lin_model, add_intercept_column(x̂); interval)
        return if !isnothing(interval)
            ŷ, lower, upper = map(vec, pred) # GLM prediction returns matrices
            (x̂, ŷ), (; lower, upper)
        else
            ŷ = vec(pred) # GLM prediction returns matrix
            (x̂, ŷ), (;)
        end
    end
    default_plottype = isempty(output.named) ? Lines : LinesFill
    plottype = Makie.plottype(output.plottype, default_plottype)
    return ProcessedLayer(output; plottype)
end

"""
    linear(; interval=automatic, dropcollinear=false, npoints=200)

Compute a linear fit of `y ~ 1 + x`. An optional named mapping `weights` determines the weights.
Use `interval` to specify what type of interval the shaded band should represent.
Valid values of interval are `:confidence` delimiting the uncertainty of the predicted
relationship, and `:prediction` delimiting estimated bounds for new data points.
By default, this analysis errors on singular (collinear) data. To avoid that,
it is possible to set `dropcollinear=true`.
`npoints` is the number of points used by Makie to draw the line
"""
linear(; options...) = transformation(LinearAnalysis(; options...))


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

@doc raw"""LimitRangeSpec defines a min/max usage limit for resources that match on kind.

    IoK8sApiCoreV1LimitRangeSpec(;
        limits=nothing,
    )

    - limits::Vector{IoK8sApiCoreV1LimitRangeItem} : Limits is the list of LimitRangeItem objects that are enforced.
"""
mutable struct IoK8sApiCoreV1LimitRangeSpec <: SwaggerModel
    limits::Any # spec type: Union{ Nothing, Vector{IoK8sApiCoreV1LimitRangeItem} } # spec name: limits

    function IoK8sApiCoreV1LimitRangeSpec(;limits=nothing)
        o = new()
        validate_property(IoK8sApiCoreV1LimitRangeSpec, Symbol("limits"), limits)
        setfield!(o, Symbol("limits"), limits)
        o
    end
end # type IoK8sApiCoreV1LimitRangeSpec

const _property_map_IoK8sApiCoreV1LimitRangeSpec = Dict{Symbol,Symbol}(Symbol("limits")=>Symbol("limits"))
const _property_types_IoK8sApiCoreV1LimitRangeSpec = Dict{Symbol,String}(Symbol("limits")=>"Vector{IoK8sApiCoreV1LimitRangeItem}")
Base.propertynames(::Type{ IoK8sApiCoreV1LimitRangeSpec }) = collect(keys(_property_map_IoK8sApiCoreV1LimitRangeSpec))
Swagger.property_type(::Type{ IoK8sApiCoreV1LimitRangeSpec }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_IoK8sApiCoreV1LimitRangeSpec[name]))}
Swagger.field_name(::Type{ IoK8sApiCoreV1LimitRangeSpec }, property_name::Symbol) =  _property_map_IoK8sApiCoreV1LimitRangeSpec[property_name]

function check_required(o::IoK8sApiCoreV1LimitRangeSpec)
    (getproperty(o, Symbol("limits")) === nothing) && (return false)
    true
end

function validate_property(::Type{ IoK8sApiCoreV1LimitRangeSpec }, name::Symbol, val)
end