text:
function layoutable(::Type{Legend},
        fig_or_scene,
        entry_groups::Node{Vector{Tuple{Optional{<:AbstractString}, Vector{LegendEntry}}}};
        bbox = nothing, kwargs...)

    topscene = get_topscene(fig_or_scene)

    default_attrs = default_attributes(Legend, topscene).attributes
    theme_attrs = subtheme(topscene, :Legend)
    attrs = merge!(merge!(Attributes(kwargs), theme_attrs), default_attrs)

    @extract attrs (
        halign, valign, padding, margin,
        titlefont, titlesize, titlehalign, titlevalign, titlevisible, titlecolor,
        labelsize, labelfont, labelcolor, labelhalign, labelvalign,
        bgcolor, framecolor, framewidth, framevisible,
        patchsize, # the side length of the entry patch area
        nbanks,
        colgap, rowgap, patchlabelgap,
        titlegap, groupgap,
        orientation, tellwidth, tellheight,
        titleposition,
        gridshalign, gridsvalign,
    )

    decorations = Dict{Symbol, Any}()

    # by default, `tellwidth = true` and `tellheight = false` for vertical legends
    # and vice versa for horizontal legends
    real_tellwidth = @lift $tellwidth === automatic ? $orientation == :vertical : $tellwidth
    real_tellheight = @lift $tellheight === automatic ? $orientation == :horizontal : $tellheight

    layoutobservables = LayoutObservables{Legend}(attrs.width, attrs.height, real_tellwidth, real_tellheight,
        halign, valign, attrs.alignmode; suggestedbbox = bbox)

    scenearea = lift(round_to_IRect2D, layoutobservables.computedbbox)

    scene = Scene(topscene, scenearea, raw = true, camera = campixel!)

    # the rectangle in which the legend is drawn when margins are removed
    legendrect = @lift(
        BBox($margin[1], width($scenearea) - $margin[2],
             $margin[3], height($scenearea) - $margin[4]))

    decorations[:frame] = poly!(scene,
        @lift(enlarge($legendrect, repeat([-$framewidth/2], 4)...)),
        color = bgcolor, strokewidth = framewidth, visible = framevisible,
        strokecolor = framecolor, raw = true, inspectable = false)

    # the grid containing all content
    grid = GridLayout(bbox = legendrect, alignmode = Outside(padding[]...))

    # while the entries are being manipulated through code, this Ref value is set to
    # true so the GridLayout doesn't update itself to save time
    manipulating_grid = Ref(false)

    on(padding) do p
        grid.alignmode = Outside(p...)
        relayout()
    end

    update_grid = Node(true)
    onany(update_grid, margin) do _, margin
        if manipulating_grid[]
            return
        end
        w = GridLayoutBase.determinedirsize(grid, GridLayoutBase.Col())
        h = GridLayoutBase.determinedirsize(grid, GridLayoutBase.Row())
        if !any(isnothing.((w, h)))
            layoutobservables.autosize[] = (w + sum(margin[1:2]), h + sum(margin[3:4]))
        end
    end

    # these arrays store all the plot objects that the legend entries need
    titletexts = Optional{Label}[]
    entrytexts = [Label[]]
    entryplots = [[AbstractPlot[]]]
    entryrects = [Box[]]

    decorations[:titletexts] = titletexts
    decorations[:entrytexts] = entrytexts
    decorations[:entryplots] = entryplots
    decorations[:entryrects] = entryrects

    function relayout()
        manipulating_grid[] = true

        rowcol(n) = ((n - 1) ÷ nbanks[] + 1, (n - 1) % nbanks[] + 1)

        for i in length(grid.content):-1:1
            GridLayoutBase.remove_from_gridlayout!(grid.content[i])
        end

        # loop through groups
        for g in 1:length(entry_groups[])
            title = titletexts[g]
            etexts = entrytexts[g]
            erects = entryrects[g]

            subgl = if orientation[] == :vertical
                if titleposition[] == :left
                    isnothing(title) || (grid[g, 1] = title)
                    grid[g, 2] = GridLayout(halign = gridshalign[], valign = gridsvalign[])
                elseif titleposition[] == :top
                    isnothing(title) || (grid[2g - 1, 1] = title)
                    grid[2g, 1] = GridLayout(halign = gridshalign[], valign = gridsvalign[])
                end
            elseif orientation[] == :horizontal
                if titleposition[] == :left
                    isnothing(title) || (grid[1, 2g-1] = title)
                    grid[1, 2g] = GridLayout(halign = gridshalign[], valign = gridsvalign[])
                elseif titleposition[] == :top
                    isnothing(title) || (grid[1, g] = title)
                    grid[2, g] = GridLayout(halign = gridshalign[], valign = gridsvalign[])
                end
            end

            for (n, (et, er)) in enumerate(zip(etexts, erects))
                i, j = orientation[] == :vertical ? rowcol(n) : reverse(rowcol(n))
                subgl[i, 2j-1] = er
                subgl[i, 2j] = et
            end

            rowgap!(subgl, rowgap[])
            for c in 1:subgl.ncols-1
                colgap!(subgl, c, c % 2 == 1 ? patchlabelgap[] : colgap[])
            end
        end

        for r in 1:grid.nrows-1
            if orientation[] == :horizontal
                if titleposition[] == :left
                    # nothing
                elseif titleposition[] == :top
                    rowgap!(grid, r, titlegap[])
                end
            elseif orientation[] == :vertical
                if titleposition[] == :left
                    rowgap!(grid, r, groupgap[])
                elseif titleposition[] == :top
                    rowgap!(grid, r, r % 2 == 1 ? titlegap[] : groupgap[])
                end
            end
        end
        for c in 1:grid.ncols-1
            if orientation[] == :horizontal
                if titleposition[] == :left
                    colgap!(grid, c, c % 2 == 1 ? titlegap[] : groupgap[])
                elseif titleposition[] == :top
                    colgap!(grid, c, groupgap[])
                end
            elseif orientation[] == :vertical
                if titleposition[] == :left
                    colgap!(grid, c, titlegap[])
                elseif titleposition[] == :top
                    # nothing here
                end
            end
        end

        # delete unused rows and columns
        trim!(grid)

        manipulating_grid[] = false
        notify(update_grid)

        # translate the legend forward so it is above the standard axis content
        # which is at zero. this will not really work if the legend should be
        # above a 3d plot, but for now this hack is ok.
        translate!(scene, (0, 0, 10))
    end

    onany(nbanks, titleposition, rowgap, colgap, patchlabelgap, groupgap, titlegap,
            titlevisible, orientation, gridshalign, gridsvalign) do args...
        relayout()
    end

    on(entry_groups) do entry_groups
        # first delete all existing labels and patches

        for t in titletexts
            !isnothing(t) && delete!(t)
        end
        empty!(titletexts)

        [delete!.(etexts) for etexts in entrytexts]
        empty!(entrytexts)

        [delete!.(erects) for erects in entrytexts]
        empty!(entryrects)

        # delete patch plots
        for eplotgroup in entryplots
            for eplots in eplotgroup
                # each entry can have a vector of patch plots
                delete!.(scene, eplots)
            end
        end
        empty!(entryplots)

        # the attributes for legend entries that the legend itself carries
        # these serve as defaults unless the legendentry gets its own value set
        preset_attrs = extractattributes(attrs, LegendEntry)

        for (title, entries) in entry_groups

            if isnothing(title)
                # in case a group has no title
                push!(titletexts, nothing)
            else
                push!(titletexts, Label(scene, text = title, font = titlefont, color = titlecolor,
                    textsize = titlesize, halign = titlehalign, valign = titlevalign, inspectable = false))
            end

            etexts = []
            erects = []
            eplots = []
            for (i, e) in enumerate(entries)
                # fill missing entry attributes with those carried by the legend
                merge!(e.attributes, preset_attrs)

                # create the label
                push!(etexts, Label(scene,
                    text = e.label, textsize = e.labelsize, font = e.labelfont,
                    color = e.labelcolor, halign = e.labelhalign, valign = e.labelvalign
                    ))

                # create the patch rectangle
                rect = Box(scene, color = e.patchcolor, strokecolor = e.patchstrokecolor,
                    strokewidth = e.patchstrokewidth,
                    width = lift(x -> x[1], e.patchsize),
                    height = lift(x -> x[2], e.patchsize))
                push!(erects, rect)

                # plot the symbols belonging to this entry
                symbolplots = AbstractPlot[]
                for element in e.elements
                    append!(symbolplots,
                        legendelement_plots!(scene, element,
                            rect.layoutobservables.computedbbox, e.attributes))
                end

                push!(eplots, symbolplots)
            end
            push!(entrytexts, etexts)
            push!(entryrects, erects)
            push!(entryplots, eplots)
        end
        relayout()
    end

    # trigger suggestedbbox
    layoutobservables.suggestedbbox[] = layoutobservables.suggestedbbox[]

    leg = Legend(fig_or_scene, layoutobservables, attrs, decorations, entry_groups)
    # trigger first relayout
    entry_groups[] = entry_groups[]
    leg
end

function legendelement_plots!(scene, element::MarkerElement, bbox::Node{FRect2D}, defaultattrs::Attributes)
    merge!(element.attributes, defaultattrs)
    attrs = element.attributes

    fracpoints = attrs.markerpoints
    points = @lift(fractionpoint.(Ref($bbox), $fracpoints))
    scat = scatter!(scene, points, color = attrs.markercolor, marker = attrs.marker,
        markersize = attrs.markersize,
        strokewidth = attrs.markerstrokewidth,
        strokecolor = attrs.markerstrokecolor, raw = true, inspectable = false)
    [scat]
end

function legendelement_plots!(scene, element::LineElement, bbox::Node{FRect2D}, defaultattrs::Attributes)
    merge!(element.attributes, defaultattrs)
    attrs = element.attributes

    fracpoints = attrs.linepoints
    points = @lift(fractionpoint.(Ref($bbox), $fracpoints))
    lin = lines!(scene, points, linewidth = attrs.linewidth, color = attrs.linecolor,
        linestyle = attrs.linestyle,
        raw = true, inspectable = false)
    [lin]
end

function legendelement_plots!(scene, element::PolyElement, bbox::Node{FRect2D}, defaultattrs::Attributes)
    merge!(element.attributes, defaultattrs)
    attrs = element.attributes

    fracpoints = attrs.polypoints
    points = @lift(fractionpoint.(Ref($bbox), $fracpoints))
    pol = poly!(scene, points, strokewidth = attrs.polystrokewidth, color = attrs.polycolor,
        strokecolor = attrs.polystrokecolor,
        raw = true, inspectable = false)
    [pol]
end

function Base.getproperty(lentry::LegendEntry, s::Symbol)
    if s in fieldnames(LegendEntry)
        getfield(lentry, s)
    else
        lentry.attributes[s]
    end
end

function Base.setproperty!(lentry::LegendEntry, s::Symbol, value)
    if s in fieldnames(LegendEntry)
        setfield!(lentry, s, value)
    else
        lentry.attributes[s][] = value
    end
end

function Base.propertynames(lentry::LegendEntry)
    [fieldnames(T)..., keys(lentry.attributes)...]
end

legendelements(le::LegendElement, legend) = LegendElement[le]
legendelements(les::AbstractArray{<:LegendElement}, legend) = LegendElement[les...]

function LegendEntry(label::AbstractString, contentelements::AbstractArray, legend; kwargs...)
    attrs = Attributes(label = label)

    kwargattrs = Attributes(kwargs)
    merge!(attrs, kwargattrs)

    elems = vcat(legendelements.(contentelements, Ref(legend))...)
    LegendEntry(elems, attrs)
end

function LegendEntry(label::AbstractString, contentelement, legend; kwargs...)
    attrs = Attributes(label = label)

    kwargattrs = Attributes(kwargs)
    merge!(attrs, kwargattrs)

    elems = legendelements(contentelement, legend)
    LegendEntry(elems, attrs)
end

function LineElement(;kwargs...)
    _legendelement(LineElement, Attributes(kwargs))
end

function MarkerElement(;kwargs...)
    _legendelement(MarkerElement, Attributes(kwargs))
end

function PolyElement(;kwargs...)
    _legendelement(PolyElement, Attributes(kwargs))
end

function _legendelement(T::Type{<:LegendElement}, a::Attributes)
    _rename_attributes!(T, a)
    T(a)
end

_renaming_mapping(::Type{LineElement}) = Dict(
    :points => :linepoints,
    :color => :linecolor,
)
_renaming_mapping(::Type{MarkerElement}) = Dict(
    :points => :markerpoints,
    :color => :markercolor,
    :strokewidth => :markerstrokewidth,
    :strokecolor => :markerstrokecolor,
)
_renaming_mapping(::Type{PolyElement}) = Dict(
    :points => :polypoints,
    :color => :polycolor,
    :strokewidth => :polystrokewidth,
    :strokecolor => :polystrokecolor,
)

function _rename_attributes!(T, a)
    m = _renaming_mapping(T)
    for (key, val) in pairs(a)
        if haskey(m, key)
            newkey = m[key]
            if haskey(a, newkey)
                error("Can't rename $key to $newkey as $newkey already exists in attributes.")
            end
            a[newkey] = pop!(a, key)
        end
    end
    a
end

function scalar_lift(attr, default)
    lift(Any, attr, default) do at, def
        Makie.is_scalar_attribute(at) ? at : def
    end
end

function legendelements(plot::Union{Lines, LineSegments}, legend)
    LegendElement[LineElement(
        color = scalar_lift(plot.color, legend.linecolor),
        linestyle = scalar_lift(plot.linestyle, legend.linestyle),
        linewidth = scalar_lift(plot.linewidth, legend.linewidth))]
end

function legendelements(plot::Scatter, legend)
    LegendElement[MarkerElement(
        color = scalar_lift(plot.color, legend.markercolor),
        marker = scalar_lift(plot.marker, legend.marker),
        markersize = scalar_lift(plot.markersize, legend.markersize),
        strokewidth = scalar_lift(plot.strokewidth, legend.markerstrokewidth),
        strokecolor = scalar_lift(plot.strokecolor, legend.markerstrokecolor),
    )]
end

function legendelements(plot::Union{Poly, Violin, BoxPlot, CrossBar, Density}, legend)
    LegendElement[PolyElement(
        color = scalar_lift(plot.color, legend.polycolor),
        strokecolor = scalar_lift(plot.strokecolor, legend.polystrokecolor),
        strokewidth = scalar_lift(plot.strokewidth, legend.polystrokewidth),
    )]
end

function legendelements(plot::Band, legend)
    # there seems to be no stroke for Band, so we set it invisible
    LegendElement[PolyElement(polycolor = scalar_lift(plot.color, legend.polystrokecolor), polystrokecolor = :transparent, polystrokewidth = 0)]
end

# if there is no specific overload available, we go through the child plots and just stack
# those together as a simple fallback
function legendelements(plot, legend)::Vector{LegendElement}
    if isempty(plot.plots)
        error("No child plot elements found in plot of type $(typeof(plot)) but also no `legendelements` method defined.")
    end
    reduce(vcat, [legendelements(childplot, legend) for childplot in plot.plots])
end

function Base.getproperty(legendelement::T, s::Symbol) where T <: LegendElement
    if s in fieldnames(T)
        getfield(legendelement, s)
    else
        legendelement.attributes[s]
    end
end

function Base.setproperty!(legendelement::T, s::Symbol, value) where T <: LegendElement
    if s in fieldnames(T)
        setfield!(legendelement, s, value)
    else
        legendelement.attributes[s][] = value
    end
end

function Base.propertynames(legendelement::T) where T <: LegendElement
    [fieldnames(T)..., keys(legendelement.attributes)...]
end

"""
    Legend(
        fig_or_scene,
        contents::AbstractArray,
        labels::AbstractArray{<:AbstractString},
        title::Optional{<:AbstractString} = nothing;
        kwargs...)

Create a legend from `contents` and `labels` where each label is associated to
one content element. A content element can be an `AbstractPlot`, an array of
`AbstractPlots`, a `LegendElement`, or any other object for which the
`legendelements` method is defined.
"""
function layoutable(::Type{Legend}, fig_or_scene,
        contents::AbstractArray,
        labels::AbstractArray{<:AbstractString},
        title::Optional{<:AbstractString} = nothing;
        kwargs...)

    if length(contents) != length(labels)
        error("Number of elements not equal: $(length(contents)) content elements and $(length(labels)) labels.")
    end

    entrygroups = Node{Vector{EntryGroup}}([])
    legend = layoutable(Legend, fig_or_scene, entrygroups; kwargs...)
    entries = [LegendEntry(label, content, legend) for (content, label) in zip(contents, labels)]
    entrygroups[] = [(title, entries)]
    legend
end

"""
    Legend(
        fig_or_scene,
        contentgroups::AbstractArray{<:AbstractArray},
        labelgroups::AbstractArray{<:AbstractArray},
        titles::AbstractArray{<:Optional{<:AbstractString}};
        kwargs...)

Create a multi-group legend from `contentgroups`, `labelgroups` and `titles`.
Each group from `contentgroups` and `labelgroups` is associated with one title
from `titles` (a title can be `nothing` to hide it).

Within each group, each content element is associated with one label. A content
element can be an `AbstractPlot`, an array of `AbstractPlots`, a `LegendElement`,
or any other object for which the `legendelements` method is defined.
"""
function layoutable(::Type{Legend}, fig_or_scene,
        contentgroups::AbstractArray{<:AbstractArray},
        labelgroups::AbstractArray{<:AbstractArray},
        titles::AbstractArray{<:Optional{<:AbstractString}};
        kwargs...)

    if !(length(titles) == length(contentgroups) == length(labelgroups))
        error("Number of elements not equal: $(length(titles)) titles, $(length(contentgroups)) content groups and $(length(labelgroups)) label groups.")
    end

    entrygroups = Node{Vector{EntryGroup}}([])
    legend = layoutable(Legend, fig_or_scene, entrygroups; kwargs...)
    entries = [[LegendEntry(l, pg, legend) for (l, pg) in zip(labelgroup, contentgroup)]
        for (labelgroup, contentgroup) in zip(labelgroups, contentgroups)]
    entrygroups[] = [(t, en) for (t, en) in zip(titles, entries)]
    legend
end

"""
    Legend(fig_or_scene, axis::Union{Axis, Scene, LScene}, title = nothing; merge = false, unique = false, kwargs...)

Create a single-group legend with all plots from `axis` that have the
attribute `label` set.

If `merge` is `true`, all plot objects with the same label will be layered on top of each other into one legend entry.
If `unique` is `true`, all plot objects with the same plot type and label will be reduced to one occurrence.
"""
function layoutable(::Type{Legend}, fig_or_scene, axis::Union{Axis, Scene, LScene}, title = nothing; merge = false, unique = false, kwargs...)
    plots, labels = get_labeled_plots(axis, merge = merge, unique = unique)
    isempty(plots) && error("There are no plots with labels in the given axis that can be put in the legend. Supply labels to plotting functions like `plot(args...; label = \"My label\")`")
    layoutable(Legend, fig_or_scene, plots, labels, title; kwargs...)
end

function get_labeled_plots(ax; merge::Bool, unique::Bool)
    lplots = filter(get_plots(ax)) do plot
        haskey(plot.attributes, :label)
    end
    labels = map(lplots) do l
        l.label[]
    end

    # filter out plots with same plot type and label
    if unique
        plots_labels = Base.unique(((p, l),) -> (typeof(p), l), zip(lplots, labels))
        lplots = first.(plots_labels)
        labels = last.(plots_labels)
    end

    if merge
        ulabels = Base.unique(labels)
        mergedplots = [[lp for (i, lp) in enumerate(lplots) if labels[i] == ul]
            for ul in ulabels]

        lplots, labels = mergedplots, ulabels
    end

    lplots, labels
end

get_plots(p::AbstractPlot) = [p]
get_plots(ax::Axis) = get_plots(ax.scene)
get_plots(lscene::LScene) = get_plots(lscene.scene)
function get_plots(scene::Scene)
    plots = AbstractPlot[]
    for p in scene.plots
        append!(plots, get_plots(p))
    end
    return plots
end

# convenience constructor for axis legend
axislegend(ax = current_axis(); kwargs...) = axislegend(ax, ax; kwargs...)

axislegend(title::AbstractString; kwargs...) = axislegend(current_axis(), current_axis(), title; kwargs...)

"""
    axislegend(ax, args...; position = :rt, kwargs...)
    axislegend(ax, args...; position = (1, 1), kwargs...)
    axislegend(ax = current_axis(); kwargs...)
    axislegend(title::AbstractString; kwargs...)

Create a legend that sits inside an Axis's plot area.

The position can be a Symbol where the first letter controls the horizontal
alignment and can be l, r or c, and the second letter controls the vertical
alignment and can be t, b or c. Or it can be a tuple where the first
element is set as the Legend's halign and the second element as its valign.
"""
function axislegend(ax, args...; position = :rt, kwargs...)
    Legend(ax.parent, args...;
        bbox = ax.scene.px_area,
        margin = (10, 10, 10, 10),
        legend_position_to_aligns(position)...,
        kwargs...)
end

function legend_position_to_aligns(s::Symbol)
    p = string(s)
    length(p) != 2 && throw(ArgumentError("Position symbol must have length == 2"))

    haligns = Dict(
        'l' => :left,
        'r' => :right,
        'c' => :center,
    )
    haskey(haligns, p[1]) || throw(ArgumentError("First letter can be l, r or c, not $(p[1])."))

    valigns = Dict(
        't' => :top,
        'b' => :bottom,
        'c' => :center,
    )
    haskey(valigns, p[2]) || throw(ArgumentError("Second letter can be b, t or c, not $(p[2])."))

    (halign = haligns[p[1]], valign = valigns[p[2]])
end

function legend_position_to_aligns(t::Tuple{Any, Any})
    (halign = t[1], valign = t[2])
end


text:
using Dates
using GRIB

DEFAULT_EPOCH = DateTime(1970, 1, 1, 0, 0)

function from_grib_date_time(date::Int, time::Int; epoch=DEFAULT_EPOCH)
    hour = time ÷ 100
    minute = time % 100
    year = date ÷ 10000
    month = date ÷ 100 % 100
    day = date % 100

    data_datetime = DateTime(year, month, day, hour, minute)

    return Dates.value(Dates.Second(data_datetime - epoch))
end

function from_grib_date_time(
        message::GRIB.Message, date_key="dataDate",
        time_key="dataTime", epoch=DEFAULT_EPOCH
    )
    date = message[date_key]
    time = message[time_key]

    return from_grib_date_time(date, time)
end

#  TODO: This probably won't work translated directly from python
#  check cases where time and step are effectively missing
function build_valid_time(time::Int, step::Int)
    step_s = step * 3600

    data = time + step_s
    dims = ()

    return dims, data
end

function build_valid_time(time::Array{Int, 1}, step::Int)
    step_s = step * 3600

    data = time .+ step_s
    dims = ("time", )

    return dims, data
end

function build_valid_time(time::Int, step::Array{Int, 1})
    step_s = step * 3600

    data = time .+ step_s
    dims = ("step", )

    return dims, data
end

function build_valid_time(time::Array{Int, 1}, step::Array{Int, 1})
    step_s = step * 3600

    if length(time) == 1 && length(step) == 1
        return build_valid_time(time[1], step[1])
    end

    #  TODO: Julia is column major, numpy is row major, not too sure what
    #  the correct approach would be here...
    data = time' .+ step_s
    dims = ("time", "step")
    return dims, data

end

#  TODO: implement other conversion methods, but some seem unused, should these
#  be implemented as well:
#   - to_grib_date_time
#   - from_grib_step
#   - to_grib_step
#   - from_grib_month

ext: jl


text:
module BasicTask

export Info

"""
    BasicTask.Info

Type contains basic information for a daemon task

"""
type Info
    id::Int64                           # Numerical identifier for specific task
    name::AbstractString                # String identifer for the type of task
    base_directory::AbstractString       # Base directory for fetching input data
    inputs::Array{AbstractString, 1}    # Names of files we are fetching
end

function Info{String <: AbstractString}(dict::Dict{String, Any})
    if haskey(dict, "id")
        id = typeof(dict["id"]) <: Int ?  dict["id"] : parse(Int64, dict["id"])
    else
        id = -1
    end

    if isempty(strip(dict["name"]))
        throw(ArgumentError("Task name can not be empty"))
    end

    # parse base directory
    if isempty(strip(dict["base_directory"]))
        throw(ArgumentError("Payload does not include a base_directory"))
    end

    # parse input list
    if typeof(dict["inputs"]) != Array{Any, 1} ||
            length(dict["inputs"]) == 0
        throw(ArgumentError("Payload does not include a input list"))
    end

    return Info(id, dict["name"], dict["base_directory"], dict["inputs"])
end

end # module BasicTask

max_forks_repo_name: seung-lab/SimpleTasks


text:
for (elty, relty, ext) in ((:Float32, :Float32, :s),
                           (:Float64, :Float64, :d),
                           (:ComplexF32, :Float32, :c),
                           (:ComplexF64, :Float64, :z))
    for (mat, sym) in ((:Matrix, "_"),
                       (:DistMatrix, "Dist_"),
                       (:DistMultiVec, "DistMultiVec_"))
        @eval begin
            function LinearAlgebra.axpy!(α::$elty, x::$mat{$elty}, y::$mat{$elty})
                ElError(ccall(($(string("ElAxpy", sym, ext)), libEl), Cuint,
                    ($elty, Ptr{Cvoid}, Ptr{Cvoid}),
                    α, x.obj, y.obj))
                return y
            end

            # Which is opposite Julia's copy! so we call it _copy! to avoid confusion
            function _copy!(src::$mat{$elty}, dest::$mat{$elty})
                ElError(ccall(($(string("ElCopy", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, Ptr{Cvoid}),
                    src.obj, dest.obj))
                dest
            end

            function LinearAlgebra.dot(x::$mat{$elty}, y::$mat{$elty})
                rval = Ref{$elty}(0)
                ElError(ccall(($(string("ElDot", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, Ptr{Cvoid}, Ref{$elty}),
                    x.obj, y.obj, rval))
                return rval[]
            end

            function fill!(x::$mat{$elty}, val::Number)
                ElError(ccall(($(string("ElFill", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, $elty),
                    x.obj, $elty(val)))
                return x
            end

            # Horizontal concatenation
            # C := [A, B]
            function hcat!(A::$mat{$elty}, B::$mat{$elty}, C::$mat{$elty})
                ElError(ccall(($(string("ElHCat", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                    A.obj, B.obj, C.obj))
                return C
            end
            hcat(A::$mat{$elty}, B::$mat{$elty}) = hcat!(A, B, $mat($elty))

            function nrm2(x::$mat{$elty})
                rval = Ref{$relty}(0)
                ElError(ccall(($(string("ElNrm2", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, Ref{$relty}),
                    x.obj, rval))
                return rval[]
            end

            function LinearAlgebra.rmul!(x::$mat{$elty}, val::Number)
                ElError(ccall(($(string("ElScale", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, $elty),
                    x.obj, $elty(val)))
                return x
            end

            # Vertical concatenation
            # C := [A; B]
            function vcat!(A::$mat{$elty}, B::$mat{$elty}, C::$mat{$elty})
                ElError(ccall(($(string("ElVCat", sym, ext)), libEl), Cuint,
                    (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cvoid}),
                    A.obj, B.obj, C.obj))
                return C
            end
            vcat(A::$mat{$elty}, B::$mat{$elty}) = vcat!(A, B, $mat($elty))
        end
    end
end


text:
function puzzle08(path::String=joinpath(@__DIR__, "..", "data/08.txt"))
    instructions = readlines(path)
    instructions = [replace(i, "inc" => "+=") for i in instructions]
    instructions = [replace(i, "dec" => "-=") for i in instructions]
    registers = unique(first.(split.(instructions, " ")))

    for register in registers
        r = Symbol(register)
        @eval $r = 0
    end

    highest = 0
    for instruction in instructions
        consequence, condition = Meta.parse.(String.(split(instruction, "if")))
        @eval $(condition) ? $(consequence) : nothing
        highest = max(highest, maximum(eval.(Meta.parse.(registers))))
    end

    partone = maximum(eval.(Meta.parse.(registers)))

    return [partone, highest]
end


text:
##############################################################################
##
## Winkler
##
##############################################################################

immutable Winkler{T1 <: PreMetric, T2 <: Real, T3 <: Real} <: PreMetric
    dist::T1
    scaling_factor::T2      # scaling factor. Default to 0.1
    boosting_limit::T3      # boost threshold. Default to 0.7
end

# restrict to distance between 0 and 1
Winkler(x) = Winkler(x, 0.1, 0.7)

function compare(dist::Winkler, s1::AbstractStringorGraphemeIterator, s2::AbstractStringorGraphemeIterator, len1::Integer, len2::Integer)
    score = compare(dist.dist, s1, s2, len1, len2)
    l = common_prefix(s1, s2, 4)[1]
    # common prefix adjustment
    if score >= dist.boosting_limit
        score += l * dist.scaling_factor * (1 - score)
    end
    return score
end


text:
function Φ(z, λ, w0)
    zR = pi*w0^2/λ
    return atan(z/zR)
end

function w(z, λ, w0)
    zR = pi*w0^2/λ
    return w0*sqrt(1+(z/zR)^2)
end

function R(z, λ, w0)
    zR = pi*w0^2/λ
    return z+zR^2/z
end

function q(z, λ, w0)
    zR = pi*w0^2/λ
    return z + zR*1im
end

function ψ(z, p, m, λ, w0)
    zR = pi*w0^2/λ
    N = abs(m) + 2p
    return (N+1)*atan(z/zR)
end

function Aeff(a)
    return 1.5*a^2
end

# gives fiber output beam waist for pcf
function fiber_beam_waist(a)
    return sqrt(1.5/pi)*a
end

function focal(r, ω0, λ0)
    return sp.pi*ω0*0.64*r/λ0
end

#=
these two functions can be used to design lenses(curved mirrors) system
equivelent to one lens (not available)
=#

function d2(f1,f2, d1)
    #=
    d2 is the distance between the first lens and the beam source.
    be careful with the order of the leses(since the equation is originally for focusing not for collimating).
    For example, the source should always be close to f2(i.e source->space->f2->space->f1->collimated beam)
    these two function work the same way for lenses and curved mirrors
    =#
    return f2*(f1-d1)/(f1+f2-d1)
end

function feff(f1,f2,d1)
    #d1 is the distance between the two lenses
    #these two function work the same way for lenses and curved mirrors
    return (f2*f1)/(f1+f2-d1)
end

#=
Example of ABCD matrex usge.
q1 = q(0, 800e-9, 8.64e-5)
abcd = [1 4; 0 1]*[1 0;-2/2 1]*[1 0.625; 0 1]*[1 0;1/0.5 1]*[1 1.5; 0 1]
the propagation is as follow:
- fiber output with waist w0 = 8.64e-5 (q1)
- 1.5m free space
- concave lens(convex mirror) with focal length(raduis) 0.5(1)
- 0.625m free space
- convex lens(concave mirror) with focal length(raduis) -1(-2)
- 4m free space
this system act as a telescope by magnifiying the initial beam to w1
=#
q1 = q(0, 400e-9, 80e-6)
# abcd = [1 15; 0 1]*[1 0; -2/8 1]*[1 4; 0 1]
abcd = [1 0.381; 0 1]*[1 0;-1/0.4 1]*[1 2.233; 0 1]*[1 0;-1/0.75 1]*[1 0.4; 0 1]*[1 0;1/0.5 1]*[1 2.2; 0 1]
q2 = (abcd[1,1]*q1 + abcd[1,2])/(abcd[2,1]*q1+abcd[2,2])
sol = 1/q2
w1 = sqrt(800e-9/(pi*imag(-sol)))
# w0 = w(2, 800e-9, 8.64e-5)

feff(1,-0.5,0.67)
# d2(0.75,-0.5, 0.342)

max_stars_repo_name: SabbahMohammed/Dev

max_issues_repo_name: SabbahMohammed/Dev

max_forks_repo_name: SabbahMohammed/Dev


text:
const make_classification = cuml.datasets.classification.make_classification
const make_regression = cuml.datasets.regression.make_regression

# Clustering
X = rand(1000, 5)
@testset "KMeans" begin
    model = KMeans()
    mach = machine(model, X)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "DBSCAN" begin
    model = DBSCAN()
    mach = machine(model, X)
    fit!(mach)
end

@testset "AgglomerativeClustering" begin
    model = AgglomerativeClustering()
    mach = machine(model, X)
    fit!(mach)
end

@testset "HDBSCAN" begin
    model = HDBSCAN()
    mach = machine(model, X)
    fit!(mach)
end

# Regression
X_py, y_py = make_regression(n_samples=200, n_features=12,
                               n_informative=7, bias=-4.2, noise=0.3)
X = RAPIDS.pyconvert(Matrix{Float32}, X_py.get())
y = RAPIDS.pyconvert(Vector{Float32}, y_py.get().flatten())

@testset "LinearRegression" begin
    model = LinearRegression()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "Ridge" begin
    model = Ridge()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "Lasso" begin
    model = Lasso()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "ElasticNet" begin
    model = ElasticNet()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "MBSGDRegressor" begin
    model = MBSGDRegressor()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "RandomForestRegressor" begin
    model = RandomForestRegressor()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "CD" begin
    model = CD()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "SVR" begin
    model = SVR()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "LinearSVR" begin
    model = LinearSVR()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "KNeighborsRegressor" begin
    model = KNeighborsRegressor()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

# Classification
X_py, y_py = make_classification(n_samples=200, n_features=4,
                           n_informative=2, n_classes=2)
X = RAPIDS.pyconvert(Matrix{Float32}, X_py.get())
y = RAPIDS.pyconvert(Vector{Float32}, y_py.get().flatten())

@testset "LogisticRegression" begin
    model = LogisticRegression()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "MBSGDClassifier" begin
    model = MBSGDClassifier()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

@testset "KNeighborsClassifier" begin
    model = KNeighborsClassifier()
    mach = machine(model, X, y)
    fit!(mach)
    preds = predict(mach, X)
end

# Dimensionality reduction
X = rand(1000, 5)

@testset "PCA" begin
    model = PCA()
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
    inverse_transform(mach, X)
end

@testset "IncrementalPCA" begin
    model = IncrementalPCA()
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
end

@testset "TruncatedSVD" begin
    model = TruncatedSVD(n_components=2)
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
    inverse_transform(mach, X)
end

@testset "UMAP" begin
    model = UMAP(n_components=2)
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
end

@testset "GaussianRandomProjection" begin
    model = GaussianRandomProjection(n_components=2)
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
end

@testset "TSNE" begin
    model = TSNE(n_components=2)
    mach = machine(model, X)
    fit!(mach)
    X_trans = transform(mach, X)
end

# Time Series
X = [1, 2, 3, 4, 5, 6,
    7, 8, 9, 10, 11, 12,
    2, 3, 4, 5, 6, 7,
    8, 9, 10, 11, 12, 13,
    3, 4, 5, 6, 7, 8, 9,
    10, 11, 12, 13, 14]
@testset "ExponentialSmoothing" begin
    model = ExponentialSmoothing()
    mach = machine(model, X)
    fit!(mach)
    forecast(mach, 4)
end


text:
# Note that this script can accept some limited command-line arguments, run
# `julia build_tarballs.jl --help` to see a usage message.
using BinaryBuilder, Pkg

name = "casacorewrapper"
version = v"0.1.0"

# Collection of sources required to complete build
sources = [
    GitSource("https://github.com/kiranshila/casascorewrapper.git", "d7574e8639c8ac55a360652e4e799a4686bec93c")
]

# Bash recipe for building across all platforms
script = raw"""
cd $WORKSPACE/srcdir/casascorewrapper/
mkdir build && cd build
cmake \
    -DCMAKE_INSTALL_PREFIX=$prefix \
    -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \
    -DCMAKE_BUILD_TYPE=Release \
    -DCASACORE_ROOT_DIR=${prefix} \
    ..
make -j${nproc}
make install
"""

# Use the same platforms from casacore
platforms = supported_platforms(exclude=(platform)-> Sys.iswindows(platform) || Sys.isfreebsd(platform))
platforms = expand_cxxstring_abis(platforms)
# Expand libgfortran versions because we need to exclude some specific combinations
filter!(p -> libc(p) != "musl", platforms)

# The products that we will ensure are always built
products = [
    LibraryProduct("libcasacorewrapper", :libcasacorewrapper)
]

# Dependencies that must be installed before this package can be built
dependencies = [
    Dependency(PackageSpec(name="casacore_jll", uuid="72fd12c2-f19b-5d3c-931a-6bbe5223e3ce"))
]

# Build the tarballs, and possibly a `build.jl` as well.
build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version = v"7")

max_stars_repo_name: staticfloat/BinaryBuilderRepos

max_issues_repo_name: staticfloat/BinaryBuilderRepos

max_forks_repo_name: staticfloat/BinaryBuilderRepos


text:
module TabularTMazeExperiment

import Flux: Descent, ADAM
import Random

using GVFHordes
using Curiosity
using MinimalRLCore
using SparseArrays
using ProgressMeter

const TTMU = Curiosity.TabularTMazeUtils
const BU = Curiosity.BaselineUtils
const SRCU = Curiosity.SRCreationUtils

default_args() =
    Dict(
        # Behaviour Items
        # "behaviour_eta" => 0.50,
        "behaviour_gamma" => 0.9,
        "behaviour_learner" => "Q",
        "behaviour_update" => "TabularRoundRobin",
        "behaviour_trace" => "AccumulatingTraces",
        "behaviour_opt" => "Auto",
        "behaviour_lambda" => 0.9,
        "behaviour_alpha_init" => 0.1,
        "exploration_param" => 0.1,
        "exploration_strategy" => "epsilon_greedy",
        "ϵ_range" => (0.4,0.1),
        "decay_period" => 1000,
        "warmup_steps" => 100,
        "behaviour_w_init" => 10.0,

        # Demon Attributes
        "demon_alpha_init" => 0.5,
        # "demon_eta" => 0.25,
        "demon_discounts" => 0.9,
        "demon_learner" => "SR",
        "demon_update" => "TB",
        "demon_interest_set" => "ttmaze",
        "demon_policy_type" => "greedy_to_cumulant",
        "demon_opt" => "Auto",
        "demon_lambda" => 0.9,
        "demon_trace"=> "AccumulatingTraces",
        "demon_beta_m" => 0.9,
        "demon_beta_v" => 0.99,

        #Shared Demon and Behaviour Attributes
        "eta" =>0.2,

        # Environment Config
        "constant_target"=> (-10,10),
        "cumulant_schedule" => "DrifterDistractor",
        "distractor" => (1.0, 5.0),
        "drifter" => (1.0, sqrt(0.01)),
        "exploring_starts" => true,
        "env_step_penalty" => 0.0,

        # Agent and Logger
        "horde_type" => "regular",
        "intrinsic_reward" => "weight_change",
        "logger_keys" => [LoggerKey.TTMAZE_ERROR, LoggerKey.TTMAZE_UNIFORM_ERROR,
                            LoggerKey.TTMAZE_OLD_ERROR, LoggerKey.TTMAZE_ROUNDROBIN_ERROR, LoggerKey.GOAL_VISITATION,
                            LoggerKey.EPISODE_LENGTH,
                            LoggerKey.TTMAZE_DIRECT_ERROR],
        "save_dir" => "TabularTMazeExperiment",
        "seed" => 1,
        "steps" => 5000,
        "use_external_reward" => true,
        "logger_interval" => 100,
        "random_first_action" => true,

    )

function construct_agent(parsed)

    feature_size = 21
    action_space = 4
    observation_size = 5
    demon_alpha_init = parsed["demon_alpha_init"]
    demon_learner = parsed["demon_learner"]
    demon_lu = parsed["demon_update"]

    behaviour_learner = parsed["behaviour_learner"]
    behaviour_lu = parsed["behaviour_update"]
    behaviour_discount = parsed["behaviour_gamma"]

    intrinsic_reward_type = parsed["intrinsic_reward"]
    behaviour_trace = parsed["behaviour_trace"]
    use_external_reward = parsed["use_external_reward"]

    #Create state constructor
    function state_constructor(observation, feature_size)
        s = spzeros(feature_size)
        s[convert(Int64,observation[1])] = 1
        return s
    end
    state_constructor_func = (obs) -> state_constructor(obs, feature_size)

    function compressed_state_constructor(obs)
        s = spzeros(4)
        if obs[1] < 5
            s[1] = 1
        elseif obs[1] < 8
            s[2] = 1
        elseif obs[1] < 15
            s[3] = 1
        elseif obs[1] < 17
            s[2] = 1
        else
            s[4] = 1
        end
        return s
    end

    behaviour_feature_projector = Curiosity.FeatureSubset(TTMU.StateActionFeatures(feature_size,action_space),1:1)
    demon_feature_projector = behaviour_feature_projector

    demons = get_horde(parsed, length(demon_feature_projector), action_space, demon_feature_projector)

    exploration_strategy = Curiosity.get_exploration_strategy(parsed, 1:action_space)

    demon_learner = Curiosity.get_linear_learner(parsed,
                                                 feature_size,
                                                 action_space,
                                                 demons,
                                                 "demon",
                                                 demon_feature_projector)

    # TODO: Behaviour horde needs access to the behaviour learner to condition the behaviour policy
    # BUT behaviour learner needs access the horde to know things like how many demons there are.
    behaviour_num_tasks = 1
    num_SFs = 4
    num_demons = if parsed["behaviour_learner"] ∈ ["GPI"]
        num_SFs * feature_size * action_space + 1
    elseif parsed["behaviour_learner"] ∈ ["Q", "FollowDemon", "RandomDemons"]
        1
    else
        println("Invalid behaviour learner. Num demons is not defined")
    end

    behaviour_learner = if parsed["behaviour_learner"] == "FollowDemon"
        demon_i = 3
        action_set = 1:action_space
        drifter_demon = GVF(GVFParamFuncs.FeatureCumulant(demon_i+1),
                            GVFParamFuncs.StateTerminationDiscount(parsed["demon_discounts"], TTMU.pseudoterm),
                            GVFParamFuncs.FunctionalPolicy((;kwargs...) -> TTMU.demon_target_policy(demon_i;kwargs...)))
        BU.FollowDemon(drifter_demon,action_set)
    elseif parsed["behaviour_learner"] == "RandomDemons"
        action_set = 1:action_space
        BU.RandomDemons(demons, action_set)
    else
            Curiosity.get_linear_learner(parsed,
                        feature_size,
                        action_space,
                        num_demons,
                        behaviour_num_tasks,
                        "behaviour",
                        behaviour_feature_projector)
    end

    behaviour_gvf = if behaviour_learner isa GPI
        #behaviour discount for immediate reward predictor for GPI should always be 0.
        TTMU.make_behaviour_gvf(0.0, state_constructor_func, behaviour_learner, exploration_strategy)
    elseif behaviour_learner isa QLearner
        TTMU.make_behaviour_gvf(behaviour_discount, state_constructor_func, behaviour_learner, exploration_strategy)
    elseif behaviour_learner isa BU.FollowDemon || behaviour_learner isa BU.RandomDemons
        TTMU.make_behaviour_gvf(behaviour_discount, state_constructor_func, behaviour_learner, exploration_strategy)
    else
        throw(ArgumentError("What other type of behaviour learner??"))
    end
    behaviour_demons = if behaviour_learner isa GPI
        SF_discounts = [GVFParamFuncs.StateTerminationDiscount(behaviour_discount, TTMU.pseudoterm) for i in 1:4]
        SF_policies = [GVFParamFuncs.FunctionalPolicy((;kwargs...) ->TTMU.demon_target_policy(i;kwargs...)) for i in 1:4]
        SF_horde = SRCU.create_SF_horde(SF_policies,SF_discounts,behaviour_feature_projector,1:4)

        pred_horde = Horde([behaviour_gvf])

        Curiosity.GVFSRHordes.SRHorde(pred_horde, SF_horde, num_SFs, behaviour_feature_projector)
    elseif behaviour_learner isa QLearner
        Horde([behaviour_gvf])
    elseif behaviour_learner isa BU.FollowDemon || behaviour_learner isa BU.RandomDemons
        Horde([behaviour_gvf])
    else
        throw(ArgumentError("goes with which horde? " ))
    end

    random_first_action = parsed["random_first_action"]
    if behaviour_learner isa TabularRoundRobin
        if random_first_action == false
            @warn "Round Robin with random first action is recommended"
        end
    elseif random_first_action == true
        @warn "Random first action is enabled"
    end

    Agent(demons,
          feature_size,
          behaviour_learner,
          behaviour_demons,
          behaviour_discount,
          demon_learner,
          observation_size,
          action_space,
          intrinsic_reward_type,
          (obs) -> state_constructor(obs, feature_size),
          use_external_reward,
          exploration_strategy,
          random_first_action)
end

function get_horde(parsed, feature_size, action_space, projected_feature_constructor)

    discount = parsed["demon_discounts"]
    pseudoterm = TTMU.pseudoterm
    num_actions = TTMU.NUM_ACTIONS
    num_demons = TTMU.NUM_DEMONS

    #TODO: Sort out the if-else block so that demon_policy_type and horde_type is not blocking eachother.
    horde = if parsed["demon_policy_type"] == "greedy_to_cumulant" && parsed["demon_learner"] != "SR"
        Horde([GVF(GVFParamFuncs.FeatureCumulant(i+1), GVFParamFuncs.StateTerminationDiscount(discount, pseudoterm), GVFParamFuncs.FunctionalPolicy((;kwargs...) -> TTMU.demon_target_policy(i;kwargs...))) for i in 1:num_demons])
    elseif parsed["demon_policy_type"] == "greedy_to_cumulant" && parsed["demon_learner"] == "SR"
        Horde([GVF(GVFParamFuncs.FeatureCumulant(i+1), GVFParamFuncs.ConstantDiscount(0.0), GVFParamFuncs.FunctionalPolicy((;kwargs...) -> TTMU.demon_target_policy(i;kwargs...))) for i in 1:num_demons])
    elseif parsed["demon_policy_type"] == "random"
        Horde([GVF(GVFParamFuncs.FeatureCumulant(i+1), GVFParamFuncs.StateTerminationDiscount(discount, pseudoterm), GVFParamFuncs.RandomPolicy(fill(1/num_actions,num_actions))) for i in 1:num_demons])
    else
        throw(ArgumentError("Not a valid policy type for demons"))
    end

    if parsed["demon_learner"] == "SR"
        num_SFs = 4
        SF_discounts = [GVFParamFuncs.StateTerminationDiscount(discount, TTMU.pseudoterm) for i in 1:4]
        SF_policies = [GVFParamFuncs.FunctionalPolicy((;kwargs...) ->TTMU.demon_target_policy(i;kwargs...)) for i in 1:4]
        SF_horde = SRCU.create_SF_horde(SF_policies,SF_discounts,projected_feature_constructor,1:4)

        horde = Curiosity.GVFSRHordes.SRHorde(horde, SF_horde, num_SFs, projected_feature_constructor)
    end

    return horde
end

function main_experiment(parsed=default_args(); progress=false, working=false)

    num_steps = parsed["steps"]
    Random.seed!(parsed["seed"])

    cumulant_schedule = TTMU.get_cumulant_schedule(parsed)
    exploring_starts = parsed["exploring_starts"]
    env = TabularTMaze(exploring_starts, cumulant_schedule)

    if "eta" in keys(parsed)
        prefixes = ["behaviour","demon"]
        for prefix in prefixes
            parsed[join([prefix, "eta"], "_")] = parsed["eta"]
        end
    end
    if "alpha_init" in keys(parsed)
        prefixes = ["behaviour","demon"]
        for prefix in prefixes
            parsed[join([prefix, "alpha_init"], "_")] = parsed["alpha_init"]
        end
    end

    agent = construct_agent(parsed)

    goal_visitations = zeros(4)

    logger_init_dict = Dict(
        LoggerInitKey.TOTAL_STEPS => num_steps,
        LoggerInitKey.INTERVAL => parsed["logger_interval"],
        LoggerInitKey.ENV => "tabular_tmaze"
    )

    Curiosity.experiment_wrapper(parsed, logger_init_dict, working) do parsed, logger
        eps = 1
        max_num_steps = num_steps
        steps = Int[]

        if progress
            p = Progress(max_num_steps)
        end

        logger_start!(logger, env, agent)
        while sum(steps) < max_num_steps
            cur_step = 0
            max_episode_steps = max_num_steps - sum(steps)
            tr, stp =
                run_episode!(env, agent, max_episode_steps) do (s, a, s_next, r, t)
                    #This is a callback for every timestep where logger can go
                    # agent is accesible in this scope

                    if t == true && working==true && t
                        goals = s_next[2:end]
                        f = findfirst(!iszero, goals)
                        goal_visitations[f] += 1
                    end

                    logger_step!(logger, env, agent, s, a, s_next, r, t)
                    cur_step+=1
                end
                logger_episode_end!(logger)
            push!(steps, stp)
            eps += 1

            if progress
                ProgressMeter.update!(p, sum(steps))
            end
        end
        if working == true
            println(goal_visitations)
        end
    end

end

end

max_stars_repo_name: MatthewMcLeod/curiosity

max_issues_repo_name: MatthewMcLeod/curiosity

max_forks_repo_name: MatthewMcLeod/curiosity


text:
###############################################################################
# Deprecations for calling impute on an Imputor with a custom AbstractContext #
###############################################################################
Base.@deprecate(
    impute(imp::Imputor, context::AbstractContext, data; kwargs...),
    impute(data, typeof(imp)(; context=context, kwargs...))
)

Base.@deprecate(
    impute!(imp::Imputor, context::AbstractContext, data; kwargs...),
    impute!(data, typeof(imp)(; context=context, kwargs...))
)

Base.@deprecate impute(imp::Imputor, data) impute(data, imp)
Base.@deprecate impute!(imp::Imputor, data) impute!(data, imp)

#####################################################################
# Deprecate all impute calls where the first argument is an Imputor #
#####################################################################
"""
    impute!(data, method::Symbol=:interp, args...; limit::Float64=0.1)

Looks up the `Imputor` type for the `method`, creates it and calls
`impute!(data, imputor::Imputor)` with it.

# Arguments
* `data`: the datset containing missing elements we should impute.
* `method::Symbol`: the imputation method to use
    (options: [`:drop`, `:fill`, `:interp`, `:locf`, `:nocb`])
* `args::Any...`: any arguments you should pass to the `Imputor` constructor.
* `limit::Float64`: missing data ratio limit/threshold (default: 0.1)
"""
function impute!(data, method::Symbol, args...; limit::Float64=0.1)
    Base.depwarn(
        """
        impute!(data, method) is deprecated.
        Please use Impute.method!(data) or impute!(data, imputor::Imputor).
        """,
        :impute!
    )
    imputor_type = imputation_methods[method]
    imputor = if length(args) > 0
        imputor_type(args...; context=Context(; limit=limit))
    else
        imputor_type(; context=Context(; limit=limit))
    end

    return impute!(data, imputor)
end

"""
    impute!(data, missing::Function, method::Symbol=:interp, args...; limit::Float64=0.1)

Creates the appropriate `Imputor` type and `Context` (using `missing` function) in order to call
`impute!(data, imputor::Imputor)` with them.

# Arguments
* `data`: the datset containing missing elements we should impute.
* `missing::Function`: the missing data function to use
* `method::Symbol`: the imputation method to use
    (options: [`:drop`, `:fill`, `:interp`, `:locf`, `:nocb`])
* `args::Any...`: any arguments you should pass to the `Imputor` constructor.
* `limit::Float64`: missing data ratio limit/threshold (default: 0.1)
"""
function impute!(data, missing::Function, method::Symbol, args...; limit::Float64=0.1)
    Base.depwarn(
        """
        impute!(data, missing, method) is deprecated. Please use impute!(data, imputor::Imputor).
        """,
        :impute!
    )
    imputor_type = imputation_methods[method]
    imputor = if length(args) > 0
        imputor_type(args...; context=Context(; is_missing=missing, limit=limit))
    else
        imputor_type(; context=Context(; is_missing=missing, limit=limit))
    end

    return impute!(data, imputor)
end

"""
    impute(data, args...; kwargs...)

Copies the `data` before calling `impute!(new_data, args...; kwargs...)`
"""
function impute(data, args...; kwargs...)
    Base.depwarn(
        """
        impute(data, args...; kwargs...) is deprecated.
        Please use Impute.method(data) or impute(data, imputor::Imputor).
        """,
        :impute
    )
    # Call `deepcopy` because we can trust that it's available for all types.
    return impute!(deepcopy(data), args...; kwargs...)
end

#################################
# Deprecate the chain functions #
#################################
"""
    chain!(data, missing::Function, imputors::Imputor...; kwargs...)

Creates a `Chain` with `imputors` and calls `impute!(imputor, missing, data; kwargs...)`
"""
function chain!(data, missing::Function, imputors::Imputor...; kwargs...)
    Base.depwarn(
        """
        chain!(data, missing, imputors...) is deprecated.
        Please use data = imp1(data) |> imp2 |> imp3
        """,
        :chain!
    )
    return chain!(data, imputors...; is_missing=missing, kwargs...)
end

"""
    chain!(data, imputors::Imputor...; kwargs...)

Creates a `Chain` with `imputors` and calls `impute!(data, imputor)`
"""
function chain!(data, imputors::Imputor...; kwargs...)
    Base.depwarn(
        """
        chain!(data, imputors...) is deprecated.
        Please use data = imp1(data) |> imp2 |> imp3
        """,
        :chain!
    )
    ctx = Context(; kwargs...)

    for imputor in imputors
        imp = typeof(imputor)(
            (isa(x, AbstractContext) ? ctx : x for x in fieldvalues(imputor))...
        )
        data = impute!(data, imp)
    end

    return data
end

"""
    chain(data, args...; kwargs...)

Copies the `data` before calling `chain!(data, args...; kwargs...)`
"""
function chain(data, args...; kwargs...)
    Base.depwarn(
        """
        chain(data, args...) is deprecated.
        Please use result = imp1(data) |> imp2 |> imp3
        """,
        :chain
    )
    # Call `deepcopy` because we can trust that it's available for all types.
    return chain!(deepcopy(data), args...; kwargs...)
end

#####################
# Misc Deprecations #
#####################
Base.@deprecate Fill(val; kwargs...) Fill(; value=val, kwargs...)
Base.@deprecate_binding Drop DropObs false

# This function is just used to support legacy behaviour and should be removed in a
# future release when we dropping accepting the limit kwarg to impute functions.
function _extract_context_kwargs(kwargs...)
    d = Dict{Symbol, Any}(kwargs...)
    limit = 1.0

    if haskey(d, :limit)
        limit = d[:limit]
        @warn(
            "Passing `limit` directly to impute functions is deprecated. " *
            "Please pass `context=Context(; limit=$limit)` in the future."
        )
        delete!(d, :limit)
    end

    if !haskey(d, :context)
        d[:context] = Context(; limit=limit)
    end

    return d
end


text:
import Pkg
Pkg.add(Pkg.PackageSpec(name="Mustache", version="0.5.12"))

import JSON2
import Mustache

template = """
#=
This file is auto-generated. Do not edit.
=#

{{#docstring}}\"\"\"{{docstring}}\"\"\"{{/docstring}}
mutable struct {{struct_name}}{{#parametric}}{T <: {{parametric}}}{{/parametric}} <: {{supertype}}
    {{#parameters}}
    {{name}}::{{data_type}}{{#comment}}  # {{{comment}}}{{/comment}}
    {{/parameters}}
    {{#inner_constructor_check}}

    function {{struct_name}}({{#parameters}}{{name}}, {{/parameters}})
        ({{#parameters}}{{name}}, {{/parameters}}) = {{inner_constructor_check}}(
            {{#parameters}}
            {{name}},
            {{/parameters}}
        )
        new({{#parameters}}{{name}}, {{/parameters}})
    end
    {{/inner_constructor_check}}
end

function {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}})
    {{#parameters}}
    {{/parameters}}
    {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}}PowerSystemInternal())
end

function {{struct_name}}(; {{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}})
    {{struct_name}}({{#parameters}}{{^internal}}{{name}}, {{/internal}}{{/parameters}})
end

{{#has_null_values}}
# Constructor for demo purposes; non-functional.

function {{struct_name}}(::Nothing)
    {{struct_name}}(;
        {{#parameters}}
        {{^internal}}
        {{name}}={{#quotes}}"{{null_value}}"{{/quotes}}{{^quotes}}{{null_value}}{{/quotes}},
        {{/internal}}
        {{/parameters}}
    )
end
{{/has_null_values}}

{{#accessors}}
\"\"\"Get {{struct_name}} {{name}}.\"\"\"
{{accessor}}(value::{{struct_name}}) = value.{{name}}
{{/accessors}}
"""

function read_json_data(filename::String)
    return open(filename) do io
        data = JSON2.read(io, Vector{Dict})
    end
end

function generate_structs(directory, data::Vector)
    struct_names = Vector{String}()
    unique_accessor_functions = Set{String}()

    for item in data
        accessors = Vector{Dict}()
        item["has_null_values"] = true
        parameters = Vector{Dict}()
        for field in item["fields"]
            param = namedtuple_to_dict(field)
            push!(parameters, param)

            accessor_name = "get_" * param["name"]
            push!(accessors, Dict("name" => param["name"], "accessor" => accessor_name))
            if accessor_name != "internal"
                push!(unique_accessor_functions, accessor_name)
            end

            if param["name"] == "internal"
                param["internal"] = true
                continue
            end

            # This controls whether a kwargs constructor will be generated.
            if !haskey(param, "null_value")
                item["has_null_values"] = false
            else
                if param["data_type"] == "String"
                    param["quotes"] = true
                end
            end
            param["struct_name"] = item["struct_name"]
        end

        item["parameters"] = parameters
        item["accessors"] = accessors

        filename = joinpath(directory, item["struct_name"] * ".jl")
        open(filename, "w") do io
            write(io, Mustache.render(template, item))
            push!(struct_names, item["struct_name"])
        end
        println("Wrote $filename")
    end

    accessors = sort!(collect(unique_accessor_functions))

    filename = joinpath(directory, "includes.jl")
    open(filename, "w") do io
        for name in struct_names
            write(io, "include(\"$name.jl\")\n")
        end
        write(io, "\n")

        for accessor in accessors
            write(io, "export $accessor\n")
        end
        println("Wrote $filename")
    end
end

function namedtuple_to_dict(tuple)
    parameters = Dict()
    for property in propertynames(tuple)
        parameters[string(property)] = getproperty(tuple, property)
    end

    return parameters
end

function generate_structs(input_file::AbstractString, output_directory::AbstractString)
    # Include each generated file.
    if !isdir(output_directory)
        mkdir(output_directory)
    end

    data = read_json_data(input_file)
    generate_structs(output_directory, data)
end

function main(args)
    if length(args) != 2
        println("Usage: julia generate_structs.jl INPUT_FILE OUTPUT_DIRECTORY")
        exit(1)
    end

    generate_structs(args[1], args[2])
end

main(ARGS)


text:
"""
Module for visualizing water networks, and potentially other graphics
"""
module WaterModelsAnalytics
    # imports
    import WaterModels
    const _WM = WaterModels
    const _IM = WaterModels._IM

    # importing only the viridis color scheme for now; might allow the color scheme to be
    # selected by the user at some point, in which case will need to import all of ColorShemes
    import ColorSchemes.viridis
    import ColorTypes.HSV
    import Memento
    import Statistics.mean
    using Printf

    import Plots

    # Python imports.
    import PyCall
    import PyCall: PyObject, @pycall
    const pgv = PyCall.PyNULL()

    # Create our module-level logger (this will get precompiled).
    const _LOGGER = Memento.getlogger(@__MODULE__)

    function __init__()
        copy!(pgv, PyCall.pyimport("pygraphviz"))

        # Register the module-level logger at runtime so users can access the logger via
        # `getlogger(WaterModelsAnalytics)` NOTE: If this line is not included, then the
        # precompiled `WaterModelsAnalytics._LOGGER` will not be registered at runtime.
        Memento.register(_LOGGER)
    end

    "Suppresses information and warning messages output by WaterModelsAnalytics. For more
    fine-grained control, use the Memento package."
    function silence()
        Memento.info(_LOGGER, "Suppressing information and warning messages for "
            * "the rest of this session. Use the Memento package for more "
            * "fine-grained control of logging.")
        Memento.setlevel!(Memento.getlogger(_IM), "error")
        Memento.setlevel!(Memento.getlogger(_WM), "error")
    end

    "Allows the user to set the logging level without the need to add Memento."
    function logger_config!(level)
        Memento.config!(Memento.getlogger("WaterModelsAnalytics"), level)
    end

    include("graph/common.jl")

    export build_graph
    export write_graph
    export colorbar
    export write_visualization
end


text:
# ---
# title: 152. Maximum Product Subarray
# id: problem152
# author: Indigo
# date: 2021-01-19
# difficulty: Medium
# categories: Array, Dynamic Programming
# link: <https://leetcode.com/problems/maximum-product-subarray/description/>
# hidden: true
# ---
#
# Given an integer array `nums`, find the contiguous subarray within an array
# (containing at least one number) which has the largest product.
#
# **Example 1:**
#
#
#
#     Input: [2,3,-2,4]
#     Output: 6
#     Explanation:  [2,3] has the largest product 6.
#
#
# **Example 2:**
#
#
#
#     Input: [-2,0,-1]
#     Output: 0
#     Explanation:  The result cannot be 2, because [-2,-1] is not a subarray.
#
#
## @lc code=start
using LeetCode

function max_product(nums::Vector{Int})
    maxF = minF = res = nums[1]
    for num in @view nums[2:end]
        maxF, minF = max(maxF * num, minF * num, num), min(minF * num, maxF * num, num)
        res = max(res, maxF)
    end
    res
end
## @lc code=end


text:
using SafeTestsets

@safetestset "Tests de Lectura de ficheros" begin
    include("lectura_tests.jl")
end

max_stars_repo_name: vntr-CC/Forward-Football

max_issues_repo_name: vntr-CC/Forward-Football

max_forks_repo_name: vntr-CC/Forward-Football


text:
# Arrays on GPU

typealias CUdeviceptr Cuint

immutable CuPtr
	p::CUdeviceptr

	CuPtr() = new(convert(CUdeviceptr, 0))
	CuPtr(p::CUdeviceptr) = new(p)
end

cubox(p::CuPtr) = cubox(p.p)

function cualloc(T::Type, len::Integer)
	a = CUdeviceptr[0]
	nbytes = int(len) * sizeof(T)
	@cucall(:cuMemAlloc, (Ptr{CUdeviceptr}, Csize_t), a, nbytes)
	return CuPtr(a[1])
end

function free(p::CuPtr)
	@cucall(:cuMemFree, (CUdeviceptr,), p.p)
end

isnull(p::CuPtr) = (p.p == 0)

#################################################
#
#  CuArray: contiguous array on GPU
#
#################################################

type CuArray{T,N}
	ptr::CuPtr
	shape::NTuple{N,Int}
	len::Int
end

function CuArray(T::Type, len::Integer)
	n = int(len)
	p = cualloc(T, n)
	CuArray{T,1}(p, (n,), n)
end

function CuArray{N}(T::Type, shape::NTuple{N,Int})
	n = prod(shape)
	p = cualloc(T, n)
	CuArray{T,N}(p, shape, n)
end

cubox(a::CuArray) = cubox(a.ptr)

length(g::CuArray) = g.len
size(g::CuArray) = g.shape
ndims{T,N}(g::CuArray{T,N}) = N
eltype{T,N}(g::CuArray{T,N}) = T

function size{T,N}(g::CuArray{T,N}, d::Integer)
	d >= 1 ? (d <= N ? g.shape[d] : 1) : error("Invalid index of dimension.")
end

function free(g::CuArray)
	if !isnull(g.ptr)
		free(g.ptr)
		g.ptr = CuPtr()
	end
end

function copy!{T}(dst::Array{T}, src::CuArray{T})
	if length(dst) != length(src)
		throw(ArgumentError("Inconsistent array length."))
	end
	nbytes = length(src) * sizeof(T)
	@cucall(:cuMemcpyDtoH, (Ptr{Void}, CUdeviceptr, Csize_t), pointer(dst), src.ptr.p, nbytes)
	return dst
end

function copy!{T}(dst::CuArray{T}, src::Array{T})
	if length(dst) != length(src)
		throw(ArgumentError("Inconsistent array length."))
	end
	nbytes = length(src) * sizeof(T)
	@cucall(:cuMemcpyHtoD, (CUdeviceptr, Ptr{Void}, Csize_t), dst.ptr.p, pointer(src), nbytes)
	return dst
end

CuArray{T,N}(a::Array{T,N}) = copy!(CuArray(T, size(a)), a)
to_host{T}(g::CuArray{T}) = copy!(Array(T, size(g)), g)


text:
@compat abstract type Estimator end

@compat abstract type LGBMEstimator <: Estimator end

type LGBMRegression <: LGBMEstimator
    booster::Booster
    model::Vector{String}
    application::String

    num_iterations::Int
    learning_rate::Float64
    num_leaves::Int
    max_depth::Int
    tree_learner::String
    num_threads::Int
    histogram_pool_size::Float64

    min_data_in_leaf::Int
    min_sum_hessian_in_leaf::Float64
    lambda_l1::Float64
    lambda_l2::Float64
    min_gain_to_split::Float64
    feature_fraction::Float64
    feature_fraction_seed::Int
    bagging_fraction::Float64
    bagging_freq::Int
    bagging_seed::Int
    early_stopping_round::Int

    max_bin::Int
    data_random_seed::Int
    init_score::String
    is_sparse::Bool
    save_binary::Bool
    categorical_feature::Vector{Int}

    is_unbalance::Bool

    metric::Vector{String}
    metric_freq::Int
    is_training_metric::Bool
    ndcg_at::Vector{Int}

    num_machines::Int
    local_listen_port::Int
    time_out::Int
    machine_list_file::String

    num_class::Int
end

"""
    LGBMRegression(; [num_iterations = 10,
                      learning_rate = .1,
                      num_leaves = 127,
                      max_depth = -1,
                      tree_learner = \"serial\",
                      num_threads = Sys.CPU_CORES,
                      histogram_pool_size = -1.,
                      min_data_in_leaf = 100,
                      min_sum_hessian_in_leaf = 10.,
                      lambda_l1 = 0.,
                      lambda_l2 = 0.,
                      min_gain_to_split = 0.,
                      feature_fraction = 1.,
                      feature_fraction_seed = 2,
                      bagging_fraction = 1.,
                      bagging_freq = 0,
                      bagging_seed = 3,
                      early_stopping_round = 0,
                      max_bin = 255,
                      data_random_seed = 1,
                      init_score = \"\",
                      is_sparse = true,
                      save_binary = false,
                      categorical_feature = Int[],
                      is_unbalance = false,
                      metric = [\"l2\"],
                      metric_freq = 1,
                      is_training_metric = false,
                      ndcg_at = Int[],
                      num_machines = 1,
                      local_listen_port = 12400,
                      time_out = 120,
                      machine_list_file = \"\"])

Return a LGBMRegression estimator.
"""
function LGBMRegression(; num_iterations = 10,
                        learning_rate = .1,
                        num_leaves = 127,
                        max_depth = -1,
                        tree_learner = "serial",
                        num_threads = Sys.CPU_CORES,
                        histogram_pool_size = -1.,
                        min_data_in_leaf = 100,
                        min_sum_hessian_in_leaf = 10.,
                        lambda_l1 = 0.,
                        lambda_l2 = 0.,
                        min_gain_to_split = 0.,
                        feature_fraction = 1.,
                        feature_fraction_seed = 2,
                        bagging_fraction = 1.,
                        bagging_freq = 0,
                        bagging_seed = 3,
                        early_stopping_round = 0,
                        max_bin = 255,
                        data_random_seed = 1,
                        init_score = "",
                        is_sparse = true,
                        save_binary = false,
                        categorical_feature = Int[],
                        is_unbalance = false,
                        metric = ["l2"],
                        metric_freq = 1,
                        is_training_metric = false,
                        ndcg_at = Int[],
                        num_machines = 1,
                        local_listen_port = 12400,
                        time_out = 120,
                        machine_list_file = "")

    @assert(in(tree_learner, ("serial", "feature", "data")),
            "Unknown tree_learner, got $tree_learner")
    foreach(m -> @assert(in(m, ("l1", "l2", "ndcg", "auc", "binary_logloss", "binary_error",
                                "multi_logloss", "multi_error")),
                         "Unknown metric, got $m"), metric)

    return LGBMRegression(Booster(), String[], "regression", num_iterations, learning_rate, num_leaves,
                          max_depth, tree_learner, num_threads, histogram_pool_size,
                          min_data_in_leaf, min_sum_hessian_in_leaf, lambda_l1, lambda_l2,
                          min_gain_to_split, feature_fraction, feature_fraction_seed,
                          bagging_fraction, bagging_freq, bagging_seed, early_stopping_round,
                          max_bin, data_random_seed, init_score,
                          is_sparse, save_binary, categorical_feature,
                          is_unbalance, metric, metric_freq,
                          is_training_metric, ndcg_at, num_machines, local_listen_port, time_out,
                          machine_list_file, 1)
end

type LGBMBinary <: LGBMEstimator
    booster::Booster
    model::Vector{String}
    application::String

    num_iterations::Int
    learning_rate::Float64
    num_leaves::Int
    max_depth::Int
    tree_learner::String
    num_threads::Int
    histogram_pool_size::Float64

    min_data_in_leaf::Int
    min_sum_hessian_in_leaf::Float64
    lambda_l1::Float64
    lambda_l2::Float64
    min_gain_to_split::Float64
    feature_fraction::Float64
    feature_fraction_seed::Int
    bagging_fraction::Float64
    bagging_freq::Int
    bagging_seed::Int
    early_stopping_round::Int

    max_bin::Int
    data_random_seed::Int
    init_score::String
    is_sparse::Bool
    save_binary::Bool
    categorical_feature::Vector{Int}

    sigmoid::Float64
    is_unbalance::Bool

    metric::Vector{String}
    metric_freq::Int
    is_training_metric::Bool
    ndcg_at::Vector{Int}

    num_machines::Int
    local_listen_port::Int
    time_out::Int
    machine_list_file::String

    num_class::Int
end

"""
    LGBMBinary(; [num_iterations = 10,
                  learning_rate = .1,
                  num_leaves = 127,
                  max_depth = -1,
                  tree_learner = \"serial\",
                  num_threads = Sys.CPU_CORES,
                  histogram_pool_size = -1.,
                  min_data_in_leaf = 100,
                  min_sum_hessian_in_leaf = 10.,
                  lambda_l1 = 0.,
                  lambda_l2 = 0.,
                  min_gain_to_split = 0.,
                  feature_fraction = 1.,
                  feature_fraction_seed = 2,
                  bagging_fraction = 1.,
                  bagging_freq = 0,
                  bagging_seed = 3,
                  early_stopping_round = 0,
                  max_bin = 255,
                  data_random_seed = 1,
                  init_score = \"\",
                  is_sparse = true,
                  save_binary = false,
                  categorical_feature = Int[],
                  sigmoid = 1.,
                  is_unbalance = false,
                  metric = [\"binary_logloss\"],
                  metric_freq = 1,
                  is_training_metric = false,
                  ndcg_at = Int[],
                  num_machines = 1,
                  local_listen_port = 12400,
                  time_out = 120,
                  machine_list_file = \"\"])

Return a LGBMBinary estimator.
"""
function LGBMBinary(; num_iterations = 10,
                    learning_rate = .1,
                    num_leaves = 127,
                    max_depth = -1,
                    tree_learner = "serial",
                    num_threads = Sys.CPU_CORES,
                    histogram_pool_size = -1.,
                    min_data_in_leaf = 100,
                    min_sum_hessian_in_leaf = 10.,
                    lambda_l1 = 0.,
                    lambda_l2 = 0.,
                    min_gain_to_split = 0.,
                    feature_fraction = 1.,
                    feature_fraction_seed = 2,
                    bagging_fraction = 1.,
                    bagging_freq = 0,
                    bagging_seed = 3,
                    early_stopping_round = 0,
                    max_bin = 255,
                    data_random_seed = 1,
                    init_score = "",
                    is_sparse = true,
                    save_binary = false,
                    categorical_feature = Int[],
                    sigmoid = 1.,
                    is_unbalance = false,
                    metric = ["binary_logloss"],
                    metric_freq = 1,
                    is_training_metric = false,
                    ndcg_at = Int[],
                    num_machines = 1,
                    local_listen_port = 12400,
                    time_out = 120,
                    machine_list_file = "")

    @assert(in(tree_learner, ("serial", "feature", "data")),
            "Unknown tree_learner, got $tree_learner")
    foreach(m -> @assert(in(m, ("l1", "l2", "ndcg", "auc", "binary_logloss", "binary_error",
                                "multi_logloss", "multi_error")),
                         "Unknown metric, got $m"), metric)

    return LGBMBinary(Booster(), String[], "binary", num_iterations, learning_rate, num_leaves,
                      max_depth, tree_learner, num_threads, histogram_pool_size, min_data_in_leaf,
                      min_sum_hessian_in_leaf, lambda_l1, lambda_l2, min_gain_to_split,
                      feature_fraction, feature_fraction_seed, bagging_fraction, bagging_freq,
                      bagging_seed, early_stopping_round, max_bin, data_random_seed,
                      init_score, is_sparse, save_binary,
                      categorical_feature, sigmoid, is_unbalance, metric,
                      metric_freq, is_training_metric, ndcg_at, num_machines, local_listen_port,
                      time_out, machine_list_file, 1)
end

type LGBMMulticlass <: LGBMEstimator
    booster::Booster
    model::Vector{String}
    application::String

    num_iterations::Int
    learning_rate::Float64
    num_leaves::Int
    max_depth::Int
    tree_learner::String
    num_threads::Int
    histogram_pool_size::Float64

    min_data_in_leaf::Int
    min_sum_hessian_in_leaf::Float64
    lambda_l1::Float64
    lambda_l2::Float64
    min_gain_to_split::Float64
    feature_fraction::Float64
    feature_fraction_seed::Int
    bagging_fraction::Float64
    bagging_freq::Int
    bagging_seed::Int
    early_stopping_round::Int

    max_bin::Int
    data_random_seed::Int
    init_score::String
    is_sparse::Bool
    save_binary::Bool
    categorical_feature::Vector{Int}

    is_unbalance::Bool

    metric::Vector{String}
    metric_freq::Int
    is_training_metric::Bool
    ndcg_at::Vector{Int}

    num_machines::Int
    local_listen_port::Int
    time_out::Int
    machine_list_file::String

    num_class::Int
end

"""
    LGBMMulticlass(; [num_iterations = 10,
                      learning_rate = .1,
                      num_leaves = 127,
                      max_depth = -1,
                      tree_learner = \"serial\",
                      num_threads = Sys.CPU_CORES,
                      histogram_pool_size = -1.,
                      min_data_in_leaf = 100,
                      min_sum_hessian_in_leaf = 10.,
                      lambda_l1 = 0.,
                      lambda_l2 = 0.,
                      min_gain_to_split = 0.,
                      feature_fraction = 1.,
                      feature_fraction_seed = 2,
                      bagging_fraction = 1.,
                      bagging_freq = 0,
                      bagging_seed = 3,
                      early_stopping_round = 0,
                      max_bin = 255,
                      data_random_seed = 1,
                      init_score = \"\",
                      is_sparse = true,
                      save_binary = false,
                      categorical_feature = Int[],
                      is_unbalance = false,
                      metric = [\"multi_logloss\"],
                      metric_freq = 1,
                      is_training_metric = false,
                      ndcg_at = Int[],
                      num_machines = 1,
                      local_listen_port = 12400,
                      time_out = 120,
                      machine_list_file = \"\",
                      num_class = 1])

Return a LGBMMulticlass estimator.
"""
function LGBMMulticlass(; num_iterations = 10,
                        learning_rate = .1,
                        num_leaves = 127,
                        max_depth = -1,
                        tree_learner = "serial",
                        num_threads = Sys.CPU_CORES,
                        histogram_pool_size = -1.,
                        min_data_in_leaf = 100,
                        min_sum_hessian_in_leaf = 10.,
                        lambda_l1 = 0.,
                        lambda_l2 = 0.,
                        min_gain_to_split = 0.,
                        feature_fraction = 1.,
                        feature_fraction_seed = 2,
                        bagging_fraction = 1.,
                        bagging_freq = 0,
                        bagging_seed = 3,
                        early_stopping_round = 0,
                        max_bin = 255,
                        data_random_seed = 1,
                        init_score = "",
                        is_sparse = true,
                        save_binary = false,
                        categorical_feature = Int[],
                        is_unbalance = false,
                        metric = ["multi_logloss"],
                        metric_freq = 1,
                        is_training_metric = false,
                        ndcg_at = Int[],
                        num_machines = 1,
                        local_listen_port = 12400,
                        time_out = 120,
                        machine_list_file = "",
                        num_class = 1)

    @assert(in(tree_learner, ("serial", "feature", "data")),
            "Unknown tree_learner, got $tree_learner")
    foreach(m -> @assert(in(m, ("l1", "l2", "ndcg", "auc", "binary_logloss", "binary_error",
                                "multi_logloss", "multi_error")),
                         "Unknown metric, got $m"), metric)

    return LGBMMulticlass(Booster(), String[], "multiclass", num_iterations, learning_rate,
                          num_leaves, max_depth, tree_learner, num_threads, histogram_pool_size,
                          min_data_in_leaf, min_sum_hessian_in_leaf, lambda_l1, lambda_l2,
                          min_gain_to_split, feature_fraction, feature_fraction_seed,
                          bagging_fraction, bagging_freq, bagging_seed, early_stopping_round,
                          max_bin, data_random_seed, init_score,
                          is_sparse, save_binary, categorical_feature,
                          is_unbalance, metric, metric_freq,
                          is_training_metric, ndcg_at, num_machines, local_listen_port, time_out,
                          machine_list_file, num_class)
end


text:
# ImageHDU methods

# Display the image datatype and dimensions
function show(io::IO, hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    bitpix = fits_get_img_type(hdu.fitsfile)
    equivbitpix = fits_get_img_equivtype(hdu.fitsfile)
    sz = fits_get_img_size(hdu.fitsfile)

    if bitpix == equivbitpix
        datainfo = string(TYPE_FROM_BITPIX[equivbitpix])
    else
        datainfo = @sprintf("%s (physical: %s)",
                            TYPE_FROM_BITPIX[equivbitpix],
                            TYPE_FROM_BITPIX[bitpix])
    end

    print(io, """
    File: $(fits_file_name(hdu.fitsfile))
    HDU: $(hdu.ext)$(fits_get_ext_info_string(hdu.fitsfile))
    Type: Image
    Datatype: $datainfo
    Datasize: $(tuple(sz...))
    """)
end

# Get image dimensions
function ndims(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    fits_get_img_dim(hdu.fitsfile)
end

# Get image size
function size(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    tuple(sz...)
end

function size(hdu::ImageHDU, i::Integer)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    sz[i]
end

# `endof` is needed so that hdu[:] can throw DimensionMismatch
# when ndim != 1, rather than no method.
length(hdu::ImageHDU) = prod(size(hdu))
endof(hdu::ImageHDU) = length(hdu::ImageHDU)

# Read a full image from an HDU
function read(hdu::ImageHDU)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)
    bitpix = fits_get_img_equivtype(hdu.fitsfile)
    data = Array{TYPE_FROM_BITPIX[bitpix]}(sz...)
    fits_read_pix(hdu.fitsfile, data)
    data
end

# _checkbounds methods copied from Julia v0.4 Base.
_checkbounds(sz, i::Integer) = 1 <= i <= sz
_checkbounds(sz, i::Colon) = true
_checkbounds(sz, r::Range{Int}) =
    (isempty(r) || (minimum(r) >= 1 && maximum(r) <= sz))

# helper functions for constructing cfitsio indexing vectors in read(hdu, ...)
_first(i::@compat(Union{Integer, Range})) = first(i)
_first(::Colon) = 1
_last(sz, i::@compat(Union{Integer, Range})) = last(i)
_last(sz, ::Colon) = sz
_step(r::Range) = step(r)
_step(::@compat(Union{Integer, Colon})) = 1

# Shape of array to create for read(hdu, ...), dropping trailing
# scalars. This is simpler than in Base because we are guaranteed that
# length(I) == length(sz).
@inline _index_shape(sz, I...) = _index_shape_dim(sz, 1, I...)
_index_shape_dim(sz, dim, I::Integer...) = ()
_index_shape_dim(sz, dim, ::Colon) = (sz[dim],)
_index_shape_dim(sz, dim, r::Range) = (length(r),)
@inline _index_shape_dim(sz, dim, ::Colon, I...) =
    tuple(sz[dim], _index_shape_dim(sz, dim+1, I...)...)
if VERSION >= v"0.5.0-dev"  # In Julia v0.5+ drop scalar-indexed dimensions.
    @inline _index_shape_dim(sz, dim, ::Integer, I...) =
        _index_shape_dim(sz, dim+1, I...)
else
    @inline _index_shape_dim(sz, dim, ::Integer, I...) =
        tuple(1, _index_shape_dim(sz, dim+1, I...)...)
end
@inline _index_shape_dim(sz, dim, r::Range, I...) =
    tuple(length(r), _index_shape_dim(sz, dim+1, I...)...)

# Read a subset of an ImageHDU
function read_internal(hdu::ImageHDU, I::@compat(Union{Range{Int}, Integer, Colon})...)
    fits_assert_open(hdu.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    sz = fits_get_img_size(hdu.fitsfile)

    # check number of indicies and bounds. Note that number of indicies and
    # array dimension must match, unlike in Arrays. Array-like behavior could
    # be supported in the future with care taken in constructing first, last,
    if length(I) != length(sz)
        throw(DimensionMismatch("number of indicies must match dimensions"))
    end
    for i=1:length(sz)
        _checkbounds(sz[i], I[i]) || throw(BoundsError())
    end

    # construct first, last and step vectors
    firsts = Clong[_first(idx) for idx in I]
    lasts = Clong[_last(sz[i], I[i]) for i=1:length(sz)]
    steps = Clong[_step(idx) for idx in I]

    # construct output array
    bitpix = fits_get_img_equivtype(hdu.fitsfile)
    data = Array{TYPE_FROM_BITPIX[bitpix]}(_index_shape(sz, I...))

    fits_read_subset(hdu.fitsfile, firsts, lasts, steps, data)
    data
end

# general method and version that returns a single value rather than 0-d array
read(hdu::ImageHDU, I::@compat(Union{Range{Int}, Int, Colon})...) =
    read_internal(hdu, I...)
read(hdu::ImageHDU, I::Int...) = read_internal(hdu, I...)[1]

# Add a new ImageHDU to a FITS object
# The following Julia data types are supported for writing images by cfitsio:
# Uint8, Int8, Uint16, Int16, Uint32, Int32, Int64, Float32, Float64
function write{T}(f::FITS, data::Array{T};
                  header::@compat(Union{Void, FITSHeader})=nothing,
                  name::@compat(Union{Void, Compat.ASCIIString})=nothing,
                  ver::@compat(Union{Void, Integer})=nothing)
    fits_assert_open(f.fitsfile)
    s = size(data)
    fits_create_img(f.fitsfile, T, [s...])
    if isa(header, FITSHeader)
        fits_write_header(f.fitsfile, header, true)
    end
    if isa(name, Compat.ASCIIString)
        fits_update_key(f.fitsfile, "EXTNAME", name)
    end
    if isa(ver, Integer)
        fits_update_key(f.fitsfile, "EXTVER", ver)
    end
    fits_write_pix(f.fitsfile, ones(Int, length(s)), length(data), data)
    nothing
end

# Copy a rectangular section of an image and write it to a new FITS
# primary image or image extension. The new image HDU is appended to
# the end of the output file; all the keywords in the input image will
# be copied to the output image. The common WCS keywords will be
# updated if necessary to correspond to the coordinates of the section.

# convert a range to a string that cfitsio understands
cfitsio_range_string(r::UnitRange) = @sprintf "%d:%d" first(r) last(r)
cfitsio_range_string(r::StepRange) =
    @sprintf "%d:%d:%d" first(r) last(r) step(r)

function copy_section(hdu::ImageHDU, dest::FITS, r::Range{Int}...)
    fits_assert_open(hdu.fitsfile)
    fits_assert_open(dest.fitsfile)
    fits_movabs_hdu(hdu.fitsfile, hdu.ext)
    fits_copy_image_section(hdu.fitsfile, dest.fitsfile,
                            join([cfitsio_range_string(ri) for ri in r], ','))
end


text:
using Stheno: BlockData, GPC, cross

@timedtestset "cross" begin
    @timedtestset "Correctness tests" begin
        rng, P, Q, gpc = MersenneTwister(123456), 2, 3, GPC()

        f1 = GP(sin, EQ(), gpc)
        f2 = GP(cos, EQ(), gpc)
        f3 = cross([f1, f2])
        f4 = cross([f1])
        f5 = cross([f2])

        x1 = collect(range(-5.0, 5.0; length=P))
        x2 = collect(range(-5.0, 5.0; length=Q))
        x3 = BlockData([x1, x2])
        x4 = BlockData([x1])
        x5 = BlockData([x2])

        # mean
        @test mean(f3(x3)) == vcat(mean(f1(x1)), mean(f2(x2)))
        @test mean(f4(x4)) == mean(f1(x1))
        @test mean(f5(x5)) == mean(f2(x2))

        # cov
        @test cov(f3(x3)) ≈ vcat(
            hcat(cov(f1(x1)), cov(f1(x1), f2(x2))),
            hcat(cov(f2(x2), f1(x1)), cov(f2(x2))),
        )
        @test cov(f3(x3), f3(x3)) == cov(f3(x3))
        @test cov(f4(x4), f4(x4)) == cov(f4(x4))
        @test cov(f5(x5), f5(x5)) == cov(f5(x5))
        @test cov(f4(x4)) == cov(f1(x1))
        @test cov(f5(x5)) == cov(f2(x2))

        # cross-cov (with block)
        @test cov(f1(x1), f2(x2)) == cov(f4(x4), f5(x5))
        @test cov(f2(x2), f1(x1)) == cov(f5(x5), f4(x4))
        @test cov(f3(x3), f4(x4)) == vcat(cov(f1(x1)), cov(f2(x2), f1(x1)))

        @test cov(f5(x5), f3(x3)) == hcat(cov(f2(x2), f1(x1)), cov(f2(x2)))

        # cross-cov (with non-block)
        @test cov(f4(x4)) == cov(f1(x1))
        @test cov(f5(x5)) == cov(f2(x2))
        @test cov(f3(x3), f4(x4)) == cov(f3(x3), f1(x1))
        @test cov(f5(x5), f3(x3)) == cov(f2(x2), f3(x3))

        @timedtestset "rand, logpdf, elbo" begin

            # Single-sample rand
            y1, y2 = rand([f1(x1), f2(x2)])
            @test length(y1) == length(x1)
            @test length(y2) == length(x2)

            # Multi-sample rand
            Y1, Y2 = rand([f1(x1), f2(x2)], 11)
            @test size(Y1) == (length(y1), 11)
            @test size(Y2) == (length(y2), 11)

            # logpdf
            @test logpdf([f1(x1), f2(x2)], [y1, y2]) ≈ logpdf(f1(x1), y1) + logpdf(f2(x2), y2)
            @test logpdf([f1(x1), f2(x2)], [y1, y2]) ≈ logpdf([f1(x1)←y1, f2(x2)←y2])

            # elbo
            @test elbo([f1(x1, 1e-3), f2(x2, 1e-3)], [y1, y2], [f1(x1), f2(x2)]) ≈
                logpdf([f1(x1, 1e-3), f2(x2, 1e-3)], [y1, y2])
            @test elbo(
                [f1(x1, 1e-3), f2(x2, 1e-3)],
                [y1, y2],
                Stheno.finites_to_block([f1(x1), f2(x2)]),
            ) ≈ logpdf([f1(x1, 1e-3), f2(x2, 1e-3)], [y1, y2])
            @test elbo(f1(x1, 1e-3), y1, [f1(x1), f2(x2)]) ≈ logpdf(f1(x1, 1e-3), y1)
        end
    end
    @timedtestset "Standardised Tests" begin
        rng, P, Q = MersenneTwister(123456), 3, 5
        x0_1, x0_2 = collect(range(-1.0, 1.0; length=P)), collect(range(2.0, 4.0; length=P))
        x1_1, x1_2 = randn(rng, Q), randn(rng, Q)
        x0, x1 = BlockData([x0_1, x0_2]), BlockData([x1_1, x1_2])
        x2, x3 = randn(rng, P), randn(2P)

        gpc = GPC()
        f1, f2 = GP(sin, EQ(), gpc), GP(cos, EQ(), gpc)
        f3 = cross([f1, f2])
        abstractgp_interface_tests(f3, f1, x0, x1, x2, x3)

        # x1, x2 = collect(range(-2.0, 2.0; length=5)), collect(range(1.2, 1.5; length=4))
        # z1, z2 = collect(range(-1.5, 0.75; length=3)), collect(range(0.89, 2.0; length=4))
        # standard_1D_tests(
        #     MersenneTwister(123456),
        #     Dict(:l1=>0.5, :l2=>2.3),
        #     θ->begin
        #         gpc = GPC()
        #         f1 = θ[:l1] * GP(sin, EQ(), gpc)
        #         f2 = θ[:l2] * GP(cos, EQ(), gpc)
        #         f3 = cross([f1, f2])
        #         return f3, f3
        #     end,
        #     BlockData([x1, x2]),
        #     BlockData([z1, z2]),
        # )
    end
    @timedtestset "finites_to_block" begin
        rng, P, Q = MersenneTwister(123456), 3, 5
        xp, xq = randn(rng, P), randn(rng, Q)
        gpc = GPC()
        f_, g_ = GP(sin, EQ(), gpc), GP(cos, EQ(), gpc)
        h_x = Stheno.finites_to_block([f_(xp, 1.0), g_(xq, 1.0)])
    end
end


text:
"""
    abstract type ItemWrapper{Item}

"""
abstract type ItemWrapper{Item} <: AbstractItem end

itemdata(wrapper::ItemWrapper) = itemdata(getwrapped(wrapper))
itemfield(wrapped::ItemWrapper) = :item

getwrapped(wrapped::ItemWrapper) = getfield(wrapped, itemfield(wrapped))

function setwrapped(wrapped::ItemWrapper, item)
    wrapped = Setfield.@set wrapped.item = item
    return wrapped
end

function apply(tfm::Transform, itemw::ItemWrapper; randstate = getrandstate(tfm))
    item = apply(tfm, getwrapped(itemw); randstate = randstate)
    itemw = setwrapped(itemw, item)
    return itemw
end


text:
using AlgebraPDF
using Test

@testset "Generate in range" begin
    d = Normalized(FGauss((a=0,b=1.1)), (2,5))
    data = generate(d, 100)
    sum(x->inrange(x,lims(d)), data) == length(data)
end


text:
module FastKmeans
# Hamerly, Greg. "Making k-means even faster."
# Proceedings of the 2010 SIAM international
# conference on data mining. Society for Industrial
# and Applied Mathematics, 2010.

using Clustering
using StaticArrays

function point_all_ctrs!(i,x,c,a,u,l)
    k = length(c)
    xx = x[i]
    cc = c[1]
    yy = xx - cc
    d1min = dot(yy,yy)
    d2min = typemax(typeof(d1min))
    ind = 1
    ai = a[i]
    @inbounds for j=1:k
        yy = xx - c[j]
        d1 = dot(yy,yy)
        d1 < d1min && (d1min = d1; ind = j)
        j == ai && continue
        d1 < d2min && (d2min = d1)
    end
    l[i] = sqrt(d2min)

    a[i] = ind
    u[i] = sqrt(d1min)
end

function initialize!(c,x,q,cp,u,l,a)
    @inbounds for i=1:length(x)
        point_all_ctrs!(i,x,c,a,u,l)
        ai = a[i]
        q[ai] += 1
        cp[ai] += x[i]
    end
end

function move_centers!(cp,q,c,p)
    k = length(c)
    pmax = zero(eltype(p))
    @inbounds for j=1:k
        cs = c[j]
        c[j] = cp[j]/q[j]
        pp = norm(cs-c[j])
        pp > pmax && (pmax=pp)
        p[j] = pp
    end
    return pmax
end

function update_bounds!(p,a,u,l)
    r = 1
    pmax = p[r]
    pmax2 = 0.0
    for j=1:length(p)
        p[j] > pmax && (pmax2 = pmax; pmax=p[j]; r = j)
    end

    @inbounds for i=1:length(u)
        u[i] += p[a[i]]
        r == a[i] ? (l[i] -= pmax2) : (l[i] -= pmax)
    end
end

function fastkmeans!(x,c, maxiter = 100)
    n = length(x)
    k = length(c)
    cp = c .* 0
    q = zeros(Int64,k)
    p = zeros(Float64,k)
    s = zeros(Float64,k)

    a = zeros(Int,n)
    l = zeros(Float64,n)
    u = zeros(Float64,n)

    initialize!(c,x,q,cp,u,l,a)

    for iter=1:maxiter
        @inbounds for j=1:k
            cc = c[j]
            dmin2 = typemax(eltype(cc))
            for jj=1:k
                jj == j && continue
                v = c[jj]-cc
                d = dot(v,v)
                d < dmin2 && (dmin2 = d)
            end
            s[j] = sqrt(dmin2)
        end

        @inbounds for i=1:n
            ap = a[i]
            xx = x[i]
            m = max(s[ap]/2,l[i])
            u[i] <= m && continue
            u[i] = norm(xx-c[ap])
            u[i] <= m && continue
            point_all_ctrs!(i,x,c,a,u,l)
            if ap != a[i]
                q[ap] -= 1
                q[a[i]] += 1
                cp[ap] -= xx
                cp[a[i]] += xx
            end
        end

        pmax = move_centers!(cp,q,c,p)
        pmax == 0.0 && break
        update_bounds!(p,a,u,l)
    end

    return a
end

function fastkmeans(x,k; maxiter = 100)
    x_matrix = hcat(x...)
    ci = Clustering.initseeds(:kmpp, x_matrix, k)
    x_s = SVector{length(x[1]),eltype(x[1])}.(x)
    c_s = SVector{length(x[1]),eltype(x[1])}.(x[ci])
    assignment = fastkmeans!(x_s, c_s, maxiter)

    return assignment, c_s
end

export fastkmeans

end # module