text:
"""
    read_nnet(fname::String; last_layer_activation = Id())

Read in neural net from a `.nnet` file and return Network struct.
The `.nnet` format is borrowed from [NNet](https://github.com/sisl/NNet).
The format assumes all hidden layers have ReLU activation.
Keyword argument `last_layer_activation` sets the activation of the last
layer, and defaults to `Id()`, (i.e. a linear output layer).
"""

function read_nnet(fname::String; last_layer_activation = Id())
    f = open(fname)
    line = readline(f)
    while occursin("//", line) #skip comments
        line = readline(f)
    end
    # number of layers
    nlayers = parse(Int64, split(line, ",")[1])
    # read in layer sizes
    layer_sizes = parse.(Int64, split(readline(f), ",")[1:nlayers+1])
    # read past additonal information
    for i in 1:5
        line = readline(f)
    end
    # i=1 corresponds to the input dimension, so it's ignored
    layers = Layer[read_layer(dim, f) for dim in layer_sizes[2:end-1]] #add ReLUact/tanh
    push!(layers, read_layer(last(layer_sizes), f, last_layer_activation))

    return Network(layers)
end

"""
    read_layer(output_dim::Int, f::IOStream, [act = ReLU()])

Read in layer from nnet file and return a `Layer` containing its weights/biases.
Optional argument `act` sets the activation function for the layer.
"""
function read_layer(output_dim::Int64, f::IOStream, act = ReLU())

    rowparse(splitrow) = parse.(Float64, splitrow[findall(!isempty, splitrow)])
     # first read in weights
     W_str_vec = [rowparse(split(readline(f), ",")) for i in 1:output_dim]
     weights = vcat(W_str_vec'...)
     # now read in bias
     bias_string = [split(readline(f), ",")[1] for j in 1:output_dim]
     bias = rowparse(bias_string)
     # activation function is set to ReLU as default
     return Layer(weights, bias, act)
end

"""
    compute_output(nnet::Network, input::Vector{Float64})

Propagate a given vector through a nnet and compute the output.
"""
function compute_output(nnet::Network, input)
    curr_value = input
    for layer in nnet.layers # layers does not include input layer (which has no weights/biases)
        curr_value = layer.activation(affine_map(layer, curr_value))
    end
    return curr_value # would another name be better?
end

"""
    get_activation(L, x::Vector)
Finds the activation pattern of a vector `x` subject to the activation function given by the layer `L`.
Returns a Vector{Bool} where `true` denotes the node is "active". In the sense of ReLU, this would be `x[i] >= 0`.
"""
get_activation(L::Layer{ReLU}, x::Vector) = x .>= 0.0
get_activation(L::Layer{Id}, args...) = trues(n_nodes(L))

"""
    get_activation(nnet::Network, x::Vector)

Given a network, find the activation pattern of all neurons at a given point x.
Returns Vector{Vector{Bool}}. Each Vector{Bool} refers to the activation pattern of a particular layer.
"""
function get_activation(nnet::Network, x::Vector{Float64})
    act_pattern = Vector{Vector{Bool}}(undef, length(nnet.layers))
    curr_value = x
    for (i, layer) in enumerate(nnet.layers)
        curr_value = affine_map(layer, curr_value)
        act_pattern[i] = get_activation(layer, curr_value)
        curr_value = layer.activation(curr_value)
    end
    return act_pattern
end

"""
    get_activation(nnet::Network, input::Hyperrectangle)

Given a network, find the activation pattern of all neurons for a given input set.
Assume ReLU.
return Vector{Vector{Int64}}.
- 1: activated
- 0: undetermined
- -1: not activated
"""
function get_activation(nnet::Network, input::Hyperrectangle)
    bounds = get_bounds(nnet, input)
    return get_activation(nnet, bounds)
end

"""
    get_activation(nnet::Network, bounds::Vector{Hyperrectangle})

Given a network, find the activation pattern of all neurons given the node-wise bounds.
Assume ReLU.
return Vector{Vector{Int64}}.
- 1: activated
- 0: undetermined
- -1: not activated
"""
function get_activation(nnet::Network, bounds::Vector{Hyperrectangle})
    act_pattern = Vector{Vector{Int}}(undef, length(nnet.layers))
    for (i, layer) in enumerate(nnet.layers)
        act_pattern[i] = get_activation(layer, bounds[i])
    end
    return act_pattern
end

function get_activation(L::Layer{ReLU}, bounds::Hyperrectangle)
    before_act_bound = approximate_affine_map(L, bounds)
    lower = low(before_act_bound)
    upper = high(before_act_bound)
    act_pattern = zeros(n_nodes(L))
    for j in 1:n_nodes(L) # For evey node
        if lower[j] > 0.0
            act_pattern[j] = 1
        elseif upper[j] < 0.0
            act_pattern[j] = -1
        end
    end
    return act_pattern
end

"""
    get_gradient(nnet::Network, x::Vector)

Given a network, find the gradient at the input x
"""
function get_gradient(nnet::Network, x::Vector)
    z = x
    gradient = Matrix(1.0I, length(x), length(x))
    for (i, layer) in enumerate(nnet.layers)
        z_hat = affine_map(layer, z)
        σ_gradient = act_gradient(layer.activation, z_hat)
        gradient = Diagonal(σ_gradient) * layer.weights * gradient
        z = layer.activation(z_hat)
    end
    return gradient
end

"""
    act_gradient(act::ReLU, z_hat::Vector{N}) where N

Computing the gradient of an activation function at point z_hat.
Currently only support ReLU and Id.
"""
act_gradient(act::ReLU, z_hat::Vector) = z_hat .>= 0.0
act_gradient(act::Id,   z_hat::Vector) = trues(length(z_hat))

"""
    get_gradient(nnet::Network, input::AbstractPolytope)

Get lower and upper bounds on network gradient for a given input set.
Return:
- `LG::Vector{Matrix}`: lower bounds
- `UG::Vector{Matrix}`: upper bounds
"""
function get_gradient(nnet::Network, input::AbstractPolytope)
    LΛ, UΛ = act_gradient_bounds(nnet, input)
    return get_gradient(nnet, LΛ, UΛ)
end

"""
    act_gradient_bounds(nnet::Network, input::AbstractPolytope)

Computing the bounds on the gradient of all activation functions given an input set.
Currently only support ReLU.
Return:
- `LΛ::Vector{Matrix}`: lower bounds
- `UΛ::Vector{Matrix}`: upper bounds
"""
function act_gradient_bounds(nnet::Network, input::AbstractPolytope)
    bounds = get_bounds(nnet, input)
    LΛ = Vector{Matrix}(undef, 0)
    UΛ = Vector{Matrix}(undef, 0)
    for (i, layer) in enumerate(nnet.layers)
        before_act_bound = approximate_affine_map(layer, bounds[i])
        lower = low(before_act_bound)
        upper = high(before_act_bound)
        l = act_gradient(layer.activation, lower)
        u = act_gradient(layer.activation, upper)
        push!(LΛ, Diagonal(l))
        push!(UΛ, Diagonal(u))
    end
    return (LΛ, UΛ)
end

"""
    get_gradient(nnet::Network, LΛ::Vector{Matrix}, UΛ::Vector{Matrix})

Get lower and upper bounds on network gradient for given gradient bounds on activations
Inputs:
- `LΛ::Vector{Matrix}`: lower bounds on activation gradients
- `UΛ::Vector{Matrix}`: upper bounds on activation gradients
Return:
- `LG::Vector{Matrix}`: lower bounds
- `UG::Vector{Matrix}`: upper bounds
"""
function get_gradient(nnet::Network, LΛ::Vector{Matrix}, UΛ::Vector{Matrix})
    n_input = size(nnet.layers[1].weights, 2)
    LG = Matrix(1.0I, n_input, n_input)
    UG = Matrix(1.0I, n_input, n_input)
    for (i, layer) in enumerate(nnet.layers)
        LG_hat, UG_hat = interval_map(layer.weights, LG, UG)
        LG = LΛ[i] * max.(LG_hat, 0) + UΛ[i] * min.(LG_hat, 0)
        UG = LΛ[i] * min.(UG_hat, 0) + UΛ[i] * max.(UG_hat, 0)
    end
    return (LG, UG)
end

"""
    get_gradient(nnet::Network, LΛ::Vector{Vector{N}}, UΛ::Vector{Vector{N}}) where N

Get lower and upper bounds on network gradient for given gradient bounds on activations
Inputs:
- `LΛ::Vector{Vector{N}}`: lower bounds on activation gradients
- `UΛ::Vector{Vector{N}}`: upper bounds on activation gradients
Return:
- `(LG, UG)` lower and upper bounds
"""
function get_gradient(nnet::Network, LΛ::Vector{Vector{N}}, UΛ::Vector{Vector{N}}) where N
    n_input = size(nnet.layers[1].weights, 2)
    LG = Matrix(1.0I, n_input, n_input)
    UG = Matrix(1.0I, n_input, n_input)
    for (i, layer) in enumerate(nnet.layers)
        LG_hat, UG_hat = interval_map(layer.weights, LG, UG)
        LG = Diagonal(LΛ[i]) * max.(LG_hat, 0) + Diagonal(UΛ[i]) * min.(LG_hat, 0)
        UG = Diagonal(LΛ[i]) * min.(UG_hat, 0) + Diagonal(UΛ[i]) * max.(UG_hat, 0)
    end
    return (LG, UG)
end

"""
    interval_map(W::Matrix, l, u)

Simple linear mapping on intervals
Inputs:
- `W::Matrix{N}`: linear mapping
- `l::Vector{N}`: lower bound
- `u::Vector{N}`: upper bound
Outputs:
- `(lbound, ubound)` (after the mapping)
"""
function interval_map(W::Matrix{N}, l::AbstractVecOrMat, u::AbstractVecOrMat) where N
    l_new = max.(W, zero(N)) * l + min.(W, zero(N)) * u
    u_new = max.(W, zero(N)) * u + min.(W, zero(N)) * l
    return (l_new, u_new)
end

"""
    get_bounds(problem::Problem)
    get_bounds(nnet::Network, input::Hyperrectangle)

This function calls maxSens to compute node-wise bounds given a input set.

Return:
- `Vector{Hyperrectangle}`: bounds for all nodes **after** activation. `bounds[1]` is the input set.
"""
function get_bounds(nnet::Network, input::Hyperrectangle, act::Bool = true) # NOTE there is another function by the same name in convDual. Should reconsider dispatch
    if act
        solver = MaxSens(0.0, true)
        bounds = Vector{Hyperrectangle}(undef, length(nnet.layers) + 1)
        bounds[1] = input
        for (i, layer) in enumerate(nnet.layers)
            bounds[i+1] = forward_layer(solver, layer, bounds[i])
        end
        return bounds
    else
       error("before activation bounds not supported yet.")
    end
end
get_bounds(problem::Problem) = get_bounds(problem.network, problem.input)

"""
    affine_map(layer, input::AbstractPolytope)

Affine transformation of a set using the weights and bias of a layer.

Inputs:
- `layer`: Layer
- `input`: input set (Hyperrectangle, HPolytope)
Return:
- `output`: set after transformation.

    affine_map(layer, input)

Inputs:
- `layer`: Layer
- `input`: Vector
Return:
- `output`: Vector after mapping
"""
affine_map(layer::Layer, input) = layer.weights*input + layer.bias
function affine_map(layer::Layer, input::AbstractPolytope)
    W, b = layer.weights, layer.bias
    return translate(b, linear_map(W, input))
end

"""
   approximate_affine_map(layer, input::Hyperrectangle)

Returns a Hyperrectangle overapproximation of the affine map of the input.
"""
function approximate_affine_map(layer::Layer, input::Hyperrectangle)
    c = affine_map(layer, input.center)
    r = abs.(layer.weights) * input.radius
    return Hyperrectangle(c, r)
end

function translate(v::Vector, H::HPolytope)
    # translate each halfpsace according to:
    # a⋅(x-v) ≤ b  ⟶  a⋅x ≤ b+a⋅v
    C, d = tosimplehrep(H)
    return HPolytope(C, d+C*v)
end
# translate(v::Vector, H::Hyperrectangle)   = Hyperrectangle(H.center .+ v, H.radius)
translate(v::Vector, V::AbstractPolytope) = tohrep(VPolytope([x+v for x in vertices_list(V)]))

"""
    split_interval(dom, i)

Split a set into two at the given index.

Inputs:
- `dom::Hyperrectangle`: the set to be split
- `i`: the index to split at
Return:
- `(left, right)::Tuple{Hyperrectangle, Hyperrectangle}`: two sets after split
"""
function split_interval(dom::Hyperrectangle, i::Int64)
    input_lower, input_upper = low(dom), high(dom)

    input_upper[i] = dom.center[i]
    input_split_left = Hyperrectangle(low = input_lower, high = input_upper)

    input_lower[i] = dom.center[i]
    input_upper[i] = dom.center[i] + dom.radius[i]
    input_split_right = Hyperrectangle(low = input_lower, high = input_upper)
    return (input_split_left, input_split_right)
end


text:
module GHCNData

using CSV
using DataDeps
using DataFrames
using Dates

include("datadeps.jl")
include("dataset_loading.jl")
include("postprocessing.jl")

export
    load_station_metadata,
    load_countries_metadata,
    load_inventories,
    load_data_file,
    convert_to_time_series,
    select_data

end

ext: jl


text:
const supported_component_types = ["bus","gen","branch","dcline","load", "connector", "nw"]

"""
    PowerModelsGraph
        graph::LightGraphs.SimpleDiGraph
        node_comp_map::Dict{Int,Tuple{String, String}}
        edge_comp_map::Dict{LightGraphs.AbstractEdge,Tuple{String, String}}
        edge_connector_map::Dict{LightGraphs.AbstractEdge,Tuple{String, String}}

A structure containing a graph of a PowerModels or PowerModelsDistribution network with
four fields: a LightGraphs.SimpleDiGraph, a map from the node ids to the components, and
 a map from the edges to the components, and a map from the edges to conenctors.
"""
mutable struct PowerModelsGraph
    graph::LightGraphs.SimpleGraph
    node_comp_map::Dict{Int,Tuple{String, String}}
    edge_comp_map::Dict{Tuple{Int,Int},Tuple{String, String}}
    edge_connector_map::Dict{Tuple{Int,Int},Tuple{String, String}}

    function PowerModelsGraph(data::Dict{String,<:Any}, node_types::Array{String,1}, edge_types::Array{String,1})

        node_count = sum(length(keys(get(data,node_type,Dict()))) for node_type in node_types)
        G = LightGraphs.SimpleGraph(node_count) # create graph

        node_comp_array = Vector{Tuple{String,String}}(undef,node_count)
        i_1 = 1
        for comp_type in node_types
            if haskey(data,comp_type)
                for comp_id in sort(collect(keys(data[comp_type]))) #sort seems to get better layout results?
                    node_comp_array[i_1] = (comp_type,comp_id)
                    i_1+=1
                end
            end
        end
        comp_node_map = Dict(zip(node_comp_array,1:node_count))
        node_comp_map = Dict(zip(1:node_count,node_comp_array))

        edge_comp_count = sum(length(keys(get(data,edge_type,Dict()))) for edge_type in edge_types)
        edge_comp_array = Vector{Tuple{String,String}}(undef,edge_comp_count)
        edge_node_array = Vector{Tuple{Int,Int}}(undef,edge_comp_count)
        i_2 = 1
        for comp_type in edge_types # add edges
            if haskey(data,comp_type)
                for (comp_id,comp) in data[comp_type]
                    edge_comp_array[i_2] = (comp_type,comp_id)
                    s = comp_node_map[("bus",string(comp["f_bus"]))]
                    d = comp_node_map[("bus",string(comp["t_bus"]))]
                    edge_node_array[i_2] = (s,d)
                    LightGraphs.add_edge!(G, s, d)
                    i_2+=1
                end
            end
        end
        edge_comp_map = Dict(zip(edge_node_array,edge_comp_array))

        # sum does not work when "bus" is only node_type
        # edge_connector_count = sum(length(keys(get(data,node_type,Dict()))) for node_type in node_types if node_type != "bus")
        edge_connector_count = 0
        for node_type in node_types
            if node_type != "bus"
                edge_connector_count+= length(keys(get(data,node_type,Dict())))
            end
        end

        edge_connector_array = Vector{Tuple{String,String}}(undef,edge_connector_count)
        edge_node_array = Vector{Tuple{Int,Int}}(undef,edge_comp_count)
        i_3 = 1
        for comp_type in node_types #add connectors
            if comp_type != "bus"
                if haskey(data,comp_type)
                    for (comp_id,comp) in data[comp_type]
                        edge_connector_array[i_3] = (comp_type,comp_id)
                        s = comp_node_map[("bus",string(comp["$(comp_type)_bus"]))]
                        d = comp_node_map[(comp_type,comp_id)]
                        edge_node_array[i_3] = (s,d)
                        LightGraphs.add_edge!(G, s, d)
                        i_3+=1
                    end
                end
            end
        end
        edge_connector_map = Dict(zip(edge_node_array,edge_connector_array))

        return new(G, node_comp_map, edge_comp_map, edge_connector_map)
    end
end

""
function PowerModelsGraph(data::Dict{String,<:Any}; node_types=["bus","gen","storage"]::Array{String,1}, edge_types=["branch","dcline","switch"]::Array{String,1})
    return PowerModelsGraph(data, node_types, edge_types)
end

"""
PowerModelsDataFrame

A structure containing a dataframe for each component type.
"""
mutable struct PowerModelsDataFrame
    metadata::DataFrames.DataFrame
    bus::DataFrames.DataFrame
    gen::DataFrames.DataFrame
    branch::DataFrames.DataFrame
    dcline::DataFrames.DataFrame
    load::DataFrames.DataFrame
    connector::DataFrames.DataFrame

    function PowerModelsDataFrame(case::Dict{String,<:Any})
        data = deepcopy(case)
        comp_dataframes = tuple((DataFrames.DataFrame() for i in 1:7)...)
        if InfrastructureModels.ismultinetwork(data)
            for (nw_id, net) in data["nw"]

                net["nw_id"]=nw_id # give each network, component its parent nw_id
                for comp_type in supported_component_types
                    for (comp_id, comp) in get(net,comp_type,Dict())
                        comp["nw_id"] = nw_id
                    end
                end

                comp_dataframes_new= _PowerModelsDataFrame(net::Dict{String,<:Any}, comp_dataframes...)
            end

            #combine toplevel and network metadata
            data["nw_id"] = "top_level"
            _metadata_to_dataframe(data, comp_dataframes[1])
        else
            ## not a multinetwork
            comp_dataframes = _PowerModelsDataFrame(data::Dict{String,<:Any}, comp_dataframes...)

        end
        new(comp_dataframes...)
    end
end

""
function _PowerModelsDataFrame(sn_net::Dict{String,<:Any}, metadata, bus, gen, branch, dcline, load, connector)

        data = deepcopy(sn_net) # prevent overwriting input data

        ## add comp_type to each component
        for comp_type in supported_component_types
            for (comp_id, comp) in get(data,comp_type,Dict())
                comp["ComponentType"] = comp_type
            end
        end

        ## Assign component DataFrames
        _metadata_to_dataframe(data, metadata)
        _comp_dict_to_dataframe(get(data,"bus", Dict{String,Any}()), bus)
        _comp_dict_to_dataframe(get(data,"gen", Dict{String,Any}()), gen)
        _comp_dict_to_dataframe(get(data,"branch", Dict{String,Any}()), branch)
        _comp_dict_to_dataframe(get(data,"dcline", Dict{String,Any}()), dcline)
        _comp_dict_to_dataframe(get(data,"load", Dict{String,Any}()), load)
        _comp_dict_to_dataframe(get(data,"connector",Dict{String,Any}()), connector)

    return (metadata,bus,gen,branch,dcline,load,connector)
end

"convert non-component data into a dataframe"
function _metadata_to_dataframe(data, metadata)
    ## Seperate componets Dicts from metadata
    metadata_key = Symbol[]
    metadata_val = Any[]
    for (k,v) in sort(collect(data); by=x->x[1])
        if typeof(v) <: Dict && InfrastructureModels._iscomponentdict(v)
            if ~(k in supported_component_types)
                Memento.warn(_PM._LOGGER, "Component type $k is not yet not supported")
            end
        else
            push!(metadata_key,Symbol(k))
            push!(metadata_val,v)
        end
    end

    metadata_dict = Dict(zip(metadata_key, metadata_val))
    DataFrames.push!(metadata, metadata_dict, cols=:union)
    return metadata
end

"convert a componet dictionary such as `bus` into a dataframe."
function _comp_dict_to_dataframe(comp_dict::Dict{String,<:Any}, df)
    if length(comp_dict) <= 0 ## Should there be an empty dataframe, or a nonexistent dataframe?
        return df
    end

    for (i, component) in comp_dict
        for (k,v) in component
            if typeof(v) <: Array || typeof(v) <: Dict
                component[k] = string(v)
            end
        end
        DataFrames.push!(df, component, cols=:union)
    end

    return df
end

"convert a componet dictionary such as `bus` into a dataframe."
function comp_dict_to_dataframe(comp_dict::Dict{String,<:Any})
    return _comp_dict_to_dataframe(comp_dict, DataFrames.DataFrame())
end


text:
@testset "912.sort-an-array.jl" begin
    @test sort_array([5, 2, 3, 1]) == [1, 2, 3, 5]
end


text:
"""
Shell for something that implements size, eltype and *
"""
struct ArrayShell{T,D} <: AbstractArray{T,D}
    A
    ArrayShell(A) = new{eltype(A),length(size(A))}(A)
end

Base.getindex(::ArrayShell) = error("AbstractArrayShell does not support getindex")

Base.size(A::ArrayShell) = size(A.A)

Base.eltype(A::ArrayShell) = eltype(A.A)

Base.print_array(io::IO,A::ArrayShell) = Base.print(io, A.A)

Base.show(io::IO,A::ArrayShell) = Base.show(io, A.A)

abstract type MyAbstractMatrix{T} <: AbstractArray{T,2} end

Base.Matrix(A::MyAbstractMatrix) = matrix_by_mul(A)

Base.print_array(io::IO,A::MyAbstractMatrix) = Base.print_array(io, Matrix(A))

Base.show(io::IO,A::MyAbstractMatrix) = Base.show(io, Matrix(A))

for (Tp,fun) in zip((:NormalizedArray,), (:normalizedarray_fun,))
    @eval begin
        struct $Tp{T,N,Axes} <:FillArrays.AbstractFill{T,N,Axes}
            axes::Axes
            @inline $Tp{T, N}(sz::Axes) where Axes<:Tuple{Vararg{AbstractUnitRange,N}} where {T, N} =
                new{T,N,Axes}(sz)
            @inline $Tp{T,0,Tuple{}}(sz::Tuple{}) where T = new{T,0,Tuple{}}(sz)
        end

        @inline $Tp{T, 0}(sz::Tuple{}) where {T} = $Tp{T,0,Tuple{}}(sz)
        @inline $Tp{T, N}(sz::Tuple{Vararg{<:Integer, N}}) where {T, N} = $Tp{T,N}(Base.OneTo.(sz))
        @inline $Tp{T, N}(sz::Vararg{<:Integer, N}) where {T, N} = $Tp{T,N}(sz)
        @inline $Tp{T}(sz::Vararg{Integer,N}) where {T, N} = $Tp{T, N}(sz)
        @inline $Tp{T}(sz::SZ) where SZ<:Tuple{Vararg{Any,N}} where {T, N} = $Tp{T, N}(sz)
        @inline $Tp(sz::Vararg{Any,N}) where N = $Tp{Float64,N}(sz)
        @inline $Tp(sz::SZ) where SZ<:Tuple{Vararg{Any,N}} where N = $Tp{Float64,N}(sz)

        @inline $Tp{T,N}(A::AbstractArray{V,N}) where{T,V,N} = $Tp{T,N}(size(A))
        @inline $Tp{T}(A::AbstractArray) where{T} = $Tp{T}(size(A))
        @inline $Tp(A::AbstractArray) = $Tp(size(A))

        @inline axes(Z::$Tp) = Z.axes
        @inline size(Z::$Tp) = length.(Z.axes)
        @inline FillArrays.getindex_value(Z::$Tp) = $fun(Z)

        AbstractArray{T}(F::$Tp{T}) where T = F
        AbstractArray{T,N}(F::$Tp{T,N}) where {T,N} = F
        AbstractArray{T}(F::$Tp) where T = $Tp{T}(F.axes)
        AbstractArray{T,N}(F::$Tp{V,N}) where {T,V,N} = $Tp{T}(F.axes)
        convert(::Type{AbstractArray{T}}, F::$Tp{T}) where T = AbstractArray{T}(F)
        convert(::Type{AbstractArray{T,N}}, F::$Tp{T,N}) where {T,N} = AbstractArray{T,N}(F)
        convert(::Type{AbstractArray{T}}, F::$Tp) where T = AbstractArray{T}(F)
        convert(::Type{AbstractArray{T,N}}, F::$Tp) where {T,N} = AbstractArray{T,N}(F)

        getindex(F::$Tp{T,0}) where T = getindex_value(F)
        function getindex(F::$Tp{T}, kj::Vararg{AbstractVector{II},N}) where {T,II<:Integer,N}
            checkbounds(F, kj...)
            Fill{T}(FillArrays.getindex_value(F),length.(kj))
        end

        function getindex(A::$Tp{T}, kr::AbstractVector{Bool}) where T
            length(A) == length(kr) || throw(DimensionMismatch())
            Fill{T}(FillArrays.getindex_value(F),count(kr))
        end
        function getindex(A::$Tp{T}, kr::AbstractArray{Bool}) where T
            size(A) == size(kr) || throw(DimensionMismatch())
            Fill{T}(FillArrays.getindex_value(F),count(kr))
        end
    end
end

normalizedarray_fun(Z::NormalizedArray{T}) where T = one(T) / convert(T,length(Z))

"A vector of the form `[1,-1,1,-1,...]`."
struct AlternatingSigns{T} <: AbstractArray{T,1}
    n   ::  Int
end

AlternatingSigns(n::Int) = AlternatingSigns{Float64}(n)

size(A::AlternatingSigns) = (A.n,)
getindex(A::AlternatingSigns{T}, i::Int) where {T} = iseven(i) ? -one(T) : one(T)

inv(D::Diagonal{T,AlternatingSigns{T}}) where {T} = D

adjoint(D::Diagonal{T,AlternatingSigns{T}}) where {T} = D

"A vector of the form `[1,1,...,1,V1,1,...1]`."
struct ScaledEntry{T} <: AbstractArray{T,1}
    n       ::  Int
    index   ::  Int
    scalar  ::  T
end

ScaledEntry(n::Int) = ScaledEntry{Float64}(n)

size(A::ScaledEntry) = (A.n,)
getindex(A::ScaledEntry{T}, i::Int) where {T} = i==A.index ? A.scalar : one(T)
conj(vc::ScaledEntry{T}) where T<:Real = vc
conj(vc::ScaledEntry{T}) where T = ScaledEntry{T}(vc.n, vc.index, conj(vc.scalar))

inv(D::Diagonal{T,ScaledEntry{T}}) where {T} = Diagonal(ScaledEntry{T}(D.diag.n, D.diag.index, inv(D.diag.scalar)))

adjoint(D::Diagonal{T,ScaledEntry{T}}) where {T} = Diagonal(ScaledEntry{T}(D.diag.n, D.diag.index, adjoint(D.diag.scalar)))

"A rectangular matrix"
struct ProjectionMatrix{I,T} <: AbstractArray{T,2}
    m       ::  Int
    n       ::  Int
    indices ::  I
end

size(A::ProjectionMatrix) = (A.m, A.n)

"""
A 2D array where every row contains equal elements.

The top row starts at index offset, the second row at step+offset.
"""
struct HorizontalBandedMatrix{T} <: MyAbstractMatrix{T}
    m       ::  Int
    n       ::  Int
    array   ::  Vector{T}
    step    ::  Int
    offset  ::  Int

    function HorizontalBandedMatrix{T}(m::Int,n::Int,array::Vector{T}, step::Int=1, offset::Int=0) where T
        @assert length(array) <= n
        @assert step <= n # apply! only works if step is smaller then L
        new(m, n, array, step, offset)
    end
end

HorizontalBandedMatrix(m::Int, n::Int, array::Vector{T}, step::Int=1, offset::Int=0) where {T} =
    HorizontalBandedMatrix{T}(m, n, array, step, offset)

similar(A::HorizontalBandedMatrix{S}, ::Type{T}) where {S,T} =
    HorizontalBandedMatrix{T}(size(A)..., similar(A.array, T), A.step, A.offset)
# The method below is moved to SparseArrays for now
similar(A::HorizontalBandedMatrix, ::Type{T}, dims::Union{Dims{1},Dims{2}}) where {T} = spzeros(T, dims...)

Base.copy(A::HorizontalBandedMatrix{T}) where T =
    HorizontalBandedMatrix{T}(size(A,1), size(A,2), Base.copy(A.array), A.step, A.offset)

isefficient(::HorizontalBandedMatrix) = true

Base.size(A::HorizontalBandedMatrix) = (A.m,A.n)
Base.size(A::HorizontalBandedMatrix, i::Int) = (i==1) ? A.m : A.n

Base.getindex(op::HorizontalBandedMatrix, i::Int, j::Int) =
    _get_horizontal_banded_index(op.array, op.step, op.offset, size(op,1), size(op,2), i, j)

Base.inv(::HorizontalBandedMatrix) = error("Inverse not implemented.")

function _get_horizontal_banded_index(array::Vector{ELT}, step::Int, offset::Int, M::Int, N::Int, i::Int, j::Int) where {ELT}
    # first transform to an index of the first column
    index = mod(j-step*(i-1)-1-offset, N)+1
    if index <= length(array)
        array[index]
    else
        convert(ELT,0)
    end
end

Base.adjoint(M::HorizontalBandedMatrix) = VerticalBandedMatrix(size(M,2), size(M,1), conj(M.array), M.step, M.offset)

function mul!(dest::AbstractVector, op::HorizontalBandedMatrix, src::AbstractVector)
    dest[:] .= 0
    L = length(src)
    aL = length(op.array)
    dL = length(dest)
    @inbounds for a_i in 1:aL
        ind = mod(a_i+op.offset-1,L)+1
        for d_i in 1:dL
            dest[d_i] += op.array[a_i]*src[ind]
            ind += op.step
            if ind > L
                ind -= L
            end
        end
    end
    dest
end

"A 2D array every column contains equal elements.

The top column starts at index offset, the second column at step+offset."
struct VerticalBandedMatrix{T} <: MyAbstractMatrix{T}
    m       ::  Int
    n       ::  Int
    array   ::  Vector{T}
    step    ::  Int
    offset  ::  Int

    function VerticalBandedMatrix{T}(m::Int,n::Int,array::Vector{T}, step::Int=1, offset::Int=0) where T
        @assert length(array) <= m
        @assert step <= m # apply! only works if step is smaller then L
        new{T}(m, n, array, step, offset)
    end
end

VerticalBandedMatrix(m::Int,n::Int,array::Vector, step::Int=1, offset::Int=0) =
    VerticalBandedMatrix{eltype(array)}(m, n, array, step, offset)

Base.copy(A::VerticalBandedMatrix{T}) where T =
    VerticalBandedMatrix{T}(size(A,1), size(A,2), Base.copy(A.array), A.step, A.offset)
similar(A::VerticalBandedMatrix{S}, ::Type{T}) where {S,T} =
    VerticalBandedMatrix{T}(size(A)..., similar(A.array, T), A.step, A.offset)
# The method below is moved to SparseArrays for now
similar(A::VerticalBandedMatrix, ::Type{T}, dims::Union{Dims{1},Dims{2}}) where {T} = spzeros(T, dims...)

isefficient(::VerticalBandedMatrix) = true

Base.size(A::VerticalBandedMatrix) = (A.m,A.n)
Base.size(A::VerticalBandedMatrix, i::Int) = (i==1) ? A.m : A.n

Base.getindex(A::VerticalBandedMatrix, i::Int, j::Int) =
    _get_vertical_banded_index(A.array, A.step, A.offset, size(A,1), size(A,2), i, j)

Base.inv(::VerticalBandedMatrix) = error("Inverse not implemented.")

function _get_vertical_banded_index(array::Vector{ELT}, step::Int, offset::Int, M::Int, N::Int, i::Int, j::Int) where {ELT}
    # first transform to an index of the first column
    index = mod(i-step*(j-1)-1-offset, M)+1
    if index <= length(array)
        array[index]
    else
        ELT(0)
    end
end

Base.adjoint(M::VerticalBandedMatrix) = HorizontalBandedMatrix(size(M,2), size(M,1), conj(M.array), M.step, M.offset)

function mul!(dest::AbstractVector, op::VerticalBandedMatrix, src::AbstractVector)
    dest[:] .= 0
    # assumes step is smaller then L
    L = length(dest)
    aL = length(op.array)
    sL = length(src)
    @inbounds for a_i in 1:aL
        ind = mod(a_i+op.offset-1,L)+1
        for s_i in 1:sL
            dest[ind] += op.array[a_i]*src[s_i]
            ind += op.step
            if ind > L
                ind -= L
            end
        end
    end

    dest
end

"""
An IndexMatrix selects/restricts a subset of coefficients based on their indices.
"""
struct IndexMatrix{T,EXTENSION,N,I} <: MyAbstractMatrix{T}
    linear_size     ::Int
    original_size   ::NTuple{N,Int}
    subindices      ::I

    function IndexMatrix{T,EXTENSION,N,I}(original_size::NTuple{N,Int}, subindices::AbstractArray) where {N,T,I,EXTENSION}
        @assert (N==1) ? eltype(subindices) == Int : eltype(subindices) == CartesianIndex{N}
        n = length(subindices)
        m = prod(original_size)
        # m == n && (@warn "IndexMatrix contains all elements, consider identity or (De)LinearizationOperator instead.")
        new{T,EXTENSION,N,I}(n, original_size, subindices)
    end
end

const ExtensionIndexMatrix{T,N,I} = IndexMatrix{T,true,N,I} where {T,N,I}
const RestrictionIndexMatrix{T,N,I} = IndexMatrix{T,false,N,I} where {T,N,I}

IndexMatrix{T,EXTENSION}(extended_size::NTuple{N,Int}, subindices::AbstractArray) where {T,EXTENSION,N} =
        IndexMatrix{T,EXTENSION,N,typeof(subindices)}(extended_size, subindices)

IndexMatrix(extended_size, subindices; T=Int, EXTENSION, options...) =
    IndexMatrix{T,EXTENSION}(extended_size, subindices)

_original_size(A::IndexMatrix) = A.original_size
_linear_size(A::IndexMatrix) = A.linear_size
subindices(A::IndexMatrix) = A.subindices
isextensionmatrix(A::IndexMatrix{T,EXTENSION,N,I}) where {T,EXTENSION,N,I} =
    EXTENSION

Base.copy(A::IndexMatrix{T,EXTENSION,N,I}) where {T,EXTENSION,N,I} =
    IndexMatrix{T,EXTENSION}(_original_size(A), Base.copy(subindices(A)))

Base.adjoint(A::IndexMatrix{T,EXTENSION,N,I}) where {T,EXTENSION,N,I} =
    IndexMatrix{T,!(EXTENSION),N,I}(_original_size(A), Base.copy(subindices(A)))

Base.size(A::IndexMatrix) = isextensionmatrix(A) ?
    (prod(_original_size(A)),_linear_size(A)) :
    (_linear_size(A),prod(_original_size(A)))

similar(A::IndexMatrix{S,EXTENSION}, ::Type{T}) where {S,T,EXTENSION} = IndexMatrix{T,EXTENSION}(_original_size(A), subindices(A))
# The method below is moved to SparseArrays for now
similar(A::IndexMatrix, ::Type{T}, dims::Union{Dims{1},Dims{2}}) where {T} = spzeros(T, dims...)

copyto!(D1::IndexMatrix, D2::AbstractArray) = error("Not possible")

isefficient(::IndexMatrix) = true

@propagate_inbounds function getindex(A::ExtensionIndexMatrix{T,1}, i::Int, j::Int) where {T}
    @boundscheck checkbounds(A,i,j)
    unsafe_array_getindex(A, i, j)
end

@propagate_inbounds getindex(A::ExtensionIndexMatrix{T,N}, i::Int, j::Int) where {T,N} =
    getindex(A, CartesianIndices(CartesianIndex(_original_size(A)))[i], j)

@propagate_inbounds function Base.getindex(A::ExtensionIndexMatrix{T,N}, i::CartesianIndex{N}, j::Int) where {T,N}
    @boundscheck checkbounds(subindices(A), j)
    @boundscheck i∈CartesianIndices(CartesianIndex(_original_size(A))) || throw(BoundsError())
    unsafe_array_getindex(A, i, j)
end

unsafe_array_getindex(A::ExtensionIndexMatrix{T,1}, i::Int, j::Int) where {T} =
     @inbounds getindex(subindices(A),j) == i ? one(T) :  zero(T)

unsafe_array_getindex(A::ExtensionIndexMatrix{T,N}, i::Int, j::Int) where {T,N} =
    unsafe_array_getindex(A, CartesianIndices(CartesianIndex(_original_size(A)))[i], j)

unsafe_array_getindex(A::ExtensionIndexMatrix{T,N}, i::CartesianIndex{N}, j::Int) where {T,N} =
    @inbounds getindex(subindices(A),j)==i ? one(T) :  zero(T)

@propagate_inbounds function getindex(A::RestrictionIndexMatrix{T,1}, i::Int, j::Int) where {T}
    @boundscheck checkbounds(A,i,j)
    unsafe_array_getindex(A, i, j)
end

@propagate_inbounds getindex(A::RestrictionIndexMatrix{T,N}, i::Int, j::Int) where {T,N} =
    getindex(A, i, CartesianIndices(CartesianIndex(_original_size(A)))[j])

@propagate_inbounds function getindex(A::RestrictionIndexMatrix{T,N}, i::Int, j::CartesianIndex{N}) where {T,N}
    @boundscheck checkbounds(subindices(A), i)
    @boundscheck j∈CartesianIndices(CartesianIndex(_original_size(A))) || throw(BoundsError())
    unsafe_array_getindex(A, i, j)
end

unsafe_array_getindex(A::RestrictionIndexMatrix{T,1}, i::Int, j::Int) where {T} =
    @inbounds getindex(subindices(A),i)==j ? one(T) :  zero(T)

unsafe_array_getindex(A::RestrictionIndexMatrix{T,N}, i::Int, j::Int) where {T,N} =
    unsafe_array_getindex(A, i, CartesianIndices(CartesianIndex(_original_size(A)))[j])

unsafe_array_getindex(A::RestrictionIndexMatrix{T,N}, i::Int, j::CartesianIndex{N}) where {T,I,N} =
    @inbounds getindex(subindices(A),i)==j  ? one(T) :  zero(T)

Base.eltype(::IndexMatrix{T}) where T = T

function (*)(A::IndexMatrix, B::AbstractMatrix)
    TS = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(B))
    C = similar(B, TS, (size(A,1),size(B,2)))

    if isextensionmatrix(A)
        fill!(C, zero(TS))
        for (i,j) in enumerate(LinearIndices(CartesianIndices(_original_size(A)))[subindices(A)])
            copyto!(C, CartesianIndices((j:j,1:size(C,2))), B, CartesianIndices(i:i,1:size(C,2)))
        end
    else
        for (i,j) in enumerate(LinearIndices(CartesianIndices(_original_size(A)))[subindices(A)])
            copyto!(C, CartesianIndices((i:i,1:size(C,2))), B, CartesianIndices((j:j,1:size(C,2))))
        end
    end
    C
end

(*)(A::IndexMatrix, B::Diagonal) = _sparse_res_mul(A, B)
(*)(A::IndexMatrix, B::VerticalBandedMatrix) = _sparse_res_mul(A, B)
(*)(A::IndexMatrix, B::HorizontalBandedMatrix) = _sparse_res_mul(A, B)

function _sparse_res_mul(A::IndexMatrix, B::Diagonal)
    @boundscheck (size(A,2)==size(B,1)) || throw(BoundsError("Sizes do not match"))
    TS = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(B))
    I = LinearIndices(CartesianIndices(CartesianIndex(_original_size(A))))[subindices(A)]
    if isextensionmatrix(A)
        sparse(I,1:length(I),TS.(parent(B)[I]),size(A,1),size(B,2))
    else
        sparse(1:length(I),I,TS.(parent(B)[I]),size(A,1),size(B,2))
    end
end

function _sparse_res_mul(A::IndexMatrix, B::AbstractArray)
    TS = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(B))
    C = similar(B, TS, (size(A,1),size(B,2)))
    @assert C isa AbstractSparseArray
    # @warn "Creation of sparse array is not yet efficient."
    mul!(C,A,B)
end

mul!(dest, A::IndexMatrix, src) =
    mul!(dest, A, src, subindices(A))
mul!(dest::AbstractVector, A::ExtensionIndexMatrix, src::AbstractVector) =
    _tensor_mul!(reshape(dest, _original_size(A)), A, src, subindices(A))
mul!(dest::AbstractVector, A::RestrictionIndexMatrix, src::AbstractVector) =
    _tensor_mul!(dest, A, reshape(src,_original_size(A)), subindices(A))
function mul!(dest::AbstractVector, A::RestrictionIndexMatrix{S,N}, src::AbstractArray{T,N}) where {S,T,N}
    @boundscheck ( (size(src) == _original_size(A)) && (length(dest)==size(A,1)))|| throw(BoundsError("Sizes do not match"))
    @inbounds _tensor_mul!(dest, A, src, subindices(A))
end
function mul!(dest::AbstractMatrix, A::RestrictionIndexMatrix, src::AbstractMatrix)
    @boundscheck ( (size(src,1) == size(A,2)) && (size(src,2)==size(dest,2)) && (size(dest,1)==size(A,1)) ) || throw(BoundsError("Sizes do not match"))
    @inbounds for i in 1:size(src,2)
        mul!( view(dest, :, i), A, view(src, :, i)  )
    end
    dest
end

function _tensor_mul!(dest, A::RestrictionIndexMatrix, src, subindices)
    for (i,j) in enumerate(subindices)
        dest[i] = src[j]
    end
    dest
end

function _tensor_mul!(dest, A::ExtensionIndexMatrix, src, subindices)
    fill!(dest, zero(eltype(A)))
    for (i,j) in enumerate(subindices)
        dest[j] = src[i]
    end
    dest
end

# function Base.inv(C::Circulant{T}) where T<:Real
#     vdft = 1 ./ C.vcvr_dft
#     cvdft = copy(vdft)
#     return Circulant(real(C.dft \ vdft), cvdft, similar(vdft), C.dft)
# end
# function Base.inv(C::Circulant)C
#     vdft = 1 ./ C.vcvr_dft
#     cvdft = copy(vdft)
#     return Circulant(C.dft \ vdft, cvdft, similar(vdft), C.dft)
# end

function mul!(out::StridedVector{T}, A::Circulant{T}, in::StridedVector, α::T, β::T) where T<:Number
    fill!(A.tmp, 0)
    copyto!(A.tmp, in)
    mul!(A.tmp, A.dft, A.tmp)
    for i in 1:length(A.tmp)
        A.tmp[i] *= A.vcvr_dft[i]
    end
    A.dft \ A.tmp
    copyto!(out, A.tmp)
end

function mul!(out::StridedVector{T}, A::Circulant{T}, in::StridedVector{T}, _::T, _::T) where {T<:Real}
    fill!(A.tmp, 0)
    copyto!(A.tmp, in)
    mul!(A.tmp, A.dft, A.tmp)
    for i in 1:length(A.tmp)
        A.tmp[i] *= A.vcvr_dft[i]
    end
    A.dft \ A.tmp
    A.tmp .= real.(A.tmp)
    copyto!(out, A.tmp)
end

Base.adjoint(C::Circulant{T}) where T<:Real  =
    Circulant{T}(real(ifft(conj.(C.vcvr_dft))))
Base.adjoint(C::Circulant{T}) where T  =
    Circulant{T}(ifft(conj.(C.vcvr_dft)))
Base.conj(C::Circulant{T}) where {T<:Complex} =
    Circulant{T}(conj(ifft(C.vcvr_dft)))


text:
"""
Perform Forward-Backward tracking by first tracking keypoints from
`previous_pyramid` to `current_pyramid` and then track resuling keypoints
in reverse order.
Then filter out those forward-backward keypoints that are too far away
from the original keypoints to be consistent.

# Arguments:

- `new_keypoints`:
    Vector where to write resulting new poisitions of keypoints.
    Should be of the same size as `keypoints` vector.
- `max_distance::Real`:
    Maximum distance in pixels between Forward-Backward tracked keypoints
    to be considered the same keypoint.
"""
function fb_tracking!(
    new_keypoints::AbstractVector{Point2f},
    previous_pyramid::LKPyramid, current_pyramid::LKPyramid,
    keypoints::AbstractVector{Point2f}, algorithm::LucasKanade;
    displacement::AbstractVector{Point2f} = fill(Point2f(0.0, 0.0), length(keypoints)),
    max_distance::Real = 0.5,
)
    isempty(keypoints) && return

    # Forward tracking.
    displacement, status, n_good = optflow!(
        displacement, previous_pyramid, current_pyramid, keypoints, algorithm)

    valid_ids = Vector{Int64}(undef, n_good) # Mapping to the original ids.
    valid_correspondences = Vector{Point2f}(undef, n_good)
    back_displacement = Vector{Point2f}(undef, n_good)

    back_pyramid_levels = 0 #algorithm.pyramid_levels
    scale = 1.0 / 2.0^back_pyramid_levels
    c = 1
    @inbounds for i in 1:length(status)
        status[i] || continue

        old_kp, Δ = keypoints[i], displacement[i]
        new_point = old_kp .+ Δ
        new_keypoints[i] = new_point

        valid_correspondences[c] = new_point
        back_displacement[c] = -Δ .* scale
        valid_ids[c] = i
        c += 1
    end

    # Backward tracking.
    back_algorithm = LucasKanade(;
        iterations=algorithm.iterations, window_size=algorithm.window_size,
        pyramid_levels=back_pyramid_levels,
        eigenvalue_threshold=algorithm.eigenvalue_threshold)
    back_displacement, back_status, n_good = optflow!(
        back_displacement, current_pyramid, previous_pyramid,
        valid_correspondences, back_algorithm)

    @inbounds for i in 1:length(back_status)
        idx = valid_ids[i]
        back_status[i] || (status[idx] = false; continue)

        new_point = valid_correspondences[i] .+ back_displacement[i]
        norm(keypoints[idx] .- new_point) ≥ max_distance &&
            (status[idx] = false; continue)
    end
    new_keypoints, status
end

function fb_tracking!(
    previous_pyramid::LKPyramid, current_pyramid::LKPyramid,
    keypoints::AbstractVector{Point2f};
    displacement::AbstractVector{Point2f} = fill(Point2f(0.0, 0.0), length(keypoints)),
    iterations::Int = 30, window_size::Int = 11, pyramid_levels::Int = 3,
    max_distance::Real = 0.5,
)
    new_keypoints = Vector{Point2f}(undef, length(keypoints))
    algorithm = LucasKanade(;iterations, window_size, pyramid_levels)
    fb_tracking!(
        new_keypoints, previous_pyramid, current_pyramid,
        keypoints, algorithm; displacement, max_distance)
end


text:
function knothash(arr::Array{Int, 1}, list::UnitRange=0:255; pos::Int=0, skip::Int=0, iterations::Int=1)
    n = length(arr)
    for iter in 1:iterations
        for i in 1:n

            L = arr[i]
            if L > length(list)
                continue
            end
            list = circshift(list, -pos)
            reverse!(list, 1, L)
            list = circshift(list, pos)
            pos += L + skip
            skip += 1

        end
    end

    return list
end

"Convert a string to bytes"
function string2bytes(s::String)
    Int.(collect(s))
end

function puzzle10(path::String=joinpath(@__DIR__, "..", "data/10.txt"))
    input = readline(path)
    lengths = parse.(Int, split(input, ","))
    list = knothash(lengths)
    partone = list[1] * list[2]

    stringofbytes = string2bytes(input)
    append!(stringofbytes, [17, 31, 73, 47, 23])
    list = knothash(stringofbytes, iterations=64)
    sparsehash = reshape(list, 16, 16)
    densehash = [reduce(xor, sparsehash[:, i]) for i in 1:size(sparsehash, 2)]
    parttwo = reduce(*, lpad.(string.(densehash, base=16), 2, "0"))

    return [partone, parttwo]

end


text:
### A Pluto.jl notebook ###
# v0.12.6

using Markdown
using InteractiveUtils

# ╔═╡ 546a0f10-1d68-11eb-042a-134a1c0d879b
using Pkg, DrWatson

# ╔═╡ 546a5c04-1d68-11eb-23bb-cb5d47d9db43
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using HTTP, JSON
end

# ╔═╡ ee516b1c-1d67-11eb-1c05-eb5995f28838
md"## Listing 1.9"

# ╔═╡ 546ad49a-1d68-11eb-1a31-c3b003680713
begin
	data = HTTP.request("GET",
		"https://ocw.mit.edu/ans7870/6/6.006/s08/lecturenotes/files/t8.shakespeare.txt")
	shakespeare = String(data.body)
	shakespeareWords = split(shakespeare)
end

# ╔═╡ 5476c1ce-1d68-11eb-20ca-31c3a9321588
begin
	jsonWords = HTTP.request("GET",
		"https://raw.githubusercontent.com/"*
		"h-Klok/StatisticsWithJuliaPlutoNotebooks/master/data/jsonCode.json")
	parsedJsonDict = JSON.parse( String(jsonWords.body))
end

# ╔═╡ 54773480-1d68-11eb-101f-41723a16b3e7
begin
	keywords = Array{String}(parsedJsonDict["words"])
	numberToShow = parsedJsonDict["numToShow"]
	wordCount = Dict([(x,count(w -> lowercase(w) == lowercase(x), shakespeareWords))
    	for x in keywords])
end

# ╔═╡ 5483aa7e-1d68-11eb-0bff-3f6ed1803a29
sortedWordCount = sort(collect(wordCount),by=last,rev=true);

# ╔═╡ 548437b6-1d68-11eb-2dea-4d5f64b16cf3
sortedWordCount[1:numberToShow]

# ╔═╡ 5490b23c-1d68-11eb-2908-e18fb1dd0aad
md"## End of listing 1.9"

# ╔═╡ Cell order:
# ╟─ee516b1c-1d67-11eb-1c05-eb5995f28838
# ╠═546a0f10-1d68-11eb-042a-134a1c0d879b
# ╠═546a5c04-1d68-11eb-23bb-cb5d47d9db43
# ╠═546ad49a-1d68-11eb-1a31-c3b003680713
# ╠═5476c1ce-1d68-11eb-20ca-31c3a9321588
# ╠═54773480-1d68-11eb-101f-41723a16b3e7
# ╠═5483aa7e-1d68-11eb-0bff-3f6ed1803a29
# ╠═548437b6-1d68-11eb-2dea-4d5f64b16cf3
# ╟─5490b23c-1d68-11eb-2908-e18fb1dd0aad


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test
import REPL, REPL.REPLCompletions
import Markdown

@testset "symbol completion" begin
    @test startswith(let buf = IOBuffer()
            Core.eval(Main, REPL.helpmode(buf, "α"))
            String(take!(buf))
        end, "\"α\" can be typed by \\alpha<tab>\n")

    @test startswith(let buf = IOBuffer()
            Core.eval(Main, REPL.helpmode(buf, "🐨"))
            String(take!(buf))
        end, "\"🐨\" can be typed by \\:koala:<tab>\n")

    @test startswith(let buf = IOBuffer()
            Core.eval(Main, REPL.helpmode(buf, "ᵞ₁₂₃¹²³α"))
            String(take!(buf))
        end, "\"ᵞ₁₂₃¹²³α\" can be typed by \\^gamma<tab>\\_123<tab>\\^123<tab>\\alpha<tab>\n")

    # Check that all symbols with several completions have a canonical mapping (#39148)
    symbols = values(REPLCompletions.latex_symbols)
    duplicates = [v for v in unique(symbols) if count(==(v), symbols) > 1]
    @test all(duplicates .∈ Ref(keys(REPLCompletions.symbols_latex_canonical)))
end

@testset "quoting in doc search" begin
    str = let buf = IOBuffer()
        Core.eval(Main, REPL.helpmode(buf, "mutable s"))
        String(take!(buf))
    end
    @test occursin("'mutable struct'", str)
    @test occursin("Couldn't find 'mutable s'", str)
end

@testset "Non-Markdown" begin
    # https://github.com/JuliaLang/julia/issues/37765
    @test isa(REPL.insert_hlines(IOBuffer(), Markdown.Text("foo")), Markdown.Text)
    # https://github.com/JuliaLang/julia/issues/37757
    @test REPL.insert_hlines(IOBuffer(), nothing) === nothing
end

@testset "fuzzy score" begin
    # https://github.com/JunoLab/FuzzyCompletions.jl/issues/7
    # shouldn't throw when there is a space in a middle of query
    @test (REPL.matchinds("a ", "a file.txt"); true)
end

max_issues_repo_name: zhuangchaoxi/julia

max_forks_repo_name: zhuangchaoxi/julia


text:
# Application Ordering (AO
# exposes 1-based indexing interface, although there is a zero based interface
# underneath
# this is WIP pending Clang getting AOs right
export AO, map_petsc_to_app!, map_app_to_petsc!
mutable struct AO{T}
  p::C.AO{T}

  function AO(p::C.AO{T}) where {T}
    o = new{T}(p)
    if (!deactivate_finalizers)
      finalizer(PetscDestroy,o)
    end
    return o
  end
end

# zero based, using arrays
function AO_(::Type{T}, app_idx::AbstractArray{PetscInt, 1},
            petsc_idx::AbstractArray{PetscInt, 1}; comm=MPI.COMM_WORLD, basic=true ) where {T}
  ao_ref = Ref{C.AO{T}}()
  if basic  # mapping is one-to-one and onto
    #chk(C.AOCreateBasic(comm, PetscInt(length(app_idx)), app_idx, petsc_idx, ao_ref))
    chk(C.AOCreateMemoryScalable(comm, PetscInt(length(app_idx)), app_idx, petsc_idx, ao_ref))
  else  # worse performance
    chk(C.AOCreateMapping(comm, PetscInt(length(app_idx)), app_idx, petsc_idx, ao_ref))
  end

  return AO(ao_ref[])
end

# zero based, using index sets
# because index sets are already zero based, this function can be exposed
# directly
function AO( app_idx::IS{T}, petsc_idx::IS{T}; basic=true ) where {T}

  ao_ref = Ref{C.AO{T}}()
  if basic  # mapping is one-to-one and onto
    chk(C.AOCreateBasicIS( app_idx.p, petsc_idx.p, ao_ref))
  else  # worse performance
    chk(C.AOCreateMappingIS(app_idx.p, petsc_idx.p, ao_ref))
  end

  return AO(ao_ref[])
end

# one based interface
function AO(::Type{T}, app_idx::AbstractArray{I1, 1},
            petsc_idx::AbstractArray{I2, 1}; comm=MPI.COMM_WORLD, basic=true ) where {T, I1 <: Integer, I2 <: Integer}

  app_idx0 = PetscInt[app_idx[i] - 1 for i=1:length(app_idx)]
  petsc_idx0 = PetscInt[petsc_idx[i] - 1 for i=1:length(petsc_idx)]
  AO_(T, app_idx0, petsc_idx0; comm=comm, basic=basic)
end

function PetscDestroy(ao::AO{T}) where {T}

  if !PetscFinalized(T)
    chk(C.AODestroy(Ref(ao.p)))
    ao.p = C.AO{T}(C_NULL)
  end
end

function isfinalized(ao::AO)
  return isfinalized(ao.p)
end

function isfinalized(ao::C.AO)
  return ao.pobj == C_NULL
end

function petscview(ao::AO{T}) where {T}
  viewer = C.PetscViewer{T}(C_NULL)
  chk(C.VecView(ao.p, viewer))
end

###############################################################################
# functions to apply index changes

function map_petsc_to_app!(ao::AO, idx::AbstractArray)

  # because idx is expected to be modified, here we can decrement in-place
  for i=1:length(idx)
    idx[i] -= 1
  end

  chk(C.AOPetscToApplication(ao.p, PetscInt(length(idx)), idx))

  # increment back to 1-based indices
   for i=1:length(idx)
    idx[i] += 1
   end
end

function map_petsc_to_app!(ao::AO, is::IS)
  chk(C.AOPetscToApplicationIS(ao.p, is.p))
end

function map_app_to_petsc!(ao::AO, idx::AbstractArray)

  # because idx is expected to be modified, here we can decrement in-place
  for i=1:length(idx)
    idx[i] -= 1
  end

  chk(C.AOApplicationToPetsc(ao.p, PetscInt(length(idx)), idx))

  # increment back to 1-based indices
   for i=1:length(idx)
    idx[i] += 1
   end
end

function map_app_to_petsc!(ao::AO, is::IS)
  chk(C.AOApplicationToPetscIS(ao.p, is.p))
end


text:
# ---
# title: 412. Fizz Buzz
# id: problem412
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories:
# link: <https://leetcode.com/problems/fizz-buzz/description/>
# hidden: true
# ---
#
# Write a program that outputs the string representation of numbers from 1 to
# _n_.
#
# But for multiples of three it should output “Fizz” instead of the number and
# for the multiples of five output “Buzz”. For numbers which are multiples of
# both three and five output “FizzBuzz”.
#
# **Example:**
#
#
#
#     n = 15,
#
#     Return:
#     [
#         "1",
#         "2",
#         "Fizz",
#         "4",
#         "Buzz",
#         "Fizz",
#         "7",
#         "8",
#         "Fizz",
#         "Buzz",
#         "11",
#         "Fizz",
#         "13",
#         "14",
#         "FizzBuzz"
#     ]
#
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
@rule GammaMixture{N}(:switch, Marginalisation) (q_out::Any, q_a::NTuple{N, Any}, q_b::NTuple{N, GammaDistributionsFamily}) where { N } = begin

    U = map(zip(q_a, q_b)) do (a, b)
        return -score(AverageEnergy(), GammaShapeRate, Val{ (:out, :α, :β) }, map((q) -> Marginal(q, false, false), (q_out, a, b)), nothing)
    end

    ρ = clamp.(softmax(U), tiny, 1.0 - tiny)
    ρ = ρ ./ sum(ρ)

    return Categorical(ρ)
end


text:
#------------------------------------------------------------------------------
"""
    OlsGMFn(Y,X)

LS of Y on X; for one dependent variable, Gauss-Markov assumptions

# Usage
(b,u,Yhat,V,R2) = OlsGMFn(Y,X)

# Input
- `Y::Vector`:    Tx1, the dependent variable
- `X::Matrix`:    Txk matrix of regressors (including deterministic ones)

# Output
- `b::Vector`:    kx1, regression coefficients
- `u::Vector`:    Tx1, residuals Y - yhat
- `Yhat::Vector`: Tx1, fitted values X*b
- `V::Matrix`:    kxk matrix, covariance matrix of b
- `R2::Number`:   scalar, R2 value

"""
function OlsGMFn(Y,X)

    T    = size(Y,1)

    b    = X\Y
    Yhat = X*b
    u    = Y - Yhat

    σ2   = var(u)
    V    = inv(X'X)*σ2
    R2   = 1 - σ2/var(Y)

    return b, u, Yhat, V, R2

end
#------------------------------------------------------------------------------

max_stars_repo_name: snowdj/FinancialTheoryMSc

max_issues_repo_name: snowdj/FinancialTheoryMSc

max_forks_repo_name: snowdj/FinancialTheoryMSc


text:
"""

### About
SparkSQL.jl is software that enables developers to use the Julia programming language with the Apache Spark data processing engine.

### Use Case
Submits *Structured Query Language* (SQL), *Data Manipulation Language* (DML) and *Data Definition Language* (DDL) statements to Apache Spark.
Has functions to move data from Spark into Julia DataFrames and Julia DataFrame data into Spark.

SparkSQL.jl delivers advanced features like dynamic horizontal autoscaling that scale compute nodes to match workload requirements.

This package supports structured and semi-structured data in Data Lakes, Lakehouses (Delta Lake, Iceberg) on premise and in the cloud.

# Available Functions
Use ? to see help for each function.
- `initJVM`: initializes the Java Virtual Machine (JVM) in Julia.
- `SparkSession`: submits application to Apache Spark cluster with config options.
- `sql`: function to submit SQL, DDL, and DML statements to Spark.
- `cache`: function to cache Spark Dataset into memory.
- `createOrReplaceTempView`: creates temporary view that lasts the duration of the session.
- `createGlobalTempView`: creates temporary view that lasts the duration of the application.
- `toJuliaDF`: move Spark data into a Julia DataFrame.
- `toSparkDS`: move Julia DataFrame data to a Spark Dataset.

The SparkSQL.jl compute node autoscaling feature is based on Kubernetes. For SparkSQL.jl on Kubernetes instructions see:
kubernetes/README.md.

# Quick Start
### Install and Setup
Download Apache Spark 3.2.0 or later and set the environmental variables for Spark and Java home:
```
export SPARK_HOME=/path/to/apache/spark
export JAVA_HOME=/path/to/java
```
If using OpenJDK 11 on Linux set processReaperUseDefaultStackSize to true:
```
export _JAVA_OPTIONS='-Djdk.lang.processReaperUseDefaultStackSize=true'
```

### Startup

Start Julia with `"JULIA_COPY_STACKS=yes"` required for JVM interop:
```
JULIA_COPY_STACKS=yes julia
```
On MacOS start Julia with "handle-signals=no":
```
JULIA_COPY_STACKS=yes julia --handle-signals=no
```
### Usage

In Julia include the DataFrames package.  Also include the Dates and Decimals packages if your Spark data contains dates or decimal numbers.
```
using SparkSQL, DataFrames, Dates, Decimals
```
Initialize the JVM and start the Spark Session:
```
initJVM()
sparkSession = SparkSession("spark://example.com:7077", "Julia SparkSQL Example App")
```
Query data from Spark and load it into a Julia Dataset.
```
stmt = sql(sparkSession, "SELECT _c0 AS columnName1, _c1 AS columnName2 FROM CSV.`/pathToFile/fileName.csv`")
createOrReplaceTempView(stmt, "TempViewName")
sqlQuery = sql(sparkSession, "SELECT columnName1, columnName2 FROM TempViewName;")
juliaDataFrame = toJuliaDF(sqlQuery)
describe(juliaDataFrame)
```
Move Julia DataFrame data into an Apache Spark Dataset.
```
sparkDataset = toSparkDS(sparkSession, juliaDataFrame,",")
createOrReplaceTempView(sparkDataset, "tempTable")
```
The Dataset is a delimited string. To generate columns use the SparkSQL "split" function.

```
sqlQuery = sql(sparkSession, "Select split(value, ',' )[0] AS columnName1, split(value, ',' )[1] AS columnName2 from tempTable")
```

# Spark Data Sources
Supported data-sources include:
- File formats including: CSV, JSON, arrow, parquet
- Data Lakes including: Hive, ORC, Avro
- Data Lake Houses: Delta Lake, Apache Iceberg.
- Cloud Object Stores: S3, Azure Blob Storage, Swift Object.

## Data Source Examples

### CSV file example:
Comma Separated Value (CSV) format.
```
stmt = sql(session, "SELECT * FROM CSV.`/pathToFile/fileName.csv`;")
```
### Parquet file example:
Apache Parquet format.
```
stmt = sql(session, "SELECT * FROM PARQUET.`/pathToFile/fileName.parquet`;")
```
### Delta Lake Example:
Delta Lake is an open-source storage layer for Spark. Delta Lake offers:

- ACID transactions on Spark: Serializable isolation levels ensure that readers never see inconsistent data.
- Scalable metadata handling: Leverages Spark’s distributed processing power to handle all the metadata for petabyte-scale tables with billions of files.

Example shows create table (DDL) statements using Delta Lake and SparkSQL:
```
sql(session, "CREATE DATABASE demo;")
sql(session, "USE demo;")
sql(session, "CREATE TABLE tb(col STRING) USING DELTA;" )
```
The Delta Lake feature requires adding the Delta Lake jar to the Spark jars folder.

"""
module SparkSQL

using JavaCall
using DataFrames
using Decimals
using Dates
using ProgressMeter

export initJVM, SparkSession, sql, cache, createOrReplaceTempView, createGlobalTempView, toJuliaDF, toSparkDS

const SparkSession = @jimport org.apache.spark.sql.SparkSession
const SparkSessionBuilder = @jimport org.apache.spark.sql.SparkSession$Builder
const Dataset = @jimport org.apache.spark.sql.Dataset
const sparkDataFrame = @jimport org.apache.spark.sql.DataFrame
const GenericRow = @jimport org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema
const Row = @jimport org.apache.spark.sql.Row
const DataTypes = @jimport org.apache.spark.sql.types.DataTypes
const StructType = @jimport org.apache.spark.sql.types.StructType
const StructField = @jimport org.apache.spark.sql.types.StructField
const ArrayList = @jimport java.util.ArrayList
const Encoder = @jimport org.apache.spark.sql.Encoder
const Encoders = @jimport org.apache.spark.sql.Encoders
const RowEncoder = @jimport org.apache.spark.sql.catalyst.encoders.RowEncoder
const RowFactory = @jimport org.apache.spark.sql.RowFactory
const ExpressionEncoder = @jimport org.apache.spark.sql.catalyst.encoders.ExpressionEncoder

jvmCast(juliaType) = juliaType
jvmCast(jvmType::JString) = unsafe_string(jvmType)
jvmCast(jvmType::JavaObject{Symbol("java.lang.Long")}) = jcall(jvmType, "longValue", jlong, ())
jvmCast(jvmType::JavaObject{Symbol("java.lang.Integer")}) = jcall(jvmType, "intValue", jint, ())
jvmCast(jvmType::JavaObject{Symbol("java.lang.Double")}) = jcall(jvmType, "doubleValue", jdouble, ())
jvmCast(jvmType::JavaObject{Symbol("java.lang.Float")}) = jcall(jvmType, "floatValue", jfloat, ())
jvmCast(jvmType::JavaObject{Symbol("java.lang.Boolean")}) = jcall(jvmType, "booleanValue", jboolean, ())
jvmCast(jvmType::JavaObject{Symbol("java.sql.Date")}) =  Dates.Date((jcall(jvmType, "toString", (JString), ())), dateformat"yyyy-mm-dd" )
jvmCast(jvmType::JavaObject{Symbol("java.sql.Timestamp")}) =  Dates.DateTime((jcall(jvmType, "toString", (JString), ())), dateformat"yyyy-mm-dd HH:MM:SS.sss")
jvmCast(jvmType::JavaObject{Symbol("org.apache.spark.sql.types.DecimalType")}) = Decimals.decimal((jcall(jvmType, "toString", (JString), ())))
jvmCast(jvmType::JavaObject{Symbol("java.math.BigDecimal")}) = parse(Decimal,(jcall(jvmType, "toString", (JString), ())) )

"""
# Purpose
Initializes the Java Virtual Machine (JVM) in Julia using the JavaCall package.

# Arguments
- leave blank to initialize the JVM without options.

# Examples
```
initJVM()
```
"""
function initJVM()
	SPARK_HOME = get(ENV,"SPARK_HOME","")
        if isempty(SPARK_HOME)
		print("JVM not initialized. Please set your SPARK_HOME ENV variable.")
        else
		for jar in readdir(joinpath(SPARK_HOME, "jars"))
			JavaCall.addClassPath(joinpath(SPARK_HOME,"jars", jar))
		end
		JavaCall.init()
        end
end

"""
# Purpose
Initializes the Java Virtual Machine (JVM) in Julia using the JavaCall package.

# Arguments
- `JVMOptions::String`: configuration options to pass to the JVM.

# Examples
```
initJVM("-XX:ParallelGCThreads=2")
```

"""
function initJVM(JVMOptions::String)
	SPARK_HOME = get(ENV,"SPARK_HOME","")
        if isempty(SPARK_HOME)
		print("JVM not initialized. Please set your SPARK_HOME ENV variable.")
        else
		for jar in readdir(joinpath(SPARK_HOME, "jars"))
			JavaCall.addClassPath(joinpath(SPARK_HOME,"jars", jar))
		end
		JavaCall.init(JVMOptions)
        end
end

"""
# Purpose
Submits application to Apache Spark cluster.

# Arguments
- `master::String`: the URL to the Spark cluster master.
- `appName::String`: the name of the application.

# Examples
Basic spark session:
```
spark = SparkSession("spark://example.com:7077", "Julia SparkSQL Example App")
```

"""
function SparkSession(master::String, appName::String)
	print("Starting Spark Session")
	builder = jcall(SparkSession,"builder", SparkSessionBuilder,())
	jcall(builder, "master", SparkSessionBuilder, (JString,), master)
	jcall(builder, "appName",SparkSessionBuilder, (JString,), appName)
	session = jcall(builder, "getOrCreate", SparkSession, ())
	return session::SparkSession
end

"""
# Purpose
Submits application to Apache Spark cluster with config options.

# Arguments
- `master::String`: the URL to the Spark cluster master.
- `appName::String`: the name of the application.
- `config::Dict{String, String}()`: one or more config option for the SparkSession.

# Examples
Basic spark session:
```
spark = SparkSession("spark://example.com:7077", "Julia SparkSQL Example App")
```

Spark Session with Hive config:
```
spark = SparkSession("spark://example.com:7077", "Julia SparkSQL Example App", Dict{String,String}("spark.sql.warehouse.dir"=>"LOCATION","spark.sql.catalogImplementation"=>"hive"))
```
Spark Session with Delta Lake config:
```
spark = SparkSession("spark://example.com:7077", "Julia SparkSQL Example App", Dict{String,String}("spark.sql.warehouse.dir"=>"LOCATION", "spark.sql.extensions"=>"io.delta.sql.DeltaSparkSessionExtension", "spark.sql.catalog.spark_catalog"=>"org.apache.spark.sql.delta.catalog.DeltaCatalog"))
```
"""
function SparkSession(master::String, appName::String, config=Dict{key::String, value::String}())
	print("Starting Spark Session")
	builder = jcall(SparkSession,"builder", SparkSessionBuilder,())
	jcall(builder, "master", SparkSessionBuilder, (JString,), master)
	jcall(builder, "appName",SparkSessionBuilder, (JString,), appName)

	for (key, value) in config
		jcall(builder, "config", SparkSessionBuilder, (JString, JString), key, value)
	end

	session = jcall(builder, "getOrCreate", SparkSession, ())
	return session::SparkSession
end

"""
# Purpose
Submits *Structured Query Language* (SQL), *Data Manipulation Language* (DML) and *Data Definition Language* (DDL) statements to Apache Spark.

# Arguments
- `session:SparkSession`: the SparkSession.  See SparkSession help for instructions on how to create in Julia.
- `sqlText::String`: the DDL, DML or SQL statements.

# DDL Supported formats:
- File formats including: CSV, JSON, arrow, parquet
- Data Lakes including: Hive, ORC, Avro
- Data Lake Houses: Delta Lake, Apache Iceberg.
- Cloud Object Stores: S3, Azure Blob Storage, Swift Object.

# Examples

## CSV file example:
Comma Separated Value (CSV) format.
```
stmt = sql(session, "SELECT * FROM CSV.`/pathToFile/fileName.csv`;")
```
## Parquet file example:
Apache Parquet format.
```
stmt = sql(session, "SELECT * FROM PARQUET.`/pathToFile/fileName.parquet`;")
```
## Delta Lake Example:
Delta Lake is an open-source storage layer for Spark. Delta Lake offers:

ACID transactions on Spark: Serializable isolation levels ensure that readers never see inconsistent data.
Scalable metadata handling: Leverages Spark’s distributed processing power to handle all the metadata for petabyte-scale tables with billions of files at ease.

To use Delta Lake you must add the Delta Lake jar to your Spark jars folder.

Example shows create table (DDL), insert (DML) and select statements (SQL) using Delta Lake and SparkSQL:
```
sql(session, "CREATE DATABASE demo;")
sql(session, "USE demo;")
sql(session, "CREATE TABLE tb(col STRING) USING DELTA;" )
```
"""
function sql(session::SparkSession, sqlText::String)
	ds = jcall(session, "sql", Dataset, (JString,), sqlText)
	return ds::Dataset
end

"""
# Purpose
Caches Spark Dataset into memory.

# Arguments
- `ds::Dataset`: Dataset to cache.

# Example
```
stmt = sql(session, "SELECT _c0 AS columnName1, _c1 AS columnName2 FROM CSV.`/pathToFile/fileName.csv`")
cache(stmt)
```
"""
function cache(ds::Dataset)
	dataset = jcall(ds, "cache", Dataset, ())
	return dataset::Dataset
end

"""
# Purpose
Creates a temporary database view on the Apache Spark Cluster for the duration of Spark session.
# Arguments
- `ds:Dataset`: the Spark Dataset to create a session database view.
- `tempViewName::String`: the name of the view.
# Example
```
stmt = sql(session, "SELECT * FROM CSV.`/pathToFile/fileName.csv`")
createOrReplaceTempView(stmt, "TempViewName")
count = sql(session, "SELECT COUNT(*) AS total FROM TempViewName;")
```
"""
function createOrReplaceTempView(ds::Dataset, tempViewName::String)
	jcall(ds, "createOrReplaceTempView", Nothing, (JString,), tempViewName)
end

"""
# Purpose
Creates a temporary database view on the Apache Spark cluster for the duration of the Spark application.
# Arguments
- `ds:Dataset`: the Spark DataSet to create a session database view.
- `tempViewName::String`: the name of the view.
# Examples
```
stmt = sql(session, "SELECT * FROM CSV.`/pathToFile/fileName.csv`")
createGlobalTempView(stmt, "TempViewName")
count = sql(session, "SELECT COUNT(*) AS total FROM TempViewName;")
```
"""
function createGlobalTempView(ds::Dataset, tempViewName::String)
	jcall(ds, "createGlobalTempView", Nothing, (JString,), tempViewName)
end

Base.length(row::GenericRow) = jcall(row, "length", jint, ())
Base.getindex(row::GenericRow, index::Integer) = jcall(row, "get", JObject, (jint,), index-1)

const toJuliaType = Dict{String, DataType}("StringType" => String, "DateType" => Date,
					"LongType" => Int64, "IntegerType" => Int32, "DoubleType" => Float64,
					"FloatType" => Float32, "BooleanType" => UInt8, "DecimalType" => Decimal,
					"TimestampType" => DateTime)

function decimalFormat(sparkType::String)
	if length(sparkType) < 7
		return sparkType
	end
	if SubString(sparkType, 1,7) == "Decimal"
		return "DecimalType"
	else
		return sparkType
	end
end

function createDataFrame(ds::Dataset)
	schema = jcall(ds, "dtypes", Array{JavaObject{Symbol("scala.Tuple2")},1}, ())
	columns = Symbol.(unsafe_string.(map(column_name -> convert(JString, jcall(column_name, "_1", JObject, ())), schema)))
	dataTypes = unsafe_string.(map(dtype -> convert(JString, jcall(dtype, "_2", JObject, ())), schema))
	juliaDataTypes = map(sparkType -> toJuliaType[decimalFormat(sparkType)], dataTypes)
	dataframe = DataFrame(columns .=> [type[] for type in juliaDataTypes])
	return dataframe::DataFrame
end

"""
# Purpose
Move Spark data into a new Julia DataFrame.

# Arguments
- `ds:Dataset`: the Apache Spark Dataset to import into Julia as a DataFrame.

# Data Types
The function supports the following data types when creating Julia DataFrames from Spark data:

- `String`
- `Date`
- `Timestamp`
- `integer`
- `long`
- `double`
- `float`
- `decimal`
- `boolean`

# Additional Requirements
You must include the DataFrames package in your Julia program.

`Using DataFrames`

Include the "Dates" and "Decimals" packages if your Spark data contains dates or decimal numbers.

`Using Dates, Decimals`

# Examples
```
stmt = sql(session, "SELECT _c0 AS columnName1, _c1 AS columnName2 FROM CSV.`/pathToFile/fileName.csv`")
createOrReplaceTempView(stmt, "TempViewName")
sqlQuery = sql(session, "SELECT columnName1, columnName2 FROM TempViewName;")
juliaDataFrame = toJuliaDF(sqlQuery)
```
"""
function toJuliaDF(ds::Dataset)

	dataframe = createDataFrame(ds)
	row = jcall(ds,"collectAsList", JavaObject{Symbol("java.util.List")},())
	showProgress = ProgressUnknown("Rows retrieved:")
        for record in JavaCall.iterator(row)
		push!(dataframe, [jvmCast(narrow(record[i])) for i in 1:length(record)])
		ProgressMeter.next!(showProgress)
	end
	ProgressMeter.finish!(showProgress)

    return dataframe::DataFrame
end

import JavaCall: iterator

"""
# Purpose
Moves Julia DataFrame data to a Dataset in Apache Spark.
# Arguments
- `session::SparkSession`: the Spark Session.
- `df::DataFrame`: the name of the Julia DataFrame to move into a Spark Dataset.

# Examples
The example demonstrates how to query Spark and move that data to a Julia dataFrame. Then, after processing in Julia, move data back into a Spark Dataset.

Spark into a Julia DataFrame:
```
stmt = sql(session, "SELECT _c0 AS columnName1, _c1 AS columnName2 FROM CSV.`/pathToFile/fileName.csv`")
createOrReplaceTempView(stmt, "TempViewName")
sqlQuery = sql(session, "SELECT columnName1, columnName2 FROM TempViewName;")
juliaDataFrame = toJuliaDF(sqlQuery)
describe(juliaDataFrame)
```
Move Julia DataFrame data into an Apache Spark Dataset.
```
sparkDataset = toSparkDS(sparkSession, juliaDataFrame)
createOrReplaceTempView(sparkDataset, "tempTable")
```
The Dataset is a delimited string. To generate columns use the SparkSQL "split" function.

```
sqlQuery = sql(sparkSession, "SELECT split(value, ',' )[0] AS columnName1, split(value, ',' )[1] AS columnName2 FROM tempTable")
```
"""
function toSparkDS(session::SparkSession, df::DataFrame)

	rows = ArrayList(())
	columnRange = 1:size(df)[2]
	showProgress = ProgressUnknown("Rows sent:")
	for record in eachrow(df)
		jcall(rows, "add", jboolean, (JObject,), join(map(i -> string(record[i]),  columnRange), ","))
		ProgressMeter.next!(showProgress)
	end

 	stringEncoder = jcall( JavaObject{Symbol("org.apache.spark.sql.Encoders")}, "STRING", JavaObject{Symbol("org.apache.spark.sql.Encoder")}, ())
	ds = jcall(session, "createDataset", Dataset, (JavaObject{Symbol("java.util.List")}, JavaObject{Symbol("org.apache.spark.sql.Encoder")}), rows, stringEncoder)
	ProgressMeter.finish!(showProgress)

	return ds::Dataset

end

"""
# Purpose
Moves Julia DataFrame data to a Dataset in Apache Spark.
# Arguments
- `session::SparkSession`: the Spark Session.
- `df::DataFrame`: the name of the Julia Dataframe to move into a Spark Dataset.
- `delimiter::String`: The delimiter to use.

# Examples
The example demonstrates how to query Spark and move that data to a Julia DataFrame. Then, after processing in Julia, move data back into a Spark Dataset.

Move data from Spark into a Julia DataFrame:
```
stmt = sql(session, "SELECT _c0 AS columnName1, _c1 AS columnName2 FROM CSV.`/pathToFile/fileName.csv`")
createOrReplaceTempView(stmt, "TempViewName")
sqlQuery = sql(session, "SELECT columnName1, columnName2 FROM TempViewName;")
juliaDataFrame = toJuliaDF(sqlQuery)
describe(juliaDataFrame)
```
Move Julia DataFrame data into an Apache Spark Dataset.
```
sparkDataset = toSparkDS(sparkSession, juliaDataFrame,",")
createOrReplaceTempView(sparkDataset, "tempTable")
```
The Dataset is a delimited string. To generate columns use the SparkSQL "split" function.

```
sqlQuery = sql(sparkSession, "SELECT split(value, ',' )[0] AS columnName1, split(value, ',' )[1] AS columnName2 FROM tempTable")
```
"""
function toSparkDS(session::SparkSession, df::DataFrame, delimiter::String)

	rows = ArrayList(())
	columnRange = 1:size(df)[2]
	showProgress = ProgressUnknown("Rows sent:")
	for record in eachrow(df)
		jcall(rows, "add", jboolean, (JObject,), join(map(i -> string(record[i]),  columnRange), delimiter))
		ProgressMeter.next!(showProgress)
	end

 	stringEncoder = jcall( JavaObject{Symbol("org.apache.spark.sql.Encoders")}, "STRING", JavaObject{Symbol("org.apache.spark.sql.Encoder")}, ())
	ds = jcall(session, "createDataset", Dataset, (JavaObject{Symbol("java.util.List")}, JavaObject{Symbol("org.apache.spark.sql.Encoder")}), rows, stringEncoder)
	ProgressMeter.finish!(showProgress)

	return ds::Dataset

end

end # module


text:
function sparse_from_coo(::Type{<:SparseMatrixCSR{Bi}}, I,J,V,m,n) where Bi
  sparsecsr(Val(Bi),I,J,V,m,n)
end

@inline function is_entry_stored(::Type{<:SparseMatrixCSR},i,j)
  true
end

function finalize_coo!(::Type{<:SparseMatrixCSR},I,J,V,m,n)
  nothing
end

function nz_index(A::SparseMatrixCSR,i0,i1)
  if !(1 <= i0 <= size(A, 1) && 1 <= i1 <= size(A, 2)); throw(BoundsError()); end
  o = getoffset(A)
  Bi = getBi(A)
  r1 = Int(A.rowptr[i0]+o)
  r2 = Int(A.rowptr[i0+1]-Bi)
  (r1 > r2) && return -1
  i1o = i1-o
  k = searchsortedfirst(colvals(A), i1o, r1, r2, Base.Order.Forward)
  ((k > r2) || (colvals(A)[k] != i1o)) ? -1 : k
end

@inline function push_coo!(::Type{<:SparseMatrixCSR},I,J,V,i,j,v)
 push!(I,i)
 push!(J,j)
 push!(V,v)
 nothing
end


text:
using Chipmunk
using Base.Test

# write your own tests here
@test begin
    try
        gravity = Vect(0, -100)
        body = Body(1, 1)

        space = Space()
        set_gravity(space, gravity)

        add_body(space, body)

        timestep = 1./60
        for i = 1:100
            step(space, timestep)
            sleep(timestep)
        end
        true
    catch exception
        false
    end
end


text:
export interest, rate

"""
    interest(amount, rate)

Calculate interest from an ;amout; and interest rate of 'rate'.
"""
function interest(amount, rate)
    return amount * (1 + rate)
end

"""
    rate(amount, interest)

Calculate interest rate based on an  'amount' and 'interest'.
"""

function rate(amount, interest)
    return interest / amount
end

"""
    sum(x::Number, y::Number) -> x + y
Calculate the sum of the two given parameters
"""
function sum(x::Number, y::Number)
    return x + y
end


text:
include("../src/BIDSTools.jl")

using .BIDSTools
using Documenter, Example

makedocs(
    sitename="BIDSTools.jl",
    modules = [BIDSTools],
    authors = "Darren Lukas, Chris Foster",
    pages = Any[
        "Home" => "index.md",
        "Module" => "module.md"
    ],
    format = Documenter.HTML(prettyurls = false),
    clean = false,
    doctest = true
)

deploydocs(
    repo = "github.com/TRIImaging/BIDSTools.jl.git",
    target = "build",
    push_preview = true
)


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

@doc raw"""A topology selector term represents the result of label queries. A null or empty topology selector term matches no objects. The requirements of them are ANDed. It provides a subset of functionality as NodeSelectorTerm. This is an alpha feature and may change in the future.

    IoK8sApiCoreV1TopologySelectorTerm(;
        matchLabelExpressions=nothing,
    )

    - matchLabelExpressions::Vector{IoK8sApiCoreV1TopologySelectorLabelRequirement} : A list of topology selector requirements by labels.
"""
mutable struct IoK8sApiCoreV1TopologySelectorTerm <: SwaggerModel
    matchLabelExpressions::Any # spec type: Union{ Nothing, Vector{IoK8sApiCoreV1TopologySelectorLabelRequirement} } # spec name: matchLabelExpressions

    function IoK8sApiCoreV1TopologySelectorTerm(;matchLabelExpressions=nothing)
        o = new()
        validate_property(IoK8sApiCoreV1TopologySelectorTerm, Symbol("matchLabelExpressions"), matchLabelExpressions)
        setfield!(o, Symbol("matchLabelExpressions"), matchLabelExpressions)
        o
    end
end # type IoK8sApiCoreV1TopologySelectorTerm

const _property_map_IoK8sApiCoreV1TopologySelectorTerm = Dict{Symbol,Symbol}(Symbol("matchLabelExpressions")=>Symbol("matchLabelExpressions"))
const _property_types_IoK8sApiCoreV1TopologySelectorTerm = Dict{Symbol,String}(Symbol("matchLabelExpressions")=>"Vector{IoK8sApiCoreV1TopologySelectorLabelRequirement}")
Base.propertynames(::Type{ IoK8sApiCoreV1TopologySelectorTerm }) = collect(keys(_property_map_IoK8sApiCoreV1TopologySelectorTerm))
Swagger.property_type(::Type{ IoK8sApiCoreV1TopologySelectorTerm }, name::Symbol) = Union{Nothing,eval(Base.Meta.parse(_property_types_IoK8sApiCoreV1TopologySelectorTerm[name]))}
Swagger.field_name(::Type{ IoK8sApiCoreV1TopologySelectorTerm }, property_name::Symbol) =  _property_map_IoK8sApiCoreV1TopologySelectorTerm[property_name]

function check_required(o::IoK8sApiCoreV1TopologySelectorTerm)
    true
end

function validate_property(::Type{ IoK8sApiCoreV1TopologySelectorTerm }, name::Symbol, val)
end


text:
module ParallelRNGs

export
    replicate

include("replicate.jl")
end # module