text:
m = AnSchorfheide()
sys = compute_system(m)
path = dirname(@__FILE__)

@testset "Testing get_jstep " begin
    @test DSGE.get_jstep(m, 1)==1
    @test DSGE.get_jstep(m, 0)==5
end

@testset "Testing n_forecast_draws" begin
    @test DSGE.n_forecast_draws(m, :init) == 1
    @test DSGE.n_forecast_draws(m, :mode) == 1
    @test DSGE.n_forecast_draws(m, :mean) == 1

    m <= Setting(:sampling_method, :MH)
    global overrides = forecast_input_file_overrides(m)
    global overrides[:full] = "$path/../reference/mhsave_.h5"
    @test DSGE.n_forecast_draws(m, :full) == 100000

    m <= Setting(:sampling_method, :SMC)
    global overrides = forecast_input_file_overrides(m)
    global overrides[:full] = "$path/../reference/smcsave_.h5"
    @test DSGE.n_forecast_draws(m, :full) == 2000
    @test DSGE.n_forecast_draws(m, :prior) == 5000
    @test_throws ArgumentError DSGE.n_forecast_draws(m, :invalid_input)
end

@testset "Testing forecast_block_inds" begin
    m <= Setting(:jstep, 1)
    m <= Setting(:forecast_block_size, 5)
    block_inds, block_inds_thin = DSGE.forecast_block_inds(m, :full)
    @test block_inds[end] == 2000:5:2000
    @test block_inds_thin[end] == 400:400
    @test_throws ErrorException block_inds, block_inds_thin = DSGE.forecast_block_inds(m, :subset)
    block_inds, block_inds_thin = DSGE.forecast_block_inds(m, :subset, subset_inds = 1:100)
    @test block_inds[end] == 100:5:100
    @test block_inds_thin[end] == 20:20
    @test_throws ArgumentError DSGE.forecast_block_inds(m, :init)
    m <= Setting(:forecast_block_size, 5000)
    @test_throws ErrorException DSGE.forecast_block_inds(m, :full)
    @test_throws ErrorException DSGE.forecast_block_inds(m, :subset, subset_inds = 1:100)
    m <= Setting(:jstep, 5)
    m <= Setting(:forecast_block_size, 99)
    @test_throws ErrorException DSGE.forecast_block_inds(m, :full)
end

@testset "Testing add_requisite_output_vars" begin
    @test DSGE.add_requisite_output_vars([:histobs, :forecastobs, :shockdecobs, :shockdecpseudo]) == [:histobs, :forecastobs, :shockdecobs, :shockdecpseudo, :bddforecastobs, :dettrendobs, :dettrendpseudo, :trendobs, :trendpseudo]
    @test DSGE.add_requisite_output_vars([:histobs, :forecastobs]) == [:histobs, :forecastobs, :bddforecastobs]
    @test DSGE.add_requisite_output_vars([:shockdecobs]) == [:shockdecobs, :dettrendobs, :trendobs]
    @test_throws ErrorException DSGE.add_requisite_output_vars([:bad_input])
end

@testset "Testing remove_meansbands_only_output_vars" begin
    @test DSGE.remove_meansbands_only_output_vars([:histut, :hist4q, :forecastut, :forecast4q,
                                                   :bddforecastut, :bddforecast4q]) == []
    @test DSGE.remove_meansbands_only_output_vars([:histstates, :histut, :hist4q, :forecastut, :forecast4q,
                                                   :bddforecastut, :bddforecast4q]) == [:histstates]

end

@testset "Testing transplant history and forecast" begin
    @test isempty(DSGE.transplant_history(zeros(0, 0), 0))
    history = ones(10, 100)
    @test size(DSGE.transplant_history(history, 99)) == (10, 99)
    forecast = ones(10, 5)
    @test size(DSGE.transplant_forecast(history, forecast, 99)) == (10, 6)

    histstates = ones(8, 100)
    forecastobs = ones(3, 5)
    @test size(DSGE.transplant_forecast_observables(histstates, forecastobs, sys, 99)) == (3, 6)
end

@testset "Testing standardize_shocks" begin
    @test DSGE.standardize_shocks(ones(3, 3), sys[:QQ]) ==ones(3,3) ./ sqrt.(diag(sys[:QQ]))
end

if haskey(ENV, "FRED_API_KEY") || isfile(joinpath(homedir(),".freddatarc"))
    df = load_data(m)
    output_vars = [:histstates, :histobs, :forecaststates, :forecastobs]
    ndraws = 10
    paras = load_draws(m, :full, 1:ndraws, verbose = :none)

    forecast_outputs = map(p -> DSGE.forecast_one_draw(m, :full, :none, output_vars,
                                                       p, df, verbose = :none),
                           paras)
    forecast_outputs = convert(Vector{Dict{Symbol, Array{Float64}}}, forecast_outputs)
    out = DSGE.assemble_block_outputs(forecast_outputs)
    @testset "Testing assemble_block_outputs" begin
        @test size(out[:histstates]) == (ndraws, n_states_augmented(m), n_mainsample_periods(m))
        @test size(out[:forecaststates]) == (ndraws, n_states_augmented(m), forecast_horizons(m))
        @test size(out[:forecastobs]) == (ndraws, n_observables(m), forecast_horizons(m))
    end

    @testset "Testing get_forecast_output_dims" begin
        @test DSGE.get_forecast_output_dims(m, :mode, :histobs)[1] == 1
        @test DSGE.get_forecast_output_dims(m, :mean, :histobs)[1] == 1
        @test DSGE.get_forecast_output_dims(m, :init, :histobs)[1] == 1
        m <= Setting(:forecast_block_size, 5)
        @test DSGE.get_forecast_output_dims(m, :full, :histobs)[1] == 400

        @test DSGE.get_forecast_output_dims(m, :mode, :histstates)[2] == n_states_augmented(m)
        @test DSGE.get_forecast_output_dims(m, :mode, :histobs)[2] == n_observables(m)
        @test DSGE.get_forecast_output_dims(m, :mode, :histpseudo)[2] == n_pseudo_observables(m)
        @test DSGE.get_forecast_output_dims(m, :mode, :histshocks)[2] == n_shocks_exogenous(m)

        @test DSGE.get_forecast_output_dims(m, :mode, :histstates)[3] == n_mainsample_periods(m)
        @test DSGE.get_forecast_output_dims(m, :mode, :forecaststates)[3] == forecast_horizons(m)

        @test DSGE.get_forecast_output_dims(m, :mode, :histshockdec)[3] == n_mainsample_periods(m)
        @test DSGE.get_forecast_output_dims(m, :mode, :irfobs)[3] == impulse_response_horizons(m)

        @test DSGE.get_forecast_output_dims(m, :mode, :trendobs) == (1, n_observables(m))

        @test DSGE.get_forecast_output_dims(m, :mode, :shockdecobs) == (1, n_observables(m), n_shockdec_periods(m), n_shocks_exogenous(m))
        @test DSGE.get_forecast_output_dims(m, :mode, :irfobs) == (1, n_observables(m), impulse_response_horizons(m), n_shocks_exogenous(m))
    end
else
    @warn "Skipping some tests because FRED_API_KEY not present"
end


text:
using BinaryBuilder, Pkg

function gen_common(rest...; openmp = true, gpu = false, kwargs...)
    version = v"2020.6.5"
    sources = BinaryBuilder.AbstractSource[
        ArchiveSource("https://github.com/AlexeyAB/darknet/archive/3708b2e47d355ba0a206fd7a06bbc5a6e38af4ff.zip", "e18a6374822fe3c9b95f2b6a4086decbdfbd1c589f2481ce5704a4384044ea6f")
    ]

    script = "OPENMP=$(Int(openmp))\nGPU=$(Int(gpu))\n" * raw"""
cd $WORKSPACE/srcdir/darknet-*

if [[ "${GPU}" -eq 1 ]]; then
    ## CUDA setup
    mkdir -p /usr/local
    ln -s "${WORKSPACE}/srcdir/cudnn" /usr/local/cudnn
    ln -s ${prefix}/cuda /usr/local/cuda

    export PATH="/usr/local/cuda/bin:${PATH}"

    atomic_patch -p1 "${WORKSPACE}/srcdir/patches/makefile-fix-cuda-paths.patch"
fi

## Note for the future: Supporting AVX will make more sense with microarchitecture support
# if [[ "${target}" = powerpc64le-* ]] || [[ "${target}" = arm* ]] || [[ "${target}" == aarch* ]] || [[ "${target}" == *-mingw* ]] then
#     # Disable AVX on powerpc, arm, aarch, windows, apple
#     export AVXENABLE=0
# else
#     # Enable everywhere else (linux)
#     export AVXENABLE=1
# fi
export AVXENABLE=0

# Make sure to have the directories, before building
# make obj backup results setchmod
make -j${nproc} \
    libdarknet.${dlext} \
    OPTS="" \
    LIBNAMESO="libdarknet.${dlext}" \
    LIBSO=1 \
    GPU=${GPU} \
    CUDNN=${GPU} \
    CUDNN_HALF=${GPU} \
    OPENCV=0 \
    DEBUG=0 \
    OPENMP=${OPENMP} \
    ZED_CAMERA=0 \
    AVX=${AVXENABLE}

mkdir -p "${libdir}"
cp libdarknet.${dlext} "${libdir}"
install_license LICENSE

rm -rf "${prefix}/cuda"
"""
    # The products that we will ensure are always built
    products = [
        # Do not automatically dlopen the library if depends on CUDA
        LibraryProduct("libdarknet", :libdarknet; dont_dlopen = gpu),
    ]

    # Dependencies that must be installed before this package can be built
    dependencies = BinaryBuilder.AbstractDependency[]
    if openmp
        append!(dependencies, [
            # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD
            # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else.
            Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)),
            Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)),
        ])
    end

    return version, sources, script, products, dependencies
end

ext: jl

max_stars_repo_name: cmhamel/Yggdrasil

max_issues_repo_name: cmhamel/Yggdrasil

max_forks_repo_name: cmhamel/Yggdrasil


text:
# vim: set et ts=2 sw=2:

# ## Meta
same(s) = s

# ## Maths
int(x)  = floor(Int,x)
any(a)  = a[ int(length(a) * rand()) + 1]
few(a,n=it.divs.few)=length(a)<n ? a : [any(a) for _ in 1:n]

# ## Strings
thing(x) = try parse(Float64,x) catch _ x end

sayln(i) = begin ay(i); println("") end

function say(i)
  s,pre="$(typeof(i)){",""
  for f in sort!([x for x in fieldnames(typeof(i))
                 if !("$x"[1] == '_')])
    g = getfield(i,f)
    s = s * pre * "$f=$g"
    pre=", "
  end
  print(s * "}")
end

# ## Files
@resumable function csv(file;zap=r"(\s+|#.*)")
  b4=""
  for line in eachline(file)
    line = replace(line,zap =>"")
    if length(line) != 0
      if line[end] == ',' # if the line ends with "," we'll
        b4 = b4 * line    # need to join it to next
      else
        @yield [thing(x) for x in split(b4*line,",")]
                b4 = "" end end end end

max_stars_repo_name: timm/misc

max_issues_repo_name: timm/misc

max_forks_repo_name: timm/misc


text:
using IJulia.CommManager
import IJulia.metadata
import JSON: json
using Patchwork
using Compat

if Pkg.installed("Reactive") !=nothing && Pkg.installed("Reactive") >= v"0.1.9"
    import Reactive: Signal, value, preserve

    function diff(s::Signal)
        prev = value(s)
        map((x) -> begin
            d = Patchwork.diff(prev, x)
            prev = x
            d
        end, s)
    end

    function send_patch(comm, d)
        jsonpatch = Patchwork.jsonfmt(d)
        if !isempty(jsonpatch)
            send_comm(comm, jsonpatch)
        end
    end

    metadata{ns, name}(x::Signal{Elem{ns, name}}) = Dict()

    @compat function Base.show{ns, name}(io::IO, ::MIME"text/html", x::Signal{Elem{ns, name}})
        id = Patchwork.pwid()

        write(io, """
""",
                  """""",
                  """
""")

        comm = Comm(:PatchStream; data=Dict(:pwid => id))
        # lift patch stream
        map((d) -> send_patch(comm, d), diff(x)) |> preserve
    end
end


text:
module TestLengths

using Phylo
using Compat.Test

@testset "routes" begin
    species = ["Dog", "Cat", "Human", "Potato"]
    tree = NamedTree(species)
    nr = addnode!(tree)
    n2 = branch!(tree, nr)
    n3 = branch!(tree, n2)
    nh = nodehistory(tree, n2)
    @test nr ∈ nh
    @test Set(nh) == Set(push!(getancestors(tree, n2), n2))
    @test nr ≠ getparent(tree, n3)
    @test n2 == getparent(tree, n3)
    @test nr ∉ getdescendants(tree, nr)
    @test n3 ∈ getdescendants(tree, nr)
    @test n3 ∉ getchildren(tree, nr)
    @test Set([nr, n2, n3]) == Set(nodehistory(tree, n3))
    br = addbranch!(tree, n3, "Dog")
    @test br ∈ branchhistory(tree, "Dog")
    br2 = addbranch!(tree, n3, "Cat")
    @test [br2, br] == branchroute(tree, "Cat", "Dog")
    br3 = addbranch!(tree, n2, "Human")
    @test addbranch!(tree, nr, "Potato", branchname = 551) == 551
    @test 551 ∈ branchroute(tree, "Dog", "Potato")
    deletebranch!(tree, 551)
    br4 = addbranch!(tree, nr, "Potato")
    @test Set(species) ⊆ Set(getdescendants(tree, nr))
    @test validate(tree)
end

end


text:
using NeighbourLists
using JuLIP, StaticArrays

# ==================================
#     Lennard-Jones Test
# ==================================

function lj_iter(nlist, V)
   Es = zeros(nsites(nlist))
   for (i, _1, r, _2) in pairs(nlist)
      Es[i] += V(r)
   end
   dE = zeros(eltype(nlist.R), nsites(nlist))
   for (i, j, r, R) in pairs(nlist)
      dV = (@D V(r)) * (R/r)
      dE[j] += dV
      dE[i] -= dV
   end
   return Es, dE
end

function lj_mapreduce{T}(nlist::PairList{T}, V)
   Es = maptosites!( (r,R) -> V(r), zeros(T, nsites(nlist)),
                        pairs(nlist) )
   dE = maptosites_d!((r,R) -> (@D V(r)), zeros(JVec{T}, nsites(nlist)),
                         pairs(nlist) )
end

function lj_nbody{T}(nlist::PairList{T}, V)
   Es = maptosites!(r -> V(r[1]),  zeros(T, nsites(nlist)),
                      nbodies(2, nlist) )
   dE = maptosites_d!(r -> (@D V(r[1])),  zeros(JVec{T}, nsites(nlist)),
                      nbodies(2, nlist) )
end

println("----------------------------------------")
println("Lennard-Jones Test")
println("----------------------------------------")

r0 = rnn(:Fe)
cutoff =  r0 * 2.7
lj = LennardJones(r0, 1.0) * C1Shift(cutoff)

for L in (2, 5, 10, 20)
   at = bulk(:Fe, cubic=true) * L
   println("Bulk Fe, Nat = $(length(at))")
   println("PairList construction:")
   @time nlist = PairList(positions(at), cutoff, cell(at), pbc(at))
   @time nlist = PairList(positions(at), cutoff, cell(at), pbc(at))
   println("Energy + Force assembly Iterator:")
   @time lj_iter(nlist, lj)
   @time lj_iter(nlist, lj)
   println("Energy + Force assembly MapReduce:")
   @time lj_mapreduce(nlist, lj)
   @time lj_mapreduce(nlist, lj)
   println("Energy + Force assembly nbody MapReduce:")
   @time lj_nbody(nlist, lj)
   @time lj_nbody(nlist, lj)
end

# # ==================================
# #     EAM Test
# # ==================================
#
#
# function eam_iter{T}(nlist::PairList{T}, V)
#    Es = zeros(T, nsites(nlist))
#    for (i, _1, r, R) in NeighbourLists.sites(nlist)
#       Es[i] += V(r, R)
#    end
#    dE = zeros(JVec{T}, nsites(nlist))
#    for (i, j, r, R) in NeighbourLists.sites(nlist)
#       dV = @D V(r, R)
#       dE[j] += dV
#       dE[i] -= sum(dV)
#    end
#    return Es, dE
# end
#
# function eam_iter!{T}(nlist::PairList{T}, V)
#    Es = zeros(T, nsites(nlist))
#    for (i, _1, r, R) in NeighbourLists.sites(nlist)
#       Es[i] += V(r, R)
#    end
#    dE = zeros(JVec{T}, nsites(nlist))
#    ndv = maximum(length(j) for (i, j, r, R) in NeighbourLists.sites(nlist))
#    dV = zeros(JVec{T}, ndv)
#    for (i, j, r, R) in NeighbourLists.sites(nlist)
#       fill!(dV, zero(JVec{T}))
#       JuLIP.Potentials.evaluate_d!(dV, V, r, R)
#       for a = 1:length(j)
#          dE[j[a]] += dV[a]
#          dE[i] -= dV[a]
#       end
#    end
#    return Es, dE
# end
#
# function eam_mapreduce{T}(nlist::PairList{T}, V)
#    Es = map!(V, zeros(T, nsites(nlist)), NeighbourLists.sites(nlist))
#    dE = map_cfd!((r, R) -> (@D V(r,R)), zeros(JVec{T}, nsites(nlist)),
#                 NeighbourLists.sites(nlist))
# end
#
# println("----------------------------------------")
# println("Analytic EAM Test")
# println("----------------------------------------")
# r0 = rnn("Fe")
# cutoff = r0 * 2.7
# ϕ = (@analytic r -> 1/r) * C1Shift(cutoff)
# ρ = (@analytic r -> exp(-r/3)) * C1Shift(cutoff)
# eam = EAM(ϕ, ρ, @analytic t -> sqrt(1+t))
#
# for L in (5, 10, 20)
#    at = bulk("Fe", cubic=true) * L
#    println("Bulk Fe, Nat = $(length(at))")
#    println("PairList construction:")
#    @time nlist = PairList(positions(at), cutoff, cell(at), pbc(at))
#    @time nlist = PairList(positions(at), cutoff, cell(at), pbc(at))
#    println("Energy + Force assembly Iterator:")
#    @time eam_iter(nlist, eam)
#    @time eam_iter(nlist, eam)
#    println("Energy + Force assembly MapReduce:")
#    @time eam_mapreduce(nlist, eam)
#    @time eam_mapreduce(nlist, eam)
#    println("Energy + Force assembly In-place Iterator:")
#    @time eam_iter!(nlist, eam)
#    @time eam_iter!(nlist, eam)
# end


text:
export @debug,
    powerset,
    getnextfreesymbol,
    resetfreesymbols

function bracketsplit(str)
    parts = []
    part_start = 1
    bracket_balance = 0
    for idx in eachindex(str)
        bracket_balance += @match str[idx] begin
        '(' => -1
        ')' => 1
        _ => 0
        end
        if bracket_balance == 0
            next_part = nextind(str, idx)
            push!(parts, str[part_start:next_part-1])
            part_start = next_part
        end
    end
    parts
end

function argumentsplit(str)
    parts = strip.(bracketsplit(str))
    comma = findfirst(parts, ",")
    if comma == 0
        [str]
    else
        arguments = [join(parts[1:comma-1])]
        while comma != 0
            next_comma = findnext(parts, ",", comma+1)
            if next_comma == 0
                push!(arguments, join(parts[comma+1:end]))
            else
                push!(arguments, join(parts[comma+1:next_comma-1]))
            end
            comma = next_comma
        end
        arguments
    end
end

function strip_and_remove_surrounding_brackets(str)
    str = strip(str)
    while length(bracketsplit(str)) == 1 && str[1] == '('
        str = str[2:end-1]
    end
    str
end

"""
    powerset(x)

Returns the powerset of x (a set of all subsets of x).
"""
function powerset(x)
    result = [Set()]
    for element in collect(x), idx = 1:length(result)
        newsubset = union(result[idx], Set([element]))
        push!(result, newsubset)
    end
    Set(result)
end

# Currently, Julia's multiple dispatch system only dispatches on the expression type for
# macros (which is most of the times Expr or Symbol). To do multiple dispatch here, we
# therefore have to work around that limitation by using the hack below. This might change
# if https://github.com/JuliaLang/julia/issues/20929 gets addressed.
macro applyrecursively(functionexpr, objectexpr, objecttype)
    typemapping = Dict(
        :CNF => :collection,
        :Clause => :collection,
        :Conjunction => :monotone_fields,
        :Negation => :monotone_fields,
        )
        dispatch_applyrecursively(functionexpr, objectexpr, Val{typemapping[objecttype]})
    end

function dispatch_applyrecursively(functionexpr, objectexpr,
                                    objecttype::Type{Val{:collection}})
    fsymbol, fargs = functionexpr.args[1],
                     functionexpr.args[2:end]
    quote
        obj = $(esc(objectexpr))
        typeof(obj)(
            [$fsymbol([arg == :_ ? element : arg
                       for arg in [$(esc.(fargs)...)]]...)
             for element in obj]
        )
    end
end

"""
    @debug variable

Print the symbol `variable`, followed by its value, followed by an empty line.
"""
macro debug(variable)
    Expr(:call,
         :println,
         Expr(:call,
              :string,
              Expr(:quote, variable)),
         " ",
         esc(variable),
        "\n"
        )
end

"""
    getnextfreesymbol()

Returns a new, unique string that can be used
for naming objects.

This does not check if a user-entered formula
already contains the returned string.
"""
function getnextfreesymbol end

"""
    resetfreesymbols()

Resets the free symbol state.

Newly generated symbols will "start over".
"""
function resetfreesymbols end

let nextfreesymbol = 0
    global getnextfreesymbol
    global resetfreesymbols
    function getnextfreesymbol()
        nextfreesymbol += 1
        string("##", nextfreesymbol)
    end
    function resetfreesymbols()
        nextfreesymbol = 0
    end
end


text:
module Configurations

export @option, from_dict, from_kwargs, from_toml, to_dict, to_toml

using OrderedCollections
using MatchCore
using Crayons.Box
using ExprTools
using TOML

struct NoDefault end

struct PartialDefault{F}
    lambda::F
    expr::Expr
end

(f::PartialDefault)(x) = f.lambda(x)
Base.show(io::IO, x::PartialDefault) = print(io, x.expr)

"const for non default fields"
const no_default = NoDefault()

"maybe of type `T` or nothing"
const Maybe{T} = Union{Nothing, T}

"""
    field_defaults(::Type)

Return default values of given option types.
"""
function field_defaults(::Type{T}) where T
    is_option(T) || error("$T is not an option type")
    return Any[field_default(T, each) for each in fieldnames(T)]
end

"""
    field_alias(::Type{T}) where T

Return all field name alias of given option types.
"""
function field_aliases(::Type{T}) where T
    is_option(T) || error("$T is not an option type")
    return Any[field_alias(T, each) for each in fieldnames(T)]
end

"""
    field_default(::Type{T}, name::Symbol)

Return the default value of field `name` of an option type `T`.
"""
function field_default(::Type{T}, name::Symbol) where {T}
    error("field_default is not defined for $T, it may not be an option type")
end

"""
    field_alias(::Type{T}, name::Symbol) where {T}

Return field name alias of given option types.
"""
function field_alias(::Type{T}, name::Symbol) where {T}
    error("field_alias is not defined for $T, it may not be an option type")
end

function alias(::Type{T}) where T
    error("alias is not defined $T, it may not be an option type")
end

"""
    option_convert(::Type{OptionType}, ::Type{ValueType}, x) where {OptionType, ValueType}

Convert `x` to type `ValueType` for option type `OptionType`. This is similar to `Base.convert`,
when creating an instance of the option type, but one can use this to avoid type piracy.
"""
option_convert(::Type, ::Type{A}, x) where {A} = nothing

option_convert(::Type, ::Type{VersionNumber}, x::String) = VersionNumber(x)

function option_convert_union(::Type{T}, ::Type{A}, x) where {T, A}
    if !(A isa Union)
        v = option_convert(T, A, x)
        v === nothing || return v
        return
    end

    v = option_convert_union(T, A.a, x)
    v === nothing || return v
    return option_convert_union(T, A.b, x)
end

"""
    toml_convert(::Type, x)

A convenient function for converting common Julia types to TOML compatible types. One
can overload the first argument to custom the behaviour for a specific option type.
"""
toml_convert(::Type, x) = x
toml_convert(::Type, x::VersionNumber) = string(x)

"""
    toml_convert(::Type{T}) where T

Curried version of `toml_convert`.
"""
toml_convert(::Type{T}) where T = x->toml_convert(T, x)

"""
    to_dict(option) -> OrderedDict

Convert an option to an `OrderedDict`.
"""
function to_dict(x)
    is_option(x) || error("argument is not an option type")
    return dictionalize(x)::OrderedDict
end

"""
    to_toml(x; sorted=false, by=identity)

Convert an instance `x` of option type to TOML and write it to `String`. See also `TOML.print`.
"""
function to_toml(x; sorted=false, by=identity)
    return sprint(to_toml, x)
end

function to_toml(io::IO, x; sorted=false, by=identity)
    return to_toml(toml_convert(typeof(x)), io, x; sorted=sorted, by=by)
end

function to_toml(filename::String, x; sorted=false, by=identity)
    return to_toml(toml_convert(typeof(x)), filename, x; sorted=sorted, by=by)
end

"""
    to_toml([f::Function], filename::String, option; sorted=false, by=identity)

Convert an instance `option` of option type to TOML and write it to `filename`. See also `TOML.print`.
"""
function to_toml(f, filename::String, x; sorted=false, by=identity)
    open(filename, "w+") do io
        to_toml(f, io, x; sorted=sorted, by=by)
    end
end

"""
    to_toml([f::Function], io::IO, option; sorted=false, by=identity)

Convert an instance `option` of option type to TOML and write it to `IO`. See also `TOML.print`.
"""
function to_toml(f, io::IO, x; sorted=false, by=identity)
    return TOML.print(f, io, to_dict(x); sorted=sorted, by=by)
end

@deprecate toml to_toml

"""
    dictionalize(x)

Convert `x` to an `OrderedDict`.
"""
function dictionalize(x)
    is_option(x) || return x
    d = OrderedDict{String, Any}()
    T = typeof(x)
    for name in fieldnames(T)
        type = fieldtype(T, name)
        value = getfield(x, name)
        if value != field_default(T, name)
            field_dict = dictionalize(value)

            # always add an alias if it's a Union
            # of multiple option types
            if is_option(value) && type isa Union
                d[string(name)] = OrderedDict{String, Any}(
                    alias(typeof(value)) => field_dict,
                )
            else
                d[string(name)] = field_dict
            end
        end
    end
    return d
end

"""
    is_option(x)

Check if `x` is an option type or not.
"""
is_option(x) = false

"""
    from_dict(::Type{T}, d::AbstractDict{String}; kw...) where T

Convert dictionary `d` to an option type `T`, the valud of valid fields of `T`
in this dictionary `d` can be override by keyword arguments.

# Example

```julia-repl
julia> @option struct OptionA
           name::String = "Sam"
           age::Int = 25
       end

julia> d = Dict{String, Any}(
           "name" => "Roger",
           "age" => 10,
       );

julia> from_dict(OptionA, d; age=25)
OptionA(;
    name = "Roger",
    age = 25,
)
```
"""
function from_dict(::Type{T}, d::AbstractDict{String}; kw...) where T
    # override dict values
    validate_keywords(T; kw...)
    from_kwargs!(d, T; kw...)
    return from_dict_validate(T, d)
end

function from_dict_validate(::Type{T}, d::AbstractDict{String}) where T
    is_option(T) || error("$T is not an option type")

    for k in keys(d)
        k == "#filename#" && continue
        k in field_aliases(T) || Symbol(k) in fieldnames(T) || error("invalid key: $k")
    end

    for (name, default) in zip(fieldnames(T), field_defaults(T))
        if default === no_default
            alias = field_alias(T, name)
            haskey(d, string(name)) && continue
            alias !== nothing && haskey(d, alias) && continue
            error("$name is required")
        end
    end

    return from_dict_inner(T, d)
end

function from_dict_inner(::Type{T}, x) where T
    error("cannot convert $x to $T")
end

function assert_union_alias(::Type{T}, name=nothing) where T
    T isa Union || return
    name === nothing && return
    name == alias(T.a) && error("duplicated alias name: $name")
    return assert_union_alias(T.b, alias(T.a))
end

# we don't process other kind of value
"""
    pick_union(::Type, x) -> type, value

Pick a type `T` and its corresponding value from a `Union`. For option
types it should be a dictionary type. The value can be furthur converted
to this type `T` via [`option_convert`](@ref) or `Base.convert`.
"""
pick_union(::Type{T}, x) where T = T, x

function pick_union(::Type{T}, d::AbstractDict{String}) where T
    if !(T isa Union)
        T === Nothing && return
        is_option(T) || return T, d
        if haskey(d, alias(T))
            return T, d[alias(T)]
        else
            return
        end
    end

    assert_union_alias(T)

    ret_a = pick_union(T.a, d)
    ret_b = pick_union(T.b, d)

    if ret_a === nothing
        return ret_b
    else
        return ret_a
    end
end

"""
    from_dict_inner(::Type{T}, d::AbstractDict{String}) where T

Internal method to convert a dictionary (subtype of `AbstractDict`)
to type `T`, this method will not check if `T` is an option type
via `is_option`, and will not validate if all the required fields
are available in the dict object.
"""
function from_dict_inner(::Type{T}, d::AbstractDict{String}) where T
    args = Any[]
    for each in fieldnames(T)
        key = string(each)
        key = haskey(d, key) ? key : field_alias(T, each)
        type = fieldtype(T, each)
        default = field_default(T, each)

        if default === no_default
            if type isa Union
                pick = pick_union(type, d[key])
                pick === nothing && error("alias name for multi-option $type is required")
                type, value = pick
            else
                value = d[key]
            end
        else
            if type isa Union && haskey(d, key)
                pick = pick_union(type, d[key])
            else
                pick = nothing
            end

            if pick === nothing
                value = get(d, key, default)
            else
                type, value = pick
            end
        end

        if is_option(type) && value isa AbstractDict{String}
            # need some assertions so we call from_dict_validate
            push!(args, from_dict_validate(type, value))
        elseif value isa AbstractDict && isempty(value) && Nothing <: type
            # empty collection
            push!(args, nothing)
        else
            v = option_convert_union(T, type, value)
            if v === nothing
                push!(args, convert(type, value))
            else
                push!(args, v)
            end
        end
    end

    return T(args...)
end

"""
    from_toml(::Type{T}, filename::String; kw...) where T

Convert a given TOML file `filename` to an option type `T`. Valid fields
can be override by keyword arguments. See also [`from_dict`](@ref).
"""
function from_toml(::Type{T}, filename::String; kw...) where T
    is_option(T) || error("not an option type")
    d = TOML.parsefile(filename)
    d["#filename#"] = filename
    return from_dict(T, d; kw...)
end

"""
    from_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T

Internal method for inserting keyword arguments to given dictionary object `d`. It will overwrite
existing keys in `d` if it is specified by keyword argument.
"""
function from_kwargs!(d::AbstractDict{String}, ::Type{T}, prefix::Maybe{Symbol} = nothing; kw...) where T
    if T isa Union
        from_kwargs!(d, T.a, prefix; kw...)
        from_kwargs!(d, T.b, prefix; kw...)
        return d
    end

    is_option(T) || return
    fnames = fieldnames(T)

    for name in fnames
        type = fieldtype(T, name)
        if prefix === nothing
            key = name
        else
            key = Symbol(prefix, :_, name)
        end

        if is_option(type) || type isa Union
            field_d = OrderedDict{String, Any}()
            from_kwargs!(field_d, type, key; kw...)
            if !isempty(field_d)
                d[string(name)] = field_d
            end
        elseif haskey(kw, key)
            d[string(name)] = kw[key]
        end
    end
    return d
end

function validate_keywords(::Type{T}; kw...) where T
    ks = keywords(T)
    hint = join(map(x->LIGHT_BLUE_FG(string(x)), ks), ", ")
    for (k, v) in kw
        k in ks || throw(ArgumentError("invalid key $(LIGHT_BLUE_FG(string(k))), possible keys are: $hint"))
    end
    return
end

"""
    keywords(::Type{T}) where T -> Vector{Symbol}

Get all the keywords of type `T`.
"""
keywords(::Type{T}) where T = collect_keywords!(Symbol[], T)

function collect_keywords!(list::Vector{Symbol}, ::Type{T}, prefix::Maybe{Symbol} = nothing) where T
    if T isa Union
        collect_keywords!(list, T.a, prefix)
        collect_keywords!(list, T.b, prefix)
        return list
    end

    is_option(T) || return list
    for name in fieldnames(T)
        type = fieldtype(T, name)
        if prefix === nothing
            key = name
        else
            key = Symbol(prefix, :_, name)
        end

        if is_option(type) || type isa Union
            collect_keywords!(list, type, key)
        else
            push!(list, key)
        end
    end
    return list
end

"""
    from_kwargs(::Type{T}; kw...) where T

Convert keyword arguments to given option type `T`. See also [`from_dict`](@ref).
"""
function from_kwargs(::Type{T}; kw...) where T
    validate_keywords(T; kw...)
    d = OrderedDict{String, Any}()
    from_kwargs!(d, T; kw...)
    return from_dict_validate(T, d)
end

"""
    Field

Type to represent a field definition in option type.
"""
struct Field
    name::Symbol
    alias::Maybe{String}
    type::Any
    default::Any
    line
end

"""
    OptionDef

Type to represent the option type definition.
"""
struct OptionDef
    name::Symbol
    alias::Union{Nothing, String}
    ismutable::Bool
    parameters::Vector{Any}
    supertype

    fields::Vector{Field}
    misc::Vector{Any}

    function OptionDef(name, alias, ismutable, params, supertype, fields::Vector{Field}, misc)
        for each in fields
            if each.name == alias
                error("fieldname is the same as alias name")
            end
        end
        new(name, alias, ismutable, params, supertype, fields, misc)
    end
end

function Base.show(io::IO, x::Field)
    indent = get(io, :indent, 0)
    print(io, " "^indent, x.name)

    if x.type !== Any
        print(io, "::", GREEN_FG(string(x.type)))
    end

    if x.default !== no_default
        print(io, " = ", x.default)
    end

    if x.alias !== nothing
        print(io, " ", MAGENTA_FG("($(x.alias))"))
    end
end

function Base.show(io::IO, x::OptionDef)
    indent = get(io, :indent, 0)

    if x.ismutable
        print(io, " "^indent, BLUE_FG("mutable "))
    end

    print(io, BLUE_FG("struct"))
    print(io, " ", x.name)
    if !isempty(x.parameters)
        print(io, "{", DARK_GRAY_FG(join(string.(x.parameters), ", ")), "}")
    end

    if x.supertype !== nothing
        print(io, " <: ", GREEN_FG(string(x.supertype)))
    end

    println(io)
    let io = IOContext(io, :indent=>indent+2)
        for (k, each) in enumerate(x.fields)
            println(io, each)
        end
    end

    print(io, " "^indent, BLUE_FG("end"))
end

function OptionDef(@nospecialize(ex), alias=nothing)
    ex isa Expr || error("invalid usage of @option")
    ex.head === :struct || error("invalid usage of @option")

    name, parameters, supertype = split_name(ex)
    fields, misc = split_body(ex)
    return OptionDef(name, alias, ex.args[1], parameters, supertype, fields, misc)
end

function has_custom_kw_fn(def::OptionDef)
    return has_custom_kw_fn(def.misc)
end

function has_custom_kw_fn(misc::Vector{Any})
    for each in misc
        is_kw_fn(each) && return true
    end
    return false
end

is_kw_fn(x) = false

# TODO: implement this directly without splitdef
function is_kw_fn(ex::Expr)
    def = splitdef(ex; throw=false)
    def === nothing && return false
    haskey(def, :args) && return false
    return haskey(def, :kwargs)
end

"""
    split_name(ex::Expr) -> name, typevars, supertype

Split the name, type parameters and supertype definition from `struct`
declaration head.
"""
function split_name(ex::Expr)
    T = ex.args[2]

    return @smatch T begin
        :($name{$(typevars...)}) => (name, typevars, nothing)
        :($name{$(typevars...)} <: $type) => (name, typevars, type)
        ::Symbol => (T, [], nothing)
        :($name <: $type) => (name, [], type)
        _ => error("invalid @option: $ex")
    end
end

"""
    split_body(ex::Expr) -> fields::Vector{Field}, misc::Vector{Any}

Split the fields of option type declaration and misc (such as inner constructors etc.).
"""
function split_body(ex::Expr)
    body = ex.args[3]
    body.head === :block || error("expect a block, got $ex")

    fields = Field[]
    misc = Any[]
    line = nothing
    alias = nothing

    for each in body.args
        @smatch each begin
            :($name::$type = $default) => begin
                push!(fields, Field(name, alias, type, default, line))
                alias = nothing
            end

            :($name::$type) => begin
                push!(fields, Field(name, alias, type, no_default, line))
                alias = nothing
            end

            Expr(:(=), name::Symbol, default) => begin
                push!(fields, Field(name, alias, Any, default, line))
                alias = nothing
            end

            ::Symbol => begin
                push!(fields, Field(each, alias, Any, no_default, line))
                alias = nothing
            end

            ::String => (alias = each)
            ::LineNumberNode => (line = each)

            _ => begin
                push!(misc, line)
                push!(misc, each)
            end
        end
    end
    return fields, misc
end

function codegen_struct_def(x::OptionDef)
    T = x.name

    if !isempty(x.parameters)
        T = Expr(:curly, T, x.parameters...)
    end

    if x.supertype !== nothing
        T = Expr(:(<:), T, x.supertype)
    end

    body = Expr(:block)
    for each in x.fields
        if each.line !== nothing
            push!(body.args, each.line)
        end

        if each.type === Any
            item = each.name
        else
            item = :($(each.name)::$(each.type))
        end
        push!(body.args, item)
    end

    for each in x.misc
        push!(body.args, each)
    end
    return Expr(:struct, x.ismutable, T, body)
end

function name_only(@nospecialize(ex))
    ex isa Expr || return ex
    ex.head === :call && return name_only(ex.args[1])
    ex.head === :curly && return name_only(ex.args[1])
    ex.head === :(<:) && return name_only(ex.args[1])
    error("unsupported expression $ex")
end

function has_symbol(@nospecialize(ex), name::Symbol)
    ex isa Symbol && return ex === name
    ex isa Expr || return false
    return any(x->has_symbol(x, name), ex.args)
end

function default_depends_on_typevars(def::OptionDef)
    typevars = map(name_only, def.parameters)
    for each in def.fields, tvar in typevars
        if has_symbol(each.default, tvar)
            return true
        end
    end
    return false
end

function field_has_typevars(def::OptionDef)
    typevars = map(name_only, def.parameters)
    for each in def.fields, tvar in typevars
        if has_symbol(each.type, tvar)
            return true
        end
    end
    return false
end

function typevars_can_be_inferred(def::OptionDef)
    field_has_typevars(def) && !default_depends_on_typevars(def)
end

function get_kw_expr(def::OptionDef)
    kwargs = []
    for each in def.fields
        if each.default === no_default
            push!(kwargs, each.name)
        else
            push!(kwargs, Expr(:kw, each.name, each.default))
        end
    end
    return kwargs
end

function codegen_kw_fn(x::OptionDef)
    isempty(x.fields) && return
    has_custom_kw_fn(x) && return
    kwargs = get_kw_expr(x)

    def = Dict(
            :name => x.name,
            :kwargs => kwargs,
            :body => Expr(:call, x.name, [each.name for each in x.fields]...)
        )

    if isempty(x.parameters)
        return combinedef(def)
    else
        typevars = map(name_only, x.parameters)
        T = Expr(:curly, x.name, typevars...)
        curly_def = Dict(
            :name => T,
            :kwargs => kwargs,
            :body => Expr(:call, T, [each.name for each in x.fields]...),
            :whereparams => x.parameters,
        )

        if typevars_can_be_inferred(x)
            return quote
                $(combinedef(def))
                $(combinedef(curly_def))
            end
        else
            return combinedef(curly_def)
        end
    end
end

option_print(io::IO, m::MIME, x) = show(io, m, x)
option_print(io::IO, ::MIME, x::AbstractDict) = show(io, x)

function option_print(io::IO, m::MIME"text/plain", x::AbstractDict{String})
    head_indent = get(io, :head_indent, 0)
    indent = get(io, :indent, 0)
    println(io, " "^head_indent, typeof(x), "(")
    for (k, v) in x
        print(io, " "^(indent+4), LIGHT_BLUE_FG("\"", k, "\""), " => ")
        option_print(IOContext(io, :indent=>(indent+4)), m, v)
        println(io, ",")
    end
    print(io, " "^indent, ")")
end

# inline printing arrays
option_print(io::IO, ::MIME, x::AbstractArray) = show(io, x)

function option_print(io::IO, m::MIME, list::Vector)
    if !(any(is_option, list) || length(list) > 4)
        return show(io, list)
    end

    head_indent = get(io, :head_indent, 0)
    indent = get(io, :indent, 0)
    println(io, " "^head_indent, "[")
    inner_io = IOContext(io, :indent=>indent+4, :head_indent=>indent+4)
    for each in list
        if is_option(each)
            show(inner_io, m, each)
        else # inline
            show(inner_io, each)
        end

        if length(list) > 1
            println(io, ", ")
        end
    end
    print(io, " "^(indent), "]")
end

function codegen_show_text(x::OptionDef)
    body = quote
        head_indent = get(io, :head_indent, 0)
        indent = get(io, :indent, 0)
        all_default = true
        print(io, " "^head_indent, $GREEN_FG(summary(x)), "(")
    end

    is_first_field = true
    for each in x.fields
        print_ex = quote
            all_default = false
            print(io, " "^(indent+4), $(LIGHT_BLUE_FG(string(each.name))), " = ")
            $(GlobalRef(Configurations, :option_print))(IOContext(io, :indent=>(indent+4)), m, x.$(each.name))
            println(io, ",")
        end

        if is_first_field
            pushfirst!(print_ex.args, :(println(io, ";")))
            is_first_field = false
        end

        if each.default !== no_default
            push!(body.args, quote
                default = $field_default(typeof(x), $(QuoteNode(each.name)))
                if default isa $PartialDefault
                    default = default(x)
                end

                if x.$(each.name) != default
                    $print_ex
                end
            end)
        else
            push!(body.args, print_ex)
        end
    end

    push!(body.args, :(all_default || print(io, " "^indent)))
    push!(body.args, :(print(io, ")")))

    if isempty(x.parameters)
        T = x.name
        def = Dict(
            :name => GlobalRef(Base, :show),
            :args => [:(io::IO), :(m::MIME"text/plain"), :(x::$T)],
            :body => body,
        )
    else
        T = Expr(:curly, x.name, map(name_only, x.parameters)...)
        def = Dict(
            :name => GlobalRef(Base, :show),
            :args => [:(io::IO), :(m::MIME"text/plain"), :(x::$T)],
            :body => body,
            :whereparams => x.parameters,
        )
    end

    return combinedef(def)
end

function codegen_is_option(x::OptionDef)
    quote
        $(GlobalRef(Configurations, :is_option))(::$(x.name)) = true
        $(GlobalRef(Configurations, :is_option))(::Type{<:$(x.name)}) = true
    end
end

function codegen_convert(x::OptionDef)
    :(Base.convert(::Type{<:$(x.name)}, d::AbstractDict{String}) = $(GlobalRef(Configurations, :from_dict))($(x.name), d))
end

replace_symbol(x::Symbol, name::Symbol, value) = x === name ? value : x
replace_symbol(x, ::Symbol, value) = x # other expressions

function replace_symbol(ex::Expr, name::Symbol, value)
    Expr(ex.head, map(x->replace_symbol(x, name, value), ex.args)...)
end

function resolve_defaults(x::OptionDef)
    map = Dict{Symbol, Any}()
    for each in x.fields
        default = each.default
        obj = gensym(:x)
        partial_default = false
        for prev in keys(map)
            if has_symbol(each.default, prev)
                if map[prev] === no_default
                    partial_default = true
                    default = replace_symbol(default, prev, :($obj.$prev))
                else
                    default = replace_symbol(default, prev, map[prev])
                end
            end
        end

        if partial_default
            def = Dict{Symbol, Any}(
                :args=>[:($obj::$(x.name))],
                :body=>default
            )
            default = combinedef(def)
            expr = Expr(:quote, each.default)
            default = :($PartialDefault($default, $expr))
        end
        map[each.name] = default
    end
    return map
end

function codegen_field_default(x::OptionDef)
    dmap = resolve_defaults(x)
    obj = gensym(:x)
    msg = Expr(:string, "type $(x.name) does not have field ", obj)
    err = :(error($msg))
    body = isempty(x.fields) ? err : Expr(:if)
    stmt = body

    for k in 1:length(x.fields)
        field = x.fields[k]
        push!(stmt.args, :($obj == $(QuoteNode(field.name))))
        push!(stmt.args, dmap[field.name])

        if k != length(x.fields)
            push!(stmt.args, Expr(:elseif))
            stmt = stmt.args[end]
        else
            push!(stmt.args, err)
        end
    end

    if isempty(x.parameters)
        def = Dict(
            :name => GlobalRef(Configurations, :field_default),
            :args => [:(::Type{$(x.name)}), :($obj::Symbol)],
            :body => body
        )
    else
        X = gensym(:T)
        def = Dict(
            :name => GlobalRef(Configurations, :field_default),
            :args => [:(::Type{$X}), :($obj::Symbol)],
            :body => body,
            :whereparams => [x.parameters..., :($X <: $(x.name){$(name_only.(x.parameters)...)})],
        )
    end

    return combinedef(def)
end

function codegen_field_alias(x::OptionDef)
    obj = gensym(:x)
    msg = Expr(:string, "type $(x.name) does not have field ", obj)
    err = :(error($msg))
    body = isempty(x.fields) ? err : Expr(:if)
    stmt = body

    for k in 1:length(x.fields)
        field = x.fields[k]
        push!(stmt.args, :($obj == $(QuoteNode(field.name))))
        push!(stmt.args, field.alias)

        if k != length(x.fields)
            push!(stmt.args, Expr(:elseif))
            stmt = stmt.args[end]
        else
            push!(stmt.args, err)
        end
    end

    if isempty(x.parameters)
        def = Dict(
            :name => GlobalRef(Configurations, :field_alias),
            :args => [:(::Type{$(x.name)}), :($obj::Symbol)],
            :body => body
        )
    else
        X = gensym(:T)
        def = Dict(
            :name => GlobalRef(Configurations, :field_alias),
            :args => [:(::Type{$X}), :($obj::Symbol)],
            :body => body,
            :whereparams => [x.parameters..., :($X <: $(x.name){$(name_only.(x.parameters)...)})],
        )
    end
    return combinedef(def)
end

function codegen_alias(x::OptionDef)
    def = Dict(
        :name => GlobalRef(Configurations, :alias),
        :args => [:(::Type{<:$(x.name)})],
        :body => x.alias
    )
    return combinedef(def)
end

"""
    create(::Type{T}; kwargs...) where T

Create an instance of option type `T` from `kwargs`. Similar
to the default keyword argument constructor, but one can use this to create
custom keyword argument constructor with extra custom keywords.
"""
function create(::Type{T}; kwargs...) where T
    error("$T is not an option type")
end

function codegen_create(x::OptionDef)
    def = Dict(
        :name => GlobalRef(Configurations, :create),
        :args => [:(::Type{T})],
        :kwargs => get_kw_expr(x),
        :whereparams => [:(T <: $(x.name))],
        :body => Expr(:call, :T, [each.name for each in x.fields]...)
    )
    return combinedef(def)
end

"""
    compare_options(a, b, xs...)::Bool

Compare option types check if they are the same.
"""
function compare_options(a, b, xs...)::Bool
    compare_options(a, b) || return false
    compare_options(b, xs...)
end

compare_options(a, b) = false

function compare_options(a::A, b::A) where {A}
    for each in fieldnames(A)
        getfield(a, each) == getfield(b, each) || return false
    end
    return true
end

function codegen_isequal(x::OptionDef)
    return :(Base.:(==)(a::$(x.name), b::$(x.name)) = $compare_options(a, b))
end

function codegen_show_toml_mime(x::OptionDef)
    :(
        function Base.show(io::IO, ::MIME"application/toml", x::$(x.name))
            return print(io, to_toml(x))
        end
    )
end

"""
generate Julia AST from `OptionDef`.
"""
function codegen(def::OptionDef)
    quote
        $(codegen_struct_def(def))
        Core.@__doc__ $(def.name)
        $(codegen_kw_fn(def))
        $(codegen_convert(def))
        $(codegen_show_text(def))
        $(codegen_is_option(def))
        $(codegen_field_default(def))
        $(codegen_field_alias(def))
        $(codegen_alias(def))
        $(codegen_isequal(def))
        $(codegen_show_toml_mime(def))
        $(codegen_create(def))
    end
end

function option_m(@nospecialize(ex), alias=nothing)
    def = OptionDef(ex, alias)

    quote
        $(codegen(def))
        nothing
    end
end

"""
    @option [alias::String] <struct def>

Define an option struct type. This will auto-generate methods that parse a given `Dict{String}`
object (the keys must be of type `String`) into an instance of the struct type you defined. One
can use `alias` string to distinguish multiple possible option type for the same field.

# Example

One can define option type via `@option` macro with or without an alias.

```julia-repl
julia> "Option A"
       @option "option_a" struct OptionA
           name::String
           int::Int = 1
       end

julia> "Option B"
       @option "option_b" struct OptionB
           opt::OptionA = OptionA(;name = "Sam")
           float::Float64 = 0.3
       end
```

and convert a dict to an option type via [`from_dict`](@ref).

```julia-repl
julia> d = Dict{String, Any}(
           "opt" => Dict{String, Any}(
               "name" => "Roger",
               "int" => 2,
           ),
           "float" => 0.33
       );

julia> option = from_dict(OptionB, d)
OptionB(;
    opt = OptionA(;
        name = "Roger",
        int = 2,
    ),
    float = 0.33,
)
```

when there are multiple possible option type for one field,
one can use the alias to distinguish them

```julia-repl
julia> @option struct OptionD
           opt::Union{OptionA, OptionB}
       end

julia> d1 = Dict{String, Any}(
               "opt" => Dict{String, Any}(
                   "option_b" => d
               )
           );

julia> from_dict(OptionD, d1)
OptionD(;
    opt = OptionB(;
        opt = OptionA(;
            name = "Roger",
            int = 2,
        ),
        float = 0.33,
    ),
)
```
"""
macro option(ex)
    esc(option_m(ex))
end

macro option(alias::String, ex)
    esc(option_m(ex, alias))
end

end


text:
# This file was generated by the Julia Swagger Code Generator
# Do not modify this file directly. Modify the swagger specification instead.

struct AdminAppsRequestsApi <: SwaggerApi
    client::Swagger.Client
end

function _swaggerinternal_adminAppsRequestsList(_api::AdminAppsRequestsApi; limit=nothing, cursor=nothing, team_id=nothing, _mediaType=nothing)
    _ctx = Swagger.Ctx(_api.client, "GET", Dict{String, Any}, "/admin.apps.requests.list", ["slackAuth"])
    Swagger.set_param(_ctx.query, "limit", limit)  # type Int32
    Swagger.set_param(_ctx.query, "cursor", cursor)  # type String
    Swagger.set_param(_ctx.query, "team_id", team_id)  # type String
    Swagger.set_header_accept(_ctx, ["application/json"])
    Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? ["application/x-www-form-urlencoded"] : [_mediaType])
    return _ctx
end

"""

List app requests for a team/workspace.
Param: limit::Int32
Param: cursor::String
Param: team_id::String
Return: Dict{String, Any}
"""
function adminAppsRequestsList(_api::AdminAppsRequestsApi; limit=nothing, cursor=nothing, team_id=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_adminAppsRequestsList(_api; limit=limit, cursor=cursor, team_id=team_id, _mediaType=_mediaType)
    Swagger.exec(_ctx)
end

function adminAppsRequestsList(_api::AdminAppsRequestsApi, response_stream::Channel; limit=nothing, cursor=nothing, team_id=nothing, _mediaType=nothing)
    _ctx = _swaggerinternal_adminAppsRequestsList(_api; limit=limit, cursor=cursor, team_id=team_id, _mediaType=_mediaType)
    Swagger.exec(_ctx, response_stream)
end

export adminAppsRequestsList


text:
# This file is a part of Julia. License is MIT: https://julialang.org/license

## generic operations on numbers ##
"""
    isinteger(x) -> Bool

Test whether `x` is numerically equal to some integer.

```jldoctest
julia> isinteger(4.0)
true
```
"""
isinteger(x::Integer) = true

"""
    iszero(x)

Return `true` if `x == zero(x)`; if `x` is an array, this checks whether
all of the elements of `x` are zero.
"""
iszero(x) = x == zero(x) # fallback method

size(x::Number) = ()
size(x::Number,d) = convert(Int,d)<1 ? throw(BoundsError()) : 1
indices(x::Number) = ()
indices(x::Number,d) = convert(Int,d)<1 ? throw(BoundsError()) : OneTo(1)
eltype(::Type{T}) where {T<:Number} = T
ndims(x::Number) = 0
ndims(::Type{<:Number}) = 0
length(x::Number) = 1
endof(x::Number) = 1
iteratorsize(::Type{<:Number}) = HasShape()

getindex(x::Number) = x
function getindex(x::Number, i::Integer)
    @_inline_meta
    @boundscheck i == 1 || throw(BoundsError())
    x
end
function getindex(x::Number, I::Integer...)
    @_inline_meta
    @boundscheck all([i == 1 for i in I]) || throw(BoundsError())
    x
end
first(x::Number) = x
last(x::Number) = x
copy(x::Number) = x  # some code treats numbers as collection-like

"""
    divrem(x, y)

The quotient and remainder from Euclidean division. Equivalent to `(div(x,y), rem(x,y))` or
`(x÷y, x%y)`.

```jldoctest
julia> divrem(3,7)
(0, 3)

julia> divrem(7,3)
(2, 1)
```
"""
divrem(x,y) = (div(x,y),rem(x,y))

"""
    fldmod(x, y)

The floored quotient and modulus after division. Equivalent to `(fld(x,y), mod(x,y))`.
"""
fldmod(x,y) = (fld(x,y),mod(x,y))
signbit(x::Real) = x < 0

"""
    sign(x)

Return zero if `x==0` and ``x/|x|`` otherwise (i.e., ±1 for real `x`).
"""
sign(x::Number) = x == 0 ? x/abs(oneunit(x)) : x/abs(x)
sign(x::Real) = ifelse(x < 0, oftype(one(x),-1), ifelse(x > 0, one(x), typeof(one(x))(x)))
sign(x::Unsigned) = ifelse(x > 0, one(x), oftype(one(x),0))
abs(x::Real) = ifelse(signbit(x), -x, x)

"""
    abs2(x)

Squared absolute value of `x`.

```jldoctest
julia> abs2(-3)
9
```
"""
abs2(x::Real) = x*x

"""
    flipsign(x, y)

Return `x` with its sign flipped if `y` is negative. For example `abs(x) = flipsign(x,x)`.

```jldoctest
julia> flipsign(5, 3)
5

julia> flipsign(5, -3)
-5
```
"""
flipsign(x::Real, y::Real) = ifelse(signbit(y), -x, +x) # the + is for type-stability on Bool
copysign(x::Real, y::Real) = ifelse(signbit(x)!=signbit(y), -x, +x)

conj(x::Real) = x
transpose(x::Number) = x
ctranspose(x::Number) = conj(x)
inv(x::Number) = one(x)/x
angle(z::Real) = atan2(zero(z), z)

"""
    widemul(x, y)

Multiply `x` and `y`, giving the result as a larger type.

```jldoctest
julia> widemul(Float32(3.), 4.)
1.200000000000000000000000000000000000000000000000000000000000000000000000000000e+01
```
"""
widemul(x::Number, y::Number) = widen(x)*widen(y)

start(x::Number) = false
next(x::Number, state) = (x, true)
done(x::Number, state) = state
isempty(x::Number) = false
in(x::Number, y::Number) = x == y

map(f, x::Number, ys::Number...) = f(x, ys...)

"""
    zero(x)

Get the additive identity element for the type of `x` (`x` can also specify the type itself).

```jldoctest
julia> zero(1)
0

julia> zero(big"2.0")
0.000000000000000000000000000000000000000000000000000000000000000000000000000000

julia> zero(rand(2,2))
2×2 Array{Float64,2}:
 0.0  0.0
 0.0  0.0
```
"""
zero(x::Number) = oftype(x,0)
zero(::Type{T}) where {T<:Number} = convert(T,0)

"""
    one(x)
    one(T::type)

Return a multiplicative identity for `x`: a value such that
`one(x)*x == x*one(x) == x`.  Alternatively `one(T)` can
take a type `T`, in which case `one` returns a multiplicative
identity for any `x` of type `T`.

If possible, `one(x)` returns a value of the same type as `x`,
and `one(T)` returns a value of type `T`.  However, this may
not be the case for types representing dimensionful quantities
(e.g. time in days), since the multiplicative
identity must be dimensionless.  In that case, `one(x)`
should return an identity value of the same precision
(and shape, for matrices) as `x`.

If you want a quantity that is of the same type as `x`, or of type `T`,
even if `x` is dimensionful, use [`oneunit`](@ref) instead.
```jldoctest
julia> one(3.7)
1.0

julia> one(Int)
1

julia> one(Dates.Day(1))
1
```
"""
one(::Type{T}) where {T<:Number} = convert(T,1)
one(x::T) where {T<:Number} = one(T)
# note that convert(T, 1) should throw an error if T is dimensionful,
# so this fallback definition should be okay.

"""
    oneunit(x::T)
    oneunit(T::Type)

Returns `T(one(x))`, where `T` is either the type of the argument or
(if a type is passed) the argument.  This differs from [`one`](@ref) for
dimensionful quantities: `one` is dimensionless (a multiplicative identity)
while `oneunit` is dimensionful (of the same type as `x`, or of type `T`).

```jldoctest
julia> oneunit(3.7)
1.0

julia> oneunit(Dates.Day)
1 day
```
"""
oneunit(x::T) where {T} = T(one(x))
oneunit(::Type{T}) where {T} = T(one(T))

_default_type(::Type{Number}) = Int

"""
    factorial(n)

Factorial of `n`. If `n` is an [`Integer`](@ref), the factorial is computed as an
integer (promoted to at least 64 bits). Note that this may overflow if `n` is not small,
but you can use `factorial(big(n))` to compute the result exactly in arbitrary precision.
If `n` is not an `Integer`, `factorial(n)` is equivalent to [`gamma(n+1)`](@ref).

```jldoctest
julia> factorial(6)
720

julia> factorial(21)
ERROR: OverflowError()
[...]

julia> factorial(21.0)
5.109094217170944e19

julia> factorial(big(21))
51090942171709440000
```
"""
factorial(x::Number) = gamma(x + 1) # fallback for x not Integer

"""
    big(T::Type)

Compute the type that represents the numeric type `T` with arbitrary precision.
Equivalent to `typeof(big(zero(T)))`.

```jldoctest
julia> big(Rational)
Rational{BigInt}

julia> big(Float64)
BigFloat

julia> big(Complex{Int})
Complex{BigInt}
```
"""
big(::Type{T}) where {T<:Number} = typeof(big(zero(T)))

max_stars_repo_name: omus/julia

max_issues_repo_name: omus/julia

max_forks_repo_name: omus/julia


text:
#=
Data should be stored in WHCN order. In other words, a 100×100 RGB image would
be a `100×100×3` array, and a batch of 50 would be a `100×100×3×50` array.
=#
using Flux

c=Flux.Conv((3,3), 3=>3)
mp=MeanPool((2,2))
@show size(mp(c(rand(224,224,3,2))))

x = rand(28,28,3,1)
dc = Flux.DepthwiseConv((3,3), 3=>2)
@show size(dc(x))

max_stars_repo_name: terasakisatoshi/juliaExer

max_issues_repo_name: terasakisatoshi/juliaExer

max_forks_repo_name: terasakisatoshi/juliaExer


text:
module ExproniconLite
export NoDefault, JLExpr, JLFor, JLIfElse, JLFunction, JLField, JLKwField, JLStruct, JLKwStruct, @expr, @test_expr, compare_expr, compare_vars, AnalysisError, is_function, is_kw_function, is_struct, is_ifelse, is_for, is_field, is_field_default, split_function, split_function_head, split_struct, split_struct_name, split_ifelse, annotations, uninferrable_typevars, has_symbol, is_literal, has_kwfn_constructor, has_plain_constructor, no_default, prettify, rm_lineinfo, flatten_blocks, name_only, rm_annotations, rm_single_block, rm_nothing, replace_symbol, subtitute, eval_interp, eval_literal, codegen_ast, codegen_ast_kwfn, codegen_ast_kwfn_plain, codegen_ast_kwfn_infer, codegen_ast_struct, codegen_ast_struct_head, codegen_ast_struct_body, construct_method_plain, construct_method_inferable, struct_name_plain, struct_name_without_inferable, xtuple, xnamedtuple, xcall, xpush, xfirst, xlast, xprint, xprintln, xmap, xmapreduce, xiterate, print_expr, sprint_expr
include("types.jl")
#= /home/roger/code/julia/Expronicon/src/expand.jl:99 =# @static if !(isdefined(#= /home/roger/code/julia/Expronicon/src/expand.jl:99 =# @__MODULE__(), :include_generated))
        function include_generated(m::Module, path::String)
            raw = read(path, String)
            ex = Base.include_string(m, "quote $(raw) end", path)
            m.eval(m.eval(ex))
            return
        end
    end
include_generated(#= /home/roger/code/julia/Expronicon/src/expand.jl:113 =# @__MODULE__(), joinpath(#= /home/roger/code/julia/Expronicon/src/expand.jl:113 =# @__DIR__(), "transform.jl"))
include_generated(#= /home/roger/code/julia/Expronicon/src/expand.jl:117 =# @__MODULE__(), joinpath(#= /home/roger/code/julia/Expronicon/src/expand.jl:117 =# @__DIR__(), "analysis.jl"))
include("codegen.jl")
include_generated(#= /home/roger/code/julia/Expronicon/src/expand.jl:117 =# @__MODULE__(), joinpath(#= /home/roger/code/julia/Expronicon/src/expand.jl:117 =# @__DIR__(), "printing.jl"))
end


text:
# advent of code 2020 day 4

function load_data(path:: String):: AbstractArray
    file = open(path)
    data_array = readlines(file)
    close(file)
    return data_array
end

function process_passport(instance, mandatory_fields, rules=nothing)

    for key in mandatory_fields
        _instance = join(instance)
        if findfirst(key, _instance) === nothing
            return 0
        end
    end

    if rules !== nothing
        for value in instance
            _key, _value = String(value[1:4]), String(value[5:end])
            if ~rules[_key](_value)
                return 0
            end
        end
    end
    return 1
end

function process_all(data_array, fields, exception_field, rules=nothing)
    mandatory_fields = []
    for _field in fields
        if ~(_field in exception_field)
            push!(mandatory_fields, _field)
        end
    end

    passport_array, passport = [], []
    let data, passport = []
        for data in data_array
            if length(data) == 0
                push!(passport_array, passport)
                passport = []
            else
                append!(passport, split(data, " "))
            end
        end
        push!(passport_array, passport)
    end

    numb_valid = 0
    for passport in passport_array
        numb_valid += process_passport(passport, mandatory_fields, rules)
    end

    return numb_valid
end

data_array = load_data("data/day4.txt")

fields = ["ecl:", "pid:", "eyr:", "hcl:", "byr:", "iyr:", "cid:", "hgt:"]

rules = Dict([("ecl:", x -> x in ["amb", "blu", "brn", "gry", "grn", "hzl", "oth"]),
              ("pid:", x -> (length(x) === 9) & (match(r"[0-9]+", x).match == x)),
              ("eyr:", x -> (length(x) === 4) && (2020 <= parse(Int, x) <= 2030)),
              ("hcl:", x -> (x[1] === '#') && (match(r"[a-f0-9]+", x[2:end]).match == x[2:end])),
              ("byr:", x -> (length(x) === 4) && (1920 <= parse(Int, x) <= 2002)),
              ("hgt:", x -> ((x[length(x)[1] - 1:end] === "in") && (59 <= parse(Int, x[1:length(x)[1]-2]) <= 76)) | ((x[length(x)[1] - 1:end] === "cm") && (150 <= parse(Int, x[1:length(x)[1]-2]) <= 193))),
              ("cid:", x -> true),
              ("iyr:", x -> (length(x) === 4) && (2010 <= parse(Int, x) <= 2020)),
              ])

exception_field = ["cid:"]

numb_valid = process_all(data_array, fields, exception_field, nothing)
println("number of valid passport with no rules: $numb_valid")

numb_valid = process_all(data_array, fields, exception_field, rules)
println("number of valid passport with strict rules: $numb_valid")


text:
export getData

"""
function jInv.ForwardShare.getData(sigma, pFor, ...)

solves forward model and computes data. getData is application specific and some guidelines
how to create getData for a new problem can be found in examples/tutorialBuildYourOwn.ipynb

Inputs:

   sigma  -  current parameters
   pFor   -  description of forward problems (ForwardProbType, Array{ForwardProbType}, Array{Future}

   Some methods of getData require further arguments.

Output:

    Dobs   - simulated data
    pFor   - modified forward problem type

"""
function getData(sigma::Union{RemoteChannel, Vector},
                 pFor::ForwardProbType,
                 Mesh2Mesh::Mesh2MeshTypes,
                 doClear::Bool=false)
#=
    computations on one worker
=#
    sig = interpGlobalToLocal(fetch(sigma),fetch(Mesh2Mesh))
    dobs,pFor   = getData(sig,pFor,doClear)
    Dobs  = remotecall(identity,myid(),dobs)
    return Dobs,pFor
end

function getData(sigma::Union{RemoteChannel, Vector},
                 pFor::RemoteChannel,
                 Mesh2Mesh::Mesh2MeshTypes,
                 doClear::Bool=false)
#=
    modify pFor on current worker (e.g., to keep factorizations)
=#
    pF = take!(pFor)
    Dobs, pF = getData(fetch(sigma), pF, fetch(Mesh2Mesh), doClear)
    put!(pFor,pF)
    return Dobs,pFor
end

function getData(
                 sigma::Vector,
                 pFor::Array{FPT},
                 Mesh2Mesh::Array{T}=ones(length(pFor)),
                 doClear::Bool=false,
                 workerList::Vector=workers()) where {FPT<:ForwardProbType,T<:Mesh2MeshTypes}
#=
    parallel forward simulation with dynamic scheduling (i.e., elements in pFor get sent to remote workers on the fly)
=#
    i=1; nextidx() = (idx = i; i+=1; idx)

    Dobs = Array{Any}(undef,length(pFor))
    workerList = intersect(workers(),workerList)
    if isempty(workerList)
        error("getData: workers do not exist!")
    end
    @sync begin
        for p=workerList
            @async begin
                while true
                    idx = nextidx()
                    if idx > length(pFor); break; end
                    Dobs[idx],pFor[idx] = remotecall_fetch(getData,p,sigma,pFor[idx],Mesh2Mesh[idx],doClear)
                end
            end
        end
    end
    return Dobs,pFor
end

function getData(
                 sigma::Vector,
                 pFor::Array{RemoteChannel},
                 Mesh2Mesh::Array{T}=ones(length(pFor)),
                 doClear::Bool=false) where {T<:Mesh2MeshTypes}
#=
    parallel forward simulation with static scheduling (i.e., pFors are distributed a-priorily)
=#

    Dobs = Array{Future}(undef, length(pFor))
    workerList = getWorkerIds(pFor)
    sigmaRef = Array{RemoteChannel}(undef, maximum(workers()))
    @sync begin
        for p=workerList
            @async begin
                sigmaRef[p] = initRemoteChannel(identity,p,sigma)  # send model to worker
                for idx=1:length(pFor)
                    if p==pFor[idx].where
                        Dobs[idx],pFor[idx] = remotecall_fetch(getData,p,sigmaRef[p],pFor[idx],Mesh2Mesh[idx],doClear)
                    end
                end
            end
        end
    end
    return Dobs,pFor
end


text:
"""
    testmode!(m)
    testmode!(m, false)

Put layers like [`Dropout`](@ref) and `BatchNorm` into testing mode (or back to
training mode with `false`).
"""
function testmode!(m, val::Bool=true)
  prefor(x -> _testmode!(x, val), m)
  return m
end

_testmode!(m, test) = nothing

"""
    Dropout(p)

A Dropout layer. For each input, either sets that input to `0` (with probability
`p`) or scales it by `1/(1-p)`. This is used as a regularisation, i.e. it
reduces overfitting during training.

Does nothing to the input once in [`testmode!`](@ref).
"""
mutable struct Dropout{F}
  p::F
  active::Bool
end

function Dropout(p)
  @assert 0 ≤ p ≤ 1
  Dropout{typeof(p)}(p, true)
end

function (a::Dropout)(x)
  a.active || return x
  y = similar(x)
  rand!(y)
  q = 1 - a.p
  @inbounds for i=1:length(y)
    y[i] = y[i] > a.p ? 1 / q : 0
  end
  return y .* x
end

_testmode!(a::Dropout, test) = (a.active = !test)


text:
module Method

using util

export computeDocFreq, MethodType, TraceSim, parseMethod, createMethod, compute_frame_weights, similarity, normalizeDocFreq,
    MlModel, compute_frame_weight, match_score, update_method_param!, compute_final_similarity!, gap_score,
    TFIDF, BM25, compute_tf, PDM, IRMethod, FaST, frame_weight, CosineTFIDF, compute_term_sim, PrefixMatch, normalize_vec

abstract type MethodType end

normalizeDocFreq(docFreq::DocFreq)::Vector{Float64}= @. (docFreq.doc_freq / docFreq.nDocs) * 100.0

function computeDocFreq(method::MethodType, docFreq)::Vector{Float64}
    return Vector{Float64}()
end

include("methods/trace_sim.jl")
# include("methods/brodie05.jl")
include("methods/pdm.jl")
include("methods/prefix_match.jl")
include("methods/IR_method.jl")
include("methods/fast.jl")

end

max_stars_repo_name: irving-muller/FaST

max_issues_repo_name: irving-muller/FaST

max_forks_repo_name: irving-muller/FaST


text:
struct AbstractPointSet{N,T} <: AbstractVector{T}

struct PointSet{N,T} <: AbstractPointSet{N,T}
    coords::Vector{SVector{N,T}}
end

getindex(ps::PointSet) = getindex(ps.coords)
length(ps::PointSet) = length(ps.coords)


text:
module MaximalCouplings

using Distributions
using Roots
using Random
import Random: rand, rand!
import Base: length, eltype
using StatsBase: Weights
using PDMats: whiten, unwhiten, AbstractPDMat
using LinearAlgebra: dot, norm

include("types.jl")
include("rand.jl")
include("prob_couple.jl")

export MaximalCoupling
export prob_couple

end # module


text:
"""
Functionality for handling the defautl conent of the various Genie files (migrations, models, controllers, etc).
"""
module FileTemplates

using Inflector

"""
    new_task(module_name::String) :: String

Default content for a new Genie task.
"""
function new_task(module_name::String) :: String
  """
  module $module_name

  using Genie, App, Toolbox

  function description()
    \"\"\"
    Description of the task here
    \"\"\"
  end

  function run_task!()
    # Build something great
  end

  end
  """
end

"""
    new_controller(controller_name::String) :: String

Default content for a new Genie controller.
"""
function new_controller(controller_name::String) :: String
  """
  module $(controller_name)Controller

  using App
  @dependencies

  end
  """
end

"""
    new_channel(channel_name::String) :: String

Default content for a new Genie channel.
"""
function new_channel(channel_name::String) :: String
  """
  module $(channel_name)Channel

  using Channels, App
  @dependencies

  function subscribe()
    Channels.subscribe(wsclient(@params), :$(lowercase(channel_name)))
    "OK"
  end

  end
  """
end

"""
    new_authorizer() :: String

Default content for a new Genie ACL YAML file.
"""
function new_authorizer() :: String
  """
  admin:
    create: all
    edit: all
    delete: all
    list: all
  editor:
    edit: all
    list: all
  writer:
    create: all
    edit: own
    delete: own
    list: own
  """
end

"""
    new_test(plural_name::String, singular_name::String) :: String

Default content for a new test file.
"""
function new_test(plural_name::String, singular_name::String) :: String
  """
  using Genie, App, App.$(plural_name)

  ### Your tests here
  @test 1 == 1
  """
end

end


text:
"""
    compute_independent_paths(env::CoordinatedMAPFEnv, task_list::Vector{AgentTask})

For Direct baseline. Computes shortest start-goal paths for each agent without any coordination.
"""
function compute_independent_paths(env::CoordinatedMAPFEnv, task_list::Vector{AgentTask})

    independent_paths = Vector{AgentPathInfo}(undef, length(task_list))

    for (agent_id, task) in enumerate(task_list)

        bidir_sp = compute_bidir_astar_euclidean(env.road_graph, task.origin, task.dest, env.road_graph_wts, env.location_list)

        path_states = AgentPathState[AgentPathState(road_vtx_id=bidir_sp.path[1], timeval=0.0)]

        cumul_time = 0.0
        for (idx, idx_next) in zip(bidir_sp.path[1:end-1], bidir_sp.path[2:end])
            cumul_time += env.road_graph_wts[idx, idx_next]/AvgSpeed
            push!(path_states, AgentPathState(road_vtx_id=idx_next, timeval=cumul_time))
        end

        independent_paths[agent_id] = AgentPathInfo(path_states=path_states, total_dist=bidir_sp.dist, total_time=cumul_time)
    end

    return independent_paths
end

struct EdgeCopyQueueType
    hop::Int64
    src::Int64
end

"""
    augment_road_graph_with_aerial_paths!(env::CoordinatedMAPFEnv, direct_drone_paths::Vector{AgentPathInfo}, decay_fn::Function)

Use aerial paths to augment road graph.
"""
function augment_road_graph_with_aerial_paths!(env::CoordinatedMAPFEnv, direct_drone_paths::Vector{AgentPathInfo}, decay_fn::Function = x -> logistic(x))

    # Iterate over drone paths
    for (aerial_id, ddp) in enumerate(direct_drone_paths)

        ddp_states = ddp.path_states

        # Temp data structure for closest distance to actual edge
        edge_copy_min_hop = Dict{Tuple{Int64,Int64},Int64}()
        edge_copy_src_to_hop = Dict{Int64,Int64}()

        # Iterate over path states and run outward search from
        for i = 1:length(ddp_states) - 1

            # First add edge on path to copy dict; must be unique
            edge_copy_min_hop[(ddp_states[i].road_vtx_id, ddp_states[i+1].road_vtx_id)] = 0
            edge_copy_src_to_hop[ddp_states[i].road_vtx_id] = 0

            # Now work outward from ddp_states[i]
            edge_copy_queue = Queue{EdgeCopyQueueType}()
            for onbr in outneighbors(env.road_graph, ddp_states[i].road_vtx_id)
                if onbr != ddp_states[i+1].road_vtx_id
                    enqueue!(edge_copy_queue, EdgeCopyQueueType(1, onbr))
                end
            end

            while ~(isempty(edge_copy_queue))

                qtop = dequeue!(edge_copy_queue)
                curr_id = qtop.src

                # Compute the min of current hop of qtop.src and MaxHop
                min_hop_for_src = get(edge_copy_src_to_hop, curr_id, MaxHop)

                # Only jump into it if it was not already in edge_copy_src_to_hop
                if qtop.hop < min_hop_for_src

                    # First update for the src itself
                    edge_copy_src_to_hop[curr_id] = qtop.hop

                    curr_hop = qtop.hop + 1

                    # Iterate over outneighbors and add to edge_copy_min_dist if curr_hop < min_hop
                    # In any case, add to queue with curr_hop
                    for onbr in outneighbors(env.road_graph, curr_id)

                        # TODO: Check later that curr_hop is necessarily better than whatever edge_copy_min_hop has
                        edge_copy_min_hop[(curr_id, onbr)] = curr_hop

                        # Enqueue onbr with curr_hop
                        enqueue!(edge_copy_queue, EdgeCopyQueueType(curr_hop, onbr))

                    end # onbr in out_neighbors(ddp_states[i])
                end # qtop.hop < min_hop_for_src
            end # ~(isempty(edge_copy_queue))
        end # i = 1:length(ddp_states) - 1

        # now iterate over edge_copy_min_hop and add copies
        for (src_dst, hop) in edge_copy_min_hop

            e = LightGraphs.Edge(src_dst[1], src_dst[2])

            # Initialize dictionary for edge if it doesn't exist
            if ~haskey(env.edge_to_copy_info, e)
                env.edge_to_copy_info[e] = EdgeCopyInfo()
            end

            reduced_wt = env.road_graph_wts[e.src, e.dst] * decay_fn(hop)
            env.edge_to_copy_info[e].aerial_id_to_weight[aerial_id] = reduced_wt

        end # (src_dst, hop) in edge_copy_min_hop

    end # ddp in direct_drone_paths

end # function augment_road_graph_with_aerial_paths!

## Define MAPF functions for ground paths
function MultiAgentPathFinding.add_constraint!(cbase::GroundMAPFConstraint, cadd::GroundMAPFConstraint)

    # iterate over edges and IDs of cadd and add new key to cbase or augment existing set
    for (edge, aerial_ids) in cadd.avoid_edge_copy_set
        if haskey(cbase.avoid_edge_copy_set, edge)
            union!(cbase.avoid_edge_copy_set[edge], aerial_ids)
        else
            cbase.avoid_edge_copy_set[edge] = deepcopy(aerial_ids)
        end
    end # (edge, aerial_ids) in cadd.avoid_edge_copy_set
end

function MultiAgentPathFinding.overlap_between_constraints(cbase::GroundMAPFConstraint, cadd::GroundMAPFConstraint)
    for common_key in intersect(keys(cbase.avoid_edge_copy_set, cadd.avoid_edge_copy_set))
        if ~(isempty(intersect(cbase.avoid_edge_copy_set[common_key], cadd.avoid_edge_copy_set[common_key])))
            return true
        end
    end
    return false
end

get_mapf_ground_action(env::CoordinatedMAPFEnv, u::GroundMAPFState, v::GroundMAPFState) = GroundMAPFAction(LightGraphs.Edge(u.road_vtx_id, v.road_vtx_id), env.unique_gmapf_states[v].aerial_ids)

MultiAgentPathFinding.get_empty_constraint(::Type{GroundMAPFConstraint}) = GroundMAPFConstraint()

function MultiAgentPathFinding.create_constraints_from_conflict(env::CoordinatedMAPFEnv, conflict::GroundMAPFConflict)

    # Iterate over conflicting edge Ids and create one element sets of edges to avoid of a drone ID
    edge_set_to_avoid = Dict{LightGraphs.Edge,Set{Int64}}()
    for (edge, aerial_ids) in conflict.conflicting_edge_aerial_ids
        edge_set_to_avoid[edge] = aerial_ids
    end

    # Now just repeat the constraint for each member of pair
    gid_pair = conflict.ground_id_pair
    constraint = Dict{Int64,GroundMAPFConstraint}(gid_pair[1] => GroundMAPFConstraint(edge_set_to_avoid), gid_pair[2] => GroundMAPFConstraint(edge_set_to_avoid))

    return [constraint]
end

function MultiAgentPathFinding.get_first_conflict(env::CoordinatedMAPFEnv, solution::Vector{PlanResult{GroundMAPFState,GroundMAPFAction,Float64}})

    # For the first two trucks that share one or more edges of same drones, return
    for (i, sol_i) in enumerate(solution[1:end-1])
        for (j, sol_j) in enumerate(solution[i+1:end])

            other_ground_id = i+j
            overlapping_aerial_edges = Vector{Tuple{LightGraphs.Edge,Set{Int64}}}(undef, 0)

            for (i_action, _) in sol_i.actions
                for (j_action, _) in sol_j.actions
                    if i_action.edge_id == j_action.edge_id && ~isempty(i_action.edge_aerial_ids) && i_action.edge_aerial_ids == j_action.edge_aerial_ids
                        push!(overlapping_aerial_edges, (i_action.edge_id, i_action.edge_aerial_ids))
                    end # i_action.edge_aerial_id != 0 && i_action == j_action
                end # (sj, (state_j, _)) in enumerate(sol_j.states[2:end])
            end # (si, (state_i, _)) in enumerate(sol_i.states[2:end])

            if ~isempty(overlapping_aerial_edges)
                env.num_global_conflicts += 1
                println("Ground conflict no. $(env.num_global_conflicts)")
                if env.num_global_conflicts > env.threshold_global_conflicts
                    throw(DomainError("Too many conflicts in Stage 1!"))
                end

                return GroundMAPFConflict((i, other_ground_id), overlapping_aerial_edges)
            end
        end # (j, sol_j) in enumerate(solution[i+1:end])
    end # (i, sol_i) in enumerate(solution[1:end-1])

    return nothing
end

function ground_mapf_heuristic(env::CoordinatedMAPFEnv, road_vtx_goal::Int64, u::GroundMAPFState)
    return distance_lat_lon_euclidean(env.location_list[u.road_vtx_id], env.location_list[road_vtx_goal])
end

function generate_ground_mapf_edge_weight(env::CoordinatedMAPFEnv, u::GroundMAPFState, v::GroundMAPFState, aerial_ids::Set{Int64})
    if isempty(aerial_ids)
        return env.road_graph_wts[u.road_vtx_id, v.road_vtx_id]
    else
        # Subtract cost from aerial IDs carried
        base_wt = env.road_graph_wts[u.road_vtx_id, v.road_vtx_id]
        uv_edge = LightGraphs.Edge(u.road_vtx_id, v.road_vtx_id)
        cumul_wt = base_wt

        # Subtract cost diff from base for each drone edge
        for aid in aerial_ids
            cumul_wt -= (base_wt - env.edge_to_copy_info[uv_edge].aerial_id_to_weight[aid])
        end

        return cumul_wt
    end
end

function get_ground_mapf_edge_weight(env::CoordinatedMAPFEnv, u::GroundMAPFState, v::GroundMAPFState)
    return (env.unique_gmapf_states[v].gval - env.unique_gmapf_states[u].gval)
end

function MultiAgentPathFinding.set_low_level_context!(::CoordinatedMAPFEnv, ::Int64, ::GroundMAPFConstraint)
end

function MultiAgentPathFinding.focal_heuristic(env::CoordinatedMAPFEnv, solution::Vector{PlanResult{GroundMAPFState,GroundMAPFAction,Float64}})

    num_potential_conflicts = 0

    for (i, sol_i) in enumerate(solution[1:end-1])
        for (j, sol_j) in enumerate(solution[i+1:end])

            num_overlapping_aerial_edges = 0

            for (i_action, _) in sol_i.actions
                for (j_action, _) in sol_j.actions
                    if i_action.edge_id == j_action.edge_id && ~isempty(i_action.edge_aerial_ids) && i_action.edge_aerial_ids == j_action.edge_aerial_ids
                        num_overlapping_aerial_edges += 1
                    end # i_action.edge_aerial_id != 0 && i_action == j_action
                end # (sj, (state_j, _)) in enumerate(sol_j.states[2:end])
            end # (si, (state_i, _)) in enumerate(sol_i.states[2:end])

            if num_overlapping_aerial_edges > 0
                num_potential_conflicts += 1
            end
        end # (j, sol_j) in enumerate(solution[i+1:end])
    end # (i, sol_i) in enumerate(solution[1:end-1])

    return num_potential_conflicts
end

"""
Underlying per-agent search over the Stage 1 MAPF graph.
"""
function MultiAgentPathFinding.low_level_search!(solver::ECBSSolver, ground_agent_id::Int64, s::GroundMAPFState, constraint::GroundMAPFConstraint, solution::Vector{PlanResult{GroundMAPFState,GroundMAPFAction,Float64}})

    env = solver.env

    # Create temp graph as thin wrapper on road graph for search
    env.ground_mapf_graph = SimpleVListGraph{GroundMAPFState}()
    empty!(env.unique_gmapf_states)

    Graphs.add_vertex!(env.ground_mapf_graph, s)
    env.unique_gmapf_states[s] = GMAPFStateInfo(1, 0.0, Set{Int64}())

    road_vtx_goal = env.ground_task_list[ground_agent_id].dest
    admissible_heuristic(u) = ground_mapf_heuristic(env, road_vtx_goal, u)
    edge_wt_fn(u, v) = get_ground_mapf_edge_weight(env, u, v)

    vis = GroundMAPFGoalVisitor(env, road_vtx_goal, constraint.avoid_edge_copy_set)

    start_idx = 1
    env.next_gmapfg_goal_idx = 0

    a_star_states = a_star_implicit_shortest_path!(env.ground_mapf_graph, edge_wt_fn, start_idx, vis, admissible_heuristic)

    @assert haskey(a_star_states.parent_indices, env.next_gmapfg_goal_idx)

    states = Tuple{GroundMAPFState,Float64}[]
    actions = Tuple{GroundMAPFAction,Float64}[]

    sp_cost = a_star_states.dists[env.next_gmapfg_goal_idx]

    sp_idxs = shortest_path_indices(a_star_states.parent_indices, env.ground_mapf_graph, start_idx, env.next_gmapfg_goal_idx)

    push!(states, (s, 0.0))

    for i = 2:length(sp_idxs)

        u_idx, v_idx =  (sp_idxs[i-1], sp_idxs[i])

        new_state = deepcopy(env.ground_mapf_graph.vertices[v_idx])
        push!(states, (new_state, a_star_states.dists[v_idx]))

        push!(actions, (get_mapf_ground_action(env, env.ground_mapf_graph.vertices[u_idx], env.ground_mapf_graph.vertices[v_idx]), a_star_states.dists[v_idx] - a_star_states.dists[u_idx]))
    end

    return PlanResult{GroundMAPFState,GroundMAPFAction,Float64}(states, actions, sp_cost, sp_cost)
end

struct GroundMAPFGoalVisitor <: Graphs.AbstractDijkstraVisitor
    env::CoordinatedMAPFEnv
    road_vtx_goal::Int64
    avoid_edge_copy_set::Dict{LightGraphs.Edge,Set{Int64}}
end

function Graphs.include_vertex!(vis::GroundMAPFGoalVisitor, u::GroundMAPFState, v::GroundMAPFState, d::Float64, nbrs::Vector{Int64})

    env = vis.env
    road_vtx_goal = vis.road_vtx_goal
    avoid_edge_copy_set = vis.avoid_edge_copy_set

    # When we expand actual goal, exit
    if v.road_vtx_id == road_vtx_goal
        env.next_gmapfg_goal_idx = env.unique_gmapf_states[v].idx
        return false
    end

    # Look up outneighbors of v.road_vtx_id in the true road graph
    # For each outneighbor, if no edge copies, just add onbr with aerial ID 0
    # Otherwise, add the outnbr with lowest cost among edge copies if not in avoid_edge_set
    idx_ctr = Graphs.num_vertices(env.ground_mapf_graph)

    for onbr in outneighbors(env.road_graph, v.road_vtx_id)

        vnbr_edge = LightGraphs.Edge(v.road_vtx_id, onbr)
        aids_to_add_with = Set{Int64}()
        is_aerial = false
        v_aids = env.unique_gmapf_states[v].aerial_ids

        if haskey(env.edge_to_copy_info, vnbr_edge)

            edge_copies_id_to_wt = env.edge_to_copy_info[vnbr_edge].aerial_id_to_weight

            aerial_ids_to_avoid = get(avoid_edge_copy_set, vnbr_edge, Set{Int64}())
            aerial_ids_to_consider = setdiff(Set{Int64}(keys(edge_copies_id_to_wt)), aerial_ids_to_avoid)

            # Find lowest weight edges up to min(capacity, aerial_ids)
            if ~(isempty(aerial_ids_to_consider))

                # NOTE: First prefer what it is already carrying
                union!(aids_to_add_with, intersect(v_aids, aerial_ids_to_consider))

                # Then remove from aerial_ids_to_consider
                setdiff!(aerial_ids_to_consider, v_aids)

                # Now choose best remaining
                if ~(isempty(aerial_ids_to_consider))
                    sorted_wt_edges = [(aid, edge_copies_id_to_wt[aid]) for aid in aerial_ids_to_consider]
                    sort!(sorted_wt_edges, by=x->x[2])

                    n_aids = min(length(sorted_wt_edges), env.car_capacity-length(aids_to_add_with))
                    for (aid, wt) in sorted_wt_edges[1:n_aids]
                        push!(aids_to_add_with, aid)
                    end
                end

                if ~isempty(aids_to_add_with)
                    is_aerial = true
                end
            end
        end

        # Check if id_to_add_with is already in ground_mapf_graph
        new_vert = GroundMAPFState(onbr, is_aerial)
        gval = d + generate_ground_mapf_edge_weight(env, v, new_vert, aids_to_add_with)
        if ~(haskey(env.unique_gmapf_states, new_vert))
            idx_ctr += 1
            Graphs.add_vertex!(env.ground_mapf_graph, new_vert)
            push!(nbrs, idx_ctr)
            env.unique_gmapf_states[new_vert] = GMAPFStateInfo(idx_ctr, gval, aids_to_add_with)
        else
            # Only add updated if gval improves
            # NOTE: Should only happen for aerial-aerial or drone-drone
            if gval < env.unique_gmapf_states[new_vert].gval
                push!(nbrs, env.unique_gmapf_states[new_vert].idx)
                env.unique_gmapf_states[new_vert].gval = gval
                env.unique_gmapf_states[new_vert].aerial_ids = aids_to_add_with
            end
        end
    end # onbr in out_neighbors(v.road_vtx_id, env.road_graph)

    return true
end

"""
    update_ground_paths_with_ground_mapf_result!(env::CoordinatedMAPFEnv, solution::Vector{PlanResult})

Using the Stage 1 MAPF solution, update the set of solution paths in the env datastructure
"""
function update_ground_paths_with_ground_mapf_result!(env::CoordinatedMAPFEnv, solution::Vector{PlanResult{GroundMAPFState,GroundMAPFAction,Float64}})

    env.ground_paths = Vector{AgentPathInfo}(undef, length(env.ground_task_list))

    for (ground_id, ground_soln) in enumerate(solution)

        ground_soln_states = ground_soln.states

        path_states = AgentPathState[AgentPathState(road_vtx_id=ground_soln_states[1][1].road_vtx_id, timeval=0.0)]

        cumul_time = 0.0
        cumul_dist = 0.0
        for (state, state_next) in zip(ground_soln_states[1:end-1], ground_soln_states[2:end])
            dist = env.road_graph_wts[state[1].road_vtx_id, state_next[1].road_vtx_id]
            cumul_dist += dist
            cumul_time += dist/AvgSpeed
            push!(path_states, AgentPathState(road_vtx_id=state_next[1].road_vtx_id, timeval=cumul_time))
        end

        env.ground_paths[ground_id] = AgentPathInfo(path_states=path_states, total_dist=cumul_dist, total_time=cumul_time)
    end
end