"""
    shifted_backward_sub!(x, R, λ, k) → x

Solve the problem (R[1:k,1:k] - λI) \\ x[1:k] in-place.
"""
function shifted_backward_sub!(x, R::AbstractMatrix{Tm}, λ, k) where {Tm<:Real}
    # Real arithmetic with quasi-upper triangular R
    @inbounds while k > 0
        if k > 1 && R[k,k-1] != zero(Tm)
            # Solve 2x2 problem
            R11, R12 = R[k-1,k-1] - λ, R[k-1,k]
            R21, R22 = R[k  ,k-1]    , R[k  ,k] - λ
            det = R11 * R22 - R21 * R12
            a1 = ( R22 * x[k-1] - R12 * x[k-0]) / det
            a2 = (-R21 * x[k-1] + R11 * x[k-0]) / det
            x[k-1] = a1
            x[k-0] = a2

            # Backward substitute
            for i = 1 : k-2
                x[i] -= R[i,k-1] * x[k-1] + R[i,k-0] * x[k-0]
            end
            k -= 2
        else
            # Solve 1x1 "problem"
            x[k] /= R[k,k] - λ

            # Backward substitute
            for i = 1 : k - 1
                x[i] -= R[i,k] * x[k]
            end

            k -= 1
        end
    end

    x
end

function shifted_backward_sub!(x, R::AbstractMatrix, λ, k)
    # Generic implementation, upper triangular R
    @inbounds while k > 0
        # Solve 1x1 "problem"
        x[k] /= R[k,k] - λ

        # Backward substitute
        for i = 1 : k - 1
            x[i] -= R[i,k] * x[k]
        end

        k -= 1
    end

    x
end

"""
    collect_eigen!(x, R, j) → k

Store the `j`th eigenvector of an upper triangular matrix `R` in `x`.
In the end `norm(x[1:k]) = 1` This function leaves x[k+1:end] untouched!
"""
function collect_eigen!(x::AbstractVector{Tv}, R::AbstractMatrix{Tm}, j::Integer) where {Tm<:Real,Tv<:Number}
    n = size(R, 2)

    @inbounds begin
        # If it's a conjugate pair with the next index, then just increment j.
        if j < n && R[j+1,j] != zero(Tv)
            j += 1
        end

        # Initialize the rhs and do the first backward substitution
        # Then do the rest of the shifted backward substitution in another function,
        # cause λ is either real or complex.
        if j > 1 && R[j,j-1] != 0
            # Complex arithmetic
            R11, R21 = R[j-1,j-1], R[j-0,j-1]
            R12, R22 = R[j-1,j-0], R[j-0,j-0]
            det = R11 * R22 - R21 * R12
            tr = R11 + R22
            λ = (tr + sqrt(complex(tr * tr - 4det))) / 2
            x[j-1] = -R12 / (R11 - λ)
            x[j-0] = one(Tv)
            for i = 1 : j-2
                x[i] = -R[i,j-1] * x[j-1] - R[i,j]
            end

            shifted_backward_sub!(x, R, λ, j-2)
        else
            # Real arithmetic
            λ = R[j,j]
            x[j] = one(Tv)
            for i = 1 : j-1
                x[i] = -R[i,j]
            end
            shifted_backward_sub!(x, R, λ, j-1)
        end

        # Normalize
        nrm = zero(real(Tv))
        for k = 1:j
            nrm += abs2(x[k])
        end
        scale = inv(√nrm)
        for k = 1:j
            x[k] *= scale
        end
    end

    return j
end

function collect_eigen!(x::AbstractVector{Tv}, R::AbstractMatrix, j::Integer) where {Tv}
    n = size(R, 2)

    @inbounds begin
        λ = R[j,j]
        x[j] = one(Tv)
        for i = 1 : j-1
            x[i] = -R[i,j]
        end

        shifted_backward_sub!(x, R, λ, j-1)

        # Normalize
        nrm = zero(real(Tv))
        for k = 1:j
            nrm += abs2(x[k])
        end
        scale = inv(√nrm)
        for k = 1:j
            x[k] *= scale
        end
    end

    return j
end


load("src/init.jl")

# Given inputs, ouputs, lambda and current weights,
# Assess the ridge loss function.
# Assume first entry of w is an intercept and do not penalize.

function ridge_error(x, y, l, w)
  (1.0 / 2.0) * sum(map(z -> z^2, y - x * w)) + (l / 2.0) * sum(map(z -> z^2, w[2:length(w)]))
end

ridge_error([1.0 2; 3 4], [1, 2], 1, [1, 1])

function ridge_gradient(x, y, l, w)
	unshift(l * w[2:length(w)], 0) - x' * (y - x * w)
end

ridge_gradient([1.0 1 2; 1 3 4], [1, 2], 1, [1, 1, 1])

function ridge_regression(x, y, l)
	function f(w)
    ridge_error(x, y, l, w)
  end

  function g(w)
    ridge_gradient(x, y, l, w)
  end

	w0 = zeros(size(x, 2))

	solution = gradient_descent(f, g, w0)

	solution.minimum
end

x = [1 1 2; 1 3 3; 1 5 6]
y = [1, 2, 2]

w = ridge_regression(x, y, 0.0)
x * w
norm(w)

w = ridge_regression(x, y, 1.0)
x * w
norm(w)

w = ridge_regression(x, y, 10.0)
x * w
norm(w)

w = ridge_regression(x, y, 100.0)
x * w
norm(w)

p = 10

x = hcat([1, 1], vcat([1:p]', [(p + 1):(2p)]'))
y = [1, 2]

w = ridge_regression(x, y, 0)
x * w
norm(w)

w = ridge_regression(x, y, 1)
x * w
norm(w)

w = ridge_regression(x, y, 10)
x * w
norm(w)


export Atan

"""
`Atan([domainType=Float64::Type,] dim_in::Tuple)`

Creates an inverse tangent non-linear operator with input dimensions `dim_in`:
```math
\\text{atan} ( \\mathbf{x} ).
```

"""
struct Atan{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function Atan(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	Atan{DomainType,N}(DomainDim)
end

Atan(DomainDim::NTuple{N,Int}) where {N} = Atan{Float64,N}(DomainDim)
Atan(DomainDim::Vararg{Int}) = Atan{Float64,length(DomainDim)}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::Atan{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= atan.(x)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T, N, A <: Atan{T,N}, TT <: AbstractArray{T,N}}
    L = J.A
    y .= conj.(1.0./(1.0.+ L.x.^2)).*b
end

fun_name(L::Atan) = "atan"

size(L::Atan) = (L.dim, L.dim)

domainType(L::Atan{T,N}) where {T,N} = T
codomainType(L::Atan{T,N}) where {T,N} = T


## ND LINEAR INTERPOLATION KERNELS

"""
    lerp(t, v0, v1)

Linear interpolation between `f(x0)=v0` and `f(x1)=v1` at the normalized coordinate t
"""
@inline lerp(t, v0, v1) = fma(t, v1, fma(-t, v0, v0))

"""
    bilinear(tx, ty, v00, v10, v01, v11)

Linear interpolation between `f(x0,y0)=v0` and `f(x1,y1)=v1` at the normalized coordinates `tx`, `ty`
"""
@inline function ndlinear(tx, ty, v00, v10, v01, v11)
    return lerp(ty, lerp(tx, v00, v10), lerp(tx, v01, v11))
end

@inline function ndlinear(t::NTuple{2,T1}, v::NTuple{4,T2}) where {T1,T2}
    return ndlinear(t[1], t[2], v[1], v[2], v[3], v[4])
end

bilinear(tx, ty, v00, v10, v01, v11) = ndlinear(tx, ty, v00, v10, v01, v11)

"""
    trilinear(tx, ty, tz, v000, v100, v001, v101, v010, v110, v011, v111)

Linear interpolation between `f(x0,y0,z0)=v0` and `f(x1,y1,z1)=v1` at the normalized coordinates `tx`, `ty`, `tz`
"""
@inline function ndlinear(tx, ty, tz, v000, v100, v001, v101, v010, v110, v011, v111)
    return lerp(
        ty,
        ndlinear(tx, tz, v000, v100, v001, v101),
        ndlinear(tx, tz, v010, v110, v011, v111),
    )
end

@inline function ndlinear(t::NTuple{3,T1}, v::NTuple{8,T2}) where {T1,T2}
    return ndlinear(t[1], t[2], t[3], v[1], v[2], v[3], v[4], v[5], v[6], v[7], v[8])
end

@inline function trilinear(tx, ty, tz, v000, v100, v001, v101, v010, v110, v011, v111)
    return ndlinear(tx, ty, tz, v000, v100, v001, v101, v010, v110, v011, v111)
end


module Power
export power_method, inverse_power_method

using LinearAlgebra

"""
Calculate the dominant eigenvalue of a matrix A and its associated eigenvector.
"""
function power_method(A::Matrix{<:Real}, x::Vector{<:Real}, n::Int)::Tuple
	y = x
	m = 1

	for _ in 0:n
		y = A * x
		m = maximum(abs.(y))
		x = (m^-1)y
	end

	return (m, y)
end

"""
Calculate the least dominant eigenvalue of a matrix A and its associated
eigenvector.
"""
function inverse_power_method(A::Matrix{<:Real}, x::Vector{<:Real}, n::Int)::Tuple
	return power_method(inv(A), x, n)
end

end


using DataDrivenDiffEq
using OrdinaryDiffEq
using LinearAlgebra
using Plots
gr()

# Create a linear , time discrete system
function linear_discrete(du, u, p, t)
    du[1] = 0.9u[1]
    du[2] = 0.9u[2] + 0.1u[1]
end

u0 = [10.0; -2.0]
tspan = (0.0, 10.0)
prob = DiscreteProblem(linear_discrete, u0, tspan)
sol = solve(prob, FunctionMap())

# Create Approximation
approx = DMD(sol[:,:])

# Create the associated problem
prob_approx = DiscreteProblem(approx, u0, tspan)
approx_sol = solve(prob_approx, FunctionMap())

# Show solutions
plot(sol)
plot!(approx_sol)

# Show error
plot((sol .- approx_sol)')
norm(sol .- approx_sol) # ≈ 2.23e-14

# Eigen Decomposition
eigen(approx)
# Eigenvalues
scatter(eigvals(approx), xlim = (-1, 1), ylim = (-1,1))

# Adapt system with new measurements
function linear_discrete_2(du, u, p, t)
    du[1] = 0.9u[1]
    du[2] = 0.9u[2] + 0.2u[1]
end

# Solve the new system
prob2 = DiscreteProblem(linear_discrete_2, u0, (0.0, 20.0))
sol2 = solve(prob2, FunctionMap())

# Split the data
x = sol2[:,1:20]
y = sol2[:,2:21]

# Update our approximation with new measurements
update!(approx, x, y)

# Lets have a look at the operator, which moves near the true value
operator(approx)

# Add time continouos system
function linear(du, u, p, t)
    du[1] = -0.9*u[1] + 0.1*u[2]
    du[2] = -0.8*u[2]
end

prob_cont = ODEProblem(linear, u0, tspan)
sol_cont = solve(prob_cont, Tsit5())

plot(sol_cont)

X = sol_cont[:,:]
DX = sol_cont(sol_cont.t, Val{1})[:,:]

# Giving the method a time step (which should be sequentially sampled)
# Enables us to get the continouos representation
approx_cont = gDMD(X, DX)
generator(approx_cont)
approx_sys = ODEProblem(approx_cont, u0, tspan)
approx_sol = solve(approx_sys, Tsit5(),  saveat = sol_cont.t)
# Lets have a look at the solution
plot(sol_cont)
plot!(approx_sol)
# And the error
plot(abs.(sol_cont .- approx_sol)')
norm(sol_cont .- approx_sol) # ≈ 1e-13


__precompile__()
# ----------------------------------------------------------------------------
module FAD # Forward Automatic Differentiation demo
#=
    see also:
        - the walkthrough at http://vladium.com/tutorials/study_julia_with_me/multiple_dispatch/

    working with this module from REPL:

    julia>include("<path>/FAD.jl")
    julia>using Main.FAD
    ... REPL can now use derivative(), ∂(), root_solve(), etc ...
=#
# ............................................................................

import Base; # needed in order to add base ops overloads

export derivative, ∂, myerf, root_solve; # for convenience of 'using Main.FAD' in REPL

# ............................................................................

struct Context <: Number
    v   ::Number
    ∂   ::Number
end
# ............................................................................
# binary ops:

# note: ':()' is necessary to reference certain functions with single-character names

Base.:(+)(lhs ::Context, rhs ::Context) = Context(lhs.v + rhs.v, lhs.∂ + rhs.∂)
Base.:(-)(lhs ::Context, rhs ::Context) = Context(lhs.v - rhs.v, lhs.∂ - rhs.∂)

Base.:(*)(lhs ::Context, rhs ::Context) = Context(lhs.v * rhs.v, lhs.v * rhs.∂ + lhs.∂ * rhs.v)
Base.:(/)(lhs ::Context, rhs ::Context) = Context(lhs.v / rhs.v, (lhs.∂ * rhs.v - lhs.v * rhs.∂) / rhs.v^2)

# unary ops:

Base.:(+)(x ::Context) = x
Base.:(-)(x ::Context) = Context(- x.v, - x.∂)

# math ops:

Base.sin(d ::Context) = Context(sin(d.v),   cos(d.v) * d.∂)
Base.cos(d ::Context) = Context(cos(d.v), - sin(d.v) * d.∂)
# ... add more as needed for your own functions ...

# ............................................................................
# conversion/promotion:

Base.promote_rule(::Type{Context}, ::Type{<: Number}) = Context
Base.convert(::Type{Context}, x ::Real) = Context(x, 0.0)

# ............................................................................
"""
    Given a function 'f' of one variable, return its derivative with respect to that variable.
"""
function derivative(f ::Function)
    return x ::Number -> f(Context(x, 1.0)).∂ # discard value, return derivative
end

∂ = derivative # add a nice-looking alias

# ............................................................................

# to get "official" erf(x), do
# pkg> add SpecialFunctions
# julia> using SpecialFunctions

function myerf(x ::Number)
    Σ = 0.0
    x² = x * x
    for k in 0 : 20 # hardcoding the number of summation terms for simplicity
        Σ += x / (factorial(k) * (2k + 1))
        x *= -x²
    end
    return 2.0 / √π * Σ
end
# ............................................................................
"""
    Newton's method for finding root(s) of 'f'
"""
function root_solve(f ::Function, x₀ ::Number; ϵ = 1e-8)
    i = 1
    while true
        ctx = f(Context(x₀, 1.0))
        println("[$i]: f($x₀)\t= $(ctx.v)")
        abs(ctx.v) < ϵ && break
        x₀ -= ctx.v / ctx.∂ # ignoring issues with division by small floats, etc for simplicity
        i += 1
    end
    return x₀
end

end # of module
# ----------------------------------------------------------------------------


# arb_hypgeom.jl

module arb_hypgeom
export one_f_two_fast, one_f_two

using ArbNumerics

"""
Implementation of the hypergeometric function 1F2 using the expansion:

    1F2(a1; {b1, b2}; z) = ∑_{k=0}^∞ (a1)_k z^k / ((b1)_k ⋅ (b2)_k ⋅ k!)

where (x)_k is the Pochhammer rising factorial = Γ(x+k)/Γ(x) with Γ(x) the Gamma function.
"""

function one_f_two(a, b, x; prec = 64) # z > 0 & n >= 0

    # Initialise precision of ArbReal to prec.
    setextrabits(0)
    setprecision(ArbReal, prec + 8)

    a = ArbReal("$a")
    b = (ArbReal("$i") for i in b)
    x = ArbReal("$x")

    k = ArbReal("0")
    result = ArbReal("0.0")
    err = eps(result)  # Machine accuracy of specified precision prec.

    risingfact(w, n) = ArbReal(ArbNumerics.gamma(w + n) / ArbNumerics.gamma(w))

    while true
        term = ArbReal(risingfact(a, k) * x^k / (prod(risingfact.(b, k)) * ArbNumerics.gamma(k + 1)))

        # Break loop if term smaller than accuracy of result.
        if abs(term) < err
            break
        end

        result += term
        k += ArbReal("1")

        # Double precision if rounding error in result exceeds accuracy specified by prec.
        if ball(result)[2] > err
            setprecision(ArbReal, precision(result) * 2)
            k = ArbReal("0")
            a = ArbReal("$a")
            b = (ArbReal("$i") for i in b)
            x = ArbReal("$x")
            result = ArbReal("0.0")
        end
    end
    ArbReal(result, bits = prec + 8)
end

"""
Implementation of specific 1F2 hypergeometric function used for integrals:

    ∫_0^1 cosh(zx) x^{2m} dx = 1F2(m+1/2; {1/2, m+3/2}; z^2/4)/(2m+1)
                             = ∑_{t=0}^∞ z^{2t} / ((2t+2m+1)⋅(2t)!)

    ∫_0^1 sinh(zx) x^{2m} dx = z ⋅ 1F2(m+1; {3/2, m+2}; z^2/4)/(2m+2)
                             = ∑_{t=0}^∞ z^{2t+1} / ((2t+2m+2)⋅(2t+1)!)

which we can combine into a generic summation:

    ∑_{t=0}^∞ z^{2t+h} / ((2t+2m+1+h)⋅(2t+h)!)

which gives the cosh version for h=0 and the sinh version for h=1. This specialised 1F2 converges faster than generic 1F2 algorithm, one_f_two.
"""

function one_f_two_fast(x, m, h; prec = 64) # z > 0 & n >= 0

    # Initialise precision of ArbReal to prec.
    p = prec
    setextrabits(0)
    setprecision(ArbReal, p)

    x = ArbReal("$x")
    m = ArbReal("$m")
    h = ArbReal("$h")

    k = ArbReal("0")
    result = ArbReal("0.0")
    term = ArbReal("1.0")
    err = eps(result)  # Machine accuracy of specified precision prec.

    while abs(midpoint(term)) > err * abs(midpoint(result))
        term = ArbReal(x^(2 * k + h) / ((2 * k + 2 * m + 1 + h) * ArbNumerics.gamma(2 * k + 1 + h)))
        result += term
        # println("term: k = ", k, "\nterm value: ", ball(ArbReal(term, bits = prec)), "\ncumulant result: ", ball(ArbReal(result, bits = prec)), "\n")
        k += 1

        # Double precision if rounding error in result exceeds accuracy specified by prec.
        if radius(result) > err * abs(midpoint(result))
            p *= 2
            setprecision(ArbReal, p)
            # println("Not precise enough. Error = ", abs(radius(result)/midpoint(result)), " > ", err, ". Increasing precision to ", p, " bits.\n")
            x = ArbReal("$x")
            m = ArbReal("$m")
            h = ArbReal("$h")
            k = ArbReal("0")
            result = ArbReal("0.0")
            term = ArbReal("1.0")
        end
    end
    # println("x: ", ArbReal(x, bits = prec), ". Final result: ", ArbReal(result, bits = prec))
    ArbReal(result, bits = prec)
end

# β = ArbReal("2.0")
# α = ArbReal("7.0")
# v = ArbReal("5.8")
# w = ArbReal("1.6")
# R = ArbReal((v^2 - w^2) / (w^2 * v))
# a = ArbReal(sqrt(β^2 / 4 + R * β * coth(β * v / 2)))
# z = ArbReal("90.61")
# m = ArbReal("1.0")
# @time c = one_f_two_fast(z, m, 0; prec = 64)
# @show(c)

end # end module

################################################################################

"""
Some plots and tests of the above functions to check that they produce the correct values to the specified precision.
"""

# using Plots
# using QuadGK
# plotly()
# Plots.PlotlyBackend()
#
# cosh_integral(z, m) = quadgk(t -> (m + 1) * cosh(z * t) * t^m, BigFloat(0), BigFloat(1))[1]
#
# # x_range = 0:200
# # m = 1
# # @time I = [cosh_integral(x, m) for x in x_range]
# # @time F = abs.([arb_hypgeom_1f2(m / 2 + 1 / 2, (1 / 2, m / 2 + 3/2), x^2 / 4.0; prec = 128) for x in x_range])
# # p = plot(x_range, F, yaxis=:log)
# # plot!(x_range, I)
# # display(p)
# # @show(I, F)
# # println(m / 2 + 1 / 2, (1 / 2, m / 2 + 3/2), x_range^2 / 4.0)
#
# """
# Conclusion: The integral is a bit faster than the hypergeometric function, but not as accurate. So, probably use 1F2 since the time difference doesn't seem to be major!
# """
#
# # @time F_test = arb_hypgeom_1f2(1/2, (1/2, 1/2), 1/2; prec = 512)
# # setprecision(BigFloat, 512)
# # t_test = parse(BigFloat, "2.17818355660857086398922206782012528343129403292165693281081574094992093930206091916837394310903919052930938811320102906774905335969974706202490049810447181898779259195640840870264051023127290510625691429186995752638681515944856002645204867940492548077249544")
# # @show(F_test, t_test)
# # @show(isequal("$F_test"[end-1], "$t_test"[end-1]))
#
# """
# The 1F2 function above can accurate produce high precision values as shown in the above test. But it can be simplified and optimized mathematically initially as shown below.
# """
#
# x_range = 0:200
# m = 1
# h = 0
# @time L = abs.([arb_hypergeom_1f2_fast(x, m, h; prec = 128) for x in x_range])
# # @time F = abs.([arb_hypgeom_1f2(m + 1 / 2, (1 / 2, m + 3/2), x^2 / 4.0; prec = 128) / (2 * m + 1) for x in x_range])
# p = plot(x_range, L, yaxis=:log)
# # plot!(x_range, F)
# display(p)
# @show(L)


# We consider as an illustrative example a simple harmonic oscillator system.
# The set propagation definitions introduced so far are illustrated using a single
# degree of freedom second order problem given by the following equation:

# ```math
# u''(t) + ω^2 u(t) = 0,
# ```
# where $ω$ is a scalar and $u(t)$ is the unknown.
#
# This problem can be associated with a spring-mass system, where $u(t)$ is the
# elongation of the spring at time $t$ and the mass and stiffness set a natural
# frequency $ω$. In this case we consider $ω = 4π$.

# Let us introduce the state variable $v(t) := u'(t)$ and define the vector
# $x(t) = [u(t), v(t)]^T$. Then we can rewrite the system in the following first order form
#
#
# ```math
# x'(t) = \begin{pmatrix} 0 & 1 \\ -\omega^2 & 0 \end{pmatrix} x(t)
#
# Two cases are considered for the initial conditions, first a single initial condition
# given by $u(0)=1$ and $v(0)=0$, and a second case where the initial conditions belong to a set.

# This problem is solved using set propagation and the results obtained for the
# single initial condition are shown in the following figures.
#
# The time step-size used is $\delta = 0.025$.
#
# The set $\Omega_0$ includes the analytic trajectory within $[0, \delta]$, and
# such set is propagated to cover the solution for further time intervals as shown below.
#
# It is worth noting that even when the initial condition is a singleton,
# the set propagation technique returns a sequence of sets, and such sequence guarantees
# an enclosure of the analytic solution at \emph{any} time $t \in [0, T] \subset \mathbb{R}$.
#
# For comparison, we also plot the analytic solution (magenta solid line) and a
# numerical solution obtained using the Newmark's method with time step $\delta/5$
# (red triangle markers).

# The result obtained when a set of initial conditions, together with few dozen
# trajectories with random initial conditions drawn from the initial set
# $\mathcal{X}_0 = \mathcal{U}_0 \times \mathcal{V}_0 = [0.9, 1.1] \times [-0.1, 0.1]$,
# where $\times$ denotes the Cartesian product.

# In both cases, it is shown that a single set-based integration covers infinitely
# many trajectories \emph{in dense time}, i.e. for all intermediate times between
# $0$ and $T$, where $T > 0$ is the time horizon, there is a reach-set that covers
# all the exact solutions.


export lse, lomax, randomindex

# Log-sum exp.
function lse(x::AbstractMatrix{<:Real}; dims = 2)
    m = maximum(x, dims = dims)
    v = exp.(x .- m)
    return log.(sum(v, dims = dims)) + m
end

# see https://www.cs.ubc.ca/~murphyk/Papers/bayesGauss.pdf
function tpdf(μ::T, σ::T, ν::T, x::V) where {T<:Real,V<:Real}
    c = gamma(ν/2 + 1/2) / gamma(ν/2)
    c *= 1/(sqrt(ν*π)*σ)
    return c * (1 + 1/ν * ((x-μ)/σ)^2)^((ν+1)/2)
end

function tlogpdf(μ::T, σ::T, ν::T, x::V) where {T<:Real,V<:Real}
    c = lgamma(ν/2 + 1/2) - lgamma(ν/2)
    c -= log(sqrt(ν*π)*σ)
    return c + ((ν+1)/2)*log(1 + 1/ν * ((x-μ)/σ)^2)
end

@inline lomax(x, α, λ) = log(α) - log(λ) + -(α + 1) * log(1 + (x / λ))

function randomindex(p::AbstractVector{T}) where T<: Real
    return randomindex(p, rand(T))
end

function randomindex(p::AbstractVector{T}, rnd::T) where T<: Real

    csum = T(0.0)
    thres = rnd * sum(p)

    for i in 1:length(p)
        @inbounds begin
            csum += p[i]
            if csum >= thres
                return Int(i)
            end
        end
    end

    return Int(length(p))
end


function test_main()

  srand(1234)

  A = rand(3,3)
  A = 0.5*(A + A')
  invA = inv(A)

  println("\nMatrix A:")
  println(A)
  println("\nInverse of A")
  println(invA)

  B = rand(3,3)
  B = 0.5*(B + B')
  invB = inv(B)

  println("\nMatrix B")
  println(B)
  println("\nInverse of B")
  println(invB)

  AB = A + B
  invAB = inv(A + B)

  println("\nMatrix A + B")
  println(AB)
  println("\nInverse of A + B")
  println(invAB)

  test_inv_Miller( A, B )

  println(rank(B))

end

function test_inv_Miller( G, H )

  g = trace( H*inv(G) )
  println("g = ", g)

  invGH_mill = inv(G) - 1/(1+g) * inv(G)*H*inv(G)

  println("\nTest Miller")
  println(invGH_mill - inv(G+H))
end

test_main()


#   Unit statistics.jl, part of PosDefManifold Package for julia language
#   v 0.3.5 - last update 6th of November 2019
#
#   MIT License
#   Copyright (c) 2019, Marco Congedo, CNRS, Grenobe, France:
#   https://sites.google.com/site/marcocongedo/home
#
#   DESCRIPTION
#   This Unit implements routines for statistics and probability.
#
#   CONTENT
#   1.  Utilities
#   2.  Probability
#   3.  Descriptive Statistocs
# __________________________________________________________________

#  ------------------------
## 1. Utilities
#  ------------------------

#  ------------------------
## 2. Probability
#  ------------------------

"""
    softmax(χ::Vector{T}) where T<:Real

 Given a real vector of ``k`` non-negative scores ``χ=c_1,...,c_k``,
 return the vector ``π=p_1,...,p_k`` of their
 [softmax](https://en.wikipedia.org/wiki/Softmax_function) probabilities,
 as per

 `` p_i=\\frac{\\textrm{e}^{c_i}}{\\sum_{i=1}^{k}\\textrm{e}^{c_i}} ``.

 ## Examples
    χ=[1.0, 2.3, 0.4, 5.0]
    π=softmax(χ)

"""
softmax(χ::Vector{T}) where T<:Real = exp.(χ) ./ 𝚺(exp.(χ))

#  -------------------------
## 3. Descriptive Statistics
#  -------------------------
"""
    mean(metric::Metric, ν::Vector{T}) where T<:RealOrComplex

 Mean of ``k`` real or complex scalars, using the specified `metric`
 of type [Metric::Enumerated type](@ref). Note that using the Fisher,
 logEuclidean and Jeffrey metric, the resulting mean
 is the scalar geometric mean. Note also that the code of this method
 is in unit *statistics.jl*, while the code for all the others is
 in unit *riemannianGeometry.jl*.

 ## Examples
    using PosDefManifold
    # Generate 10 random numbers distributed as a chi-square with 2 df.
    ν=[randχ²(2) for i=1:10]
    arithmetic_mean=mean(Euclidean, ν)
    geometric_mean=mean(Fisher, ν)
    harmonic_mean=mean(invEuclidean, ν)
    harmonic_mean<=geometric_mean<=arithmetic_mean # AGH inequality

"""
function mean(metric::Metric, ν::Vector{T}) where T<:RealOrComplex
    if      metric == Euclidean     return mean(ν)
    elseif  metric == invEuclidean  return inv(mean(inv, ν))
    elseif  metric in (Fisher,
                       logEuclidean,
                       Jeffrey)     return exp(mean(log, ν))
    elseif  metric == logdet0
        @warn "function statistics.mean (scalar mean) not implemented for metric $metric"
    elseif  metric == ChoEuclidean  return (mean(√, ν))^2
    elseif  metric == logCholesky   return exp((mean(log, sqrt.(ν))))^2
    elseif  metric == VonNeumann
        @warn "function statistics.mean and .geodesic not defined for metric $metric"
    elseif  metric == Wasserstein   return (mean(√, ν))^(-0.5)
    else
        @error "in RiemannianGeometry.mean function: the chosen 'metric' does not exist"
    end # if metric
end

"""
    std(metric::Metric, ν::Vector{T};
        corrected::Bool=true,
        mean=nothing) where T<:RealOrComplex

 Standard deviation of ``k`` real or complex scalars,
 using the specified `metric`
 of type [Metric::Enumerated type](@ref) and the
 specified `mean` if provided.

 Only the Euclidean and Fisher
 metric are supported by this function. Using the Euclidean
 metric return the output of standard Julia
 [std](https://docs.julialang.org/en/v1/stdlib/Statistics/#Statistics.std)
 function. Using the Fisher metric return the scalar geometric standard deviation,
 which is defined such as,

 ``\\sigma=\\text{exp}\\Big(\\sqrt{k^{-1}\\sum_{i=1}^{k}\\text{ln}^2(v_i/\\mu})\\Big)``.

 If `corrected` is `true`, then the sum is scaled with ``k-1``,
 whereas if it is `false` the sum is scaled with ``k``.

 ## Examples
    using PosDefManifold
    # Generate 10 random numbers distributed as a chi-square with 2 df.
    ν=[randχ²(2) for i=1:10]
    arithmetic_sd=std(Euclidean, ν) # mean not provided
    geometric_mean=mean(Fisher, ν)
    geometric_sd=std(Fisher, ν, mean=geometric_mean) # mean provided

"""
function std(metric::Metric, ν::Vector{T};
             corrected::Bool=true,
             mean=nothing) where T<:RealOrComplex

    if      metric == Euclidean     return std(ν; corrected=corrected, mean=mean)

    elseif  metric == Fisher
            mean==nothing ? μ=mean(Fisher, ν) : μ=mean
            if corrected return exp(√(𝚺(log(w/μ)^2 for w in ν)/(length(ν)-1)))
            else         return exp(√(𝛍(log(w/μ)^2 for w in ν))) end

    elseif  metric in (invEuclidean, logEuclidean, Jeffrey,
                        logdet0, ChoEuclidean, logCholesky,
                        VonNeumann, Wasserstein)
        @warn "function statistics.mean (scalar mean) not implemented for metric $metric"
    else
        @error "in RiemannianGeometry.mean function: the chosen 'metric' does not exist"
    end # if metric
end


# f(x)=elu(x)=x*[x>0] + (exp(x)-1)*[x<0]

alpha=1.0
function Felu(x)
    y=zeros(size(x))
    for i=1:length(x)
        if x[i]>0
            y[i]=x[i]
        else
            y[i]=alpha*(exp(x[i])-1.0)
        end
    end
    return (y,nothing)
end

function Felu_inplace(value::Array,auxvalue,x) # inplace
    for i=1:length(x)
        if x[i]>0
            value[i]=x[i]
        else
            value[i]=alpha*(exp(x[i])-1.0)
        end
    end
end

function Delu(derivativeIDX,f_c,faux_c,grad_c,grad_n,x)
    tmp=ones(size(x))
    for i=1:length(x)
        if x[i]<0
            tmp[i]=f_c[i]+alpha
        end
    end
    axpy!(1.0,tmp.*grad_c,grad_n)
end

if 1==0 # TODO GPU version

    function Felu_inplace(value::CudaArray,auxvalue,x::CudaArray...) # inplace
    end

    function Delu(derivativeIDX,f_c,faux_c,grad_c,grad_n,x::CudaArray...)
    end

end

Derivative[Felu]=Delu # Define dictionary lookup
Inplace[Felu]=Felu_inplace

elu(n::ADnode)=ADnode(Felu,n)

###elu(A::ADtrans)=transpose(elu(node[A.parent])) # elu(A')=(elu(A))' TODO:check

export elu


#=
  Where is my bag problem.

  From
  "Probabilistic Reasoning Under Uncertainty with Bayesian Networks"
  https://www.bayesia.com/2018-03-02-probabilistic-reasoning-under-uncertainty

  (Also see The BaysiaLabBook "Bayesian Networks BayesiaLab" v1, page 51f.)

  Travel from Singapore -> Tokyo -> Los Angeles.
  1) The departure of Singapore -> Tokyo is delayed
  2) Just made the Tokyo -> LA trip. There's a 50/50 change the luggage is on the plane to LA.
  3) Arriving at LA, waiting for the luggage in 5 minutes, the luggage has not been seen.
  What is the probability that the bag was in the Tokyo -> LA plane?

  Answer: The probability that the bag was with the LA plane is 33% (as in the talk).

  From Judea Pearl: The Book of Why.

  For p(bag_on_plane) = 0.5
    parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
  bag_on_plane    0.3364    0.4725     0.0047    0.0053   8953.8759    1.0001     1581.1188

  For p(bag_on_plane) = 0.8
    parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
  bag_on_plane    0.6641    0.4723     0.0047    0.0051   8363.8461    0.9999     1469.6619

  Cf ~/blog/where_is_my_bag.blog
     ~/webppl/where_is_my_bag.wppl

=#
using Turing, StatsPlots, Distributions
include("jl_utils.jl")

@model function where_is_my_bag(maxTime=10,p=0.5,time=5.0,bag_on_carousel=false)

    # We know that the bag was on the plane with 50% probability.
    # Variant: https://www.youtube.com/watch?v=c71pnonOMkI&t=1074s it's 80/20.
    bag_on_plane ~ flip(p)

    # We assume uniform time step
    time ~ DiscreteUniform(1,maxTime)

    # Probability that the bag is on the carousel given a specific time
    # (if it was on the plane, that is).
    # The probability that the bag is on the carousel on a given time
    # is directly proportional on the time:
    #     time / maxTime
    bag_on_carousel ~ bag_on_plane == true ? flip(1.0*time/maxTime) : Dirac(0.0)

    # He waited 5 minutes without seeing the luggage.
    # Note: It's faster if we observe via the parameters
    # true ~ Dirac(time == 5.0)
    # true ~ Dirac(bag_on_carousel == false)

end

maxTime = 10
p = 0.5
# p = 0.8
time = 5.0
bag_on_carousel = false
model = where_is_my_bag(maxTime,p,time,bag_on_carousel)

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))


"""
    PeriodicKernel(r::AbstractVector)
    PeriodicKernel(dims::Int)
    PeriodicKernel(T::DataType, dims::Int)

Periodic Kernel as described in http://www.inference.org.uk/mackay/gpB.pdf eq. 47.
```
    κ(x,y) = exp( - 0.5 sum_i(sin (π(x_i - y_i))/r_i))
```
"""
struct PeriodicKernel{T} <: SimpleKernel
    r::Vector{T}
    function PeriodicKernel(; r::AbstractVector{T} = ones(Float64, 1)) where {T<:Real}
        @assert all(r .> 0)
        new{T}(r)
    end
end

PeriodicKernel(dims::Int) = PeriodicKernel(Float64, dims)

PeriodicKernel(T::DataType, dims::Int = 1) = PeriodicKernel(r = ones(T, dims))

metric(κ::PeriodicKernel) = Sinus(κ.r)

kappa(κ::PeriodicKernel, d::Real) = exp(- 0.5d)

Base.show(io::IO, κ::PeriodicKernel) = print(io, "Periodic Kernel, length(r) = ", length(κ.r), ")")


using Plots
using LinearAlgebra

dx = 0.001
L = pi
x = L * (-1+dx:dx:1)
n = length(x)
nquart = Int(floor(n/4))

# Define hat function
f = zeros(n);
f[nquart:2*nquart] = 4*(1:nquart+1)/n
f[2*nquart+1:3*nquart] = (-4*(0:nquart-1).+1)/n

p1 = plot(x, f)

# Compute Fourier series

A0 = sum(f) * dx

fFS = A0/2
A = zeros(20)
B = zeros(20)

for k = 1:20
    A[k] = sum(f.*cos.(pi*k*x/L)) * dx
    B[k] = sum(f.*sin.(pi*k*x/L)) * dx
    global fFS = A[k]*cos.(k*pi*x/L) .+ B[k]*sin.(k*pi*x/L) .+ fFS
    plot!(x, fFS)
end

fFS2 = A0/2

kmax = 100
A2 = zeros(kmax+1)
B2 = zeros(kmax+1)
ERR = zeros(kmax+1)

A2[1] = A0/2
ERR[1] = norm(f-fFS)
p2 = plot(x, f)

for k = 1:kmax
    A2[k+1] = sum(f.*cos.(pi*k*x/L)) * dx
    B2[k+1] = sum(f.*sin.(pi*k*x/L)) * dx
    plot!(x, B2[k] * sin.(2k*pi*x/L))
    global fFS2 = A2[k+1]*cos.(k*pi*x/L) .+ B2[k+1]*sin.(k*pi*x/L) .+ fFS2
    ERR[k+1] = norm(f-fFS2)/norm(f)
end

p3 = plot(0:kmax, A2)
p4 = plot(0:kmax, ERR, yaxis = :log)


using Plots

function branin(x, a=1, b=5.1/(4π^2), c=5/π, r=6, s=10, t=1/(8π))
    return a * (x[2] - b * x[1]^2 + c * x[1] - r)^2 + s*(1-t) * cos(x[1]) + s;
end

# plot

contour(-50:50, -50:50, (x, y)->branin([x, y]))


using PSO
using Base.Test
using FactCheck

facts("Example minimization of 4th-order banana function") do
    function myfunc(x)
        x1, x2 = x[1], x[2]
        return x1^4 - 2*x2*x1^2 + x2^2 + x1^2 - 2*x1 + 5
    end

    lb = [-3.0, -1.0]
    ub = [2.0, 6.0]

    context("no constraints") do
        xopt1, fopt1 = pso(myfunc, lb, ub)
        x, y = xopt1
        @fact x --> roughly(1.0; atol=1e-3)
        @fact y --> roughly(1.0; atol=1e-3)
        @fact fopt1 --> roughly(4.0, atol=1e-8)

        println("The optimum is at:")
        println("    $(xopt1)")
        println("Optimal function value:")
        println("    myfunc: $(fopt1)")
    end

    function mycon(x)
        x1, x2 = x[1], x[2]
        return [-(x1 + 0.25)^2 + 0.75*x2]
    end

    context("with constraint") do
        # xopt2, fopt2 = pso(myfunc, initial, bounds, num_particles, maxiter)
        xopt2, fopt2 = pso(myfunc, lb, ub, constraints=mycon, swarmsize=1000)
        x, y = xopt2
        @fact x --> roughly(0.5; atol=1e-2)
        @fact y --> roughly(0.75; atol=1e-1)
        @fact fopt2 --> roughly(4.5, atol=1e-3)

        println("The optimum is at:")
        println("    $(xopt2)")
        println("Optimal function value:")
        println("    myfunc: $(fopt2)")
        println("    mycon : $(mycon(xopt2))")
    end
end

facts("Engineering example: minimization of twobar truss weight, subject to") do
    println("  Yield Stress <= 100 kpsi")
    println("  Yield Stress <= Buckling Stress")
    println("  Deflection   <= 0.25 inches")

    function weight(x, B, ρ, E, P)
        H, d, t = x  # all in inches
        return ρ*2*π*d*t*√((B/2)^2 + H^2)
    end

    function stress(x, B, ρ, E, P)
        H, d, t = x  # all in inches
        return (P * √((B/2)^2 + H^2)) / (2*t*π*d*H)
    end

    function buckling_stress(x, B, ρ, E, P)
        H, d, t = x  # all in inches
        return (π^2 * E * (d^2 + t^2)) / (8 * ((B/2)^2 + H^2))
    end

    function deflection(x, B, ρ, E, P)
        H, d, t = x  # all in inches
        return (P * (√((B/2)^2 + H^2))^3) / (2*t*π*d*H^2 * E)
    end

    function mycons(x, B, ρ, E, P)
        strs = stress(x, B, ρ, E, P)
        buck = buckling_stress(x, B, ρ, E, P)
        defl = deflection(x, B, ρ, E, P)
        return [100 - strs, buck - strs, 0.25 - defl]
    end

    B = 60  # inches
    rho = 0.3  # lb/in^3
    E = 30000  # kpsi
    P = 66  # lb (force)
    args = (B, rho, E, P)
    lb = [10.0, 1.0, 0.01]
    ub = [30.0, 3.0, 0.25]
    context("") do
        xopt4, fopt4 = pso(weight, lb, ub, constraints=mycons, args=args, swarmsize=1000)
        x, y, z = xopt4
        @fact x --> roughly(29.0; atol=1.0)
        @fact y --> roughly(2.5; atol=1.0)
        @fact fopt4 --> roughly(11.88, atol=1e-2)

        println("The optimum is at:")
        println("    $(xopt4)")
        println("Optimal function values:")
        println("    weight         : $(fopt4)")
        println("Constraint functions:")
        println("    stress         : $(stress(xopt4, args...))")
        println("    buckling stress: $(buckling_stress(xopt4, args...))")
        println("    deflection     : $(deflection(xopt4, args...))")
    end
end


# Define exit time
A = 600

# Starting amount of money (S0)
k = 500

# Define probability of winning
p = .49
q = 1 - p

# Define b vector
b = zeros(A + 1)
b[A+1] = 1

# Define M matrix
M = zeros(A+1, A+1)
M[1,1] = 1
M[A+1,A+1] = 1

for i=2:A
    global M, q, p
    M[i,i] = -1
    M[i,i-1] = q
    M[i,i+1] = p
end

# Compute exit probabilities by computing M^(-1)b
u = M\b

# Get probability of interest
using Printf
@printf("Probability of walking away with \$%g: %f", k,
 u[k+1])


export hom, hom_check

"""
`hom_check(G,H,d)` checks if the dictionary `d` is a graph homomorphism
from `G` to `H`.
"""
function hom_check(G::SimpleGraph, H::SimpleGraph, d::Dict)::Bool
    for e in G.E
        v, w = e
        x = d[v]
        y = d[w]
        if !H[x, y]
            return false
        end
    end
    return true
end

"""
`hom(G,H)` finds a graph homomorphism from `G` to `H`.
"""
function hom(G::SimpleGraph{S}, H::SimpleGraph{T}) where {S,T}

    m = Model(get_solver())

    @variable(m, P[G.V, H.V], Bin)

    for v in G.V
        @constraint(m, sum(P[v, x] for x in H.V) == 1)
    end

    for e in G.E
        u, v = e
        for x in H.V
            for y in H.V
                if !H[x, y]
                    @constraint(m, P[u, x] + P[v, y] <= 1)
                end
            end
        end
    end

    optimize!(m)
    status = Int(termination_status(m))

    if status != 1
        error("No homomorphism")
    end

    PP = Int.(value.(P))

    result = Dict{S,T}()

    for v in G.V
        for x in H.V
            if PP[v, x] > 0
                result[v] = x
            end
        end
    end

    return result

end


"""
    euler(f,a,b,y0,M)
berisi program untuk mencari solusi persamaan differensial `y'=f(t,y)`
dengan masalah nilai awal `y(a) = y0` pada interval `[a, b]`. Program ini secara default
berisi 5 masukan, yaitu fungsi `f(t,y)`, titik ujung interval penyelesaian `[a,b]`, nilai awal
`y0`, dan jumlah sub-interval `M`.
# Example
```jl
julia> f(t,y) = (t-y)/2;

julia> sol = euler(f,0,3,1,6)
7×2 Array{Float64,2}:
 0.0  1.0
 0.5  0.75
 1.0  0.6875
 1.5  0.765625
 2.0  0.949219
 2.5  1.21191
 3.0  1.53394
```
return solusi masalah nilai awal `sol`.
"""
function euler(f,a,b,y0,M)
    M = Int(M)
    h = (b-a)/M;
    T = a:h:b;
    Y = Array{Float64}(undef,length(T),1)
    Y[1] = y0;
    # Mulai langkah iterasi euler
    for k = 1:M
        # Rumus iterasi euler
        Y[k+1] = Y[k]+h*f(T[k],Y[k])
    end
    sol = [T Y]
end


doc"""
    Skellam(μ1, μ2)

A *Skellam distribution* describes the difference between two independent [`Poisson`](:func:`Poisson`) variables, respectively with rate `μ1` and `μ2`.

$P(X = k) = e^{-(\mu_1 + \mu_2)} \left( \frac{\mu_1}{\mu_2} \right)^{k/2} I_k(2 \sqrt{\mu_1 \mu_2}) \quad \text{for integer } k$

where $I_k$ is the modified Bessel function of the first kind.

```julia
Skellam(mu1, mu2)   # Skellam distribution for the difference between two Poisson variables,
                    # respectively with expected values mu1 and mu2.

params(d)           # Get the parameters, i.e. (mu1, mu2)
```

External links:

* [Skellam distribution on Wikipedia](http://en.wikipedia.org/wiki/Skellam_distribution)
"""
immutable Skellam <: DiscreteUnivariateDistribution
    μ1::Float64
    μ2::Float64

    function Skellam(μ1::Real, μ2::Real)
        @check_args(Skellam, μ1 > zero(μ1) && μ2 > zero(μ2))
        new(μ1, μ2)
    end
    Skellam(μ::Real) = Skellam(μ, μ)
    Skellam() = new(1.0, 1.0)
end

@distr_support Skellam -Inf Inf

### Parameters

params(d::Skellam) = (d.μ1, d.μ2)

### Statistics

mean(d::Skellam) = d.μ1 - d.μ2

var(d::Skellam) = d.μ1 + d.μ2

skewness(d::Skellam) = mean(d) / (var(d)^1.5)

kurtosis(d::Skellam) = 1.0 / var(d)

### Evaluation

function logpdf(d::Skellam, x::Int)
    μ1, μ2 = params(d)
    - (μ1 + μ2) + (x / 2.0) * log(μ1 / μ2) + log(besseli(x, 2.0 * sqrt(μ1) * sqrt(μ2)))
end

pdf(d::Skellam, x::Int) = exp(logpdf(d, x))

function mgf(d::Skellam, t::Real)
    μ1, μ2 = params(d)
    exp(-(μ1 + μ2) + μ1 * exp(t) + μ2 * exp(-t))
end

function cf(d::Skellam, t::Real)
    μ1, μ2 = params(d)
    exp(-(μ1 + μ2) + μ1 * cis(t) + μ2 * cis(-t))
end

### Sampling

rand(d::Skellam) = rand(Poisson(d.μ1)) - rand(Poisson(d.μ2))


# Parts of this code were taken / derived from NetworkX. See LICENSE for
# licensing details.

"""
    pagerank(g, α=0.85, n=100, ϵ=1.0e-6)

Calculate the [PageRank](https://en.wikipedia.org/wiki/PageRank) of the
directed graph `g` parameterized by damping factor `α`, number of
iterations `n`, and convergence threshold `ϵ`. Return a vector representing
the centrality calculated for each node in `g`, or an error if convergence
is not reached within `n` iterations.
"""
function pagerank end

@traitfn function pagerank(g::::IsDirected, α=0.85, n=100, ϵ=1.0e-6)
    A = adjacency_matrix(g,:in,Float64)
    S = vec(sum(A,1))
    S = 1./S
    S[find(S .== Inf)]=0.0
    M = A' # need a separate line due to bug #17456 in julia
    M = (Diagonal(S) * M)'
    N = Int(nv(g))
    x = fill(1.0/N, N)
    p = fill(1.0/N, N)
    dangling_weights = p
    is_dangling = find(S .== 0)

    for _ in 1:n
        xlast = x
        x = α * (M * x + sum(x[is_dangling]) * dangling_weights) + (1 - α) * p
        err = sum(abs, (x - xlast))
        if (err < N * ϵ)
            return x
        end
    end
    error("Pagerank did not converge after $n iterations.")
end


# Sigma can be used to approximate the volume of an n-dimensional ball
using Sigma
using Base.Test

## Ground Truth
# unit n-sphere volume (n-ball surface area)
S(n) = n == 0 ? 2 : 2π*V(n-1)

# unit n-ball volume
V(n) = n == 0 ? 1 : S(n-1) / n

# ratio of n_ball volume to enclosing unit_sphere
ratio_V(n) = V(n) / 2^n

## Returns the probability that any x,y,z,.. within a unit
## n dim box will be within the enclosed sphere.
## Approximates volume of nball ground-truth
function unit_n_ball(num_dims::Integer)
  ncube = mvuniform(-1.0, 1.0, num_dims)
  sphere = sqrt(sum([x^2 for x in ncube])) < 1.0
  ncube, sphere
end

## Prob that point within n dim box will be element of smaller cube
## of side length 1 10th
function unit_n_box(lenratio::Float64, num_dims::Integer)
  @assert 0 < lenratio <= 1
  bigcube = mvuniform(-1.0, 1.0, num_dims)
  smallcube  = [(x > -lenratio) & (x < lenratio) for x in bigcube]
  smallcube_cond = apply(&, smallcube)
  bigcube, smallcube_cond
end

model, condition = unit_n_ball(10)
rand(model, condition)

model, condition = unit_n_box(0.5, 10)
rand(model, condition)


@testset "Affine Parametric Array construction" begin
    @linvars x y z

    vars = [x, y, z]

    A = AffineParametricArray([x+1 y+2;x+y+z+1 x-z], vars)
    A1 = AffineParametricArray([1 2;3 4], vars)
    B = AffineParametricArray([x+1 y+2;x+y+z+1 x-z+1], vars)

    @test A.vars == vars
    @test A.coeffs == [[1 0;1 1], [0 1;1 0], [0 0;1 -1], [1 2;1 0]]
    @test A1.coeffs == [[0 0;0 0], [0 0;0 0], [0 0;0 0], [1 2;3 4]]

    @test A[1, 2] == y + 2
    @test A[:, 1] == [x+1, x+y+z+1]

    A1[1, 2] = x
    @test A1 == AffineParametricArray([1 x;3 4], vars)
    A1[:, 1] = [x+1, z-1]
    @test A1 == AffineParametricArray([x+1 x;z-1 4], vars)

    @test A() == [x+1 y+2;x+y+z+1 x-z]
    @test A([1..2, 2..3, 3..4]) == [2..3 4..5; 7..10 -3.. -1]
end

@testset "Affine parametric array operations" begin
    @linvars x y z
    vars = [x, y, z]

    A = AffineParametricArray([x+1 y+2;x+y+z+1 x-z], vars)
    B = AffineParametricArray([x+1 y+2;x+y+z+1 x-z+1], vars)

    @test A == A
    @test A != B

    @test +A == A
    @test -A == AffineParametricArray([-x-1 -y-2; -x-y-z-1 -x+z], vars)
    @test A + B == AffineParametricArray([2x+2 2y+4; 2x+2y+2z+2 2x-2z+1], vars)
    @test A - B == AffineParametricArray([0 0;0 -1], vars)

    C = [2 1;1 1]
    @test A + C == AffineParametricArray([x+3 y+3;x+y+z+2 x-z+1], vars)
    @test C + A == AffineParametricArray([x+3 y+3;x+y+z+2 x-z+1], vars)

    @test A * C == AffineParametricArray([2x+y+4 x+y+3;3x+2y+z+2 2x+y+1], vars)
    @test C * A == AffineParametricArray([3x+y+z+3 x+2y-z+4;2x+y+z+2 x+y-z+2], vars)
    @test C \ A == inv(C) * A

    @test A * 2 == AffineParametricArray([2x+2 2y+4;2x+2y+2z+2 2x-2z], vars)
    @test 2 * A == AffineParametricArray([2x+2 2y+4;2x+2y+2z+2 2x-2z], vars)
    @test A / 2 == AffineParametricArray([0.5x+0.5 0.5y+1;0.5x+0.5y+0.5z+0.5 0.5x-0.5z], vars)
end


# we write a function that returns a random vector from [0,1]^d with exponential density
function randExp(d, lam)

	# we generate a uniformly random vector
	y = rand(d)

	# we transform (elementwise) to random numbers in [exp(-lambda),1]
	y = exp(-lam) .+ (1-exp(-lam))*y

	# we now get numbers with exponential density
	x = -log.(y)/lam
	return x
end


### A Pluto.jl notebook ###
# v0.17.4

using Markdown
using InteractiveUtils

# ╔═╡ c12ef471-40b9-4b3e-81b6-6d10fa4e5f04
begin
	bigbreak = HTML("<br>"^4);
	using PlutoUI
	using Test
end

# ╔═╡ 58863e50-8029-11ec-28da-35397c3e293f
md"""
# Loops
Topics:
1. `while` loops
2. `for` loops

## `while` loops
The syntax for a while is
```julia
while *condition*
    *loop body*
end
```
For example, we could use while to count or to iterate over an array.

"""

# ╔═╡ e75a73ad-c49a-4d0d-a548-e96ed41cc5cc
let
	n = 0
	while n < 10
	    n += 1
	    println(n)
	end
	n
end

# ╔═╡ bfe2dd76-54a3-479e-9f94-efaecde8266b
with_terminal() do
	myfriends = ["Ted", "Robyn", "Barney", "Lily", "Marshall"]

	i = 1
	while i <= length(myfriends)
	    friend = myfriends[i]
	    print("Hi $friend, it's great to see you!\n")
	    i += 1
	end
end

# ╔═╡ 35079b58-796c-4239-a201-cd6f55c16ca7
begin
end

# ╔═╡ ecd9ce51-6596-4a58-97d1-fbdee32ff1d8
bigbreak

# ╔═╡ 1e88489d-f123-4171-b406-47f6da914674
md"""
## `for` loops

The syntax for a for loop is
```julia
for *var* in *loop iterable*
    *loop body*
end
```
We could use a for loop to generate the same results as either of the examples above:
"""

# ╔═╡ 22ddbdaf-9483-417c-a702-317daa8c6537
with_terminal() do
	for n ∈ 1:10

		println(n)

	end
end

# ╔═╡ 3a38e1ef-7454-498d-af5d-876a9fa6e963
with_terminal() do
	myfriends = ["Ted", "Robyn", "Barney", "Lily", "Marshall"]

	for friend in myfriends
	    println("Hi $friend, it's great to see you!")
	end
end

# ╔═╡ c146bf2e-b890-482d-9b45-4fcfcf37eb93
Z = [10 30; 20 40]

# ╔═╡ 8e72a3b6-d276-46b9-b28e-3a9ab7530467
with_terminal() do

	for i in CartesianIndices(Z)
		println(i)
	end
end

# ╔═╡ f0390b57-a52e-4669-b992-2acd54f484da
bigbreak

# ╔═╡ f5a82c3e-47c6-4712-9c7e-7ff3fc1e5203
md"""
Now let's use for loops to create some addition tables, where the value of every entry is the sum of its row and column indices.

Note that we iterate over this array via column-major loops in order to get the best performance!

First, we initialize an array with zeros.

"""

# ╔═╡ 44c04d48-4e24-4034-b188-c92ba252ff22
begin
	m, n = 5, 5
	A = fill(0.0, (m, n))
end

# ╔═╡ 7a662dce-97fa-4c1c-ba60-51f6f147e808
begin
	for j in 1:n
	    for i in 1:m
	        A[i, j] = i + j
	    end
	end
	A
end

# ╔═╡ e2308376-31cd-4418-932c-8f166b4a316f
md"
Here's some syntactic sugar for the same nested for loop"

# ╔═╡ 0ae376b8-8451-46d7-9bb4-cc259a5b4557
B = fill(0, (m, n))

# ╔═╡ bf437047-2de0-4a5c-93d8-1ecab06dcda6
begin
	for j in 1:n, i in 1:m
	    B[i, j] = i + j
	end
	B
end

# ╔═╡ 4441001f-fd92-4f6b-9e5c-6daee5274d54
md"""The more "Julia" way to create this addition table would have been with an array comprehension."""

# ╔═╡ 2b9190e9-28c5-456f-97f4-b3b59b2d4957
C = [i + j for i in 1:m, j in 1:n]

# ╔═╡ 569ab1ca-623f-4569-b128-6a4fe4208d1c
bigbreak

# ╔═╡ d0ef189a-635e-4811-9c71-206e4a84fd81
md"""
### Generators

If you don't want to have the data in your memory, generators are your tool!
"""

# ╔═╡ e7afb4e9-f843-4113-ad67-f2eef26fa443
G = ( 10*i + j   for i in 1:5, j in 1:5    if i < j)

# ╔═╡ 5012a7a4-a37d-49df-9fcc-68a18295f5bd
G

# ╔═╡ bb5b8711-a987-4921-ac15-ffd3e71e903f
md"They can be iterated, which is enough for many utility functions, like
```julia
sum(), prod(), minimum(), maximum(), map(), broadcast(), ...
```
"

# ╔═╡ bcec978b-6da0-4720-a7fe-a65333233c2d
collect(G)

# ╔═╡ f19d75e2-a933-4534-b638-a724248671c4
sum(G)

# ╔═╡ 8d26881f-094c-42e6-95ea-93637646d210
minimum(G)

# ╔═╡ 50b79c47-09d1-41aa-8b60-5fc1dd114c29
map(x -> x^2, G)

# ╔═╡ d1782a41-8ea5-456c-a513-7fec9457c9ad
G .^ 2  # equivalent to the map command above

# ╔═╡ d0effe30-adf5-4cd2-9bd5-a99138a33e0e
with_terminal() do
	for i in 1:10
		print(i, " ")
	end
end

# ╔═╡ 99eefdf8-3ea5-41fb-92cc-5655b5e92fc2
bigbreak

# ╔═╡ f77c41cd-a36e-4ec0-968c-f1842b703032
md"""
## Exercises

4.1

Loop over integers between 1 and 100 and print their squares.
"""

# ╔═╡ 7021bc95-71bf-4861-98d2-a3f358e5aae6
with_terminal() do
	# write your code here
end

# ╔═╡ 33e4917e-ec98-44f5-85e4-b8f0c617f03c
bigbreak

# ╔═╡ e53d967c-d166-4208-a237-d4b692ab7431
md"""
4.2

Add to the code above a bit to create a dictionary, squares that holds integers and their squares as key, value pairs such that

```julia
squares[10] == 100
```
"""

# ╔═╡ 8a9b4ff0-1ebc-4546-9ac8-663c77d748c0
begin
	squares = missing
end

# ╔═╡ ebab307a-1a97-48da-89f2-940cb7936c6e
begin
	@assert squares[10] == 100
	@assert squares[11] == 121
end

# ╔═╡ d5ba15fa-255c-4bef-bb9d-d165232cd009
bigbreak

# ╔═╡ e7eeb88a-2e15-487a-b9f4-177b81d52b09
md"""
4.3

Use an array comprehension to create an an array `squares_arr` that stores the squares for all integers between 1 and 100.

"""

# ╔═╡ 5c13e9a8-491b-4486-ac35-c54ed91791c7
begin
	squares_arr = Int64[]
end

# ╔═╡ 01e7c0a1-9d4d-4f20-8625-6f8c08eeb418
begin
	@assert length(squares_arr) == 100
	@assert sum(squares_arr) == 338350
end

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[compat]
PlutoUI = "~0.7.32"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

julia_version = "1.7.1"
manifest_format = "2.0"

[[deps.AbstractPlutoDingetjes]]
deps = ["Pkg"]
git-tree-sha1 = "8eaf9f1b4921132a4cff3f36a1d9ba923b14a481"
uuid = "6e696c72-6542-2067-7265-42206c756150"
version = "1.1.4"

[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[deps.ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.11.0"

[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[deps.Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[deps.FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.4"

[[deps.Hyperscript]]
deps = ["Test"]
git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9"
uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91"
version = "0.0.4"

[[deps.HypertextLiteral]]
git-tree-sha1 = "2b078b5a615c6c0396c77810d92ee8c6f470d238"
uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2"
version = "0.9.3"

[[deps.IOCapture]]
deps = ["Logging", "Random"]
git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
version = "0.2.2"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[deps.JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[deps.LinearAlgebra]]
deps = ["Libdl", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[deps.Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"

[[deps.Parsers]]
deps = ["Dates"]
git-tree-sha1 = "0b5cfbb704034b5b4c1869e36634438a047df065"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.2.1"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[deps.PlutoUI]]
deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"]
git-tree-sha1 = "ae6145ca68947569058866e443df69587acc1806"
uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8"
version = "0.7.32"

[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[deps.REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[deps.SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[deps.Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"

[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╟─c12ef471-40b9-4b3e-81b6-6d10fa4e5f04
# ╟─58863e50-8029-11ec-28da-35397c3e293f
# ╠═e75a73ad-c49a-4d0d-a548-e96ed41cc5cc
# ╠═bfe2dd76-54a3-479e-9f94-efaecde8266b
# ╠═35079b58-796c-4239-a201-cd6f55c16ca7
# ╟─ecd9ce51-6596-4a58-97d1-fbdee32ff1d8
# ╟─1e88489d-f123-4171-b406-47f6da914674
# ╠═22ddbdaf-9483-417c-a702-317daa8c6537
# ╠═3a38e1ef-7454-498d-af5d-876a9fa6e963
# ╠═c146bf2e-b890-482d-9b45-4fcfcf37eb93
# ╠═8e72a3b6-d276-46b9-b28e-3a9ab7530467
# ╟─f0390b57-a52e-4669-b992-2acd54f484da
# ╟─f5a82c3e-47c6-4712-9c7e-7ff3fc1e5203
# ╠═44c04d48-4e24-4034-b188-c92ba252ff22
# ╠═7a662dce-97fa-4c1c-ba60-51f6f147e808
# ╟─e2308376-31cd-4418-932c-8f166b4a316f
# ╠═0ae376b8-8451-46d7-9bb4-cc259a5b4557
# ╠═bf437047-2de0-4a5c-93d8-1ecab06dcda6
# ╟─4441001f-fd92-4f6b-9e5c-6daee5274d54
# ╠═2b9190e9-28c5-456f-97f4-b3b59b2d4957
# ╟─569ab1ca-623f-4569-b128-6a4fe4208d1c
# ╟─d0ef189a-635e-4811-9c71-206e4a84fd81
# ╠═e7afb4e9-f843-4113-ad67-f2eef26fa443
# ╠═5012a7a4-a37d-49df-9fcc-68a18295f5bd
# ╟─bb5b8711-a987-4921-ac15-ffd3e71e903f
# ╠═bcec978b-6da0-4720-a7fe-a65333233c2d
# ╠═f19d75e2-a933-4534-b638-a724248671c4
# ╠═8d26881f-094c-42e6-95ea-93637646d210
# ╠═50b79c47-09d1-41aa-8b60-5fc1dd114c29
# ╠═d1782a41-8ea5-456c-a513-7fec9457c9ad
# ╠═d0effe30-adf5-4cd2-9bd5-a99138a33e0e
# ╟─99eefdf8-3ea5-41fb-92cc-5655b5e92fc2
# ╟─f77c41cd-a36e-4ec0-968c-f1842b703032
# ╠═7021bc95-71bf-4861-98d2-a3f358e5aae6
# ╟─33e4917e-ec98-44f5-85e4-b8f0c617f03c
# ╟─e53d967c-d166-4208-a237-d4b692ab7431
# ╠═8a9b4ff0-1ebc-4546-9ac8-663c77d748c0
# ╠═ebab307a-1a97-48da-89f2-940cb7936c6e
# ╟─d5ba15fa-255c-4bef-bb9d-d165232cd009
# ╟─e7eeb88a-2e15-487a-b9f4-177b81d52b09
# ╠═5c13e9a8-491b-4486-ac35-c54ed91791c7
# ╠═01e7c0a1-9d4d-4f20-8625-6f8c08eeb418
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


# The prime factors of 13195 are 5, 7, 13 and 29.
# What is the largest prime factor of the number 600851475143 ?

num = BigInt(600851475143)
#num = BigInt(13195)

is_prime(n) = !any(n%i == 0 for i in 2:BigInt(floor(sqrt(n))))

function largest_factor(num)
    #step_ = (num % 2 == 0) ? -1 : -2
    num_leftover = num
    for p in (BigInt(p) for p=2:num_leftover if is_prime(p))
        if p == num_leftover
            println(p)
            return p
        elseif num_leftover % p == 0
            num_leftover = BigInt(num_leftover / p)
        end
    end
end
largest_factor(num)
println()


using Distributions
using PyPlot
using MAT

srand(0) #seed

function generate_gaussian_data(N::Int64, D::Int64, K::Int64)
	x = randn(D,N)
	tpi = rand(Dirichlet(ones(K)))
	tzn = rand(Multinomial(N,tpi))  #number of points in each cluster, e.g. [2, 2, 6]
	tz = zeros(N)

	tmean = zeros(D,K)
	tcov = zeros(D,D,K)

	ind = 1
	println(tzn)
	for i=1:length(tzn)
		indices = ind:ind+tzn[i]-1
		tz[indices] = i
		tmean[:,i] = rand(MvNormal(zeros(D), 100*eye(D)))  #sample cluster mean
		tcov[:,:,i] = rand(InverseWishart(D+2, eye(D)))    #sample cluster covariance
		# T = chol(slice(tcov,:,:,i))
		# x[:,indices] = broadcast(+, T'*x[:,indices], tmean[:,i]);
		d = MvNormal(tmean[:,i], tcov[:,:,i])  #form cluster distribution
		for j=indices
			x[:,j] = rand(d)  #generate cluster point
		end
		ind += tzn[i]
	end
	x, tz, tmean, tcov
end

N = 1000
D = 2
K = 6
latent, latent_z, latent_mu, latent_cov = generate_gaussian_data(N, D, K)

figure()
scatter(latent[1,:], latent[2,:], c=latent_z)

file = matopen("clusters.mat", "w")
write(file, "data", latent)
write(file, "labels", latent_z)
close(file)


# # [Linear Time Discrete System](@id linear_discrete)
#
# We will start by estimating the underlying dynamical system of a time discrete process based on some measurements via [Dynamic Mode Decomposition](https://arxiv.org/abs/1312.0041) on a simple linear system of the form ``u(k+1) = A u(k)``.
#
# At first, we simulate the correspoding system using `OrdinaryDiffEq.jl` and generate a [`DiscreteDataDrivenProblem`](@ref DataDrivenProblem) from the simulated data.

using DataDrivenDiffEq
using ModelingToolkit
using LinearAlgebra
using OrdinaryDiffEq
#md using Plots

A = [0.9 -0.2; 0.0 0.2]
u0 = [10.0; -10.0]
tspan = (0.0, 11.0)

f(u,p,t) = A*u

sys = DiscreteProblem(f, u0, tspan)
sol = solve(sys, FunctionMap());

# Next we transform our simulated solution into a [`DataDrivenProblem`](@ref). Given that the solution knows its a discrete solution, we can simply write

prob = DataDrivenProblem(sol)

# And plot the solution and the problem

#md plot(sol, label = string.([:x₁ :x₂]))
#md scatter!(prob)

# To estimate the underlying operator in the states ``x_1, x_2``, we `solve` the estimation problem using the [`DMDSVD`](@ref) algorithm for approximating the operator. First, we will have a look at the [`DataDrivenSolution`](@ref)

res = solve(prob, DMDSVD(), digits = 1)
#md println(res) # hide

# We see that the system has been recovered correctly, indicated by the small error and high AIC score of the result. We can confirm this by looking at the resulting [`Basis`](@ref)

system = result(res)
using Symbolics

#md println(system) # hide

# And also plot the prediction of the recovered dynamics

#md plot(res)

# Or a have a look at the metrics of the result

#md metrics(res)

# To have a look at the representation of the operator as a `Matrix`, we can simply call

#md Matrix(system)

# to see that the operator is indeed our initial `A`. Since we have a linear representation, we can gain further insights into the stability of the dynamics via its eigenvalues

#md eigvals(system)

# And plot the stability margin of the discrete System

#md φ = 0:0.01π:2π
#md plot(sin.(φ), cos.(φ), xlabel = "Real", ylabel = "Im", label = "Stability margin", color = :red, linestyle = :dash)
#md scatter!(real(eigvals(system)), imag(eigvals(system)), label = "Eigenvalues", color = :black, marker = :cross)

# Similarly, we could use a sparse regression to derive our system from our data. We start by defining a [`Basis`](@ref)

using ModelingToolkit

@parameters t
@variables x[1:2](t)

basis = Basis(x, x, independent_variable = t, name = :LinearBasis)
#md print(basis) #hide

# Afterwards, we simply `solve` the already defined problem with our `Basis` and a `SparseOptimizer`

sparse_res = solve(prob, basis, STLSQ())
#md println(sparse_res) #hide

# Which holds the same equations
sparse_system = result(sparse_res)
#md println(sparse_system) #hide

# Again, we can have a look at the result

#md plot(
#md     plot(prob), plot(sparse_res), layout = (1,2)
#md )

# Both results can be converted into a `DiscreteProblem`

@named sys = DiscreteSystem(equations(sparse_system), get_iv(sparse_system),states(sparse_system), parameters(sparse_system))
#md println(sys) #hide

# And simulated using `OrdinaryDiffEq.jl` using the (known) initial conditions and the parameter mapping of the estimation.

x0 = [x[1] => u0[1], x[2] => u0[2]]
ps = parameter_map(sparse_res)

discrete_prob = DiscreteProblem(sys, x0, tspan, ps)
estimate = solve(discrete_prob, FunctionMap());

# And look at the result
#md plot(sol, color = :black)
#md plot!(estimate, color = :red, linestyle = :dash)

#md # ## [Copy-Pasteable Code](@id linear_discrete_copy_paste)
#md #
#md # ```julia
#md # @__CODE__
#md # ```

## Test the result #src
for r_ in [res, sparse_res] #src
    @test all(l2error(r_) .<= 1e-10) #src
    @test all(aic(r_) .>= 1e10) #src
    @test all(determination(r_) .≈ 1.0) #src
end  #src
@test Array(sol) ≈ Array(estimate) #src


#=
example/fft.jl

1D FFT example

Because FFT returns complex numbers, we must use T=ComplexF32 here
for LinearMap to work properly.

Because the usual `fft` is not the unitary FFT, we need a factor of N
for the adjoint.
=#

using FFTW
using Test: @test

N = 8
A = LinearMapAA(fft, y -> N*ifft(y), (N, N), (name="fft",), T=ComplexF32)
@show A[:,2]

@test Matrix(A') == Matrix(A)' # test the adjoint

# timing tests to confirm that LinearMapAA does not have overhead

using BenchmarkTools

N = 2^11
L = LinearMap{ComplexF32}(fft, y -> N*ifft(y), N, N)
A = LinearMapAA(fft, y -> N*ifft(y), (N, N), (name="fft",), T=ComplexF32)

x = rand(N)
y = rand(Float32, N) + 1im * rand(Float32, N)
y2 = rand(Float32, N) + 1im * rand(Float32, N)

if true
	mul!(y, L, x)
	mul!(y2, A, x)
	@test isapprox(y, y2)
end

if true # essentially identical
	@btime x = ($L)' * $y
	@btime x = ($A)' * $y
end

if true # essentially identical
	@btime y = $L * $x
	@btime y = $A * $x
end

if true # essentially identical
	@btime mul!($y, $L, $x)
	@btime mul!($y, $A, $x)
end

nothing


export ad, mad, mad_core

"""
`mad_model(G)` returns the solved linear program whose optimum
value is `mad(G)`.
"""
function mad_model(G::SimpleGraph)
    EE = elist(G)
    VV = vlist(G)

    n = length(VV)
    m = length(EE)

    MOD = Model(get_solver())

    # variables
    @variable(MOD, z >= 0)
    @variable(MOD, x[i = 1:n, j = 1:m; in(VV[i], EE[j])] >= 0)

    # vertex constraints
    for i = 1:n
        v = VV[i]
        # get all edges that have v as an end point
        j_iterator = (j for j = 1:m if EE[j][1] == v || EE[j][2] == v)
        j_list = collect(j_iterator)
        @constraint(MOD, sum(x[i, j] for j in j_list) <= z)
    end

    # edge constraints
    for j = 1:m
        v, w = EE[j]
        i1 = first(findall(x -> x == v, VV))
        i2 = first(findall(x -> x == w, VV))
        @constraint(MOD, x[i1, j] + x[i2, j] == 2)
    end

    @objective(MOD, Min, z)
    optimize!(MOD)
    return MOD, x, VV, EE
end

"""
`mad(G)` computes the maximum average degree of `G`.
See also `mad_core`.
"""
function mad(G::SimpleGraph)
    if cache_check(G, :mad)
        return cache_recall(G, :mad)
    end
    MOD, x, VV, EE = mad_model(G)
    mval = objective_value(MOD)
    cache_save(G, :mad, mval)
    return mval
end

"""
`ad(G)` is the average degree of a vertex in `G`.
"""
ad(G::SimpleGraph) = 2 * NE(G) / NV(G)

"""
`mad_core(G)` returns a subgraph `H` of `G` for which
`ad(H)==mad(G)`.
"""
function mad_core(G::SimpleGraph)
    if cache_check(G, :mad_core)
        return cache_recall(G, :mad_core)
    end
    T = eltype(G)
    GG = deepcopy(G)
    while true
        if NV(GG) == 0
            error("Bad graph or something went wrong")
        end
        # solve the LP for GG
        MOD, x, VV, EE = mad_model(GG)
        n = length(VV)
        m = length(EE)
        err = 0.1 / n
        mval = objective_value(MOD)

        # if balanced, return
        if abs(ad(GG) - mval) <= err
            cache_save(G, :mad_core, GG)
            return GG
        end

        # otherwise, find and destroy defective vertices
        for i = 1:n
            v = VV[i]
            total = 0.0
            for j = 1:m
                e = EE[j]
                if in(v, e)
                    val = value(x[i, j])
                    total += val
                end
            end
            if total < mval - err   # this is a defective vertex
                delete!(GG, v)
            end
        end
    end # end while
    error("This can't happen (but it did)")
    cache_save(G, :mad_core, GG)
    return GG
end


#=

# 101: 1D Laplace equation
([source code](SOURCE_URL))

Let $\Omega=(\gamma_1,\gamma_2)$ with $\gamma_1=0$, $\gamma_2=1$.
This is the simplest second order boundary value problem (BVP)
for a partial differential equation  (PDE):

```math
-\Delta u =0\\
u(\gamma_1)=g_1\\
u(\gamma_2)=g_2.
```

We replace the Dirichlet boundary condition by a Robin boundary
condition with a penalty parameter $\frac{1}{\varepsilon}$:

```math
\nabla u(\gamma_1) + \frac{1}{\varepsilon}(u(\gamma_1)-g_1)=0  \\
-\nabla u(\gamma_2) + \frac{1}{\varepsilon}(u(\gamma_2)-g_2)
=0
```

This penalty method for the implementation of Dirichlet
boundary conditions is used throughout VoronoiFVM.

In order to discretize it, we choose collocation points
$\gamma_1=x_1 < x_2 < \dots < x_n=\gamma_2$.

For instance, we can choose 6 collocation points in $(0,1)$:
From these, we create a discretization grid structure
for working with the method.

This implicitely creates a number of control volumes $\omega_k $ around each
discretization point $x_k$: Let
$\sigma_{k,k+1}=\frac{x_k+x_{k+1}}{2}$. Then $\omega_1=(\gamma_1,\sigma_{1,2})$,
$\omega_k= (\sigma_{k-1,k}, \sigma_{k,k+1})$ for $k=2\dots n-1$, $\omega_{n}=(\sigma_{n-1,n},\gamma_2)$.

```
 x1    x2    x3    x4    x5    x6
 o-----o-----o-----o-----o-----o
 |--|-----|-----|-----|-----|--|
  ω1  ω2     ω3    ω4    ω5  ω6
```

For each $\omega_k$, we integrate the equation

```math
\begin{aligned}
0&=\int_{\omega_k} -\Delta u d\omega=  -\int_{\partial \omega_k} \nabla u ds\\
&= \begin{cases}
u'(\sigma_{1,2}) - u'(0)& k=1\\
u'(\sigma_{k,k+1}) - u'(\sigma_{k-1,k}) & 1<k<n\\
u'(1)- u'(\sigma_{n,n+1})&k=n
\end{cases}\\
&\approx \begin{cases}
\frac{1}{x_2-x_1} g(u_1,u_2) + \frac{1}{\varepsilon}(u_1-0)& k=1\\
\frac{1}{x_k-x_{k-1}}g(u_k,u_{k-1}) -\frac{1}{x_{k+1}-x_{k}}g(u_{k+1},u_{k}) & 1<k<n\\
\frac{1}{\varepsilon}(u_n-1)+ \frac{1}{x_n-x_{n-1}} g(u_{n},u_{n-1})&k=n
\end{cases}
\end{aligned}
```

In the last equation, we wrote $u_k=u(x_k)$ and $g(u_k,u_l)=u_k-u_l$. For
the interior interfaces between control volumes, we replaced $u'$ by a
difference quotient. In the boundary control volumes, we replaced $u'$  by the boundary
conditions.

In the example below, we fix a number of species and  write a Julia function describing $g$,
we create a physics record, and a finite volume system with one unknown species and
a dense matrix to describe it's degrees of freedom
(the matrix used  to calculate the solution is sparse).
We give the species the number 1 and enable it for grid region number one 1.
Then, we set boundary conditions for species 1 at $\gamma_1, \gamma_2$.

We create a zero initial value and a solution vector
and initialize them.

With these data, we solve the system.

We wrap this example and all later ones
into a module structure. This allows to load
all of them at once into the REPL without name
clashes. We shouldn't forget the corresponding end
statement.

=#

module Example101_Laplace1D

using VoronoiFVM

## Flux function which describes the flux
## between neigboring control volumes
function g!(f,u0,edge)
    u=unknowns(edge,u0)
    f[1]=u[1,1]-u[1,2]
end

function main()

    nspecies=1 ## Number of species
    ispec=1    ## Index of species we are working with

    ## Create a one dimensional discretization grid
    ## Each grid cell belongs to a region marked by a region number
    ## By default, there is only one region numbered with 1
    X=collect(0:0.2:1)
    grid=VoronoiFVM.Grid(X)

    ## Create a physics structure
    physics=VoronoiFVM.Physics(num_species=nspecies,flux=g!)

    ## Create a finite volume system
    sys=VoronoiFVM.System(grid,physics)

    ## Enable species 1 in region 1
    enable_species!(sys,ispec,[1])

    ## Set boundary conditions at boundary regions 1 and 2
    boundary_dirichlet!(sys,ispec,1,0.0)
    boundary_dirichlet!(sys,ispec,2,1.0)

    ## Create & initialize array for solution and initial value
    inival=unknowns(sys,inival=0)
    solution=unknowns(sys)

    ## Solve stationary problem
    solve!(solution,inival,sys)

    ## Return test value
    return solution[3]
end

## Called by unit test

function test()
    main() ≈ 0.4
end

end


function recurring_decimal_length(n)
    d = 1
    while true
        if powermod(10, d, n) == 1
            return d
        end
        d += 1
    end
end

function longest_recurring_decimal(upper_limit)
    longest = 0; arg = 0

    for i in 2:upper_limit
        if i % 2 == 0 || i % 5 == 0
            continue
        end

        len = recurring_decimal_length(i)
        if len > longest
            longest = len; arg = i
        end
    end

    return arg
end

println(longest_recurring_decimal(1000))


using Random
using LinearAlgebra
"""
dir_sample(dir_type::String, d::Int, m::Int, seed::Int)

"dir_sample"
samples "m" directions in dimension "d" of type "dir_type"

Inputs:
(1) dir_type: a string, the direction type
(2) d: an integer, the dimension in which the directions are sampled
(3) m: an integer, the number of directions to generate
(4) seed: an integer, used for random sampling

Output:
an m x d matrix, where each row is a direction in R^d
"""
function dir_sample(dir_type::String, d::Int, m::Int, seed::Int,
    theta = pi/4)
    ## set the seed for random number generation
    Random.seed!(seed)
    if dir_type=="pos"
        directions = zeros(0,d)
        num_row = 0
        while num_row<m
            direction = randn(1,d)
            if sum(direction.>0)==d
                directions = [directions; direction]
                num_row+=1
            end
        end
        return directions
    elseif dir_type=="unif"
        directions = randn(m,d)
        return directions
    elseif dir_type=="angle"
        directions = zeros(0,d)
        num_row = 0
        vec_1 = ones(d)/sqrt(d) ## the vector [1,1,...,1]\in\R^d
        num_row = 0
        while num_row<m
            direction = randn(1,d)
            direction = direction/norm(direction)
            if acos(dot(vec_1, direction'))<theta
                directions = [directions; direction]
                num_row+=1
            end
        end
        return directions
    end
end

# ## sample usage code
# dir_type = "pos" ## "unif"
# d = 2
# m = 100
# seed = 1
# A = dir_sample(dir_type::String, d::Int, m::Int, seed::Int)

# using Plots
# pyplot()

# plot()
# for i = 1:size(A,1)
#     plot!([0, A[i,1]], [0,A[i,2]], legend = false)
# end
# plot!()

"""
"UnifRandBall"
generates a d x N matrix, whose columns represent points,
randomly uniformly sampled in the unit ball of R^d
"""
function UnifRandBall(d::Int,N::Int)
    Collect = zeros(d,0) ## Set an (d x 0) empty matrix for storing
    i = 0 ## number of collected points
    while i<N
        pt = 2*rand(d).-1 ## (uniformly) generate a random point inside [-1,1]^d
        if norm(pt)<1 ## if the point is inside the unit ball, collect it
            i = i+1
            Collect = [Collect pt]
        end
    end
    return Collect
end

"""
pt_sample(pt_type::String, d::Int, n::Int, seed::Int)

"pt_sample"
samples "n" points in R^d of type "pt_type"

Inputs:
(1) pt_type: a string, the point type
(2) d: an integer, the dimension in which the directions are sampled
(3) n: an integer, the number of directions to generate
(4) seed: an integer, used for random sampling
(5) r(optional): the reciprocal of the curvature of the de-sitter space, between 0 and 1, exclusive

Output:
a d x n matrix, where each row is a direction in R^d
"""
function pt_sample(pt_type::String, d::Int, n::Int, seed::Int,
        r=0.1::Float64)
    ## set the seed for random number generation
    Random.seed!(seed)
    if pt_type=="uniform"
        pts = UnifRandBall(d,n)
        return pts
    elseif pt_type=="sphere"
        pts = UnifRandBall(d,n)
        for a = 1:n
            pts[:,a] = pts[:,a]/norm(pts[:,a])
        end
        return pts
    elseif pt_type=="de-sitter"
        c = 1/r ## the constant/curvature c in x_1^2+...+x_(d-1)^2-x_d^2 = 1/c^2
        pts = zeros(d,0)
        a = 0
        while a<n
            new_pt = UnifRandBall(d-1,1)
            x_d_a2 = sum(new_pt.^2)-1/c^2
            if x_d_a2>0
                new_pt = [new_pt;(-1)^a*sqrt(x_d_a2)] ## (-1)^a is used to cover both the upper and lower piece
                pts = [pts new_pt]
                a+=1
            end
        end
        return pts
    end
end

# ## sample usage code
# pt_type = "sphere" ## "uniform"
# d = 2
# n = 100
# seed = 1
# X = pt_sample(pt_type::String, d::Int, n::Int, seed::Int)

# using Plots
# pyplot()

# scatter(X[1,:], X[2,:], legend = false)


### Irving Gomez Mendez
### January 13th 2019

using Pkg
using Plots; pyplot()
using LaTeXStrings
using Distributions
using StatsPlots
using Random
using StatsBase

# Apply a function to each value of an array and returns a new array
# containing the resulting values:
roundmap(vect, b) = map(x -> round(x, digits = b), vect)

# La densidad de la distribución Weibull de mínimos (de 2 parámetros) está dada por la expresión:

L"f(t;\eta,\beta)=\frac{\beta}{\eta}\left(\frac{t}{\eta}\right)^{\beta-1}
e^{-(t/\eta)^\beta}1\!\!1\,\,_{(0<t<\infty)}"

# Grafica f(x;θ) para varios valores de θ. ¿Existe alguna combinación de los parámetros
# que logre que f sea simétrica alrededor de la moda?

# Definimos la función de densidad
# eta parametro de escala
# beta parametrode forma
dweib_min(x, eta = 1, beta = 1) = beta/eta*(x/eta)^(beta-1)*exp(-(x/eta)^beta)

# Hacemos una secuencia de valores para x
xx = range(0.001, stop = 3, length = 100)

# La siguiente linea mandara un error
yy = dweib_min(xx)

# La manera correcta de hacerlo es
yy = map(dweib_min, xx)
roundmap(yy,3)

plot(xx, yy,
    seriestype = :scatter,
    label = "",
    m = (0.9, :RebeccaPurple, 5),
    background_color = :Lavender
    )
plot!(dweib_min,
    line = (:tomato, 1, 2),
    label = "Weibull(1,1)")

plot(dweib_min, 0, 3,
    label = "Weibull(1,1)",
    background_color = :Lavender,
    lw = 2,
    color = :RebeccaPurple)

# StatsPlots.jl extends Distributions.jl by adding a type recipe for its
# distribution types, so they can be directly interpreted as plotting data:

plot(Weibull(1,1), 0, 3,
    label = "Weibull(1,1)",
    background_color = :Lavender,
    lw = 2,
    color = :RebeccaPurple)

# Si cambiamos el parámetro de forma a 2

plot(Weibull(2,1), 0, 3,
    label = "Weibull(1,2)",
    background_color = :Lavender,
    lw = 2,
    ylabel = L"f(x,θ)",
    color = :RebeccaPurple)

# Con varios valores de beta (2, 3, 5)

plot!(Weibull(3,1),
    label = "Weibull(1,3)",
    lw = 2,
    ls = :dash,
    color = :orange)
plot!(Weibull(5,1),
    label = "Weibull(1,5)",
    lw = 2,
    ls = :dot,
    color = :green)

# Obtener información sobre funciones y sobre atributos de plot

?plot
plotattr("linestyle")

## Teorema Integral de Probabilidad

#Si U∼Unifcont[0,1], entonces la v.a. F^(-1)(U) tiene la misma distribución que X

# Funcion cuantiles
qweib_min(x, η, β) = η*(log(1/(1-x)))^(1/β)

# Función que genera una muestra (simple) de la distribucion Weibull
function rweib_min(n, η, β)
    u = rand(n) #Generamos una muestra uniforme
    out = map(x -> qweib_min(x, η, β), u)
    return(out)
end

Random.seed!(111)
muestra = rweib_min(50, 2, 6.2)

histogram(muestra,
    label = "Empirical Weibull(2,6.2)",
    normalized = true,
    background_color = :Lavender,
    fillalpha = 0.33,
    color = :RebeccaPurple)

plot!(Weibull(6.2,2),
    label = "Weibull(2,6.2)",
    lw = 2)

Empirical = ecdf(muestra)

Empirical(1.23)

plot(sort(muestra), map(Empirical, sort(muestra)),
    label = "Empirical Weibull(2,6.2)",
    ylabel = L"F_n(x,θ)",
    background_color = :Lavender,
    linetype = :steppost,
    lw = 2,
    color = :RebeccaPurple)

pweib_min(x, η, β) = cdf(Weibull(β, η), x)

plot!(x -> pweib_min(x, 2, 6.2),
    label = "Weibull(2,6.2)",
    lw = 2)

muestra2 = rand(Weibull(6.2,2), 50)


#=

# 102: 1D Stationary convection-diffusion equation
([source code](SOURCE_URL))

Solve the equation

```math
-\nabla ( D \nabla u - v u) = 0
```
in $\Omega=(0,1)$ with boundary condition $u(0)=0$ and $u(1)=1$.
$v$ could be e.g. the velocity
of a moving medium or the gradient of an electric field.

This is a convection dominant second order boundary value problem which obeys
a local and a global maximum principle:
the solution which is bounded by the values at the boundary and has no local extrema in the
interior.
If $v$ is large compared to $D$, a boundary layer is observed.

The maximum principle of the solution can only be guaranteed it the discretization is
performed accordingly: the flux function must monotonically increase in the first argument
and monotonically decrease in the second argument.

The example describes three possible ways to define the flux function and demonstrates
the impact on the qualitative properties of the solution.

=#

module Example102_StationaryConvectionDiffusion1D
using Printf
using VoronoiFVM
using ExtendableGrids
using GridVisualize

## Central difference flux. The velocity term is discretized using the
## average of the solution in the endpoints of the grid. If the local Peclet
## number v*h/D>1, the monotonicity property is lost.  Grid refinement
## can fix this situation by decreasing $h$.

function central_flux!(f,u0,edge,data)
    u=unknowns(edge,u0)
    f_diff=data.D*(u[1,1]-u[1,2])
    vh=project(edge,data.v)
    f[1]=f_diff+vh*(u[1,1]+u[1,2])/2
end

## The simple upwind flux corrects the monotonicity properties essentially
## via brute force and loses one order of convergence for small $h$ compared
## to the central flux.

function upwind_flux!(f,u0,edge,data)
    u=unknowns(edge,u0)
    fdiff=data.D*(u[1,]-u[1,2])
    vh=project(edge,data.v)
    if vh>0
        f[1]=fdiff+vh*u[1,1]
    else
        f[1]=fdiff+vh*u[1,2]
    end
end

## The exponential fitting flux has the proper monotonicity properties and
## kind of interpolates in a clever way between central
## and upwind flux. It can be derived by solving the two-point boundary value problem
## at the grid interval analytically.

## Bernoulli function used in the exponential fitting discretization
function bernoulli(x)
    if abs(x)<nextfloat(eps(typeof(x)))
        return 1
    end
    return x/(exp(x)-1)
end

function exponential_flux!(f,u0,edge,data)
    u=unknowns(edge,u0)
    vh=project(edge,data.v)
    Bplus= data.D*bernoulli(vh/data.D)
    Bminus=data.D*bernoulli(-vh/data.D)
    f[1]=Bminus*u[1,1]-Bplus*u[1,2]
end

function calculate(grid,data,flux,verbose)

    sys=VoronoiFVM.System(grid,VoronoiFVM.Physics(flux=flux, data=data))

    ## Add species 1 to region 1
    enable_species!(sys,1,[1])

    ## Set boundary conditions
    boundary_dirichlet!(sys,1,1,0.0)
    boundary_dirichlet!(sys,1,2,1.0)

    ## Create a solution array
    inival=unknowns(sys,inival=0.5)
    solution=unknowns(sys)

    ## Create solver control info
    control=VoronoiFVM.NewtonControl()
    control.verbose=verbose

    ## Stationary solution of the problem
    solve!(solution,inival,sys, control=control)
    return solution
end

function main(;n=10,Plotter=nothing,verbose=false,D=0.01,v=1.0)

    ## Create a one-dimensional discretization
    h=1.0/convert(Float64,n)
    grid=VoronoiFVM.Grid(collect(0:h:1))

    data=(v=[v],D=D)

    # Calculate three stationary solutions with different ways to calculate flux
    solution_exponential=calculate(grid,data,exponential_flux!,verbose)
    solution_upwind=calculate(grid,data,upwind_flux!,verbose)
    solution_central=calculate(grid,data,central_flux!,verbose)

    # Visualize solutions using GridVisualize.jl
    p=GridVisualizer(Plotter=Plotter,layout=(3,1))
    scalarplot!(p[1,1],grid,solution_exponential[1,:],title="exponential")
    scalarplot!(p[2,1],grid,solution_upwind[1,:],title="upwind")
    scalarplot!(p[3,1],grid,solution_central[1,:],title="centered",show=true)

    ## Return test value
    return sum(solution_exponential)+sum(solution_upwind)+sum(solution_central)
end

function test()
    testval=2.523569744561089
    main() ≈ testval
end

end


"""
    linreg(X, y)

    Objective function of Linear Regression on data matrix X and label vector y.
"""
linreg(X::Array{Float64,2}, y::Array{Float64,1}) = Objective(
    w::Array{Float64,1} -> _linreg_f(X, y, w),  # f
    w::Array{Float64,1} -> _linreg_g(X, y, w),  # g
    f_b = (w::Array{Float64,1}, b::Array{Int,1}) -> _linreg_f(X, y, w, b),
    g_b = (w::Array{Float64,1}, b::Array{Int,1}) -> _linreg_g(X, y, w, b),
    dim = length(y)
)

function _linreg_f(X::Array{Float64,2}, y::Array{Float64,1}, w::Array{Float64,1},
                   b::Array{Int,1}=Int[])
    if length(b) == 0
        return vecnorm(y - X*w)^2 / (2*length(y))
    else
        return vecnorm(y[b] - X[b,:]*w)^2 / (2*length(b))
    end
end

function _linreg_g(X::Array{Float64,2}, y::Array{Float64,1}, w::Array{Float64,1},
                   b::Array{Int,1}=Int[])
    if length(b) == 0
        return vec(X' * (X*w - y)) ./ length(y)
    else
        return vec(X[b,:]' * (X[b,:]*w - y[b])) ./ length(b)
    end
end

"""
    linreg_predict(w, X)

    Predict the examples in X with Linear Regression under parameter vector w.
"""
linreg_predict(w::Array{Float64,1}, X::Array{Float64,2}) = sign(vec(X*w))


function calc_fan(dims...; mode = "fan_in")
  @assert mode in ["fan_in", "fan_out"]
  num_dims = length(dims)
  num_dims < 2 && throw("Number of dimensions should be more than 1!")

  fan_out = dims[1]
  fan_in = dims[2]
  # if num_dims > 2
  #   receptive_field = prod(dims[1:end-2])
  #   fan_in  *= receptive_field
  #   fan_out *= receptive_field
  # end

  mode == "fan_in" ? fan_in : fan_out
end

function calc_gain(nonlinearity, a = nothing)
  linear_fns = ["linear", "conv1d", "conv2d", "conv3d", "conv_transpose1d", "conv_transpose2d", "conv_transpose3d"]
  if nonlinearity in linear_fns || nonlinearity == "sigmoid" || nonlinearity == "σ"
    return 1.0
  elseif nonlinearity == "tanh"
    return 5.0 / 3
  elseif nonlinearity == "relu"
    return √2.0
  elseif nonlinearity == "leakyrelu"
    if a == nothing
      negative_slope = 0.01
    elseif !(a isa Bool) && a isa Integer || a isa AbstractFloat
      # true/false are instances of Integer, hence check above
      negative_slope = a
    else
      throw("negative_slope $a not a valid number.")
    end
    return √(2.0 / (1 + negative_slope ^ 2))
  else
    throw("Unsupported nonlinearity $nonlinearity.")
  end
end

function kaiming_uniform(dims...; a = 0, mode = "fan_in", nonlinearity = "leakyrelu")
  fan = calc_fan(dims...; mode = mode)
  gain = calc_gain(nonlinearity, a)
  (rand(dims...) .- 0.5) .* √(12.0 / fan) .* gain
end


@testset "Risk Measures" begin
    sample = [0:100...]

    @testset "VaR" begin
        @test VaR(sample,.9) ≈ 90
        @test VaR(sample,.9,rev=true) ≈ 10
        @test VaR(sample,.1) ≈ VaR(sample,1-.9)
        @test VaR(sample,.1) == ValueAtRisk(sample,.1)
    end

    @testset "CTE" begin
        @test CTE(sample,.9) ≈ sum(90:100) / length(90:100)
        @test CTE(sample,.1) ≈ sum(10:100) / length(10:100)
        @test CTE(sample,.15) >= CTE(sample,.1) # monotonicity
        @test CTE(sample,.9,rev=true) ≈ sum(0:10) / length(0:10)
        @test CTE(sample,.9) == ConditionalTailExpectation(sample,.9)
    end

end


using SatisfiabilityInterface
using Symbolics
using Symbolics: Sym

function all_different(v)
    vv = vec(v)
    n = length(vv)

    return [vv[i] ≠ vv[j] for i in 1:n for j in i+1:n]
end

function make_matrix(name, m, n)
    return [Num(Sym{Real}(name, i, j)) for i in 1:m, j in 1:n]
end

n = 9
M = make_matrix(:M, n, n)

## initial condition from https://www.juliaopt.org/notebooks/JuMP-Sudoku.html

initial = [
    5 3 -1 -1 7 -1 -1 -1 -1
    6 -1 -1 1 9 5 -1 -1 -1
    -1 9 8 -1 -1 -1 -1 6 -1
    8 -1 -1 -1 6 -1 -1 -1 3
    4 -1 -1 8 -1 3 -1 -1 1
    7 -1 -1 -1 2 -1 -1 -1 6
    -1 6 -1 -1 -1 -1 2 8 -1
    -1 -1 -1 4 1 9 -1 -1 5
    -1 -1 -1 -1 8 -1 -1 7 9
    ]

constraints = [
    [M[i, j] ∈ 1:9 for i in 1:n for j in 1:n]

    [all_different(M[:, i]) for i in 1:n]
    [all_different(M[i, :]) for i in 1:n]

    [all_different(M[ 3i+1 : 3i+3, 3j+1 : 3j+3 ]) for i in 0:2 for j in 0:2]

    [M[i, j] == initial[i, j] for i in 1:9 for j in 1:9 if initial[i, j] > 0]

] |> Iterators.flatten |> collect

prob = DiscreteCSP(constraints)

@time status, results = solve(prob);

prob.varmap

results

output = reshape([last(x) for x in results], 9, 9)'

output == [
    5  3  4  6  7  8  9  1  2
    6  7  2  1  9  5  3  4  8
    1  9  8  3  4  2  5  6  7
    8  5  9  7  6  1  4  2  3
    4  2  6  8  5  3  7  9  1
    7  1  3  9  2  4  8  5  6
    9  6  1  5  3  7  2  8  4
    2  8  7  4  1  9  6  3  5
    3  4  5  2  8  6  1  7  9
]

all_solutions(prob)


function rowcol(s)
    # Find row and col of the seatid

    rowpart = s[1:7]
    rowpart = replace(rowpart, "B" => "1")
    rowpart = replace(rowpart, "F" => "0")
    colpart = s[8:end]
    colpart = replace(colpart, "R" => "1")
    colpart = replace(colpart, "L" => "0")
    rowbin, colbin = parse(UInt8, rowpart, base=2), parse(UInt8, colpart, base=2)
    Int.((rowbin & UInt(127), colbin & UInt(7)))
end

seatid((row, col)) = row * 8 + col
# Every seat also has a unique seat ID: multiply the row by 8, then add the column

# First Half

puzzeinput = readlines("inputs/day05.txt")
maximum(map(seatid ∘ rowcol, puzzeinput))

# Second Half

function missingid(strs)

    rowcols = map(rowcol, strs)
    filter!(rc -> rc[1] > 0 && rc[1] < 127, rowcols)
    # Removes very front and very back rows, because your seat wasn't
    # at the very front or back

    sorted_ids = map(seatid, rowcols) |> sort
    for (i, id) in enumerate(sorted_ids)
        if sorted_ids[i + 1] ≠ id + 1
            # Missing ID found

            return id + 1
        end
    end
end

puzzeinput = readlines("inputs/day05.txt")
missingid(puzzeinput)


# Truss Finite Element Analysis

using LinearAlgebra

## Local Stiffness Matrix"

function L(member::Dict{Integer,Array{Integer}}, node::Dict{Integer,Array{Float64}})
	l = Dict{Integer, Float64}()
	for e = 1:length(member)
		l[e] = hypot(node[member[e][2]][1]-node[member[e][1]][1], node[member[e][2]][2]-node[member[e][1]][2])
	end
	return l
end

function cosϕ(member::Dict{Integer,Array{Integer}}, node::Dict{Integer,Array{Float64}})
	cosphi = Dict{Integer, Float64}()
	for e = 1:length(member)
		hor = node[member[e][2]][1]-node[member[e][1]][1]
		hyp = hypot(node[member[e][2]][1]-node[member[e][1]][1], node[member[e][2]][2]-node[member[e][1]][2])
		cosphi[e] = hor/hyp
	end
	return cosphi
end

function sinϕ(member::Dict{Integer,Array{Integer}}, node::Dict{Integer,Array{Float64}})
	sinphi = Dict{Integer, Float64}()
	for e = 1:length(member)
		hor = node[member[e][2]][2]-node[member[e][1]][2]
		hyp = hypot(node[member[e][2]][1]-node[member[e][1]][1], node[member[e][2]][2]-node[member[e][1]][2])
		sinphi[e] = hor/hyp
	end
	return sinphi
end

function AxialLocalK(E::Dict, A::Dict, Member::Dict{Integer,Array{Integer}}, Node::Dict{Integer,Array{Float64}})
	k = Dict{}()
	Length = L(Member,Node)
	Cosϕ = cosϕ(Member,Node)
	Sinϕ = sinϕ(Member,Node)
	for e = 1:length(Member)
		k[e] = E[e]*A[e]/Length[e]*
	[[Cosϕ[e]^2, Sinϕ[e]*Cosϕ[e], -Cosϕ[e]^2, -Sinϕ[e]*Cosϕ[e]],
	[Sinϕ[e]*Cosϕ[e], Sinϕ[e]^2, -Sinϕ[e]*Cosϕ[e], -Sinϕ[e]^2],
	[-Cosϕ[e]^2, -Sinϕ[e]*Cosϕ[e], Cosϕ[e]^2, Sinϕ[e]*Cosϕ[e]],
	[-Sinϕ[e]*Cosϕ[e], -Sinϕ[e]^2, Sinϕ[e]*Cosϕ[e], Sinϕ[e]^2]]
	end
	return k
end

function AxialGlobalK(k::Dict,member::Dict, node::Dict)
	K = zeros(length(node)*2, length(node)*2)
	for e = 1:length(member)
		a = member[e][1] * 2 - 1
		b = member[e][1] * 2
		c = member[e][2] * 2 - 1
		d = member[e][2] * 2
		K[a,a] += k[e][1][1]
		K[a,b] += k[e][1][2]
		K[a,c] += k[e][1][3]
		K[a,d] += k[e][1][4]
		K[b,a] += k[e][2][1]
		K[b,b] += k[e][2][2]
		K[b,c] += k[e][2][3]
		K[b,d] += k[e][2][4]
		K[c,a] += k[e][3][1]
		K[c,b] += k[e][3][2]
		K[c,c] += k[e][3][3]
		K[c,d] += k[e][3][4]
		K[d,a] += k[e][4][1]
		K[d,b] += k[e][4][2]
		K[d,c] += k[e][4][3]
		K[d,d] += k[e][4][4]
	end
	return K
end

### Determine free nodal directions
function FreeNodalDir(u::Dict)
	Free = Integer[]
	for i = 1:length(u)
		if ismissing(u[i][1])
			push!(Free, i*2-1)
		end
		if ismissing(u[i][2])
			push!(Free, i*2)
		end
	end
	return Free
end

### Augmented Global Stiffness Matrix
function AugmentGlobal(Free::Array,K::Array)
	K_Aug = zeros(length(Free),length(Free))
	for m = 1:length(Free)
		for n = 1:length(Free)
			global K_aug
			K_Aug[m,n] = K[Free[m],Free[n]]
		end
	end
	K_Aug
end

## Global Force Matrix"
function GlobalForce(free::Array,f::Dict)
	Force = zeros(length(free))
	for m = 1:length(free)
		if free[m]%2 != 0
			Force[m]= f[0.5*free[m]+0.5][1]
		end
		if free[m]%2 == 0
			Force[m]= f[0.5*free[m]][2]
		end
	end
	return Force
end


##                                                                            ##
##              Problem set #2 for Structural modelling course                ##
##                               Nicolás Urdaneta                             ##
##                                                                            ##

clearconsole()
#=
Pkg.add("Optim")
Pkg.add("HTTP")
Pkg.add("GLM")
=#

using LinearAlgebra, Random, Statistics, DataFrames, CSV, FreqTables, Distributions, Optim, HTTP, GLM

## 1. Basic optimization in Julia

f(x) = -x[1]^4-10x[1]^3-2x[1]^2-3x[1]-2
negf(x) =-copy(f(x))
startval = rand(1)
result = optimize(negf, startval, LBFGS())
println(result)

## 2. Optimization with data

url = "https://raw.githubusercontent.com/OU-PhD-Econometrics/fall-2020/master/ProblemSets/PS1-julia-intro/nlsw88.csv"
df = CSV.read(HTTP.get(url).body)
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.married.==1

function ols(beta, X, y)
	ssr = (y.-X*beta)'*(y.-X*beta)
	return ssr
end

beta_hat_ols = optimize(b-> ols(b,X,y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations = 100_000, show_trace = true))

println(beta_hat_ols.minimizer)

	## Just checking:

	bols = inv(X'X)*X'y
	df.white = df.race.==1
	bols_lm = lm(@formula(married ~ age+white+collgrad), df)

## 3. Optimization logit lokelihood

# Log likelihood
# y(lnP) + (1-y)ln(1-P)

function logit_ll(alpha, X, Y)

	ll_i = (y.==1).*log.(exp.(X*alpha)./(1 .+exp.(X*alpha))).+(y.==0).*log.(1 ./(1 .+exp.(X*alpha)))

	ll = -sum(ll_i)

	return ll

end

alpha_hat_optim = optimize(a -> logit_ll(a, X, y), rand(size(X,2)), LBFGS(), Optim.Options(g_tol=1e-6, iterations=100_000, show_trace=true))
println(alpha_hat_optim.minimizer)

## 4. Check logit using glm

alpha_hat_glm = glm(@formula(married ~ age + white + collgrad), df, Binomial(), LogitLink())
println(alpha_hat_glm)

## 5. Multinomial logit

	 # Join categories
freqtable(df, :occupation) # note small number of obs in some occupations
df = dropmissing(df, :occupation)
df[df.occupation.==8 ,:occupation] .= 7
df[df.occupation.==9 ,:occupation] .= 7
df[df.occupation.==10,:occupation] .= 7
df[df.occupation.==11,:occupation] .= 7
df[df.occupation.==12,:occupation] .= 7
df[df.occupation.==13,:occupation] .= 7
freqtable(df, :occupation) # problem solved

	# Redefine X and Y as we changed their rows
X = [ones(size(df,1),1) df.age df.race.==1 df.collgrad.==1]
y = df.occupation

function mlogit(alpha, X, Y)

	K = size(X,2)
	J = length(unique(y))
	N = size(X,1)

	bigY = zeros(N,J)
	for j=1:J
		bigY[:,j] = y.==j
	end

	bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

	num = zeros(N,J)
	dem = zeros(N)
	for j=1:J
		num[:,j] = exp.(X*bigAlpha[:,j])
		dem .+= num[:,j]
	end

	P = num ./ repeat(dem,1,J)

	ll = -sum( bigY.*log.(P))

end

alpha_zero = zeros(6*size(X,2))
alpha_rand = rand(6*size(X,2))
alpha_true = [.1910213,-.0335262,.5963968,.4165052,-.1698368,-.0359784,1.30684,-.430997,.6894727,-.0104578,.5231634,-1.492475,-2.26748,-.0053001,1.391402,-.9849661,-1.398468,-.0142969,-.0176531,-1.495123,.2454891,-.0067267,-.5382892,-3.78975]

alpha_start = alpha_true.*rand(size(alpha_true))
println(size(alpha_true))

alpha_hat_optim = optimize(a -> mlogit(a, X, y), alpha_start, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
alpha_hat_mle = alpha_hat_optim.minimizer
println(alpha_hat_mle)

## Stanrd erorrs

	## Adjust mlogit function
function mlogit_hessian(alpha, X, Y)

	K = size(X,2)
	J = length(unique(y))
	N = size(X,1)

	bigY = zeros(N,J)
	for j=1:J
		bigY[:,j] = y.==j
	end

	bigAlpha = [reshape(alpha,K,J-1) zeros(K)]

	T = promote_type(eltype(X),eltype(alpha)) # this line is new
	num = zeros(T,N,J) ## Added T first
	dem = zeros(T,N) ## Added T first
	for j=1:J
		num[:,j] = exp.(X*bigAlpha[:,j])
		dem .+= num[:,j]
	end

	P = num ./ repeat(dem,1,J)

	ll = -sum( bigY.*log.(P))

end

# declare that the objective function is twice differentiable
td = TwiceDifferentiable(b -> mlogit_for_h(b, X, y), alpha_start; autodiff = :forward)
# run the optimizer
alpha_hat_optim_ad = optimize(td, alpha_zero, LBFGS(), Optim.Options(g_tol = 1e-5, iterations=100_000, show_trace=true, show_every=50))
alpha_hat_mle_ad = alpha_hat_optim_ad.minimizer
# evaluate the Hessian at the estimates
H  = Optim.hessian!(td, alpha_hat_mle_ad)
# standard errors = sqrt(diag(inv(H))) [usually it's -H but we've already multiplied the obj fun by -1]
alpha_hat_mle_ad_se = sqrt.(diag(inv(H)))
println([alpha_hat_mle_ad alpha_hat_mle_ad_se]) # these standard errors match Stata


#=
  From
  "Probabilistic logic programming and its applications"
  Luc De Raedt, Leuven
  https://www.youtube.com/watch?v=3lnVBqxjC88
  @ 3:44

  """
  Mike has a bag of marbles with 4 white, 8 blue, and
  6 red marbles. He pulls out one marble from the bag
  and it is red. What is the probability that the
  second marble he pulls out of the bag is white?

  The answer is 0.234931.
  """

  draw1: The probability of drawing white is the probability for 1.0
  Distributions of variable draw1
  blue       =>   47204  (0.472040)
  red        =>   29283  (0.292830)
  white      =>   23513  (0.235130)

  Cf ~/cplint/bag_of_marbles.pl
     ~/blog/bag_of_marbles.blog
     ~/psi/bag_of_marbles.psi
     ~/webppl/bag_of_marbles.wppl

  Also see bag_of_marbles2.jl for an alternative approach (using recursion).

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function bag_of_marbles()

    white = 1
    blue  = 2
    red   = 3
    colors = [white,blue,red]

    start = [4,8,6];

    draw0 ~ Categorical(simplex([start[white],start[blue],start[red]]))

    if draw0 == white
        draw1 ~ Categorical(simplex([start[white]-1,start[blue],start[red]]))
    elseif draw0==blue
        draw1 ~ Categorical(simplex([start[white],start[blue]-1,start[red]]))
    else
        # red
        draw1 ~ Categorical([start[white],start[blue],start[red]-1]|>simplex);
    end

    true ~ Dirac(draw0 == red)

end

model = bag_of_marbles()
num_chns = 4

# chns = sample(model, Prior(), 100_000)
chns = sample(model, MH(), 100_000)

# chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(), 10_000)

# Note: IS don't generate chns the same way as MH, PG, and SMC!
# chns = sample(model, IS(), 10_000)

display(chns)

show_var_dist_pct(chns,:draw0,["white","blue","red"])
println("\ndraw1: The probability of drawing white is the probability for 1.0")
show_var_dist_pct(chns,:draw1,["white","blue","red"])


using Distributions
using Plots

grid_size = 40
proba_grid = range(0, 1, length=grid_size)

## Code block 2.5
# prior = ones((grid_size))
# prior = collect(proba_grid) .>= 0.5
prior = exp.(-5 * abs.(collect(proba_grid) .- 0.5))

plot(
    proba_grid, prior, title="40 Points", xlabel="Probability of water",
    ylabel="Prior probability", lw=3
)

likelihood = [pdf(Binomial(9, p), 6) for p in proba_grid]
posterior = prior .* likelihood
posterior_proba = posterior / sum(posterior)

## Code block 2.4
plot(
    proba_grid, posterior_proba, title="40 Points", xlabel="Probability of water",
    ylabel="Posterior probability", lw=3
)


# SQRT FUNCTION
# (x+1)^0.5
function sum_sqrt(x::Array{Float64,1})
    return sum(sqrt(x+1))
end

function d_sum_sqrt(x::Array{Float64,1})
    return 0.5 * (x+1).^(-0.5)
end

function dd_sum_sqrt(x::Array{Float64,1})
    return spdiagm(-0.25 * (x+1).^(-1.5))
end

function nl_sqrt(num_var::Int64)
    return class_nl_function(num_var,sum_sqrt,d_sum_sqrt,dd_sum_sqrt);
end

# LOG function
function sum_log(x::Array{Float64,1})
    return sum(log(x+1))
end

function d_sum_log(x::Array{Float64,1})
    return (x+1).^(-1.0)
end

function dd_sum_log(x::Array{Float64,1})
    return spdiagm(-(x+1).^(-2.0))
end

function nl_log(num_var::Int64)
    return class_nl_function(num_var::Int64,sum_log,d_sum_log,dd_sum_log);
end

# Quadratic
# c'*x + 0.5 * x*Q*x + b
function nl_quad(c::AbstractArray, Q::SparseMatrixCSC{Float64,Int64}, b::Float64)
    function value(x::Array{Float64,1})
        return (c' * x)[1] + 0.5 * (x' * Q * x)[1] + b
    end

    function gradient(x::Array{Float64,1})
        return (c + Q * x)[:]
    end

    function hessian(x::Array{Float64,1})
        return Q
    end

    return class_nl_function(length(c), value, gradient, hessian)
end

function sphere_function(center_point::Float64,radius::Float64)
    b = dot(center_point,center_point) - radius
    c = -2 * center_point

    return nl_quad(c, speye(length(c)), b)
end